diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index b8f8b61b20d..d5b9b1501bb 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -1051,6 +1051,14 @@ class RecursiveMakeBackend(CommonBackend): if not obj.dupe_manifest: raise + for base, pattern, dest in obj.pattern_installs: + try: + self._install_manifests['tests'].add_pattern_symlink(base, + pattern, dest) + except ValueError: + if not obj.dupe_manifest: + raise + for dest in obj.external_installs: try: self._install_manifests['tests'].add_optional_exists(dest) diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 9983c13081c..66408c96699 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -361,6 +361,10 @@ class TestManifest(SandboxDerived): # path is relative from the tests root directory. 'installs', + # A list of pattern matching installs to perform. Entries are + # (base, pattern, dest). + 'pattern_installs', + # Where all files for this manifest flavor are installed in the unified # test package directory. 'install_prefix', @@ -400,6 +404,7 @@ class TestManifest(SandboxDerived): self.manifest_relpath = relpath self.dupe_manifest = dupe_manifest self.installs = {} + self.pattern_installs = [] self.tests = [] self.external_installs = set() diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 6387a52b3c6..d0dd1e62c34 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -15,8 +15,6 @@ from mach.mixin.logging import LoggingMixin import mozpack.path as mozpath import manifestparser -from mozpack.files import FileFinder - from .data import ( ConfigFileSubstitution, Defines, @@ -395,8 +393,6 @@ class TreeMetadataEmitter(LoggingMixin): out_dir = mozpath.join(install_prefix, manifest_reldir) - finder = FileFinder(base=manifest_dir, find_executables=False) - # "head" and "tail" lists. # All manifests support support-files. # @@ -421,22 +417,9 @@ class TreeMetadataEmitter(LoggingMixin): for pattern in value.split(): # We only support globbing on support-files because # the harness doesn't support * for head and tail. - # - # While we could feed everything through the finder, we - # don't because we want explicitly listed files that - # no longer exist to raise an error. The finder is also - # slower than simple lookup. if '*' in pattern and thing == 'support-files': - paths = [f[0] for f in finder.find(pattern)] - if not paths: - raise SandboxValidationError('%s support-files ' - 'wildcard in %s returns no results.' % ( - pattern, path)) - - for f in paths: - full = mozpath.normpath(mozpath.join(manifest_dir, f)) - obj.installs[full] = mozpath.join(out_dir, f) - + obj.pattern_installs.append( + (manifest_dir, pattern, out_dir)) else: full = mozpath.normpath(mozpath.join(manifest_dir, pattern)) diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini index 8dada2a4e5f..f6a5351e94e 100644 --- a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini @@ -1,3 +1,4 @@ [DEFAULT] +support-files = support/** [xpcshell.js] diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 4c4046de679..199718ca593 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -382,6 +382,18 @@ class TestRecursiveMakeBackend(BackendTester): self.assertEqual(len(o['xpcshell.js']), 1) + def test_test_manifest_pattern_matches_recorded(self): + """Pattern matches in test manifests' support-files should be recorded.""" + env = self._consume('test-manifests-written', RecursiveMakeBackend) + m = InstallManifest(path=os.path.join(env.topobjdir, + '_build_manifests', 'install', 'tests')) + + # This is not the most robust test in the world, but it gets the job + # done. + entries = [e for e in m._dests.keys() if '**' in e] + self.assertEqual(len(entries), 1) + self.assertIn('support/**', entries[0]) + def test_xpidl_generation(self): """Ensure xpidl files and directories are written out.""" env = self._consume('xpidl', RecursiveMakeBackend) diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index aafb4f6eaf4..772afc1a869 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -248,10 +248,8 @@ class TestEmitterBasic(unittest.TestCase): 'installs': { 'a11y.ini', 'test_a11y.js', - # From ** wildcard. - 'a11y-support/foo', - 'a11y-support/dir1/bar', }, + 'pattern-installs': 1, }, 'browser.ini': { 'flavor': 'browser-chrome', @@ -319,6 +317,9 @@ class TestEmitterBasic(unittest.TestCase): self.assertIn(path, m['installs']) + if 'pattern-installs' in m: + self.assertEqual(len(o.pattern_installs), m['pattern-installs']) + def test_test_manifest_unmatched_generated(self): reader = self.reader('test-manifest-unmatched-generated') diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py index 462e0f9c82f..7e7959a2b01 100644 --- a/python/mozbuild/mozpack/manifests.py +++ b/python/mozbuild/mozpack/manifests.py @@ -6,11 +6,11 @@ from __future__ import unicode_literals from contextlib import contextmanager -from .copier import FilePurger from .files import ( AbsoluteSymlinkFile, ExistingFile, File, + FileFinder, ) import mozpack.path as mozpath @@ -63,8 +63,15 @@ class InstallManifest(object): the FileCopier. No error is raised if the destination path does not exist. - Versions 1 and 2 of the manifest format are similar. Version 2 added - optional path support. + patternsymlink -- Paths matched by the expression in the source path + will be symlinked to the destination directory. + + patterncopy -- Similar to patternsymlink except files are copied, not + symlinked. + + Version 1 of the manifest was the initial version. + Version 2 added optional path support + Version 3 added support for pattern entries. """ FIELD_SEPARATOR = '\x1f' @@ -72,6 +79,8 @@ class InstallManifest(object): COPY = 2 REQUIRED_EXISTS = 3 OPTIONAL_EXISTS = 4 + PATTERN_SYMLINK = 5 + PATTERN_COPY = 6 def __init__(self, path=None, fileobj=None): """Create a new InstallManifest entry. @@ -94,7 +103,7 @@ class InstallManifest(object): def _load_from_fileobj(self, fileobj): version = fileobj.readline().rstrip() - if version not in ('1', '2'): + if version not in ('1', '2', '3'): raise UnreadableInstallManifest('Unknown manifest version: ' % version) @@ -106,7 +115,7 @@ class InstallManifest(object): record_type = int(fields[0]) if record_type == self.SYMLINK: - dest, source= fields[1:] + dest, source = fields[1:] self.add_symlink(source, dest) continue @@ -125,6 +134,16 @@ class InstallManifest(object): self.add_optional_exists(path) continue + if record_type == self.PATTERN_SYMLINK: + _, base, pattern, dest = fields[1:] + self.add_pattern_symlink(base, pattern, dest) + continue + + if record_type == self.PATTERN_COPY: + _, base, pattern, dest = fields[1:] + self.add_pattern_copy(base, pattern, dest) + continue + raise UnreadableInstallManifest('Unknown record type: %d' % record_type) @@ -158,7 +177,7 @@ class InstallManifest(object): It is an error if both are specified. """ with _auto_fileobj(path, fileobj, 'wb') as fh: - fh.write('2\n') + fh.write('3\n') for dest in sorted(self._dests): entry = self._dests[dest] @@ -198,6 +217,29 @@ class InstallManifest(object): """ self._add_entry(dest, (self.OPTIONAL_EXISTS,)) + def add_pattern_symlink(self, base, pattern, dest): + """Add a pattern match that results in symlinks being created. + + A ``FileFinder`` will be created with its base set to ``base`` + and ``FileFinder.find()`` will be called with ``pattern`` to discover + source files. Each source file will be symlinked under ``dest``. + + Filenames under ``dest`` are constructed by taking the path fragment + after ``base`` and concatenating it with ``dest``. e.g. + + /foo/bar.h -> /foo/bar.h + """ + self._add_entry(mozpath.join(base, pattern, dest), + (self.PATTERN_SYMLINK, base, pattern, dest)) + + def add_pattern_copy(self, base, pattern, dest): + """Add a pattern match that results in copies. + + See ``add_pattern_symlink()`` for usage. + """ + self._add_entry(mozpath.join(base, pattern, dest), + (self.PATTERN_COPY, base, pattern, dest)) + def _add_entry(self, dest, entry): if dest in self._dests: raise ValueError('Item already in manifest: %s' % dest) @@ -231,5 +273,23 @@ class InstallManifest(object): registry.add(dest, ExistingFile(required=False)) continue + if install_type in (self.PATTERN_SYMLINK, self.PATTERN_COPY): + _, base, pattern, dest = entry + finder = FileFinder(base, find_executables=False) + paths = [f[0] for f in finder.find(pattern)] + + if install_type == self.PATTERN_SYMLINK: + cls = AbsoluteSymlinkFile + else: + cls = File + + for path in paths: + source = mozpath.join(base, path) + dest = mozpath.join(dest, path) + + registry.add(dest, cls(source)) + + continue + raise Exception('Unknown install type defined in manifest: %d' % install_type) diff --git a/python/mozbuild/mozpack/test/test_manifests.py b/python/mozbuild/mozpack/test/test_manifests.py index 667972d3f08..87e3add84dc 100644 --- a/python/mozbuild/mozpack/test/test_manifests.py +++ b/python/mozbuild/mozpack/test/test_manifests.py @@ -29,8 +29,10 @@ class TestInstallManifest(TestWithTmpDir): m.add_copy('c_source', 'c_dest') m.add_required_exists('e_dest') m.add_optional_exists('o_dest') + m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest') + m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest') - self.assertEqual(len(m), 4) + self.assertEqual(len(m), 6) self.assertIn('s_dest', m) self.assertIn('c_dest', m) self.assertIn('e_dest', m) @@ -48,12 +50,20 @@ class TestInstallManifest(TestWithTmpDir): with self.assertRaises(ValueError): m.add_optional_exists('o_dest') + with self.assertRaises(ValueError): + m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest') + + with self.assertRaises(ValueError): + m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest') + def _get_test_manifest(self): m = InstallManifest() m.add_symlink(self.tmppath('s_source'), 's_dest') m.add_copy(self.tmppath('c_source'), 'c_dest') m.add_required_exists('e_dest') m.add_optional_exists('o_dest') + m.add_pattern_symlink('ps_base', '*', 'ps_dest') + m.add_pattern_copy('pc_base', '**', 'pc_dest') return m @@ -67,18 +77,12 @@ class TestInstallManifest(TestWithTmpDir): with open(p, 'rb') as fh: c = fh.read() - self.assertEqual(c.count('\n'), 5) + self.assertEqual(c.count('\n'), 7) lines = c.splitlines() - self.assertEqual(len(lines), 5) + self.assertEqual(len(lines), 7) - self.assertEqual(lines[0], '2') - self.assertEqual(lines[1], '2\x1fc_dest\x1f%s' % - self.tmppath('c_source')) - self.assertEqual(lines[2], '3\x1fe_dest') - self.assertEqual(lines[3], '4\x1fo_dest') - self.assertEqual(lines[4], '1\x1fs_dest\x1f%s' % - self.tmppath('s_source')) + self.assertEqual(lines[0], '3') m2 = InstallManifest(path=p) self.assertEqual(m, m2) @@ -98,8 +102,28 @@ class TestInstallManifest(TestWithTmpDir): self.assertEqual(len(r), 4) self.assertEqual(r.paths(), ['c_dest', 'e_dest', 'o_dest', 's_dest']) + def test_pattern_expansion(self): + source = self.tmppath('source') + os.mkdir(source) + os.mkdir('%s/base' % source) + os.mkdir('%s/base/foo' % source) + + with open('%s/base/foo/file1' % source, 'a'): + pass + + with open('%s/base/foo/file2' % source, 'a'): + pass + + m = InstallManifest() + m.add_pattern_symlink('%s/base' % source, '**', 'dest') + + c = FileCopier() + m.populate_registry(c) + self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2']) + def test_or(self): m1 = self._get_test_manifest() + orig_length = len(m1) m2 = InstallManifest() m2.add_symlink('s_source2', 's_dest2') m2.add_copy('c_source2', 'c_dest2') @@ -107,7 +131,7 @@ class TestInstallManifest(TestWithTmpDir): m1 |= m2 self.assertEqual(len(m2), 2) - self.assertEqual(len(m1), 6) + self.assertEqual(len(m1), orig_length + 2) self.assertIn('s_dest2', m1) self.assertIn('c_dest2', m1)