From 0c8af7b874893bab6b3a179864ca87dd25a8a65e Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 3 Sep 2013 22:51:58 -0700 Subject: [PATCH] Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium --- Makefile.in | 1 + js/src/Makefile.in | 1 + .../action/process_install_manifest.py | 9 +++-- .../mozbuild/backend/recursivemake.py | 34 +++++++---------- python/mozbuild/mozbuild/frontend/data.py | 6 ++- python/mozbuild/mozbuild/frontend/emitter.py | 3 +- .../test/backend/test_recursivemake.py | 38 +++++-------------- 7 files changed, 38 insertions(+), 54 deletions(-) diff --git a/Makefile.in b/Makefile.in index 3d3bc0f5bce..923f7c263a2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -99,6 +99,7 @@ TIER_precompile_CUSTOM := 1 include $(topsrcdir)/config/rules.mk default all alldep:: + $(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include) $(call BUILDSTATUS,TIERS $(TIERS)) $(foreach tier,$(TIERS),$(call SUBMAKE,tier_$(tier))) diff --git a/js/src/Makefile.in b/js/src/Makefile.in index 2439e81e72b..8e6996e1e48 100644 --- a/js/src/Makefile.in +++ b/js/src/Makefile.in @@ -177,6 +177,7 @@ MOZILLA_DTRACE_SRC = $(srcdir)/devtools/javascript-trace.d endif default:: + $(call py_action,process_install_manifest,--no-remove $(DIST)/include _build_manifests/install/dist_include) ifneq (,$(CROSS_COMPILE)$(filter-out WINNT OS2,$(OS_ARCH))) # nsinstall doesn't get built until we enter config/ in the exports phase, diff --git a/python/mozbuild/mozbuild/action/process_install_manifest.py b/python/mozbuild/mozbuild/action/process_install_manifest.py index 33468703145..9c5aec14196 100644 --- a/python/mozbuild/mozbuild/action/process_install_manifest.py +++ b/python/mozbuild/mozbuild/action/process_install_manifest.py @@ -13,14 +13,14 @@ COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \ 'Removed {rm_files} files and {rm_dirs} directories.' -def process_manifest(destdir, *paths): +def process_manifest(destdir, paths, remove_unaccounted=True): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) - return copier.copy(destdir) + return copier.copy(destdir, remove_unaccounted=remove_unaccounted) if __name__ == '__main__': @@ -29,10 +29,13 @@ if __name__ == '__main__': parser.add_argument('destdir', help='Destination directory.') parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).') + parser.add_argument('--no-remove', action='store_true', + help='Do not remove unaccounted files from destination.') args = parser.parse_args() - result = process_manifest(args.destdir, *args.manifests) + result = process_manifest(args.destdir, args.manifests, + remove_unaccounted=not args.no_remove) print(COMPLETE.format(dest=args.destdir, existing=result.existing_files_count, diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 4527e3cf477..765f329d004 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -149,11 +149,8 @@ class RecursiveMakeBackend(CommonBackend): self.backend_input_files.add(os.path.join(self.environment.topobjdir, 'config', 'autoconf.mk')) - self._install_manifests = dict() - self._purge_manifests = dict( dist_bin=PurgeManifest(relpath='dist/bin'), - dist_include=PurgeManifest(relpath='dist/include'), dist_private=PurgeManifest(relpath='dist/private'), dist_public=PurgeManifest(relpath='dist/public'), dist_sdk=PurgeManifest(relpath='dist/sdk'), @@ -163,6 +160,7 @@ class RecursiveMakeBackend(CommonBackend): self._install_manifests = dict( dist_idl=InstallManifest(), + dist_include=InstallManifest(), ) def _update_from_avoid_write(self, result): @@ -368,27 +366,22 @@ class RecursiveMakeBackend(CommonBackend): ' '.join(obj.parallel_external_make_dirs)) def _process_exports(self, obj, exports, backend_file, namespace=""): + # This may not be needed, but is present for backwards compatibility + # with the old make rules, just in case. + if not obj.dist_install: + return + strings = exports.get_strings() if namespace: - if strings: - backend_file.write('EXPORTS_NAMESPACES += %s\n' % namespace) - export_name = 'EXPORTS_%s' % namespace namespace += '/' - else: - export_name = 'EXPORTS' - # Iterate over the list of export filenames, printing out an EXPORTS - # declaration for each. - if strings: - backend_file.write('%s += %s\n' % (export_name, ' '.join(strings))) + for s in strings: + source = os.path.normpath(os.path.join(obj.srcdir, s)) + dest = '%s%s' % (namespace, os.path.basename(s)) + self._install_manifests['dist_include'].add_symlink(source, dest) - for s in strings: - source = os.path.normpath(os.path.join(obj.srcdir, s)) - if not os.path.isfile(source): - raise Exception('File listed in EXPORTS does not exist: %s' % source) - - p = '%s%s' % (namespace, s) - self._purge_manifests['dist_include'].add(p) + if not os.path.exists(source): + raise Exception('File listed in EXPORTS does not exist: %s' % source) children = exports.get_children() for subdir in sorted(children): @@ -405,7 +398,8 @@ class RecursiveMakeBackend(CommonBackend): for idl in manager.idls.values(): self._install_manifests['dist_idl'].add_symlink(idl['source'], idl['basename']) - self._purge_manifests['dist_include'].add('%s.h' % idl['root']) + self._install_manifests['dist_include'].add_optional_exists('%s.h' + % idl['root']) build_files.add(mozpath.join('headers', '%s.h' % idl['root'])) for module in manager.modules: diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 0ca88f36cdf..9db3d89079c 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -161,11 +161,13 @@ class Exports(SandboxDerived): this object fills that role. It just has a reference to the underlying HierarchicalStringList, which is created when parsing EXPORTS. """ - __slots__ = ('exports') + __slots__ = ('exports', 'dist_install') - def __init__(self, sandbox, exports): + def __init__(self, sandbox, exports, dist_install=True): SandboxDerived.__init__(self, sandbox) self.exports = exports + self.dist_install = dist_install + class IPDLFile(SandboxDerived): """Describes an individual .ipdl source file.""" diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 4979e8b02c5..3e8bd677582 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -154,7 +154,8 @@ class TreeMetadataEmitter(LoggingMixin): exports = sandbox.get('EXPORTS') if exports: - yield Exports(sandbox, exports) + yield Exports(sandbox, exports, + dist_install=not sandbox.get('NO_DIST_INSTALL', False)) program = sandbox.get('PROGRAM') if program: diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index ff411d2701c..a2d64e8b31c 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -234,33 +234,16 @@ class TestRecursiveMakeBackend(BackendTester): self.assertEqual(found, val) def test_exports(self): - """Ensure EXPORTS is written out correctly.""" + """Ensure EXPORTS is handled properly.""" env = self._consume('exports', RecursiveMakeBackend) - backend_path = os.path.join(env.topobjdir, 'backend.mk') - lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] - - self.assertEqual(lines, [ - 'MOZBUILD_DERIVED := 1', - 'NO_MAKEFILE_RULE := 1', - 'NO_SUBMAKEFILES_RULE := 1', - 'EXPORTS += foo.h', - 'EXPORTS_NAMESPACES += mozilla', - 'EXPORTS_mozilla += mozilla1.h mozilla2.h', - 'EXPORTS_NAMESPACES += mozilla/dom', - 'EXPORTS_mozilla/dom += dom1.h dom2.h', - 'EXPORTS_NAMESPACES += mozilla/gfx', - 'EXPORTS_mozilla/gfx += gfx.h', - 'EXPORTS_NAMESPACES += nspr/private', - 'EXPORTS_nspr/private += pprio.h', - ]) - - # EXPORTS files should appear in the dist_include purge manifest. - m = PurgeManifest(path=os.path.join(env.topobjdir, - '_build_manifests', 'purge', 'dist_include')) - self.assertIn('foo.h', m.entries) - self.assertIn('mozilla/mozilla1.h', m.entries) - self.assertIn('mozilla/dom/dom2.h', m.entries) + # EXPORTS files should appear in the dist_include install manifest. + m = InstallManifest(path=os.path.join(env.topobjdir, + '_build_manifests', 'install', 'dist_include')) + self.assertEqual(len(m), 7) + self.assertIn('foo.h', m) + self.assertIn('mozilla/mozilla1.h', m) + self.assertIn('mozilla/dom/dom2.h', m) def test_xpcshell_manifests(self): """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly.""" @@ -297,8 +280,8 @@ class TestRecursiveMakeBackend(BackendTester): self.assertIn('bar.idl', m) self.assertIn('foo.idl', m) - m = PurgeManifest(path=os.path.join(purge_dir, 'dist_include')) - self.assertIn('foo.h', m.entries) + m = InstallManifest(path=os.path.join(install_dir, 'dist_include')) + self.assertIn('foo.h', m) p = os.path.join(env.topobjdir, 'config/makefiles/xpidl') self.assertTrue(os.path.isdir(p)) @@ -325,7 +308,6 @@ class TestRecursiveMakeBackend(BackendTester): expected = [ 'dist_bin', - 'dist_include', 'dist_private', 'dist_public', 'dist_sdk',