From b6fa6a247d112faf49547acb6a38251f528ba8e9 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Mon, 23 Sep 2013 13:17:41 -0400 Subject: [PATCH] Bug 907789 - part 3 - build dom/bindings/ in "unified" mode; r=gps --- dom/bindings/Makefile.in | 3 +- .../mozbuild/backend/recursivemake.py | 84 ++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/dom/bindings/Makefile.in b/dom/bindings/Makefile.in index e9cf828115a..0bd7a036924 100644 --- a/dom/bindings/Makefile.in +++ b/dom/bindings/Makefile.in @@ -60,7 +60,7 @@ globalgen_targets := \ # would break |make FooBinding.o(bj)|. Ah, well. ifneq (export TestExampleInterface-example TestExampleProxyInterface-example,$(MAKECMDGOALS)) CPPSRCS = \ - $(linked_binding_cpp_files) \ + $(unified_binding_cpp_files) \ $(linked_generated_events_cpp_files) \ $(filter %.cpp, $(globalgen_targets)) \ BindingUtils.cpp \ @@ -258,6 +258,7 @@ GARBAGE += \ parser.out \ $(wildcard *-example.h) \ $(wildcard *-example.cpp) \ + $(unified_binding_cpp_files) \ .BindingGen \ .all-webidl-file-list \ .generated-events-webidl-files \ diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index d7269e60188..de48c155767 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -8,6 +8,7 @@ import errno import itertools import logging import os +import re import types from collections import namedtuple @@ -510,6 +511,69 @@ class RecursiveMakeBackend(CommonBackend): self._update_from_avoid_write(root.close()) self._update_from_avoid_write(root_deps.close()) + def _add_unified_build_rules(self, makefile, files, output_directory, + unified_prefix='Unified', + extra_dependencies=[], + unified_files_makefile_variable='unified_files', + include_curdir_build_rules=True): + files_per_unified_file = 16 + + explanation = "\n" \ + "# We build files in 'unified' mode by including several files\n" \ + "# together into a single source file. This cuts down on\n" \ + "# compilation times and debug information size. %d was chosen as\n" \ + "# a reasonable compromise between clobber rebuild time, incremental\n" \ + "# rebuild time, and compiler memory usage.\n" % files_per_unified_file + makefile.add_statement(explanation) + + def unified_files(): + "Return an iterator of (unified_filename, source_filenames) tuples." + # Our last returned list of source filenames may be short, and we + # don't want the fill value inserted by izip_longest to be an + # issue. So we do a little dance to filter it out ourselves. + dummy_fill_value = ("dummy",) + def filter_out_dummy(iterable): + return itertools.ifilter(lambda x: x != dummy_fill_value, + iterable) + + # From the itertools documentation, slightly modified: + def grouper(n, iterable): + "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return itertools.izip_longest(fillvalue=dummy_fill_value, *args) + + for i, unified_group in enumerate(grouper(files_per_unified_file, + sorted(files))): + just_the_filenames = list(filter_out_dummy(unified_group)) + yield '%s%d.cpp' % (unified_prefix, i), just_the_filenames + + all_sources = ' '.join(source for source, _ in unified_files()) + makefile.add_statement('%s := %s\n' % (unified_files_makefile_variable, + all_sources)) + + regen_cmds = ['for f in $(filter %.cpp,$^); do echo "#include \\"$$f\\""; done > $@'] + for unified_file, source_filenames in unified_files(): + rule = makefile.create_rule([unified_file]) + rule.add_dependencies(extra_dependencies + source_filenames) + rule.add_commands(regen_cmds) + + # The rule we just defined is only for cases where the cpp files get + # blown away and we need to regenerate them. The rule doesn't correctly + # handle source files being added/removed/renamed. Therefore, we + # generate them here also to make sure everything's up-to-date. + with FileAvoidWrite(os.path.join(output_directory, unified_file)) as f: + f.write('\n'.join(['#include "%s"' % s for s in source_filenames])) + + if include_curdir_build_rules: + makefile.add_statement('\n' + '# Make sometimes gets confused between "foo" and "$(CURDIR)/foo".\n' + '# Help it out by explicitly specifiying dependencies.') + makefile.add_statement('all_absolute_unified_files := \\\n' + ' $(addprefix $(CURDIR)/,$(%s))' + % unified_files_makefile_variable) + rule = makefile.create_rule(['$(all_absolute_unified_files)']) + rule.add_dependencies(['$(CURDIR)/%: %']) + def consume_finished(self): CommonBackend.consume_finished(self) @@ -585,8 +649,8 @@ class RecursiveMakeBackend(CommonBackend): self.summary.managed_count += 1 # Write out master lists of WebIDL source files. - webidls = FileAvoidWrite(os.path.join(self.environment.topobjdir, - 'dom', 'bindings', 'webidlsrcs.mk')) + bindings_dir = os.path.join(self.environment.topobjdir, 'dom', 'bindings') + webidls = FileAvoidWrite(os.path.join(bindings_dir, 'webidlsrcs.mk')) mk = mozmakeutil.Makefile() @@ -600,7 +664,21 @@ class RecursiveMakeBackend(CommonBackend): write_var('generated_webidl_files', self._generated_webidl_sources) write_var('preprocessed_webidl_files', self._preprocessed_webidl_sources) - mk.dump(webidls) + all_webidl_files = itertools.chain(iter(self._webidl_sources), + iter(self._generated_events_webidl_sources), + iter(self._generated_webidl_sources), + iter(self._preprocessed_webidl_sources)) + all_webidl_files = [os.path.basename(x) for x in all_webidl_files] + all_webidl_sources = [re.sub(r'\.webidl$', 'Binding.cpp', x) for x in all_webidl_files] + + self._add_unified_build_rules(mk, all_webidl_sources, + bindings_dir, + unified_prefix='UnifiedBindings', + unified_files_makefile_variable='unified_binding_cpp_files') + + # Assume that Somebody Else has responsibility for correctly + # specifying removal dependencies for |all_webidl_sources|. + mk.dump(webidls, removal_guard=False) self._update_from_avoid_write(webidls.close()) self.summary.managed_count += 1