Bug 784841 - Part 16: Use moz.build files to build the tree; r=ted, glandium

This commit is contained in:
Gregory Szorc 2013-02-25 12:47:11 -08:00
parent 540e3b0cd2
commit 39ec7e4cdf
13 changed files with 262 additions and 34 deletions

View File

@ -6,15 +6,24 @@
# drop-in replacement for autoconf 2.13's config.status, with features
# borrowed from autoconf > 2.5, and additional features.
import logging
import os
import sys
from optparse import OptionParser
from mach.logging import LoggingManager
from mozbuild.backend.configenvironment import ConfigEnvironment
from mozbuild.backend.recursivemake import RecursiveMakeBackend
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader
from Preprocessor import Preprocessor
log_manager = LoggingManager()
# Basic logging facility
verbose = False
def log(string):
@ -83,6 +92,12 @@ def config_status(topobjdir = '.', topsrcdir = '.',
env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
non_global_defines=non_global_defines, substs=substs)
reader = BuildReader(env)
emitter = TreeMetadataEmitter(env)
backend = RecursiveMakeBackend(env)
# This won't actually do anything because of the magic of generators.
definitions = emitter.emit(reader.read_topsrcdir())
if options.recheck:
# Execute configure from the top object directory
if not os.path.isabs(topsrcdir):
@ -99,11 +114,21 @@ def config_status(topobjdir = '.', topsrcdir = '.',
files = []
# Default to display messages when giving --file or --headers on the
# command line.
log_level = logging.INFO
if options.files or options.headers or options.verbose:
global verbose
verbose = True
log_level = logging.DEBUG
log_manager.add_terminal_logging(level=log_level)
log_manager.enable_unstructured()
if not options.files and not options.headers:
print >>sys.stderr, "creating config files and headers..."
backend.consume(definitions)
files = [os.path.join(topobjdir, f) for f in files]
headers = [os.path.join(topobjdir, f) for f in headers]

View File

@ -733,11 +733,6 @@ ifdef MOZ_DEBUG
JAVAC_FLAGS += -g
endif
ifdef TIERS
DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
endif
CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
# MDDEPDIR is the subdirectory where dependency files are stored

View File

@ -10,6 +10,37 @@ ifndef topsrcdir
$(error topsrcdir was not set))
endif
# Integrate with mozbuild-generated make files. We first verify that no
# variables provided by the automatically generated .mk files are
# present. If they are, this is a violation of the separation of
# responsibility between Makefile.in and mozbuild files.
_MOZBUILD_EXTERNAL_VARIABLES := \
DIRS \
PARALLEL_DIRS \
TEST_DIRS \
TIERS \
TOOL_DIRS \
$(NULL)
ifndef EXTERNALLY_MANAGED_MAKE_FILE
$(foreach var,$(_MOZBUILD_EXTERNAL_VARIABLES),$(if $($(var)),\
$(error Variable $(var) is defined in Makefile. It should only be defined in moz.build files.),\
))
# Import the automatically generated backend file. If this file doesn't exist,
# the backend hasn't been properly configured. We want this to be a fatal
# error, hence not using "-include".
ifndef STANDALONE_MAKEFILE
GLOBAL_DEPS += backend.mk
include backend.mk
endif
endif
ifdef TIERS
DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
endif
ifndef MOZILLA_DIR
MOZILLA_DIR = $(topsrcdir)
endif
@ -1147,9 +1178,29 @@ $(JAVA_LIBRARY): $(addprefix $(_JAVA_DIR)/,$(JAVA_SRCS:.java=.class)) $(GLOBAL_D
GARBAGE_DIRS += $(_JAVA_DIR)
###############################################################################
# Update Makefiles
# Update Files Managed by Build Backend
###############################################################################
ifdef MOZBUILD_DERIVED
# If this Makefile is derived from moz.build files, substitution for all .in
# files is handled by SUBSTITUTE_FILES. This includes Makefile.in.
ifneq ($(SUBSTITUTE_FILES),,)
$(SUBSTITUTE_FILES): % : $(srcdir)/%.in $(DEPTH)/config/autoconf.mk
@$(PYTHON) $(DEPTH)/config.status -n --file=$@
@$(TOUCH) $@
endif
# Detect when the backend.mk needs rebuilt. This will cause a full scan and
# rebuild. While relatively expensive, it should only occur once per recursion.
ifneq ($(BACKEND_INPUT_FILES),,)
backend.mk: $(BACKEND_INPUT_FILES)
@$(PYTHON) $(DEPTH)/config.status -n
@$(TOUCH) $@
endif
endif # MOZBUILD_DERIVED
ifndef NO_MAKEFILE_RULE
Makefile: Makefile.in
@$(PYTHON) $(DEPTH)/config.status -n --file=Makefile

View File

@ -6,15 +6,24 @@
# drop-in replacement for autoconf 2.13's config.status, with features
# borrowed from autoconf > 2.5, and additional features.
import logging
import os
import sys
from optparse import OptionParser
from mach.logging import LoggingManager
from mozbuild.backend.configenvironment import ConfigEnvironment
from mozbuild.backend.recursivemake import RecursiveMakeBackend
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader
from Preprocessor import Preprocessor
log_manager = LoggingManager()
# Basic logging facility
verbose = False
def log(string):
@ -83,6 +92,12 @@ def config_status(topobjdir = '.', topsrcdir = '.',
env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
non_global_defines=non_global_defines, substs=substs)
reader = BuildReader(env)
emitter = TreeMetadataEmitter(env)
backend = RecursiveMakeBackend(env)
# This won't actually do anything because of the magic of generators.
definitions = emitter.emit(reader.read_topsrcdir())
if options.recheck:
# Execute configure from the top object directory
if not os.path.isabs(topsrcdir):
@ -99,11 +114,21 @@ def config_status(topobjdir = '.', topsrcdir = '.',
files = []
# Default to display messages when giving --file or --headers on the
# command line.
log_level = logging.INFO
if options.files or options.headers or options.verbose:
global verbose
verbose = True
log_level = logging.DEBUG
log_manager.add_terminal_logging(level=log_level)
log_manager.enable_unstructured()
if not options.files and not options.headers:
print >>sys.stderr, "creating config files and headers..."
backend.consume(definitions)
files = [os.path.join(topobjdir, f) for f in files]
headers = [os.path.join(topobjdir, f) for f in headers]

View File

@ -733,11 +733,6 @@ ifdef MOZ_DEBUG
JAVAC_FLAGS += -g
endif
ifdef TIERS
DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
endif
CREATE_PRECOMPLETE_CMD = $(PYTHON) $(call core_abspath,$(topsrcdir)/config/createprecomplete.py)
# MDDEPDIR is the subdirectory where dependency files are stored

View File

@ -10,6 +10,37 @@ ifndef topsrcdir
$(error topsrcdir was not set))
endif
# Integrate with mozbuild-generated make files. We first verify that no
# variables provided by the automatically generated .mk files are
# present. If they are, this is a violation of the separation of
# responsibility between Makefile.in and mozbuild files.
_MOZBUILD_EXTERNAL_VARIABLES := \
DIRS \
PARALLEL_DIRS \
TEST_DIRS \
TIERS \
TOOL_DIRS \
$(NULL)
ifndef EXTERNALLY_MANAGED_MAKE_FILE
$(foreach var,$(_MOZBUILD_EXTERNAL_VARIABLES),$(if $($(var)),\
$(error Variable $(var) is defined in Makefile. It should only be defined in moz.build files.),\
))
# Import the automatically generated backend file. If this file doesn't exist,
# the backend hasn't been properly configured. We want this to be a fatal
# error, hence not using "-include".
ifndef STANDALONE_MAKEFILE
GLOBAL_DEPS += backend.mk
include backend.mk
endif
endif
ifdef TIERS
DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_dirs))
STATIC_DIRS += $(foreach tier,$(TIERS),$(tier_$(tier)_staticdirs))
endif
ifndef MOZILLA_DIR
MOZILLA_DIR = $(topsrcdir)
endif
@ -1147,9 +1178,29 @@ $(JAVA_LIBRARY): $(addprefix $(_JAVA_DIR)/,$(JAVA_SRCS:.java=.class)) $(GLOBAL_D
GARBAGE_DIRS += $(_JAVA_DIR)
###############################################################################
# Update Makefiles
# Update Files Managed by Build Backend
###############################################################################
ifdef MOZBUILD_DERIVED
# If this Makefile is derived from moz.build files, substitution for all .in
# files is handled by SUBSTITUTE_FILES. This includes Makefile.in.
ifneq ($(SUBSTITUTE_FILES),,)
$(SUBSTITUTE_FILES): % : $(srcdir)/%.in $(DEPTH)/config/autoconf.mk
@$(PYTHON) $(DEPTH)/config.status -n --file=$@
@$(TOUCH) $@
endif
# Detect when the backend.mk needs rebuilt. This will cause a full scan and
# rebuild. While relatively expensive, it should only occur once per recursion.
ifneq ($(BACKEND_INPUT_FILES),,)
backend.mk: $(BACKEND_INPUT_FILES)
@$(PYTHON) $(DEPTH)/config.status -n
@$(TOUCH) $@
endif
endif # MOZBUILD_DERIVED
ifndef NO_MAKEFILE_RULE
Makefile: Makefile.in
@$(PYTHON) $(DEPTH)/config.status -n --file=Makefile

View File

@ -2,9 +2,6 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Python 2.5 needs this for the with statement.
from __future__ import with_statement
import collections
import gyp
import gyp.common
@ -35,6 +32,8 @@ topsrcdir = %(topsrcdir)s
srcdir = %(srcdir)s
VPATH = %(srcdir)s
EXTERNALLY_MANAGED_MAKE_FILE := 1
"""
COMMON_FOOTER = """

View File

@ -4,6 +4,7 @@
from __future__ import unicode_literals
import errno
import logging
import os
@ -18,35 +19,94 @@ from ..util import FileAvoidWrite
class BackendMakeFile(object):
"""Represents a generated backend.mk file.
This is both a wrapper around FileAvoidWrite as well as a container that
This is both a wrapper around a file handle as well as a container that
holds accumulated state.
It's worth taking a moment to explain the make dependencies. The
generated backend.mk as well as the Makefile.in (if it exists) are in the
GLOBAL_DEPS list. This means that if one of them changes, all targets
in that Makefile are invalidated. backend.mk also depends on all of its
input files.
It's worth considering the effect of file mtimes on build behavior.
Since we perform an "all or none" traversal of moz.build files (the whole
tree is scanned as opposed to individual files), if we were to blindly
write backend.mk files, the net effect of updating a single mozbuild file
in the tree is all backend.mk files have new mtimes. This would in turn
invalidate all make targets across the whole tree! This would effectively
undermine incremental builds as any mozbuild change would cause the entire
tree to rebuild!
The solution is to not update the mtimes of backend.mk files unless they
actually change. We use FileAvoidWrite to accomplish this. However, this
puts us in a somewhat complicated position when it comes to tree recursion.
As you are recursing the tree, the first time you come across a backend.mk
that is out of date, a full tree build will be incurred. In typical make
build systems, we would touch the out-of-date target (backend.mk) to ensure
its mtime is newer than all its dependencies - even if the contents did
not change. However, we can't rely on just this approach. During recursion,
the first trigger of backend generation will cause only that backend.mk to
update. If there is another backend.mk that is also out of date according
to mtime but whose contents were not changed, when we recurse to that
directory, make will trigger another full backend generation! This would
be completely redundant and would slow down builds! This is not acceptable.
We work around this problem by having backend generation update the mtime
of backend.mk if they are older than their inputs - even if the file
contents did not change. This is essentially a middle ground between
always updating backend.mk and only updating the backend.mk that was out
of date during recursion.
"""
def __init__(self, srcdir, objdir):
self.srcdir = srcdir
self.objdir = objdir
self.path = os.path.join(objdir, 'backend.mk')
# Filenames that influenced the content of this file.
self.inputs = set()
# Filenames that are automatically generated by the build backend.
self.outputs = set()
self.fh = FileAvoidWrite(os.path.join(objdir, 'backend.mk'))
self.fh = FileAvoidWrite(self.path)
self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
self.fh.write('\n')
self.fh.write('MOZBUILD_DERIVED := 1\n')
# SUBSTITUTE_FILES handles Makefile.in -> Makefile conversion.
self.fh.write('NO_MAKEFILE_RULE := 1\n')
def write(self, buf):
self.fh.write(buf)
def close(self):
if len(self.inputs):
self.fh.write('BACKEND_INPUT_FILES += %s\n' % ' '.join(self.inputs))
if len(self.outputs):
self.fh.write('BACKEND_OUTPUT_FILES += %s\n' % ' '.join(self.outputs))
if self.inputs:
l = ' '.join(sorted(self.inputs))
self.fh.write('BACKEND_INPUT_FILES += %s\n' % l)
self.fh.close()
if not self.inputs:
return
# Update mtime iff any of its input files are newer. See class notes
# for why we do this.
existing_mtime = os.path.getmtime(self.path)
def mtime(path):
try:
return os.path.getmtime(path)
except OSError as e:
if e.errno == errno.ENOENT:
return 0
raise
input_mtime = max(mtime(path) for path in self.inputs)
if input_mtime > existing_mtime:
os.utime(self.path, None)
class RecursiveMakeBackend(BuildBackend):
"""Backend that integrates with the existing recursive make build system.
@ -72,13 +132,18 @@ class RecursiveMakeBackend(BuildBackend):
backend_file = self._backend_files.get(obj.srcdir,
BackendMakeFile(obj.srcdir, obj.objdir))
# Define the paths that will trigger a backend rebuild. We always
# add autoconf.mk because that is proxy for CONFIG. We can't use
# config.status because there is no make target for that!
autoconf_path = os.path.join(obj.topobjdir, 'config', 'autoconf.mk')
backend_file.inputs.add(autoconf_path)
backend_file.inputs |= obj.sandbox_all_paths
if isinstance(obj, DirectoryTraversal):
self._process_directory_traversal(obj, backend_file)
elif isinstance(obj, ConfigFileSubstitution):
backend_file.inputs.add(obj.input_path)
backend_file.outputs.add(obj.output_path)
backend_file.write('SUBSTITUTE_FILES += %s\n' % obj.relpath)
self.environment.create_config_file(obj.output_path)
self._backend_files[obj.srcdir] = backend_file
@ -99,8 +164,8 @@ class RecursiveMakeBackend(BuildBackend):
self.log(logging.DEBUG, 'create_makefile', {'path': out_path},
'Generating makefile: {path}')
self.environment.create_config_file(out_path)
bf.outputs.add(out_path)
bf.write('SUBSTITUTE_FILES += Makefile\n')
bf.close()
def _process_directory_traversal(self, obj, backend_file):
@ -118,6 +183,9 @@ class RecursiveMakeBackend(BuildBackend):
fh.write('tier_%s_staticdirs += %s\n' % (
tier, ' '.join(obj.tier_static_dirs[tier])))
static = ' '.join(obj.tier_static_dirs[tier])
fh.write('EXTERNAL_DIRS += %s\n' % static)
if obj.dirs:
fh.write('DIRS := %s\n' % ' '.join(obj.dirs))

View File

@ -102,6 +102,7 @@ class ConfigFileSubstitution(SandboxDerived):
__slots__ = (
'input_path',
'output_path',
'relpath',
)
def __init__(self, sandbox):
@ -109,4 +110,5 @@ class ConfigFileSubstitution(SandboxDerived):
self.input_path = None
self.output_path = None
self.relpath = None

View File

@ -55,6 +55,7 @@ class TreeMetadataEmitter(object):
sub = ConfigFileSubstitution(sandbox)
sub.input_path = os.path.join(sandbox['SRCDIR'], '%s.in' % path)
sub.output_path = os.path.join(sandbox['OBJDIR'], path)
sub.relpath = path
yield sub
def _emit_directory_traversal_from_sandbox(self, sandbox):

View File

@ -102,8 +102,17 @@ class MozbuildSandbox(Sandbox):
d['SRCDIR'] = os.path.join(config.topsrcdir, reldir).replace(os.sep, '/').rstrip('/')
d['OBJDIR'] = os.path.join(topobjdir, reldir).replace(os.sep, '/').rstrip('/')
d['CONFIG'] = ReadOnlyDefaultDict(config.substs,
global_default=None)
# config.status does not yet use unicode. However, mozbuild expects
# unicode everywhere. So, decode binary into unicode as necessary.
# Bug 844509 tracks a better way to do this.
substs = {}
for k, v in config.substs.items():
if not isinstance(v, text_type):
v = v.decode('utf-8', 'strict')
substs[k] = v
d['CONFIG'] = ReadOnlyDefaultDict(substs, global_default=None)
# Register functions.
for name, func in FUNCTIONS.items():

View File

@ -5,6 +5,7 @@
from __future__ import unicode_literals
import os
import time
from mozunit import main
@ -62,21 +63,23 @@ class TestRecursiveMakeBackend(BackendTester):
p = os.path.join(env.topobjdir, 'backend.mk')
lines = [l.strip() for l in open(p, 'rt').readlines()[1:-2]]
lines = [l.strip() for l in open(p, 'rt').readlines()[2:-1]]
self.assertEqual(lines, [
'MOZBUILD_DERIVED := 1',
'NO_MAKEFILE_RULE := 1',
'DIRS := dir1',
'PARALLEL_DIRS := dir2',
'TEST_DIRS := dir3',
'SUBSTITUTE_FILES += Makefile',
])
def test_no_mtime_bump(self):
def test_mtime_no_change(self):
"""Ensure mtime is not updated if file content does not change."""
env = self._consume('stub0', RecursiveMakeBackend)
makefile_path = os.path.join(env.topobjdir, 'Makefile')
backend_path = os.path.join(env.topobjdir, 'backend.mk')
makefile_mtime = os.path.getmtime(makefile_path)
backend_mtime = os.path.getmtime(backend_path)
@ -93,12 +96,15 @@ class TestRecursiveMakeBackend(BackendTester):
env = self._consume('external_make_dirs', RecursiveMakeBackend)
backend_path = os.path.join(env.topobjdir, 'backend.mk')
lines = [l.strip() for l in open(backend_path, 'rt').readlines()[1:-2]]
lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:-1]]
self.assertEqual(lines, [
'MOZBUILD_DERIVED := 1',
'NO_MAKEFILE_RULE := 1',
'DIRS := dir',
'PARALLEL_DIRS := p_dir',
'DIRS += external',
'PARALLEL_DIRS += p_external',
'SUBSTITUTE_FILES += Makefile',
])
def test_substitute_config_files(self):

View File

@ -104,6 +104,7 @@ class TestEmitterBasic(unittest.TestCase):
self.assertIsInstance(objs[2], ConfigFileSubstitution)
topobjdir = os.path.abspath(reader.config.topobjdir)
self.assertEqual(objs[1].relpath, 'foo')
self.assertEqual(os.path.normpath(objs[1].output_path),
os.path.normpath(os.path.join(topobjdir, 'foo')))
self.assertEqual(os.path.normpath(objs[2].output_path),