From 59053e50c00cce3b73e5daf3da342f0256e29810 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Sat, 2 Aug 2014 08:02:30 +0900 Subject: [PATCH] Bug 1046533 - Completely wrap subconfigures. r=mshal While bug 903369 added some kind of wrapping, msys mangling on Windows made it hard to make the python wrapper invoke subconfigures itself. This change overcomes this, allowing to run subconfigures entirely independently of the main configure if necessary, or to do more fancy checks without having to resort to m4 and shell. --- build/autoconf/hooks.m4 | 49 ++++----- build/clang-plugin/configure | 2 +- build/subconfigure.py | 200 ++++++++++++++++++++++++++-------- build/win32/dumpenv4python.pl | 19 ++++ 4 files changed, 196 insertions(+), 74 deletions(-) create mode 100644 build/win32/dumpenv4python.pl diff --git a/build/autoconf/hooks.m4 b/build/autoconf/hooks.m4 index 98233f534ba..b178131eabc 100644 --- a/build/autoconf/hooks.m4 +++ b/build/autoconf/hooks.m4 @@ -17,50 +17,45 @@ define([AC_INIT_PREPARE], MOZ_CONFIG_LOG_TRAP ]) -dnl Disable the trap when running sub-configures. -define(GEN_MOZ_AC_OUTPUT_SUBDIRS, [ -define([_MOZ_AC_OUTPUT_SUBDIRS], [ -patsubst($@, [$srcdir/$ac_config_dir], [$srcdir/$moz_config_srcdir]) -]) -]) -GEN_MOZ_AC_OUTPUT_SUBDIRS(defn([AC_OUTPUT_SUBDIRS])) - define([AC_OUTPUT_SUBDIRS], -[trap '' EXIT -for moz_config_dir in $1; do +[for moz_config_dir in $1; do + _CONFIG_SHELL=${CONFIG_SHELL-/bin/sh} case "$moz_config_dir" in *:*) - moz_config_srcdir=$(echo $moz_config_dir | awk -F: '{print [$]1}') - moz_config_dir=$(echo $moz_config_dir | awk -F: '{print [$]2}') + objdir=$(echo $moz_config_dir | awk -F: '{print [$]2}') ;; *) - moz_config_srcdir=$moz_config_dir + objdir=$moz_config_dir ;; esac - _CONFIG_SHELL=${CONFIG_SHELL-/bin/sh} + + dumpenv="true | " case "$host" in *-mingw*) _CONFIG_SHELL=$(cd $(dirname $_CONFIG_SHELL); pwd -W)/$(basename $_CONFIG_SHELL) if test ! -e "$_CONFIG_SHELL" -a -e "${_CONFIG_SHELL}.exe"; then _CONFIG_SHELL="${_CONFIG_SHELL}.exe" fi + dnl Yes, this is horrible. But since msys doesn't preserve environment + dnl variables and command line arguments as they are when transitioning + dnl from msys (this script) to python (below), we have to resort to hacks, + dnl storing the environment and command line arguments from a msys process + dnl (perl), and reading it from python. + dumpenv="$PERL $srcdir/build/win32/dumpenv4python.pl $ac_configure_args | " ;; esac - if test -d "$moz_config_dir"; then - (cd "$moz_config_dir"; eval $PYTHON $_topsrcdir/build/subconfigure.py dump "$_CONFIG_SHELL" $ac_configure_args) - else - mkdir -p "$moz_config_dir" - fi - _save_cache_file="$cache_file" - ifelse($2,,cache_file="$moz_config_dir/config.cache",cache_file="$2") - cache_file="$(cd $(dirname "$cache_file"); pwd -W 2>/dev/null || pwd)/$(basename "$cache_file")" - _MOZ_AC_OUTPUT_SUBDIRS($moz_config_dir) - cache_file="$_save_cache_file" - (cd "$moz_config_dir"; $PYTHON $_topsrcdir/build/subconfigure.py adjust $ac_sub_configure) -done + eval $dumpenv $PYTHON $_topsrcdir/build/subconfigure.py --prepare "$srcdir" "$moz_config_dir" "$_CONFIG_SHELL" $ac_configure_args ifelse($2,,,--cache-file="$2") -MOZ_CONFIG_LOG_TRAP + dnl Execute subconfigure, unless --no-recursion was passed to configure. + if test "$no_recursion" != yes; then + trap '' EXIT + if ! $PYTHON $_topsrcdir/build/subconfigure.py "$objdir"; then + exit 1 + fi + MOZ_CONFIG_LOG_TRAP + fi +done ]) dnl Print error messages in config.log as well as stderr diff --git a/build/clang-plugin/configure b/build/clang-plugin/configure index 4d71391a9ea..104fefca9b4 100755 --- a/build/clang-plugin/configure +++ b/build/clang-plugin/configure @@ -3,7 +3,7 @@ PLATFORM=`uname` # Default srcdir to this directory -srcdir= +srcdir=$(dirname $0) for option; do case "$option" in diff --git a/build/subconfigure.py b/build/subconfigure.py index 0690e936111..135a2e8c1da 100644 --- a/build/subconfigure.py +++ b/build/subconfigure.py @@ -6,12 +6,15 @@ # files and subsequently restore their timestamp if they haven't changed. import argparse +import errno import os import re import subprocess import sys import pickle +import mozpack.path as mozpath + class File(object): def __init__(self, path): self._path = path @@ -61,27 +64,22 @@ PRECIOUS_VARS = set([ # There's no reason not to do the latter automatically instead of failing, # doing the cleanup (which, on buildbots means a full clobber), and # restarting from scratch. -def maybe_clear_cache(args): - parser = argparse.ArgumentParser() - parser.add_argument('--target', type=str) - parser.add_argument('--host', type=str) - parser.add_argument('--build', type=str) - args, others = parser.parse_known_args(args) - env = dict(os.environ) +def maybe_clear_cache(data): + env = dict(data['env']) for kind in ('target', 'host', 'build'): - arg = getattr(args, kind) + arg = data[kind] if arg is not None: env['%s_alias' % kind] = arg # configure can take variables assignments in its arguments, and that # overrides whatever is in the environment. - for arg in others: + for arg in data['args']: if arg[:1] != '-' and '=' in arg: key, value = arg.split('=', 1) env[key] = value comment = re.compile(r'^\s+#') cache = {} - with open('config.cache') as f: + with open(data['cache-file']) as f: for line in f: if not comment.match(line) and '=' in line: key, value = line.rstrip(os.linesep).split('=', 1) @@ -97,72 +95,182 @@ def maybe_clear_cache(args): is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set' value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None if value != env.get(precious): - print 'Removing config.cache because of %s value change from:' \ - % precious + print 'Removing %s because of %s value change from:' \ + % (data['cache-file'], precious) print ' %s' % (value if value is not None else 'undefined') print 'to:' print ' %s' % env.get(precious, 'undefined') - os.remove('config.cache') + os.remove(data['cache-file']) return -def dump(dump_file, shell, args): - if os.path.exists('config.cache'): - maybe_clear_cache(args) - if not os.path.exists('config.status'): - if os.path.exists(dump_file): - os.remove(dump_file) - return +def split_template(s): + """Given a "file:template" string, returns "file", "template". If the string + is of the form "file" (without a template), returns "file", "file.in".""" + if ':' in s: + return s.split(':', 1) + return s, '%s.in' % s - config_files = [File('config.status')] + +def get_config_files(data): + config_status = mozpath.join(data['objdir'], 'config.status') + if not os.path.exists(config_status): + return [], [] + + configure = mozpath.join(data['srcdir'], 'configure') + config_files = [(config_status, configure)] + command_files = [] # Scan the config.status output for information about configuration files # it generates. config_status_output = subprocess.check_output( - [shell, '-c', './config.status --help'], + [data['shell'], '-c', '%s --help' % config_status], stderr=subprocess.STDOUT).splitlines() state = None for line in config_status_output: if line.startswith('Configuration') and line.endswith(':'): - state = 'config' - elif not line.startswith(' '): + if line.endswith('commands:'): + state = 'commands' + else: + state = 'config' + elif not line.strip(): state = None - elif state == 'config': - for f in (couple.split(':')[0] for couple in line.split()): - if os.path.isfile(f): - config_files.append(File(f)) + elif state: + for f, t in (split_template(couple) for couple in line.split()): + f = mozpath.join(data['objdir'], f) + t = mozpath.join(data['srcdir'], t) + if state == 'commands': + command_files.append(f) + else: + config_files.append((f, t)) - with open(dump_file, 'wb') as f: - pickle.dump(config_files, f) + return config_files, command_files -def adjust(dump_file, configure): - if not os.path.exists(dump_file): - return +def prepare(data_file, srcdir, objdir, shell, args): + parser = argparse.ArgumentParser() + parser.add_argument('--target', type=str) + parser.add_argument('--host', type=str) + parser.add_argument('--build', type=str) + parser.add_argument('--cache-file', type=str) + # The --srcdir argument is simply ignored. It's a useless autoconf feature + # that we don't support well anyways. This makes it stripped from `others` + # and allows to skip setting it when calling the subconfigure (configure + # will take it from the configure path anyways). + parser.add_argument('--srcdir', type=str) - config_files = [] + # Msys likes to break environment variables and command line arguments, + # so read those from stdin, as they are passed from the configure script + # when necessary (on windows). + # However, for some reason, $PATH is not handled like other environment + # variables, and msys remangles it even when giving it is already a msys + # $PATH. Fortunately, the mangling/demangling is just find for $PATH, so + # we can just take the value from the environment. Msys will convert it + # back properly when calling subconfigure. + input = sys.stdin.read() + if input: + data = {a: b for [a, b] in eval(input)} + environ = {a: b for a, b in data['env']} + environ['PATH'] = os.environ['PATH'] + args = data['args'] + else: + environ = os.environ + + args, others = parser.parse_known_args(args) + + data = { + 'target': args.target, + 'host': args.host, + 'build': args.build, + 'args': others, + 'shell': shell, + 'srcdir': srcdir, + 'env': environ, + } + + if args.cache_file: + data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(), + args.cache_file)) + else: + data['cache-file'] = mozpath.join(objdir, 'config.cache') try: - with open(dump_file, 'rb') as f: - config_files = pickle.load(f) - except Exception: - pass + os.makedirs(objdir) + except OSError as e: + if e.errno != errno.EEXIST: + raise - for f in config_files: + with open(os.path.join(objdir, data_file), 'wb') as f: + pickle.dump(data, f) + + +def run(data_file, objdir): + with open(os.path.join(objdir, data_file), 'rb') as f: + data = pickle.load(f) + + data['objdir'] = objdir + + cache_file = data['cache-file'] + if os.path.exists(cache_file): + maybe_clear_cache(data) + + config_files, command_files = get_config_files(data) + contents = [] + for f, t in config_files: + contents.append(File(f)) + + # AC_CONFIG_COMMANDS actually only registers tags, not file names + # but most commands are tagged with the file name they create. + # However, a few don't, or are tagged with a directory name (and their + # command is just to create that directory) + for f in command_files: + if os.path.isfile(f): + contents.append(File(f)) + + configure = mozpath.join(data['srcdir'], 'configure') + command = [data['shell'], configure] + for kind in ('target', 'build', 'host'): + if data.get(kind) is not None: + command += ['--%s=%s' % (kind, data[kind])] + command += data['args'] + command += ['--cache-file=%s' % cache_file] + + print 'configuring in %s' % os.path.relpath(objdir, os.getcwd()) + print 'running %s' % ' '.join(command) + sys.stdout.flush() + ret = subprocess.call(command, cwd=objdir, env=data['env']) + + for f in contents: # Still touch config.status if configure is newer than its original # mtime. - if configure and os.path.basename(f.path) == 'config.status' and \ + if os.path.basename(f.path) == 'config.status' and \ os.path.getmtime(configure) > f.mtime: continue f.update_time() - os.remove(dump_file) + return ret -CONFIG_DUMP = 'config_files.pkl' +CONFIGURE_DATA = 'configure.pkl' + +def main(args): + if args[0] != '--prepare': + if len(args) != 1: + raise Exception('Usage: %s relativeobjdir' % __file__) + return run(CONFIGURE_DATA, args[0]) + + topsrcdir = os.path.abspath(args[1]) + subdir = args[2] + # subdir can be of the form srcdir:objdir + if ':' in subdir: + srcdir, subdir = subdir.split(':', 1) + else: + srcdir = subdir + srcdir = os.path.join(topsrcdir, srcdir) + objdir = os.path.abspath(subdir) + + return prepare(CONFIGURE_DATA, srcdir, objdir, args[3], args[4:]) + if __name__ == '__main__': - if sys.argv[1] == 'dump': - dump(CONFIG_DUMP, sys.argv[2], sys.argv[3:]) - elif sys.argv[1] == 'adjust': - adjust(CONFIG_DUMP, sys.argv[2] if len(sys.argv) > 2 else None) + sys.exit(main(sys.argv[1:])) diff --git a/build/win32/dumpenv4python.pl b/build/win32/dumpenv4python.pl new file mode 100644 index 00000000000..34a1135a189 --- /dev/null +++ b/build/win32/dumpenv4python.pl @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# See build/autoconf/hooks.m4 + +use Data::Dumper; + +$Data::Dumper::Terse = 1; +$Data::Dumper::Indent = 0; + +# We can't use perl hashes because Mozilla-Build's perl is 5.6, and perl +# 5.6's Data::Dumper doesn't have Pair to change ' => ' into ' : '. +@data = ( + ['env', [map { [$_, $ENV{$_}] } keys %ENV]], + ['args', \@ARGV], +); + +print Dumper \@data;