Bug 1035599: Merge the mozilla-central and comm-central objdir, r=gps

This change merges mozilla-central and comm-central into having a single
topobjdir file but retaining two topsrcdirs (and two build systems). This state
is hopefully only the first part of a series of changes that eliminate the
comm-central build system partial clone completely.

--HG--
extra : rebase_source : 02aa2c4551df405d9783ac85cc41fe90b67bf057
extra : amend_source : d3cc677d59603648165bf65afa28413f2c40b2fd

Makefile.in

@@ -328,7 +328,8 @@ ifeq ($(MOZ_WIDGET_TOOLKIT),gtk3)
 toolkit/library/target: widget/gtk/mozgtk/gtk3/target
 endif
 ifdef MOZ_LDAP_XPCOM
-toolkit/library/target: ../ldap/target
+ldap/target: config/external/nss/target mozglue/build/target
+toolkit/library/target: ldap/target
 endif
 ifndef MOZ_FOLD_LIBS
 ifndef MOZ_NATIVE_SQLITE

configure.in

@@ -2524,10 +2524,10 @@ if test "$GNU_CC" -a "$OS_TARGET" != WINNT; then
   *)
     case $GCC_VERSION in
     4.4*|4.6*)
-      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(topsrcdir)/config/gcc_hidden_dso_handle.h'
+      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(MOZILLA_DIR)/config/gcc_hidden_dso_handle.h'
       ;;
     *)
-      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(topsrcdir)/config/gcc_hidden.h'
+      VISIBILITY_FLAGS='-I$(DIST)/system_wrappers -include $(MOZILLA_DIR)/config/gcc_hidden.h'
       ;;
     esac
     WRAP_SYSTEM_INCLUDES=1
@@ -9034,10 +9034,13 @@ unset CONFIG_FILES
 
 # Run all configure scripts specified by a subconfigure
 if test -n "$_subconfigure_subdir"; then
+  _save_srcdir="$srcdir"
+  srcdir="$srcdir/.."
   _save_ac_configure_args="$ac_configure_args"
   ac_configure_args="$_subconfigure_config_args"
   AC_OUTPUT_SUBDIRS("$_subconfigure_subdir")
   ac_configure_args="$_save_ac_configure_args"
+  srcdir="$_save_srcdir"
 fi
 
 # No need to run subconfigures when building with LIBXUL_SDK_DIR

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -89,7 +89,8 @@ class BackendMakeFile(object):
     actually change. We use FileAvoidWrite to accomplish this.
     """
 
-    def __init__(self, srcdir, objdir, environment, topobjdir):
+    def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir):
+        self.topsrcdir = topsrcdir
         self.srcdir = srcdir
         self.objdir = objdir
         self.relobjdir = mozpath.relpath(objdir, topobjdir)
@@ -329,7 +330,7 @@ class RecursiveMakeBackend(CommonBackend):
         if obj.objdir not in self._backend_files:
             self._backend_files[obj.objdir] = \
                 BackendMakeFile(obj.srcdir, obj.objdir, obj.config,
-                    self.environment.topobjdir)
+                    obj.topsrcdir, self.environment.topobjdir)
         backend_file = self._backend_files[obj.objdir]
 
         CommonBackend.consume_object(self, obj)
@@ -696,7 +697,7 @@ class RecursiveMakeBackend(CommonBackend):
                 obj = self.Substitution()
                 obj.output_path = makefile
                 obj.input_path = makefile_in
-                obj.topsrcdir = bf.environment.topsrcdir
+                obj.topsrcdir = backend_file.topsrcdir
                 obj.topobjdir = bf.environment.topobjdir
                 obj.config = bf.environment
                 self._create_makefile(obj, stub=stub)

python/mozbuild/mozbuild/frontend/reader.py

@@ -136,34 +136,25 @@ class MozbuildSandbox(Sandbox):
         topsrcdir = config.topsrcdir
         norm_topsrcdir = mozpath.normpath(topsrcdir)
 
+        self.external_source_dirs = []
+        external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
+        for external in external_dirs:
+            external = mozpath.normpath(external)
+
+            if not os.path.isabs(external):
+                external = mozpath.join(config.topsrcdir, external)
+
+            external = mozpath.normpath(external)
+            self.external_source_dirs.append(external)
+
+
         if not path.startswith(norm_topsrcdir):
-            external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
-            for external in external_dirs:
-                external = mozpath.normpath(external)
-
-                if not os.path.isabs(external):
-                    external = mozpath.join(config.topsrcdir, external)
-
-                external = mozpath.normpath(external)
-
+            for external in self.external_source_dirs:
                 if not path.startswith(external):
                     continue
 
                 topsrcdir = external
 
-                # This is really hacky and should be replaced with something
-                # more robust. We assume that if an external source directory
-                # is in play that the main build system is built in a
-                # subdirectory of its topobjdir. Therefore, the topobjdir of
-                # the external source directory is the parent of our topobjdir.
-                topobjdir = mozpath.dirname(topobjdir)
-
-                # This is suboptimal because we load the config.status multiple
-                # times. We should consider caching it, possibly by moving this
-                # code up to the reader.
-                config = ConfigEnvironment.from_config_status(
-                    mozpath.join(topobjdir, 'config.status'))
-                self.config = config
                 break
 
         self.topsrcdir = topsrcdir
@@ -196,8 +187,8 @@ class MozbuildSandbox(Sandbox):
             extra_vars = self.metadata.get('exports', dict())
             self._globals.update(extra_vars)
 
-    def exec_file(self, path, filesystem_absolute=False):
-        """Override exec_file to normalize paths and restrict file loading.
+    def normalize_path(self, path, filesystem_absolute=False, srcdir=None):
+        """Normalizes paths.
 
         If the path is absolute, behavior is governed by filesystem_absolute.
         If filesystem_absolute is True, the path is interpreted as absolute on
@@ -207,31 +198,45 @@ class MozbuildSandbox(Sandbox):
         If the path is not absolute, it will be treated as relative to the
         currently executing file. If there is no currently executing file, it
         will be treated as relative to topsrcdir.
-
-        Paths will be rejected if they do not fall under topsrcdir.
         """
         if os.path.isabs(path):
-            if not filesystem_absolute:
-                path = mozpath.normpath(mozpath.join(self.topsrcdir,
-                    path[1:]))
-
+            # If the path isn't in Unix-style, this is going to be problematic.
+            assert path[0] == '/'
+            if filesystem_absolute:
+                return path
+            for root in [self.topsrcdir] + self.external_source_dirs:
+                # mozpath.join would ignore the self.topsrcdir argument if we
+                # passed in the absolute path, so omit the leading /
+                p = mozpath.normpath(mozpath.join(root, path[1:]))
+                if os.path.exists(p):
+                    return p
+            # mozpath.join would ignore the self.topsrcdir argument if we passed
+            # in the absolute path, so omit the leading /
+            return mozpath.normpath(mozpath.join(self.topsrcdir, path[1:]))
+        elif srcdir:
+            return mozpath.normpath(mozpath.join(srcdir, path))
+        elif len(self._execution_stack):
+            return mozpath.normpath(mozpath.join(
+                mozpath.dirname(self._execution_stack[-1]), path))
         else:
-            if len(self._execution_stack):
-                path = mozpath.normpath(mozpath.join(
-                    mozpath.dirname(self._execution_stack[-1]),
-                    path))
-            else:
-                path = mozpath.normpath(mozpath.join(
-                    self.topsrcdir, path))
+            return mozpath.normpath(mozpath.join(self.topsrcdir, path))
+
+    def exec_file(self, path, filesystem_absolute=False):
+        """Override exec_file to normalize paths and restrict file loading.
+
+        Paths will be rejected if they do not fall under topsrcdir or one of
+        the external roots.
+        """
 
         # realpath() is needed for true security. But, this isn't for security
         # protection, so it is omitted.
-        normalized_path = mozpath.normpath(path)
+        normalized_path = self.normalize_path(path,
+            filesystem_absolute=filesystem_absolute)
         if not is_read_allowed(normalized_path, self.config):
             raise SandboxLoadError(list(self._execution_stack),
                 sys.exc_info()[2], illegal_path=path)
 
-        Sandbox.exec_file(self, path)
+        Sandbox.exec_file(self, normalized_path)
 
     def _add_java_jar(self, name):
         """Add a Java JAR build target."""
@@ -874,16 +879,19 @@ class BuildReader(object):
                             'times in %s' % (d, tier), sandbox)
                     recurse_info[d] = {'tier': tier,
                                        'parent': sandbox['RELATIVEDIR'],
+                                       'check_external': True,
                                        'var': 'DIRS'}
 
         for relpath, child_metadata in recurse_info.items():
-            child_path = mozpath.join(curdir, relpath, 'moz.build')
+            if 'check_external' in child_metadata:
+                relpath = '/' + relpath
+            child_path = sandbox.normalize_path(mozpath.join(relpath,
+                'moz.build'), srcdir=curdir)
 
             # Ensure we don't break out of the topsrcdir. We don't do realpath
             # because it isn't necessary. If there are symlinks in the srcdir,
             # that's not our problem. We're not a hosted application: we don't
             # need to worry about security too much.
-            child_path = mozpath.normpath(child_path)
             if not is_read_allowed(child_path, sandbox.config):
                 raise SandboxValidationError(
                     'Attempting to process file outside of allowed paths: %s' %

python/mozbuild/mozbuild/test/frontend/test_sandbox.py

@@ -227,8 +227,7 @@ add_tier_dir('t1', 'bat')
         with self.assertRaises(SandboxLoadError) as se:
             sandbox.exec_file('relative.build')
 
-        expected = mozpath.join(test_data_path, 'moz.build')
-        self.assertEqual(se.exception.illegal_path, expected)
+        self.assertEqual(se.exception.illegal_path, '../moz.build')
 
     def test_include_error_stack(self):
         # Ensure the path stack is reported properly in exceptions.