Bug 1132771 - Support and test for reading without a config object; r=glandium

We want the ability to read data from any moz.build file without needing
a full build configuration (running configure). This will enable tools
to consume metadata by merely having a copy of the source code and
nothing more.

This commit creates the EmptyConfig object. It is a config object that -
as its name implies - is empty. It will be used for reading moz.build
files in "no config" mode.

Many moz.build files make assumptions that variables in CONFIG are
defined and that they are strings. We create the EmptyValue type that
behaves like an empty unicode string. Since moz.build files also do some
type checking, we carve an exemption for EmptyValue, just like we do for
None.

We add a test to verify that reading moz.build files in "no config" mode
works. This required some minor changes to existing moz.build files to
make them work in the new execution mode.

build/gyp.mozbuild

@@ -95,7 +95,7 @@ flavors = {
     'NetBSD': 'netbsd',
     'OpenBSD': 'openbsd',
 }
-gyp_vars['OS'] = flavors[os]
+gyp_vars['OS'] = flavors.get(os)
 
 arches = {
     'x86_64': 'x64',

config/tests/test_mozbuild_reading.py

@@ -10,7 +10,10 @@ import unittest
 from mozunit import main
 
 from mozbuild.base import MozbuildObject
-from mozbuild.frontend.reader import BuildReader
+from mozbuild.frontend.reader import (
+    BuildReader,
+    EmptyConfig,
+)
 
 
 class TestMozbuildReading(unittest.TestCase):
@@ -24,6 +27,12 @@ class TestMozbuildReading(unittest.TestCase):
         os.environ.clear()
         os.environ.update(self._old_env)
 
+    def _mozbuilds(self, reader):
+        if not hasattr(self, '_mozbuild_paths'):
+            self._mozbuild_paths = set(reader.all_mozbuild_paths())
+
+        return self._mozbuild_paths
+
     def test_filesystem_traversal_reading(self):
         """Reading moz.build according to filesystem traversal works.
 
@@ -34,11 +43,27 @@ class TestMozbuildReading(unittest.TestCase):
         mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
         config = mb.config_environment
         reader = BuildReader(config)
-        all_paths = set(reader.all_mozbuild_paths())
+        all_paths = self._mozbuilds(reader)
         paths, contexts = reader.read_relevant_mozbuilds(all_paths)
         self.assertEqual(set(paths), all_paths)
         self.assertGreaterEqual(len(contexts), len(paths))
 
+    def test_filesystem_traversal_no_config(self):
+        """Reading moz.build files via filesystem traversal mode with no build config.
+
+        This is similar to the above test except no build config is applied.
+        This will likely fail in more scenarios than the above test because a
+        lot of moz.build files assumes certain variables are present.
+        """
+        here = os.path.abspath(os.path.dirname(__file__))
+        root = os.path.normpath(os.path.join(here, '..', '..'))
+        config = EmptyConfig(root)
+        reader = BuildReader(config)
+        all_paths = self._mozbuilds(reader)
+        paths, contexts = reader.read_relevant_mozbuilds(all_paths)
+        self.assertEqual(set(paths.keys()), all_paths)
+        self.assertGreaterEqual(len(contexts), len(paths))
+
 
 if __name__ == '__main__':
     main()

ipc/glue/moz.build

@@ -87,7 +87,7 @@ elif CONFIG['OS_ARCH'] in ('DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD'):
     UNIFIED_SOURCES += [
         'ProcessUtils_bsd.cpp'
     ]
-elif CONFIG['OS_ARCH'] in ('Darwin'):
+elif CONFIG['OS_ARCH'] == 'Darwin':
     UNIFIED_SOURCES += [
         'ProcessUtils_mac.mm'
     ]

moz.build

@@ -66,7 +66,7 @@ if CONFIG['COMPILE_ENVIRONMENT'] and not CONFIG['LIBXUL_SDK']:
         DIRS += ['config/external/icu']
     DIRS += ['js/src']
 
-if not CONFIG['JS_STANDALONE']:
+if not CONFIG['JS_STANDALONE'] and CONFIG['MOZ_BUILD_APP']:
     # Bring in the configuration for the configured application.
     include('/' + CONFIG['MOZ_BUILD_APP'] + '/app.mozbuild')
 

netwerk/build/moz.build

@@ -33,7 +33,8 @@ LOCAL_INCLUDES += [
 ]
 
 protocols = CONFIG['NECKO_PROTOCOLS'].copy()
-protocols.remove("about")
+if 'about' in protocols:
+    protocols.remove('about')
 LOCAL_INCLUDES += sorted([
     '/netwerk/protocol/%s' % d for d in protocols
 ])

python/mozbuild/mozbuild/frontend/reader.py

@@ -36,6 +36,7 @@ from collections import (
 from io import StringIO
 
 from mozbuild.util import (
+    EmptyValue,
     memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
@@ -82,6 +83,48 @@ def log(logger, level, action, params, formatter):
     logger.log(level, formatter, extra={'action': action, 'params': params})
 
 
+class EmptyConfig(object):
+    """A config object that is empty.
+
+    This config object is suitable for using with a BuildReader on a vanilla
+    checkout, without any existing configuration. The config is simply
+    bootstrapped from a top source directory path.
+    """
+    class PopulateOnGetDict(ReadOnlyDefaultDict):
+        """A variation on ReadOnlyDefaultDict that populates during .get().
+
+        This variation is needed because CONFIG uses .get() to access members.
+        Without it, None (instead of our EmptyValue types) would be returned.
+        """
+        def get(self, key, default=None):
+            return self[key]
+
+    def __init__(self, topsrcdir):
+        self.topsrcdir = topsrcdir
+        self.topobjdir = ''
+
+        self.substs = self.PopulateOnGetDict(EmptyValue, {
+            # These 2 variables are used semi-frequently and it isn't worth
+            # changing all the instances.
+            b'MOZ_APP_NAME': b'empty',
+            b'MOZ_CHILD_PROCESS_NAME': b'empty',
+            # Set manipulations are performed within the moz.build files. But
+            # set() is not an exposed symbol, so we can't create an empty set.
+            b'NECKO_PROTOCOLS': set(),
+            # Needed to prevent js/src's config.status from loading.
+            b'JS_STANDALONE': b'1',
+        })
+        udict = {}
+        for k, v in self.substs.items():
+            if isinstance(v, str):
+                udict[k.decode('utf-8')] = v.decode('utf-8')
+            else:
+                udict[k] = v
+        self.substs_unicode = self.PopulateOnGetDict(EmptyValue, udict)
+        self.defines = self.substs
+        self.external_source_dir = None
+
+
 def is_read_allowed(path, config):
     """Whether we are allowed to load a mozbuild file at the specified path.
 

python/mozbuild/mozbuild/util.py

@@ -18,6 +18,7 @@ import os
 import stat
 import sys
 import time
+import types
 
 from collections import (
     defaultdict,
@@ -50,6 +51,17 @@ def hash_file(path, hasher=None):
     return h.hexdigest()
 
 
+class EmptyValue(unicode):
+    """A dummy type that behaves like an empty string and sequence.
+
+    This type exists in order to support
+    :py:class:`mozbuild.frontend.reader.EmptyConfig`. It should likely not be
+    used elsewhere.
+    """
+    def __init__(self):
+        super(EmptyValue, self).__init__()
+
+
 class ReadOnlyDict(dict):
     """A read-only dictionary."""
     def __init__(self, *args, **kwargs):
@@ -254,9 +266,9 @@ class ListMixin(object):
         return super(ListMixin, self).__setslice__(i, j, sequence)
 
     def __add__(self, other):
-        # Allow None is a special case because it makes undefined variable
+        # Allow None and EmptyValue is a special case because it makes undefined
-        # references in moz.build behave better.
+        # variable references in moz.build behave better.
-        other = [] if other is None else other
+        other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other
         if not isinstance(other, list):
             raise ValueError('Only lists can be appended to lists.')
 
@@ -265,7 +277,7 @@ class ListMixin(object):
         return new_list
 
     def __iadd__(self, other):
-        other = [] if other is None else other
+        other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other
         if not isinstance(other, list):
             raise ValueError('Only lists can be appended to lists.')