Bug 1134072 - Support for sub-contexts; r=glandium

As content in moz.build files has grown, it has become clear that
storing everything in one global namespace (the "context") per moz.build
file will not scale. This approach (which is carried over from
Makefile.in patterns) limits our ability to do things like declare
multiple instances of things (like libraries) per file.

A few months ago, templates were introduced to moz.build files. These
started the process of introducing separate contexts / containers in
each moz.build file. But it stopped short of actually emitting multiple
contexts per container. Instead, results were merged with the main
context.

This patch takes sub-contexts to the next level.

Introduced is the "SubContext" class. It is a Context derived from
another context. SubContexts are special in that they are context
managers. With the context manager is entered, the SubContext becomes
the main context associated with the executing sandbox, temporarily
masking the existence of the main context. This means that UPPERCASE
variable accesses and writes will be handled by the active SubContext.
This allows SubContext instances to define different sets of variables.

When a SubContext is spawned, it is attached to the sandbox executing
it. The moz.build reader will now emit not only the main context, but
also every SubContext that was derived from it.

To aid with the creation and declaration of sub-contexts, we introduce
the SUBCONTEXTS variable. This variable holds a list of classes that
define sub-contexts.

Sub-contexts behave a lot like templates. Their class names becomes the
symbol name in the sandbox.

python/mozbuild/mozbuild/frontend/context.py

@@ -60,11 +60,11 @@ class Context(KeyedDefaultDict):
     lots of empty/default values, you have a data structure with only the
     values that were read or touched.
 
-    Instances of variables classes are created by invoking class_name(),
-    except when class_name derives from ContextDerivedValue, in which
-    case class_name(instance_of_the_context) is invoked.
-    A value is added to those calls when instances are created during
-    assignment (setitem).
+    Instances of variables classes are created by invoking ``class_name()``,
+    except when class_name derives from ``ContextDerivedValue`` or
+    ``SubContext``, in which case ``class_name(instance_of_the_context)`` or
+    ``class_name(self)`` is invoked. A value is added to those calls when
+    instances are created during assignment (setitem).
 
     allowed_variables is a dict of the variables that can be set and read in
     this context instance. Keys in this dict are the strings representing keys
@@ -84,6 +84,7 @@ class Context(KeyedDefaultDict):
         self._all_paths = []
         self.config = config
         self.execution_time = 0
+        self._sandbox = None
         KeyedDefaultDict.__init__(self, self._factory)
 
     def push_source(self, path):
@@ -272,6 +273,35 @@ class TemplateContext(Context):
         return Context._validate(self, key, value, True)
 
 
+class SubContext(Context, ContextDerivedValue):
+    """A Context derived from another Context.
+
+    Sub-contexts are intended to be used as context managers.
+
+    Sub-contexts inherit paths and other relevant state from the parent
+    context.
+    """
+    def __init__(self, parent):
+        assert isinstance(parent, Context)
+
+        Context.__init__(self, allowed_variables=self.VARIABLES,
+                         config=parent.config)
+
+        # Copy state from parent.
+        for p in parent.source_stack:
+            self.push_source(p)
+        self._sandbox = parent._sandbox
+
+    def __enter__(self):
+        if not self._sandbox or self._sandbox() is None:
+            raise Exception('a sandbox is required')
+
+        self._sandbox().push_subcontext(self)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._sandbox().pop_subcontext(self)
+
+
 class FinalTargetValue(ContextDerivedValue, unicode):
     def __new__(cls, context, value=""):
         if not value:
@@ -366,6 +396,27 @@ def ContextDerivedTypedList(type, base_class=List):
     return _TypedList
 
 
+# This defines functions that create sub-contexts.
+#
+# Values are classes that are SubContexts. The class name will be turned into
+# a function that when called emits an instance of that class.
+#
+# Arbitrary arguments can be passed to the class constructor. The first
+# argument is always the parent context. It is up to each class to perform
+# argument validation.
+SUBCONTEXTS = [
+]
+
+for cls in SUBCONTEXTS:
+    if not issubclass(cls, SubContext):
+        raise ValueError('SUBCONTEXTS entry not a SubContext class: %s' % cls)
+
+    if not hasattr(cls, 'VARIABLES'):
+        raise ValueError('SUBCONTEXTS entry does not have VARIABLES: %s' % cls)
+
+SUBCONTEXTS = {cls.__name__: cls for cls in SUBCONTEXTS}
+
+
 # This defines the set of mutable global variables.
 #
 # Each variable is a tuple of:

python/mozbuild/mozbuild/frontend/reader.py

@@ -62,6 +62,7 @@ from .context import (
     VARIABLES,
     DEPRECATION_HINTS,
     SPECIAL_VARIABLES,
+    SUBCONTEXTS,
     TemplateContext,
 )
 
@@ -140,12 +141,14 @@ class MozbuildSandbox(Sandbox):
             return SPECIAL_VARIABLES[key][0](self._context)
         if key in FUNCTIONS:
             return self._create_function(FUNCTIONS[key])
+        if key in SUBCONTEXTS:
+            return self._create_subcontext(SUBCONTEXTS[key])
         if key in self.templates:
             return self._create_template_function(self.templates[key])
         return Sandbox.__getitem__(self, key)
 
     def __setitem__(self, key, value):
-        if key in SPECIAL_VARIABLES or key in FUNCTIONS:
+        if key in SPECIAL_VARIABLES or key in FUNCTIONS or key in SUBCONTEXTS:
             raise KeyError()
         if key in self.exports:
             self._context[key] = value
@@ -310,6 +313,14 @@ class MozbuildSandbox(Sandbox):
 
         self.templates[name] = func, code, self._context.current_path
 
+    @memoize
+    def _create_subcontext(self, cls):
+        """Return a function object that creates SubContext instances."""
+        def fn(*args, **kwargs):
+            return cls(self._context, *args, **kwargs)
+
+        return fn
+
     @memoize
     def _create_function(self, function_def):
         """Returns a function object for use within the sandbox for the given
@@ -1003,11 +1014,12 @@ class BuildReader(object):
 
         for gyp_context in gyp_contexts:
             context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir))
+            sandbox.subcontexts.append(gyp_context)
 
         yield context
 
-        for gyp_context in gyp_contexts:
-            yield gyp_context
+        for subcontext in sandbox.subcontexts:
+            yield subcontext
 
         # Traverse into referenced files.
 

python/mozbuild/mozbuild/frontend/sandbox.py

@@ -22,6 +22,7 @@ from __future__ import unicode_literals
 import copy
 import os
 import sys
+import weakref
 
 from contextlib import contextmanager
 
@@ -116,12 +117,22 @@ class Sandbox(dict):
         assert isinstance(self._builtins, ReadOnlyDict)
         assert isinstance(context, Context)
 
-        self._context = context
+        # Contexts are modeled as a stack because multiple context managers
+        # may be active.
+        self._active_contexts = [context]
+
+        # Seen sub-contexts. Will be populated with other Context instances
+        # that were related to execution of this instance.
+        self.subcontexts = []
 
         # We need to record this because it gets swallowed as part of
         # evaluation.
         self._last_name_error = None
 
+    @property
+    def _context(self):
+        return self._active_contexts[-1]
+
     def exec_file(self, path):
         """Execute code at a path in the sandbox.
 
@@ -153,6 +164,9 @@ class Sandbox(dict):
         if path:
             self._context.push_source(path)
 
+        old_sandbox = self._context._sandbox
+        self._context._sandbox = weakref.ref(self)
+
         # We don't have to worry about bytecode generation here because we are
         # too low-level for that. However, we could add bytecode generation via
         # the marshall module if parsing performance were ever an issue.
@@ -190,9 +204,31 @@ class Sandbox(dict):
             raise SandboxExecutionError(self._context.source_stack, exc[0],
                 exc[1], exc[2])
         finally:
+            self._context._sandbox = old_sandbox
             if path:
                 self._context.pop_source()
 
+    def push_subcontext(self, context):
+        """Push a SubContext onto the execution stack.
+
+        When called, the active context will be set to the specified context,
+        meaning all variable accesses will go through it. We also record this
+        SubContext as having been executed as part of this sandbox.
+        """
+        self._active_contexts.append(context)
+        if context not in self.subcontexts:
+            self.subcontexts.append(context)
+
+    def pop_subcontext(self, context):
+        """Pop a SubContext off the execution stack.
+
+        SubContexts must be pushed and popped in opposite order. This is
+        validated as part of the function call to ensure proper consumer API
+        use.
+        """
+        popped = self._active_contexts.pop()
+        assert popped == context
+
     def __getitem__(self, key):
         if key.isupper():
             try:

python/mozbuild/mozbuild/sphinx.py

@@ -101,6 +101,21 @@ def special_reference(v, func, typ, doc):
 
 def format_module(m):
     lines = []
+
+    for subcontext, cls in sorted(m.SUBCONTEXTS.items()):
+        lines.extend([
+            '.. _mozbuild_subcontext_%s:' % subcontext,
+            '',
+            'Sub-Context: %s' % subcontext,
+            '=============' + '=' * len(subcontext),
+            '',
+            prepare_docstring(cls.__doc__)[0],
+            '',
+        ])
+
+        for k, v in sorted(cls.VARIABLES.items()):
+            lines.extend(variable_reference(k, *v))
+
     lines.extend([
         'Variables',
         '=========',

python/mozbuild/mozbuild/test/frontend/test_context.py

@@ -11,6 +11,7 @@ from mozbuild.frontend.context import (
     Context,
     FUNCTIONS,
     SPECIAL_VARIABLES,
+    SUBCONTEXTS,
     VARIABLES,
 )
 
@@ -256,6 +257,12 @@ class TestSymbols(unittest.TestCase):
         for func, typ, doc in SPECIAL_VARIABLES.values():
             self._verify_doc(doc)
 
+        for name, cls in SUBCONTEXTS.items():
+            self._verify_doc(cls.__doc__)
+
+            for name, v in cls.VARIABLES.items():
+                self._verify_doc(v[2])
+
 
 if __name__ == '__main__':
     main()