Bug 1168607 - Use mozpack Finders for reading moz.build files; r=glandium

Sometimes moz.build data may not come from the local filesystem. To
support defining moz.build data in other backing stores, we switch
moz.build reading to use mozpack's *Finder classes for I/O.

The use of a FileFinder bound to / is sub-optimal. We should ideally
be creating a finder bound to topsrcdir. However, doing this would
require refactoring a lot of path handling in the reader, as that code
makes many assumptions that paths are absolute. This would be excellent
follow-up work.

While I was here, I cleaned up some linter warnings for unused imports.

On my machine, this commit results in a slight slowdown of moz.build
reading. Before, `mach build-backend` was consistently reporting 1.99s
for reading 2,572 moz.build files. After, it is consistently reporting
2.07s, an increase of 4%. This is likely due to a) function call
overhead b) the cost of instantiating a few thousand File instances
c) FileFinder performing an os.path.exists() before returning File
instances. I believe the regression is tolerable.

python/mozbuild/mozbuild/frontend/reader.py

@@ -39,7 +39,6 @@ from mozbuild.util import (
     EmptyValue,
     memoize,
     ReadOnlyDefaultDict,
-    ReadOnlyDict,
 )
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
@@ -53,6 +52,7 @@ from .data import (
 )
 
 from .sandbox import (
+    default_finder,
     SandboxError,
     SandboxExecutionError,
     SandboxLoadError,
@@ -172,10 +172,10 @@ class MozbuildSandbox(Sandbox):
     metadata is a dict of metadata that can be used during the sandbox
     evaluation.
     """
-    def __init__(self, context, metadata={}):
+    def __init__(self, context, metadata={}, finder=default_finder):
         assert isinstance(context, Context)
 
-        Sandbox.__init__(self, context)
+        Sandbox.__init__(self, context, finder=finder)
 
         self._log = logging.getLogger(__name__)
 
@@ -391,7 +391,8 @@ class MozbuildSandbox(Sandbox):
                 'special_variables': self.metadata.get('special_variables', {}),
                 'subcontexts': self.metadata.get('subcontexts', {}),
                 'templates': self.metadata.get('templates', {})
-            })
+            }, finder=self._finder)
+
             template.exec_in_sandbox(sandbox, *args, **kwargs)
 
             # This is gross, but allows the merge to happen. Eventually, the
@@ -853,12 +854,13 @@ class BuildReader(object):
     each sandbox evaluation. Its return value is ignored.
     """
 
-    def __init__(self, config):
+    def __init__(self, config, finder=default_finder):
         self.config = config
 
         self._log = logging.getLogger(__name__)
         self._read_files = set()
         self._execution_stack = []
+        self._finder = finder
 
     def read_topsrcdir(self):
         """Read the tree of linked moz.build files.
@@ -1091,7 +1093,8 @@ class BuildReader(object):
             config.external_source_dir = None
 
         context = Context(VARIABLES, config)
-        sandbox = MozbuildSandbox(context, metadata=metadata)
+        sandbox = MozbuildSandbox(context, metadata=metadata,
+                                  finder=self._finder)
         sandbox.exec_file(path)
         context.execution_time = time.time() - time_start
 
@@ -1127,7 +1130,7 @@ class BuildReader(object):
             non_unified_sources = set()
             for s in gyp_dir.non_unified_sources:
                 source = SourcePath(context, s)
-                if not os.path.exists(source.full_path):
+                if not self._finder.get(source.full_path):
                     raise SandboxValidationError('Cannot find %s.' % source,
                         context)
                 non_unified_sources.add(source)
@@ -1215,7 +1218,7 @@ class BuildReader(object):
 
         @memoize
         def exists(path):
-            return os.path.exists(path)
+            return self._finder.get(path) is not None
 
         def itermozbuild(path):
             subpath = ''

python/mozbuild/mozbuild/frontend/sandbox.py

@@ -19,15 +19,16 @@ user-friendly error messages in the case of errors.
 
 from __future__ import unicode_literals
 
-import copy
 import os
 import sys
 import weakref
 
-from contextlib import contextmanager
-
 from mozbuild.util import ReadOnlyDict
-from context import Context
+from .context import Context
+from mozpack.files import FileFinder
+
+
+default_finder = FileFinder('/', find_executables=False)
 
 
 def alphabetical_sorted(iterable, cmp=None, key=lambda x: x.lower(),
@@ -108,7 +109,7 @@ class Sandbox(dict):
         'int': int,
     })
 
-    def __init__(self, context, builtins=None):
+    def __init__(self, context, builtins=None, finder=default_finder):
         """Initialize a Sandbox ready for execution.
         """
         self._builtins = builtins or self.BUILTINS
@@ -132,6 +133,8 @@ class Sandbox(dict):
         # Current literal source being executed.
         self._current_source = None
 
+        self._finder = finder
+
     @property
     def _context(self):
         return self._active_contexts[-1]
@@ -143,11 +146,8 @@ class Sandbox(dict):
         """
         assert os.path.isabs(path)
 
-        source = None
-
         try:
-            with open(path, 'rt') as fd:
-                source = fd.read()
+            source = self._finder.get(path).read()
         except Exception as e:
             raise SandboxLoadError(self._context.source_stack,
                 sys.exc_info()[2], read_error=path)