Bug 772828 - Part a: add support for RESOURCE_FILES in moz.build; r=mshal

config/config.mk

@@ -61,6 +61,7 @@ _MOZBUILD_EXTERNAL_VARIABLES := \
   NO_DIST_INSTALL \
   PARALLEL_DIRS \
   PROGRAM \
+  RESOURCE_FILES \
   SDK_HEADERS \
   SIMPLE_PROGRAMS \
   TEST_DIRS \

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -37,6 +37,7 @@ from ..frontend.data import (
     LibraryDefinition,
     LocalInclude,
     Program,
+    Resources,
     SandboxDerived,
     SandboxWrapped,
     SimpleProgram,
@@ -413,6 +414,9 @@ class RecursiveMakeBackend(CommonBackend):
         elif isinstance(obj, Exports):
             self._process_exports(obj, obj.exports, backend_file)
 
+        elif isinstance(obj, Resources):
+            self._process_resources(obj, obj.resources, backend_file)
+
         elif isinstance(obj, JARManifest):
             backend_file.write('JAR_MANIFEST := %s\n' % obj.path)
 
@@ -873,28 +877,68 @@ class RecursiveMakeBackend(CommonBackend):
         for tier in set(self._may_skip.keys()) - affected_tiers:
             self._may_skip[tier].add(backend_file.relobjdir)
 
-    def _process_exports(self, obj, exports, backend_file, namespace=""):
-        # This may not be needed, but is present for backwards compatibility
-        # with the old make rules, just in case.
-        if not obj.dist_install:
-            return
+    def _process_hierarchy(self, obj, element, namespace, action):
+        """Walks the ``HierarchicalStringList`` ``element`` and performs
+        ``action`` on each string in the heirarcy.
 
-        strings = exports.get_strings()
+        ``action`` is a callback to be invoked with the following arguments:
+        - ``source`` - The path to the source file named by the current string
+        - ``dest``   - The relative path, including the namespace, of the
+                       destination file.
+        """
+        strings = element.get_strings()
         if namespace:
             namespace += '/'
 
         for s in strings:
             source = mozpath.normpath(mozpath.join(obj.srcdir, s))
             dest = '%s%s' % (namespace, mozpath.basename(s))
+            flags = None
+            if '__getitem__' in dir(element):
+                flags = element[s]
+            action(source, dest, flags)
+
+        children = element.get_children()
+        for subdir in sorted(children):
+            self._process_hierarchy(obj, children[subdir],
+                namespace=namespace + subdir,
+                action=action)
+
+    def _process_exports(self, obj, exports, backend_file):
+        # This may not be needed, but is present for backwards compatibility
+        # with the old make rules, just in case.
+        if not obj.dist_install:
+            return
+
+        def handle_export(source, dest, flags):
             self._install_manifests['dist_include'].add_symlink(source, dest)
 
             if not os.path.exists(source):
                 raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
-        children = exports.get_children()
-        for subdir in sorted(children):
-            self._process_exports(obj, children[subdir], backend_file,
-                namespace=namespace + subdir)
+        self._process_hierarchy(obj, exports,
+                                namespace="",
+                                action=handle_export)
+
+    def _process_resources(self, obj, resources, backend_file):
+        dep_path = mozpath.join(self.environment.topobjdir, '_build_manifests', '.deps', 'install')
+        def handle_resource(source, dest, flags):
+            if flags.preprocess:
+                if dest.endswith('.in'):
+                    dest = dest[:-3]
+                dep_file = mozpath.join(dep_path, mozpath.basename(source) + '.pp')
+                self._install_manifests['dist_bin'].add_preprocess(source, dest, dep_file, marker='%', defines=obj.defines)
+            else:
+                self._install_manifests['dist_bin'].add_symlink(source, dest)
+
+            if not os.path.exists(source):
+                raise Exception('File listed in RESOURCE_FILES does not exist: %s' % source)
+
+        # Resources need to go in the 'res' subdirectory of $(DIST)/bin, so we
+        # specify a root namespace of 'res'.
+        self._process_hierarchy(obj, resources,
+                                namespace='res',
+                                action=handle_resource)
 
     def _process_installation_target(self, obj, backend_file):
         # A few makefiles need to be able to override the following rules via

python/mozbuild/mozbuild/frontend/data.py

@@ -204,6 +204,24 @@ class Exports(SandboxDerived):
         self.exports = exports
         self.dist_install = dist_install
 
+class Resources(SandboxDerived):
+    """Sandbox container object for RESOURCE_FILES, which is a HierarchicalStringList,
+    with an extra ``.preprocess`` property on each entry.
+
+    The local defines plus anything in ACDEFINES are stored in ``defines`` as a
+    dictionary, for any files that need preprocessing.
+    """
+    __slots__ = ('resources', 'defines')
+
+    def __init__(self, sandbox, resources, defines=None):
+        SandboxDerived.__init__(self, sandbox)
+        self.resources = resources
+        defs = {}
+        defs.update(sandbox.config.defines)
+        if defines:
+            defs.update(defines)
+        self.defines = defs
+
 
 class IPDLFile(SandboxDerived):
     """Describes an individual .ipdl source file."""

python/mozbuild/mozbuild/frontend/emitter.py

@@ -37,6 +37,7 @@ from .data import (
     PreprocessedWebIDLFile,
     Program,
     ReaderSummary,
+    Resources,
     SandboxWrapped,
     SimpleProgram,
     TestWebIDLFile,
@@ -309,6 +310,10 @@ class TreeMetadataEmitter(LoggingMixin):
         if defines:
             yield Defines(sandbox, defines)
 
+        resources = sandbox.get('RESOURCE_FILES')
+        if resources:
+            yield Resources(sandbox, resources, defines)
+
         program = sandbox.get('PROGRAM')
         if program:
             yield Program(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])

python/mozbuild/mozbuild/frontend/sandbox_symbols.py

@@ -20,6 +20,7 @@ from __future__ import unicode_literals
 from collections import OrderedDict
 from mozbuild.util import (
     HierarchicalStringList,
+    HierarchicalStringListWithFlagsFactory,
     StrictOrderingOnAppendList,
     StrictOrderingOnAppendListWithFlagsFactory,
 )
@@ -328,7 +329,6 @@ VARIABLES = {
 
         This variable contains a list of system libaries to link against.
         """, None),
-
     'RCFILE': (unicode, unicode,
         """The program .rc file.
 
@@ -341,6 +341,28 @@ VARIABLES = {
         This variable can only be used on Windows.
         """, None),
 
+    'RESOURCE_FILES': (HierarchicalStringListWithFlagsFactory({'preprocess': bool}), list,
+        """List of resources to be exported, and in which subdirectories.
+
+        ``RESOURCE_FILES`` is used to list the resource files to be exported to
+        ``dist/bin/res``, but it can be used for other files as well. This variable
+        behaves as a list when appending filenames for resources in the top-level
+        directory. Files can also be appended to a field to indicate which
+        subdirectory they should be exported to. For example, to export
+        ``foo.res`` to the top-level directory, and ``bar.res`` to ``fonts/``,
+        append to ``RESOURCE_FILES`` like so::
+
+           RESOURCE_FILES += ['foo.res']
+           RESOURCE_FILES.fonts += ['bar.res']
+
+        Added files also have a 'preprocess' attribute, which will cause the
+        affected file to be run through the preprocessor, using any ``DEFINES``
+        set. It is used like this::
+
+           RESOURCE_FILES.fonts += ['baz.res.in']
+           RESOURCE_FILES.fonts['baz.res.in'].preprocess = True
+        """, None),
+
     'SDK_LIBRARY': (StrictOrderingOnAppendList, list,
         """Elements of the distributed SDK.
 

python/mozbuild/mozbuild/test/backend/data/resources/moz.build

@@ -0,0 +1,13 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+RESOURCE_FILES += ['bar.res.in', 'foo.res']
+RESOURCE_FILES.cursors += ['cursor.cur']
+RESOURCE_FILES.fonts += ['font1.ttf', 'font2.ttf']
+RESOURCE_FILES.fonts.desktop += ['desktop1.ttf', 'desktop2.ttf']
+RESOURCE_FILES.fonts.mobile += ['mobile.ttf']
+RESOURCE_FILES.tests += ['extra.manifest', 'test.manifest']
+
+RESOURCE_FILES['bar.res.in'].preprocess = True
+RESOURCE_FILES.tests['extra.manifest'].preprocess = True
+RESOURCE_FILES.tests['test.manifest'].preprocess = True

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -356,6 +356,22 @@ class TestRecursiveMakeBackend(BackendTester):
         self.assertIn('mozilla/mozilla1.h', m)
         self.assertIn('mozilla/dom/dom2.h', m)
 
+    def test_resources(self):
+        """Ensure RESOURCE_FILES is handled properly."""
+        env = self._consume('resources', RecursiveMakeBackend)
+
+        # RESOURCE_FILES should appear in the dist_bin install manifest.
+        m = InstallManifest(path=os.path.join(env.topobjdir,
+            '_build_manifests', 'install', 'dist_bin'))
+        self.assertEqual(len(m), 10)
+        self.assertIn('res/foo.res', m)
+        self.assertIn('res/fonts/font1.ttf', m)
+        self.assertIn('res/fonts/desktop/desktop2.ttf', m)
+
+        self.assertIn('res/bar.res', m)
+        self.assertIn('res/tests/test.manifest', m)
+        self.assertIn('res/tests/extra.manifest', m)
+
     def test_test_manifests_files_written(self):
         """Ensure test manifests get turned into files."""
         env = self._consume('test-manifests-written', RecursiveMakeBackend)

python/mozbuild/mozbuild/test/common.py

@@ -34,5 +34,7 @@ class MockConfig(object):
         self.substs_unicode = ReadOnlyDict({k.decode('utf-8'): v.decode('utf-8',
             'replace') for k, v in self.substs.items()})
 
+        self.defines = self.substs
+
     def child_path(self, p):
         return os.path.join(self.topsrcdir, p)

python/mozbuild/mozbuild/test/frontend/data/resources/moz.build

@@ -0,0 +1,30 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+DEFINES['FOO'] = True
+DEFINES['BAR'] = 'BAZ'
+
+RESOURCE_FILES += ['foo.res']
+RESOURCE_FILES += ['bar.res', 'baz.res']
+RESOURCE_FILES.mozilla += ['mozilla1.res']
+RESOURCE_FILES.mozilla += ['mozilla2.res']
+RESOURCE_FILES.mozilla.dom += ['dom1.res']
+RESOURCE_FILES.mozilla.dom += ['dom2.res', 'dom3.res']
+RESOURCE_FILES.mozilla.gfx += ['gfx.res']
+RESOURCE_FILES.vpx = ['mem.res']
+RESOURCE_FILES.vpx += ['mem2.res']
+RESOURCE_FILES.nspr.private = ['pprio.res', 'pprthred.res']
+RESOURCE_FILES.overwrite = ['old.res']
+RESOURCE_FILES.overwrite = ['new.res']
+
+RESOURCE_FILES += ['foo_p.res.in']
+RESOURCE_FILES['foo_p.res.in'].preprocess=True
+
+RESOURCE_FILES += ['bar_p.res.in', 'baz_p.res.in']
+RESOURCE_FILES['bar_p.res.in'].preprocess=True
+RESOURCE_FILES['baz_p.res.in'].preprocess=True
+
+RESOURCE_FILES.mozilla += ['mozilla1_p.res.in']
+RESOURCE_FILES.mozilla += ['mozilla2_p.res.in']
+RESOURCE_FILES.mozilla['mozilla1_p.res.in'].preprocess=True
+RESOURCE_FILES.mozilla['mozilla2_p.res.in'].preprocess=True

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -20,6 +20,7 @@ from mozbuild.frontend.data import (
     LocalInclude,
     Program,
     ReaderSummary,
+    Resources,
     SimpleProgram,
     TestManifest,
     VariablePassthru,
@@ -218,6 +219,63 @@ class TestEmitterBasic(unittest.TestCase):
         overwrite = exports._children['overwrite']
         self.assertEqual(overwrite.get_strings(), ['new.h'])
 
+    def test_resources(self):
+        reader = self.reader('resources')
+        objs = self.read_topsrcdir(reader)
+
+        expected_defines = reader.config.defines
+        expected_defines.update({
+            'FOO': True,
+            'BAR': 'BAZ',
+        })
+
+        self.assertEqual(len(objs), 2)
+        self.assertIsInstance(objs[0], Defines)
+        self.assertIsInstance(objs[1], Resources)
+
+        self.assertEqual(objs[1].defines, expected_defines)
+
+        resources = objs[1].resources
+        self.assertEqual(resources.get_strings(), ['foo.res', 'bar.res', 'baz.res',
+                                                   'foo_p.res.in', 'bar_p.res.in', 'baz_p.res.in'])
+        self.assertFalse(resources['foo.res'].preprocess)
+        self.assertFalse(resources['bar.res'].preprocess)
+        self.assertFalse(resources['baz.res'].preprocess)
+        self.assertTrue(resources['foo_p.res.in'].preprocess)
+        self.assertTrue(resources['bar_p.res.in'].preprocess)
+        self.assertTrue(resources['baz_p.res.in'].preprocess)
+
+        self.assertIn('mozilla', resources._children)
+        mozilla = resources._children['mozilla']
+        self.assertEqual(mozilla.get_strings(), ['mozilla1.res', 'mozilla2.res',
+                                                 'mozilla1_p.res.in', 'mozilla2_p.res.in'])
+        self.assertFalse(mozilla['mozilla1.res'].preprocess)
+        self.assertFalse(mozilla['mozilla2.res'].preprocess)
+        self.assertTrue(mozilla['mozilla1_p.res.in'].preprocess)
+        self.assertTrue(mozilla['mozilla2_p.res.in'].preprocess)
+
+        self.assertIn('dom', mozilla._children)
+        dom = mozilla._children['dom']
+        self.assertEqual(dom.get_strings(), ['dom1.res', 'dom2.res', 'dom3.res'])
+
+        self.assertIn('gfx', mozilla._children)
+        gfx = mozilla._children['gfx']
+        self.assertEqual(gfx.get_strings(), ['gfx.res'])
+
+        self.assertIn('vpx', resources._children)
+        vpx = resources._children['vpx']
+        self.assertEqual(vpx.get_strings(), ['mem.res', 'mem2.res'])
+
+        self.assertIn('nspr', resources._children)
+        nspr = resources._children['nspr']
+        self.assertIn('private', nspr._children)
+        private = nspr._children['private']
+        self.assertEqual(private.get_strings(), ['pprio.res', 'pprthred.res'])
+
+        self.assertIn('overwrite', resources._children)
+        overwrite = resources._children['overwrite']
+        self.assertEqual(overwrite.get_strings(), ['new.res'])
+
     def test_program(self):
         reader = self.reader('program')
         objs = self.read_topsrcdir(reader)

python/mozbuild/mozbuild/test/test_util.py

@@ -23,6 +23,7 @@ from mozbuild.util import (
     resolve_target_to_make,
     MozbuildDeletionError,
     HierarchicalStringList,
+    HierarchicalStringListWithFlagsFactory,
     StrictOrderingOnAppendList,
     StrictOrderingOnAppendListWithFlagsFactory,
     UnsortedError,
@@ -356,5 +357,64 @@ class TestStrictOrderingOnAppendListWithFlagsFactory(unittest.TestCase):
             l['b'].baz = False
 
 
+class TestHierarchicalStringListWithFlagsFactory(unittest.TestCase):
+    def test_hierarchical_string_list_with_flags_factory(self):
+        cls = HierarchicalStringListWithFlagsFactory({
+            'foo': bool,
+            'bar': int,
+        })
+
+        l = cls()
+        l += ['a', 'b']
+
+        with self.assertRaises(Exception):
+            l['a'] = 'foo'
+
+        with self.assertRaises(Exception):
+            c = l['c']
+
+        self.assertEqual(l['a'].foo, False)
+        l['a'].foo = True
+        self.assertEqual(l['a'].foo, True)
+
+        with self.assertRaises(TypeError):
+            l['a'].bar = 'bar'
+
+        self.assertEqual(l['a'].bar, 0)
+        l['a'].bar = 42
+        self.assertEqual(l['a'].bar, 42)
+
+        l['b'].foo = True
+        self.assertEqual(l['b'].foo, True)
+
+        with self.assertRaises(AttributeError):
+            l['b'].baz = False
+
+        l.x += ['x', 'y']
+
+        with self.assertRaises(Exception):
+            l.x['x'] = 'foo'
+
+        with self.assertRaises(Exception):
+            c = l.x['c']
+
+        self.assertEqual(l.x['x'].foo, False)
+        l.x['x'].foo = True
+        self.assertEqual(l.x['x'].foo, True)
+
+        with self.assertRaises(TypeError):
+            l.x['x'].bar = 'bar'
+
+        self.assertEqual(l.x['x'].bar, 0)
+        l.x['x'].bar = 42
+        self.assertEqual(l.x['x'].bar, 42)
+
+        l.x['y'].foo = True
+        self.assertEqual(l.x['y'].foo, True)
+
+        with self.assertRaises(AttributeError):
+            l.x['y'].baz = False
+
+
 if __name__ == '__main__':
     main()

python/mozbuild/mozbuild/util.py

@@ -332,22 +332,15 @@ class MozbuildDeletionError(Exception):
     pass
 
 
-def StrictOrderingOnAppendListWithFlagsFactory(flags):
-    """Returns a StrictOrderingOnAppendList-like object, with optional
-    flags on each item.
+def FlagsFactory(flags):
+    """Returns a class which holds optional flags for an item in a list.
 
     The flags are defined in the dict given as argument, where keys are
     the flag names, and values the type used for the value of that flag.
 
-    Example:
-        FooList = StrictOrderingOnAppendListWithFlagsFactory({
-            'foo': bool, 'bar': unicode
-        })
-        foo = FooList(['a', 'b', 'c'])
-        foo['a'].foo = True
-        foo['b'].bar = 'bar'
+    The resulting class is used by the various <TypeName>WithFlagsFactory
+    functions below.
     """
-
     assert isinstance(flags, dict)
     assert all(isinstance(v, type) for v in flags.values())
 
@@ -379,10 +372,28 @@ def StrictOrderingOnAppendListWithFlagsFactory(flags):
         def __delattr__(self, name):
             raise MozbuildDeletionError('Unable to delete attributes for this object')
 
+    return Flags
+
+
+def StrictOrderingOnAppendListWithFlagsFactory(flags):
+    """Returns a StrictOrderingOnAppendList-like object, with optional
+    flags on each item.
+
+    The flags are defined in the dict given as argument, where keys are
+    the flag names, and values the type used for the value of that flag.
+
+    Example:
+        FooList = StrictOrderingOnAppendListWithFlagsFactory({
+            'foo': bool, 'bar': unicode
+        })
+        foo = FooList(['a', 'b', 'c'])
+        foo['a'].foo = True
+        foo['b'].bar = 'bar'
+    """
     class StrictOrderingOnAppendListWithFlags(StrictOrderingOnAppendList):
         def __init__(self, iterable=[]):
             StrictOrderingOnAppendList.__init__(self, iterable)
-            self._flags_type = Flags
+            self._flags_type = FlagsFactory(flags)
             self._flags = dict()
 
         def __getitem__(self, name):
@@ -473,6 +484,58 @@ class HierarchicalStringList(object):
                     'Expected a list of strings, not an element of %s' % type(v))
 
 
+def HierarchicalStringListWithFlagsFactory(flags):
+    """Returns a HierarchicalStringList-like object, with optional
+    flags on each item.
+
+    The flags are defined in the dict given as argument, where keys are
+    the flag names, and values the type used for the value of that flag.
+
+    Example:
+        FooList = HierarchicalStringListWithFlagsFactory({
+            'foo': bool, 'bar': unicode
+        })
+        foo = FooList(['a', 'b', 'c'])
+        foo['a'].foo = True
+        foo['b'].bar = 'bar'
+        foo.sub = ['x, 'y']
+        foo.sub['x'].foo = False
+        foo.sub['y'].bar = 'baz'
+    """
+    class HierarchicalStringListWithFlags(HierarchicalStringList):
+        __flag_slots__ = ('_flags_type', '_flags')
+
+        def __init__(self):
+            HierarchicalStringList.__init__(self)
+            self._flags_type = FlagsFactory(flags)
+            self._flags = dict()
+
+        def __setattr__(self, name, value):
+            if name in self.__flag_slots__:
+                return object.__setattr__(self, name, value)
+            HierarchicalStringList.__setattr__(self, name, value)
+
+        def __getattr__(self, name):
+            if name in self.__flag_slots__:
+                return object.__getattr__(self, name)
+            return HierarchicalStringList.__getattr__(self, name)
+
+        def __getitem__(self, name):
+            if name not in self._flags:
+                if name not in self._strings:
+                    raise KeyError("'%s'" % name)
+                self._flags[name] = self._flags_type()
+            return self._flags[name]
+
+        def __setitem__(self, name, value):
+            raise TypeError("'%s' object does not support item assignment" %
+                            self.__class__.__name__)
+
+        def _get_exportvariable(self, name):
+            return self._children.setdefault(name, HierarchicalStringListWithFlags())
+
+    return HierarchicalStringListWithFlags
+
 class LockFile(object):
     """LockFile is used by the lock_file method to hold the lock.