Bug 1136383 - enable specifying method names for GENERATED_FILES scripts; r=gps

The inputs to scripts for GENERATED_FILES are restricted to filenames
only.  We have several examples in the tree, however, where a script
takes non-filename arguments.  For converting those cases to use
GENERATED_FILES, we first need to provide some way of "injecting"
non-filename arguments into the script.

This commit adds a method for doing that, by extending the .script flag
on GENERATED_FILES to include an optional method name:

  f = GENERATED_FILES['foo']
  f.script = 'script.py:make_foo'

will invoke the make_foo function found in script.py instead of the
function named main.

python/mozbuild/mozbuild/action/file_generate.py

@@ -20,6 +20,8 @@ def main(argv):
                                      add_help=False)
     parser.add_argument('python_script', metavar='python-script', type=str,
                         help='The Python script to run')
+    parser.add_argument('method_name', metavar='method-name', type=str,
+                        help='The method of the script to invoke')
     parser.add_argument('output_file', metavar='output-file', type=str,
                         help='The file to generate')
     parser.add_argument('additional_arguments', metavar='arg', nargs='*',
@@ -31,15 +33,16 @@ def main(argv):
     with open(script, 'r') as fh:
         module = imp.load_module('script', fh, script,
                                  ('.py', 'r', imp.PY_SOURCE))
-    if not hasattr(module, 'main'):
-        print('Error: script "{0}" is missing a main method'.format(script),
+    method = args.method_name
+    if not hasattr(module, method):
+        print('Error: script "{0}" is missing a {1} method'.format(script, method),
               file=sys.stderr)
         return 1
 
     ret = 1
     try:
         with FileAvoidWrite(args.output_file) as output:
-            ret = module.main(output, *args.additional_arguments)
+            ret = module.__dict__[method](output, *args.additional_arguments)
     except IOError as e:
         print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
         traceback.print_exc()

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -409,11 +409,12 @@ class RecursiveMakeBackend(CommonBackend):
             backend_file.write('GENERATED_FILES += %s\n' % obj.output)
             if obj.script:
                 backend_file.write("""{output}: {script}{inputs}
-\t$(call py_action,file_generate,{script} {output}{inputs})
+\t$(call py_action,file_generate,{script} {method} {output}{inputs})
 
 """.format(output=obj.output,
            inputs=' ' + ' '.join(obj.inputs) if obj.inputs else '',
-           script=obj.script))
+           script=obj.script,
+           method=obj.method))
 
         elif isinstance(obj, TestHarnessFiles):
             self._process_test_harness_files(obj, backend_file)

python/mozbuild/mozbuild/frontend/context.py

@@ -526,6 +526,14 @@ VARIABLES = {
         supported for passing to scripts, and that all arguments provided
         to the script should be filenames relative to the directory in which
         the moz.build file is located.
+
+        To enable using the same script for generating multiple files with
+        slightly different non-filename parameters, alternative entry points
+        into ``script`` can be specified::
+
+          GENERATED_FILES += ['bar.c']
+          bar = GENERATED_FILES['bar.c']
+          bar.script = 'generate.py:make_bar'
         """, 'export'),
 
     'DEFINES': (OrderedDict, dict,

python/mozbuild/mozbuild/frontend/data.py

@@ -858,13 +858,15 @@ class GeneratedFile(ContextDerived):
 
     __slots__ = (
         'script',
+        'method',
         'output',
         'inputs',
     )
 
-    def __init__(self, context, script, output, inputs):
+    def __init__(self, context, script, method, output, inputs):
         ContextDerived.__init__(self, context)
         self.script = script
+        self.method = method
         self.output = output
         self.inputs = inputs
 

python/mozbuild/mozbuild/frontend/emitter.py

@@ -524,7 +524,13 @@ class TreeMetadataEmitter(LoggingMixin):
                 flags = generated_files[f]
                 output = f
                 if flags.script:
-                    script = mozpath.join(context.srcdir, flags.script)
+                    method = "main"
+                    # Deal with cases like "C:\\path\\to\\script.py:function".
+                    if not flags.script.endswith('.py') and ':' in flags.script:
+                        script, method = flags.script.rsplit(':', 1)
+                    else:
+                        script = flags.script
+                    script = mozpath.join(context.srcdir, script)
                     inputs = [mozpath.join(context.srcdir, i) for i in flags.inputs]
 
                     if not os.path.exists(script):
@@ -542,8 +548,9 @@ class TreeMetadataEmitter(LoggingMixin):
                                 % (f, i), context)
                 else:
                     script = None
+                    method = None
                     inputs = []
-                yield GeneratedFile(context, script, output, inputs)
+                yield GeneratedFile(context, script, method, output, inputs)
 
         test_harness_files = context.get('TEST_HARNESS_FILES')
         if test_harness_files:

python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build

@@ -5,7 +5,7 @@
 GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ]
 
 bar = GENERATED_FILES['bar.c']
-bar.script = 'generate-bar.py'
+bar.script = 'generate-bar.py:baz'
 
 foo = GENERATED_FILES['foo.c']
 foo.script = 'generate-foo.py'

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -379,11 +379,11 @@ class TestRecursiveMakeBackend(BackendTester):
         expected = [
             'GENERATED_FILES += bar.c',
             'bar.c: %s/generate-bar.py' % env.topsrcdir,
-            '$(call py_action,file_generate,%s/generate-bar.py bar.c)' % env.topsrcdir,
+            '$(call py_action,file_generate,%s/generate-bar.py baz bar.c)' % env.topsrcdir,
             '',
             'GENERATED_FILES += foo.c',
             'foo.c: %s/generate-foo.py %s/foo-data' % (env.topsrcdir, env.topsrcdir),
-            '$(call py_action,file_generate,%s/generate-foo.py foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir),
+            '$(call py_action,file_generate,%s/generate-foo.py main foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir),
             '',
             'GENERATED_FILES += quux.c',
         ]

python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ['bar.c']
+
+bar = GENERATED_FILES['bar.c']
+bar.script = TOPSRCDIR + '/script.py:make_bar'
+bar.inputs = []

python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build

@@ -0,0 +1,13 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += [ 'bar.c', 'foo.c' ]
+
+bar = GENERATED_FILES['bar.c']
+bar.script = 'script.py:make_bar'
+bar.inputs = []
+
+foo = GENERATED_FILES['foo.c']
+foo.script = 'script.py'
+foo.inputs = []

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -194,6 +194,37 @@ class TestEmitterBasic(unittest.TestCase):
         expected = ['bar.c', 'foo.c']
         for o, expected_filename in zip(objs, expected):
             self.assertEqual(o.output, expected_filename)
+            self.assertEqual(o.script, None)
+            self.assertEqual(o.method, None)
+            self.assertEqual(o.inputs, [])
+
+    def test_generated_files_method_names(self):
+        reader = self.reader('generated-files-method-names')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 2)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+
+        expected = ['bar.c', 'foo.c']
+        expected_method_names = ['make_bar', 'main']
+        for o, expected_filename, expected_method in zip(objs, expected, expected_method_names):
+            self.assertEqual(o.output, expected_filename)
+            self.assertEqual(o.method, expected_method)
+            self.assertEqual(o.inputs, [])
+
+    def test_generated_files_absolute_script(self):
+        reader = self.reader('generated-files-absolute-script')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 1)
+
+        o = objs[0]
+        self.assertIsInstance(o, GeneratedFile)
+        self.assertEqual(o.output, 'bar.c')
+        self.assertRegexpMatches(o.script, 'script.py$')
+        self.assertEqual(o.method, 'make_bar')
+        self.assertEqual(o.inputs, [])
 
     def test_generated_files_no_script(self):
         reader = self.reader('generated-files-no-script')