From 02e43668cb786803ecfc6fc3b2f6d7fee80ace09 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Tue, 24 Feb 2015 16:36:39 -0500 Subject: [PATCH] Bug 1136383 - enable specifying method names for GENERATED_FILES scripts; r=gps The inputs to scripts for GENERATED_FILES are restricted to filenames only. We have several examples in the tree, however, where a script takes non-filename arguments. For converting those cases to use GENERATED_FILES, we first need to provide some way of "injecting" non-filename arguments into the script. This commit adds a method for doing that, by extending the .script flag on GENERATED_FILES to include an optional method name: f = GENERATED_FILES['foo'] f.script = 'script.py:make_foo' will invoke the make_foo function found in script.py instead of the function named main. --- .../mozbuild/mozbuild/action/file_generate.py | 9 ++++-- .../mozbuild/backend/recursivemake.py | 5 +-- python/mozbuild/mozbuild/frontend/context.py | 8 +++++ python/mozbuild/mozbuild/frontend/data.py | 4 ++- python/mozbuild/mozbuild/frontend/emitter.py | 11 +++++-- .../backend/data/generated-files/moz.build | 2 +- .../test/backend/test_recursivemake.py | 4 +-- .../generated-files-absolute-script/moz.build | 9 ++++++ .../generated-files-absolute-script/script.py | 0 .../generated-files-method-names/moz.build | 13 ++++++++ .../generated-files-method-names/script.py | 0 .../mozbuild/test/frontend/test_emitter.py | 31 +++++++++++++++++++ 12 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py diff --git a/python/mozbuild/mozbuild/action/file_generate.py b/python/mozbuild/mozbuild/action/file_generate.py index bddf9dbc534..6708bf1f365 100644 --- a/python/mozbuild/mozbuild/action/file_generate.py +++ b/python/mozbuild/mozbuild/action/file_generate.py @@ -20,6 +20,8 @@ def main(argv): add_help=False) parser.add_argument('python_script', metavar='python-script', type=str, help='The Python script to run') + parser.add_argument('method_name', metavar='method-name', type=str, + help='The method of the script to invoke') parser.add_argument('output_file', metavar='output-file', type=str, help='The file to generate') parser.add_argument('additional_arguments', metavar='arg', nargs='*', @@ -31,15 +33,16 @@ def main(argv): with open(script, 'r') as fh: module = imp.load_module('script', fh, script, ('.py', 'r', imp.PY_SOURCE)) - if not hasattr(module, 'main'): - print('Error: script "{0}" is missing a main method'.format(script), + method = args.method_name + if not hasattr(module, method): + print('Error: script "{0}" is missing a {1} method'.format(script, method), file=sys.stderr) return 1 ret = 1 try: with FileAvoidWrite(args.output_file) as output: - ret = module.main(output, *args.additional_arguments) + ret = module.__dict__[method](output, *args.additional_arguments) except IOError as e: print('Error opening file "{0}"'.format(e.filename), file=sys.stderr) traceback.print_exc() diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 0c4eb38d757..cd90ff1f66e 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -409,11 +409,12 @@ class RecursiveMakeBackend(CommonBackend): backend_file.write('GENERATED_FILES += %s\n' % obj.output) if obj.script: backend_file.write("""{output}: {script}{inputs} -\t$(call py_action,file_generate,{script} {output}{inputs}) +\t$(call py_action,file_generate,{script} {method} {output}{inputs}) """.format(output=obj.output, inputs=' ' + ' '.join(obj.inputs) if obj.inputs else '', - script=obj.script)) + script=obj.script, + method=obj.method)) elif isinstance(obj, TestHarnessFiles): self._process_test_harness_files(obj, backend_file) diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index 28abf734dfd..c12ccfa192a 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -526,6 +526,14 @@ VARIABLES = { supported for passing to scripts, and that all arguments provided to the script should be filenames relative to the directory in which the moz.build file is located. + + To enable using the same script for generating multiple files with + slightly different non-filename parameters, alternative entry points + into ``script`` can be specified:: + + GENERATED_FILES += ['bar.c'] + bar = GENERATED_FILES['bar.c'] + bar.script = 'generate.py:make_bar' """, 'export'), 'DEFINES': (OrderedDict, dict, diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index d3e6970b2be..abe4e8902a0 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -858,13 +858,15 @@ class GeneratedFile(ContextDerived): __slots__ = ( 'script', + 'method', 'output', 'inputs', ) - def __init__(self, context, script, output, inputs): + def __init__(self, context, script, method, output, inputs): ContextDerived.__init__(self, context) self.script = script + self.method = method self.output = output self.inputs = inputs diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 6141f7a88fb..a9d01889c3b 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -524,7 +524,13 @@ class TreeMetadataEmitter(LoggingMixin): flags = generated_files[f] output = f if flags.script: - script = mozpath.join(context.srcdir, flags.script) + method = "main" + # Deal with cases like "C:\\path\\to\\script.py:function". + if not flags.script.endswith('.py') and ':' in flags.script: + script, method = flags.script.rsplit(':', 1) + else: + script = flags.script + script = mozpath.join(context.srcdir, script) inputs = [mozpath.join(context.srcdir, i) for i in flags.inputs] if not os.path.exists(script): @@ -542,8 +548,9 @@ class TreeMetadataEmitter(LoggingMixin): % (f, i), context) else: script = None + method = None inputs = [] - yield GeneratedFile(context, script, output, inputs) + yield GeneratedFile(context, script, method, output, inputs) test_harness_files = context.get('TEST_HARNESS_FILES') if test_harness_files: diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build index e377cb6763b..e5e90983691 100644 --- a/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build +++ b/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build @@ -5,7 +5,7 @@ GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ] bar = GENERATED_FILES['bar.c'] -bar.script = 'generate-bar.py' +bar.script = 'generate-bar.py:baz' foo = GENERATED_FILES['foo.c'] foo.script = 'generate-foo.py' diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index d733325c9af..c9ee468fbd1 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -379,11 +379,11 @@ class TestRecursiveMakeBackend(BackendTester): expected = [ 'GENERATED_FILES += bar.c', 'bar.c: %s/generate-bar.py' % env.topsrcdir, - '$(call py_action,file_generate,%s/generate-bar.py bar.c)' % env.topsrcdir, + '$(call py_action,file_generate,%s/generate-bar.py baz bar.c)' % env.topsrcdir, '', 'GENERATED_FILES += foo.c', 'foo.c: %s/generate-foo.py %s/foo-data' % (env.topsrcdir, env.topsrcdir), - '$(call py_action,file_generate,%s/generate-foo.py foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir), + '$(call py_action,file_generate,%s/generate-foo.py main foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir), '', 'GENERATED_FILES += quux.c', ] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build new file mode 100644 index 00000000000..e840e09419a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build @@ -0,0 +1,9 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ['bar.c'] + +bar = GENERATED_FILES['bar.c'] +bar.script = TOPSRCDIR + '/script.py:make_bar' +bar.inputs = [] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build new file mode 100644 index 00000000000..def78740284 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build @@ -0,0 +1,13 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c' ] + +bar = GENERATED_FILES['bar.c'] +bar.script = 'script.py:make_bar' +bar.inputs = [] + +foo = GENERATED_FILES['foo.c'] +foo.script = 'script.py' +foo.inputs = [] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 4d6300e7bd2..68244f10c9f 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -194,6 +194,37 @@ class TestEmitterBasic(unittest.TestCase): expected = ['bar.c', 'foo.c'] for o, expected_filename in zip(objs, expected): self.assertEqual(o.output, expected_filename) + self.assertEqual(o.script, None) + self.assertEqual(o.method, None) + self.assertEqual(o.inputs, []) + + def test_generated_files_method_names(self): + reader = self.reader('generated-files-method-names') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 2) + for o in objs: + self.assertIsInstance(o, GeneratedFile) + + expected = ['bar.c', 'foo.c'] + expected_method_names = ['make_bar', 'main'] + for o, expected_filename, expected_method in zip(objs, expected, expected_method_names): + self.assertEqual(o.output, expected_filename) + self.assertEqual(o.method, expected_method) + self.assertEqual(o.inputs, []) + + def test_generated_files_absolute_script(self): + reader = self.reader('generated-files-absolute-script') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + + o = objs[0] + self.assertIsInstance(o, GeneratedFile) + self.assertEqual(o.output, 'bar.c') + self.assertRegexpMatches(o.script, 'script.py$') + self.assertEqual(o.method, 'make_bar') + self.assertEqual(o.inputs, []) def test_generated_files_no_script(self): reader = self.reader('generated-files-no-script')