Bug 1136383 - enable specifying method names for GENERATED_FILES scripts; r=gps
authorNathan Froyd <froydnj@mozilla.com>
Tue, 24 Feb 2015 16:36:39 -0500
changeset 249272 4314d56097d748eede3c062681a2624df8d64fd7
parent 249271 2877cc2ddca8dba15b5ea1e722e5ad80661acffd
child 249273 9f70f885338c0726de548d26b962e8e627875ea0
push id7860
push userjlund@mozilla.com
push dateMon, 30 Mar 2015 18:46:02 +0000
treeherdermozilla-aurora@8ac636cd51f3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1136383
milestone39.0a1
Bug 1136383 - enable specifying method names for GENERATED_FILES scripts; r=gps The inputs to scripts for GENERATED_FILES are restricted to filenames only. We have several examples in the tree, however, where a script takes non-filename arguments. For converting those cases to use GENERATED_FILES, we first need to provide some way of "injecting" non-filename arguments into the script. This commit adds a method for doing that, by extending the .script flag on GENERATED_FILES to include an optional method name: f = GENERATED_FILES['foo'] f.script = 'script.py:make_foo' will invoke the make_foo function found in script.py instead of the function named main.
python/mozbuild/mozbuild/action/file_generate.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build
python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py
python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build
python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/python/mozbuild/mozbuild/action/file_generate.py
+++ b/python/mozbuild/mozbuild/action/file_generate.py
@@ -15,36 +15,39 @@ import traceback
 
 from mozbuild.util import FileAvoidWrite
 
 def main(argv):
     parser = argparse.ArgumentParser('Generate a file from a Python script',
                                      add_help=False)
     parser.add_argument('python_script', metavar='python-script', type=str,
                         help='The Python script to run')
+    parser.add_argument('method_name', metavar='method-name', type=str,
+                        help='The method of the script to invoke')
     parser.add_argument('output_file', metavar='output-file', type=str,
                         help='The file to generate')
     parser.add_argument('additional_arguments', metavar='arg', nargs='*',
                         help="Additional arguments to the script's main() method")
 
     args = parser.parse_args(argv)
 
     script = args.python_script
     with open(script, 'r') as fh:
         module = imp.load_module('script', fh, script,
                                  ('.py', 'r', imp.PY_SOURCE))
-    if not hasattr(module, 'main'):
-        print('Error: script "{0}" is missing a main method'.format(script),
+    method = args.method_name
+    if not hasattr(module, method):
+        print('Error: script "{0}" is missing a {1} method'.format(script, method),
               file=sys.stderr)
         return 1
 
     ret = 1
     try:
         with FileAvoidWrite(args.output_file) as output:
-            ret = module.main(output, *args.additional_arguments)
+            ret = module.__dict__[method](output, *args.additional_arguments)
     except IOError as e:
         print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
         traceback.print_exc()
         return 1
     return ret
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -404,21 +404,22 @@ class RecursiveMakeBackend(CommonBackend
 
         elif isinstance(obj, Exports):
             self._process_exports(obj, obj.exports, backend_file)
 
         elif isinstance(obj, GeneratedFile):
             backend_file.write('GENERATED_FILES += %s\n' % obj.output)
             if obj.script:
                 backend_file.write("""{output}: {script}{inputs}
-\t$(call py_action,file_generate,{script} {output}{inputs})
+\t$(call py_action,file_generate,{script} {method} {output}{inputs})
 
 """.format(output=obj.output,
            inputs=' ' + ' '.join(obj.inputs) if obj.inputs else '',
-           script=obj.script))
+           script=obj.script,
+           method=obj.method))
 
         elif isinstance(obj, TestHarnessFiles):
             self._process_test_harness_files(obj, backend_file)
 
         elif isinstance(obj, Resources):
             self._process_resources(obj, obj.resources, backend_file)
 
         elif isinstance(obj, JsPreferenceFile):
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -521,16 +521,24 @@ VARIABLES = {
         generate.py with a open (for writing) file object for bar.c, and
         the string ``datafile-for-bar``. In a similar fashion, the main
         method of generate.py will also be called with an open
         (for writing) file object for foo.c and the string
         ``datafile-for-foo``. Please note that only string arguments are
         supported for passing to scripts, and that all arguments provided
         to the script should be filenames relative to the directory in which
         the moz.build file is located.
+
+        To enable using the same script for generating multiple files with
+        slightly different non-filename parameters, alternative entry points
+        into ``script`` can be specified::
+
+          GENERATED_FILES += ['bar.c']
+          bar = GENERATED_FILES['bar.c']
+          bar.script = 'generate.py:make_bar'
         """, 'export'),
 
     'DEFINES': (OrderedDict, dict,
         """Dictionary of compiler defines to declare.
 
         These are passed in to the compiler as ``-Dkey='value'`` for string
         values, ``-Dkey=value`` for numeric values, or ``-Dkey`` if the
         value is True. Note that for string values, the outer-level of
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -853,23 +853,25 @@ class FinalTargetFiles(ContextDerived):
         self.target = target
 
 
 class GeneratedFile(ContextDerived):
     """Represents a generated file."""
 
     __slots__ = (
         'script',
+        'method',
         'output',
         'inputs',
     )
 
-    def __init__(self, context, script, output, inputs):
+    def __init__(self, context, script, method, output, inputs):
         ContextDerived.__init__(self, context)
         self.script = script
+        self.method = method
         self.output = output
         self.inputs = inputs
 
 
 class ClassPathEntry(object):
     """Represents a classpathentry in an Android Eclipse project."""
 
     __slots__ = (
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -519,17 +519,23 @@ class TreeMetadataEmitter(LoggingMixin):
                 dist_install=not context.get('NO_DIST_INSTALL', False))
 
         generated_files = context.get('GENERATED_FILES')
         if generated_files:
             for f in generated_files:
                 flags = generated_files[f]
                 output = f
                 if flags.script:
-                    script = mozpath.join(context.srcdir, flags.script)
+                    method = "main"
+                    # Deal with cases like "C:\\path\\to\\script.py:function".
+                    if not flags.script.endswith('.py') and ':' in flags.script:
+                        script, method = flags.script.rsplit(':', 1)
+                    else:
+                        script = flags.script
+                    script = mozpath.join(context.srcdir, script)
                     inputs = [mozpath.join(context.srcdir, i) for i in flags.inputs]
 
                     if not os.path.exists(script):
                         raise SandboxValidationError(
                             'Script for generating %s does not exist: %s'
                             % (f, script), context)
                     if os.path.splitext(script)[1] != '.py':
                         raise SandboxValidationError(
@@ -537,18 +543,19 @@ class TreeMetadataEmitter(LoggingMixin):
                             % (f, script), context)
                     for i in inputs:
                         if not os.path.exists(i):
                             raise SandboxValidationError(
                                 'Input for generating %s does not exist: %s'
                                 % (f, i), context)
                 else:
                     script = None
+                    method = None
                     inputs = []
-                yield GeneratedFile(context, script, output, inputs)
+                yield GeneratedFile(context, script, method, output, inputs)
 
         test_harness_files = context.get('TEST_HARNESS_FILES')
         if test_harness_files:
             srcdir_files = defaultdict(list)
             srcdir_pattern_files = defaultdict(list)
             objdir_files = defaultdict(list)
 
             for path, strings in test_harness_files.walk():
--- a/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build
@@ -1,12 +1,12 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ]
 
 bar = GENERATED_FILES['bar.c']
-bar.script = 'generate-bar.py'
+bar.script = 'generate-bar.py:baz'
 
 foo = GENERATED_FILES['foo.c']
 foo.script = 'generate-foo.py'
 foo.inputs = ['foo-data']
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -374,21 +374,21 @@ class TestRecursiveMakeBackend(BackendTe
         env = self._consume('generated-files', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
             'GENERATED_FILES += bar.c',
             'bar.c: %s/generate-bar.py' % env.topsrcdir,
-            '$(call py_action,file_generate,%s/generate-bar.py bar.c)' % env.topsrcdir,
+            '$(call py_action,file_generate,%s/generate-bar.py baz bar.c)' % env.topsrcdir,
             '',
             'GENERATED_FILES += foo.c',
             'foo.c: %s/generate-foo.py %s/foo-data' % (env.topsrcdir, env.topsrcdir),
-            '$(call py_action,file_generate,%s/generate-foo.py foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir),
+            '$(call py_action,file_generate,%s/generate-foo.py main foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir),
             '',
             'GENERATED_FILES += quux.c',
         ]
 
         self.maxDiff = None
         self.assertEqual(lines, expected)
 
     def test_resources(self):
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build
@@ -0,0 +1,9 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ['bar.c']
+
+bar = GENERATED_FILES['bar.c']
+bar.script = TOPSRCDIR + '/script.py:make_bar'
+bar.inputs = []
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build
@@ -0,0 +1,13 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += [ 'bar.c', 'foo.c' ]
+
+bar = GENERATED_FILES['bar.c']
+bar.script = 'script.py:make_bar'
+bar.inputs = []
+
+foo = GENERATED_FILES['foo.c']
+foo.script = 'script.py'
+foo.inputs = []
new file mode 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -189,16 +189,47 @@ class TestEmitterBasic(unittest.TestCase
 
         self.assertEqual(len(objs), 2)
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)
 
         expected = ['bar.c', 'foo.c']
         for o, expected_filename in zip(objs, expected):
             self.assertEqual(o.output, expected_filename)
+            self.assertEqual(o.script, None)
+            self.assertEqual(o.method, None)
+            self.assertEqual(o.inputs, [])
+
+    def test_generated_files_method_names(self):
+        reader = self.reader('generated-files-method-names')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 2)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+
+        expected = ['bar.c', 'foo.c']
+        expected_method_names = ['make_bar', 'main']
+        for o, expected_filename, expected_method in zip(objs, expected, expected_method_names):
+            self.assertEqual(o.output, expected_filename)
+            self.assertEqual(o.method, expected_method)
+            self.assertEqual(o.inputs, [])
+
+    def test_generated_files_absolute_script(self):
+        reader = self.reader('generated-files-absolute-script')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 1)
+
+        o = objs[0]
+        self.assertIsInstance(o, GeneratedFile)
+        self.assertEqual(o.output, 'bar.c')
+        self.assertRegexpMatches(o.script, 'script.py$')
+        self.assertEqual(o.method, 'make_bar')
+        self.assertEqual(o.inputs, [])
 
     def test_generated_files_no_script(self):
         reader = self.reader('generated-files-no-script')
         with self.assertRaisesRegexp(SandboxValidationError,
             'Script for generating bar.c does not exist'):
             objs = self.read_topsrcdir(reader)
 
     def test_generated_files_no_inputs(self):