Bug 1041941 - Add support for templates in moz.build. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Sun, 24 Aug 2014 09:11:05 +0900
changeset 224208 313a81600191c795d8e6108dd1f5c53e04b15c00
parent 224207 0898cd98bb28c9369336182d231081894828b6c3
child 224209 62b9f6d4328bb3609e731c694aabf9ad2ba9406d
push id583
push userbhearsum@mozilla.com
push dateMon, 24 Nov 2014 19:04:58 +0000
treeherdermozilla-release@c107e74250f4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1041941
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1041941 - Add support for templates in moz.build. r=gps
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/data/templates/templates.mozbuild
python/mozbuild/mozbuild/test/frontend/test_sandbox.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -945,22 +945,23 @@ for name, (storage_type, input_types, do
     if storage_type == list:
         raise RuntimeError('%s has a "list" storage type. Use "List" instead.'
             % name)
 
 # The set of functions exposed to the sandbox.
 #
 # Each entry is a tuple of:
 #
-#  (method attribute, (argument types), docs)
+#  (function returning the corresponding function from a given sandbox,
+#   (argument types), docs)
 #
 # The first element is an attribute on Sandbox that should be a function type.
 #
 FUNCTIONS = {
-    'include': ('_include', (str,),
+    'include': (lambda self: self._include, (str,),
         """Include another mozbuild file in the context of this one.
 
         This is similar to a ``#include`` in C languages. The filename passed to
         the function will be read and its contents will be evaluated within the
         context of the calling file.
 
         If a relative path is given, it is evaluated as relative to the file
         currently being processed. If there is a chain of multiple include(),
@@ -977,57 +978,61 @@ FUNCTIONS = {
 
            include('sibling.build')
 
         Include ``foo.build`` from a path within the top source directory::
 
            include('/elsewhere/foo.build')
         """),
 
-    'add_java_jar': ('_add_java_jar', (str,),
+    'add_java_jar': (lambda self: self._add_java_jar, (str,),
         """Declare a Java JAR target to be built.
 
         This is the supported way to populate the JAVA_JAR_TARGETS
         variable.
 
         The parameters are:
         * dest - target name, without the trailing .jar. (required)
 
         This returns a rich Java JAR type, described at
         :py:class:`mozbuild.frontend.data.JavaJarData`.
         """),
 
-    'add_android_eclipse_project': ('_add_android_eclipse_project', (str, str),
+    'add_android_eclipse_project': (
+        lambda self: self._add_android_eclipse_project, (str, str),
         """Declare an Android Eclipse project.
 
         This is one of the supported ways to populate the
         ANDROID_ECLIPSE_PROJECT_TARGETS variable.
 
         The parameters are:
         * name - project name.
         * manifest - path to AndroidManifest.xml.
 
         This returns a rich Android Eclipse project type, described at
         :py:class:`mozbuild.frontend.data.AndroidEclipseProjectData`.
         """),
 
-    'add_android_eclipse_library_project': ('_add_android_eclipse_library_project', (str,),
+    'add_android_eclipse_library_project': (
+        lambda self: self._add_android_eclipse_library_project, (str,),
         """Declare an Android Eclipse library project.
 
         This is one of the supported ways to populate the
         ANDROID_ECLIPSE_PROJECT_TARGETS variable.
 
         The parameters are:
         * name - project name.
 
         This returns a rich Android Eclipse project type, described at
         :py:class:`mozbuild.frontend.data.AndroidEclipseProjectData`.
         """),
 
-    'add_tier_dir': ('_add_tier_directory', (str, [str, list], bool, bool, str),
+    'add_tier_dir': (
+        lambda self: self._add_tier_directory,
+        (str, [str, list], bool, bool, str),
         """Register a directory for tier traversal.
 
         This is the preferred way to populate the TIERS variable.
 
         Tiers are how the build system is organized. The build process is
         divided into major phases called tiers. The most important tiers are
         "platform" and "apps." The platform tier builds the Gecko platform
         (typically outputting libxul). The apps tier builds the configured
@@ -1052,17 +1057,17 @@ FUNCTIONS = {
            add_tier_dir('app', ['components', 'base'])
 
         Register a directory as having external content (no dependencies,
         and traversed with export, libs, and tools subtiers::
 
            add_tier_dir('base', 'bar', external=True)
         """),
 
-    'export': ('_export', (str,),
+    'export': (lambda self: self._export, (str,),
         """Make the specified variable available to all child directories.
 
         The variable specified by the argument string is added to the
         environment of all directories specified in the DIRS and TEST_DIRS
         variables. If those directories themselves have child directories,
         the variable will be exported to all of them.
 
         The value used for the variable is the final value at the end of the
@@ -1079,29 +1084,76 @@ FUNCTIONS = {
         ^^^^^^^^^^^^^
 
         To make all children directories install as the given extension::
 
           XPI_NAME = 'cool-extension'
           export('XPI_NAME')
         """),
 
-    'warning': ('_warning', (str,),
+    'warning': (lambda self: self._warning, (str,),
         """Issue a warning.
 
         Warnings are string messages that are printed during execution.
 
         Warnings are ignored during execution.
         """),
 
-    'error': ('_error', (str,),
+    'error': (lambda self: self._error, (str,),
         """Issue a fatal error.
 
         If this function is called, processing is aborted immediately.
         """),
+
+    'template': (lambda self: self._template_decorator, (),
+        """Decorator for template declarations.
+
+        Templates are a special kind of functions that can be declared in
+        mozbuild files. Uppercase variables assigned in the function scope
+        are considered to be the result of the template.
+
+        Contrary to traditional python functions:
+           - return values from template functions are ignored,
+           - template functions don't have access to the global scope.
+
+        Example template
+        ^^^^^^^^^^^^^^^^
+
+        The following ``Program`` template sets two variables ``PROGRAM`` and
+        ``USE_LIBS``. ``PROGRAM`` is set to the argument given on the template
+        invocation, and ``USE_LIBS`` to contain "mozglue"::
+
+           @template
+           def Program(name):
+               PROGRAM = name
+               USE_LIBS += ['mozglue']
+
+        Template invocation
+        ^^^^^^^^^^^^^^^^^^^
+
+        A template is invoked in the form of a function call::
+
+           Program('myprog')
+
+        The result of the template, being all the uppercase variable it sets
+        is mixed to the existing set of variables defined in the mozbuild file
+        invoking the template::
+
+           FINAL_TARGET = 'dist/other'
+           USE_LIBS += ['mylib']
+           Program('myprog')
+           USE_LIBS += ['otherlib']
+
+        The above mozbuild results in the following variables set:
+
+           - ``FINAL_TARGET`` is 'dist/other'
+           - ``USE_LIBS`` is ['mylib', 'mozglue', 'otherlib']
+           - ``PROGRAM`` is 'myprog'
+
+        """),
 }
 
 # Special variables. These complement VARIABLES.
 #
 # Each entry is a tuple of:
 #
 #  (function returning the corresponding value from a given context, type, docs)
 #
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -13,27 +13,30 @@ this file, which is represented by the S
 to fill a Context, representing the output of an individual mozbuild file. The
 
 The BuildReader contains basic logic for traversing a tree of mozbuild files.
 It does this by examining specific variables populated during execution.
 """
 
 from __future__ import print_function, unicode_literals
 
+import inspect
 import logging
 import os
 import sys
 import time
+import tokenize
 import traceback
 import types
 
 from collections import OrderedDict
 from io import StringIO
 
 from mozbuild.util import (
+    memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
 )
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 
 from mozpack.files import FileFinder
 import mozpack.path as mozpath
@@ -121,22 +124,25 @@ class MozbuildSandbox(Sandbox):
         Sandbox.__init__(self, context)
 
         self._log = logging.getLogger(__name__)
 
         self.metadata = dict(metadata)
         exports = self.metadata.get('exports', {})
         self.exports = set(exports.keys())
         context.update(exports)
+        self.templates = self.metadata.setdefault('templates', {})
 
     def __getitem__(self, key):
         if key in SPECIAL_VARIABLES:
             return SPECIAL_VARIABLES[key][0](self._context)
         if key in FUNCTIONS:
-            return getattr(self, FUNCTIONS[key][0])
+            return FUNCTIONS[key][0](self)
+        if key in self.templates:
+            return self._create_template_function(self.templates[key])
         return Sandbox.__getitem__(self, key)
 
     def __setitem__(self, key, value):
         if key in SPECIAL_VARIABLES or key in FUNCTIONS:
             raise KeyError()
         if key in self.exports:
             self._context[key] = value
             self.exports.remove(key)
@@ -293,16 +299,116 @@ class MozbuildSandbox(Sandbox):
 
     def _warning(self, message):
         # FUTURE consider capturing warnings in a variable instead of printing.
         print('WARNING: %s' % message, file=sys.stderr)
 
     def _error(self, message):
         raise SandboxCalledError(self._execution_stack, message)
 
+    def _template_decorator(self, func):
+        """Registers template as expected by _create_template_function.
+
+        The template data consists of:
+        - the function object as it comes from the sandbox evaluation of the
+          template declaration.
+        - its code, modified as described in the comments of this method.
+        - the path of the file containing the template definition.
+        """
+
+        if not inspect.isfunction(func):
+            raise Exception('`template` is a function decorator. You must '
+                'use it as `@template` preceding a function declaration.')
+
+        name = func.func_name
+
+        if name in self.templates:
+            raise KeyError(
+                'A template named "%s" was already declared in %s.' % (name,
+                self.templates[name][2]))
+
+        if name.islower() or name.isupper() or name[0].islower():
+            raise NameError('Template function names must be CamelCase.')
+
+        lines, firstlineno = inspect.getsourcelines(func)
+        first_op = None
+        generator = tokenize.generate_tokens(iter(lines).next)
+        # Find the first indent token in the source of this template function,
+        # which corresponds to the beginning of the function body.
+        for typ, s, begin, end, line in generator:
+            if typ == tokenize.OP:
+                first_op = True
+            if first_op and typ == tokenize.INDENT:
+                break
+        if typ != tokenize.INDENT:
+            # This should never happen.
+            raise Exception('Could not find the first line of the template %s' %
+                func.func_name)
+        # The code of the template in moz.build looks like this:
+        # m      def Foo(args):
+        # n          FOO = 'bar'
+        # n+1        (...)
+        #
+        # where,
+        # - m is firstlineno - 1,
+        # - n is usually m + 1, but in case the function signature takes more
+        # lines, is really m + begin[0] - 1
+        #
+        # We want that to be replaced with:
+        # m       if True:
+        # n           FOO = 'bar'
+        # n+1         (...)
+        #
+        # (this is simpler than trying to deindent the function body)
+        # So we need to prepend with n - 1 newlines so that line numbers
+        # are unchanged.
+        code = '\n' * (firstlineno + begin[0] - 3) + 'if True:\n'
+        code += ''.join(lines[begin[0] - 1:])
+
+        self.templates[name] = func, code, self._execution_stack[-1]
+
+    @memoize
+    def _create_template_function(self, template):
+        """Returns a function object for use within the sandbox for the given
+        template.
+
+        When a moz.build file contains a reference to a template call, the
+        sandbox needs a function to execute. This is what this method returns.
+        That function creates a new sandbox for execution of the template.
+        After the template is executed, the data from its execution is merged
+        with the context of the calling sandbox.
+        """
+        func, code, path = template
+
+        def template_function(*args, **kwargs):
+            context = Context(VARIABLES, self._context.config)
+            context.add_source(self._execution_stack[-1])
+            for p in self._context.all_paths:
+                context.add_source(p)
+
+            sandbox = MozbuildSandbox(context, self.metadata)
+            for k, v in inspect.getcallargs(func, *args, **kwargs).items():
+                sandbox[k] = v
+
+            sandbox.exec_source(code, path)
+
+            # The sandbox will do all the necessary checks for these merges.
+            for key, value in context.items():
+                if isinstance(value, dict):
+                    self[key].update(value)
+                elif isinstance(value, list):
+                    self[key] += value
+                else:
+                    self[key] = value
+
+            for p in context.all_paths:
+                self._context.add_source(p)
+
+        return template_function
+
 
 class SandboxValidationError(Exception):
     """Represents an error encountered when validating sandbox results."""
     def __init__(self, message, context):
         Exception.__init__(self, message)
         self.context = context
 
     def __str__(self):
@@ -841,16 +947,19 @@ class BuildReader(object):
         for var, var_dirs in dirs:
             for d in var_dirs:
                 if d in recurse_info:
                     raise SandboxValidationError(
                         'Directory (%s) registered multiple times in %s' % (
                             d, var), context)
 
                 recurse_info[d] = {}
+                if 'templates' in sandbox.metadata:
+                    recurse_info[d]['templates'] = dict(
+                        sandbox.metadata['templates'])
                 if 'exports' in sandbox.metadata:
                     sandbox.recompute_exports()
                     recurse_info[d]['exports'] = dict(sandbox.metadata['exports'])
 
         # We also have tiers whose members are directories.
         if 'TIERS' in context:
             if not read_tiers:
                 raise SandboxValidationError(
@@ -860,16 +969,19 @@ class BuildReader(object):
                 # We don't descend into external directories because external by
                 # definition is external to the build system.
                 for d in values['regular']:
                     if d in recurse_info:
                         raise SandboxValidationError(
                             'Tier directory (%s) registered multiple '
                             'times in %s' % (d, tier), context)
                     recurse_info[d] = {'check_external': True}
+                    if 'templates' in sandbox.metadata:
+                        recurse_info[d]['templates'] = dict(
+                            sandbox.metadata['templates'])
 
         for relpath, child_metadata in recurse_info.items():
             if 'check_external' in child_metadata:
                 relpath = '/' + relpath
             child_path = sandbox.normalize_path(mozpath.join(relpath,
                 'moz.build'), srcdir=curdir)
 
             # Ensure we don't break out of the topsrcdir. We don't do realpath
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/templates/templates.mozbuild
@@ -0,0 +1,21 @@
+@template
+def Template(foo, bar=[]):
+    SOURCES += foo
+    DIRS += bar
+
+@template
+def TemplateError(foo):
+    ILLEGAL = foo
+
+@template
+def TemplateGlobalVariable():
+    SOURCES += illegal
+
+@template
+def TemplateGlobalUPPERVariable():
+    SOURCES += DIRS
+
+@template
+def TemplateInherit(foo):
+    USE_LIBS += ['foo']
+    Template(foo)
--- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
@@ -115,25 +115,25 @@ class TestSandbox(unittest.TestCase):
         e = se.exception
         self.assertIsInstance(e.exc_value, KeyError)
 
         e = se.exception.exc_value
         self.assertEqual(e.args[0], 'Cannot reassign builtins')
 
 
 class TestMozbuildSandbox(unittest.TestCase):
-    def sandbox(self, data_path=None):
+    def sandbox(self, data_path=None, metadata={}):
         config = None
 
         if data_path is not None:
             config = MockConfig(mozpath.join(test_data_path, data_path))
         else:
             config = MockConfig()
 
-        return MozbuildSandbox(Context(VARIABLES, config))
+        return MozbuildSandbox(Context(VARIABLES, config), metadata)
 
     def test_default_state(self):
         sandbox = self.sandbox()
         sandbox._context.add_source(sandbox.normalize_path('moz.build'))
         config = sandbox._context.config
 
         self.assertEqual(sandbox['TOPSRCDIR'], config.topsrcdir)
         self.assertEqual(sandbox['TOPOBJDIR'], config.topobjdir)
@@ -353,10 +353,144 @@ add_tier_dir('t1', 'bat')
     def test_invalid_exports_set_base(self):
         sandbox = self.sandbox()
 
         with self.assertRaises(SandboxExecutionError) as se:
             sandbox.exec_source('EXPORTS = "foo.h"')
 
         self.assertEqual(se.exception.exc_type, ValueError)
 
+    def test_templates(self):
+        sandbox = self.sandbox(data_path='templates')
+
+        # Templates need to be defined in actual files because of
+        # inspect.getsourcelines.
+        sandbox.exec_file('templates.mozbuild')
+
+        sandbox2 = self.sandbox(metadata={'templates': sandbox.templates})
+        source = '''
+Template([
+    'foo.cpp',
+])
+'''
+        sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        self.assertEqual(sandbox2._context, {
+            'SOURCES': ['foo.cpp'],
+            'DIRS': [],
+        })
+
+        sandbox2 = self.sandbox(metadata={'templates': sandbox.templates})
+        source = '''
+SOURCES += ['qux.cpp']
+Template([
+    'bar.cpp',
+    'foo.cpp',
+],[
+    'foo',
+])
+SOURCES += ['hoge.cpp']
+'''
+        sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        self.assertEqual(sandbox2._context, {
+            'SOURCES': ['qux.cpp', 'bar.cpp', 'foo.cpp', 'hoge.cpp'],
+            'DIRS': ['foo'],
+        })
+
+        source = '''
+TemplateError([
+    'foo.cpp',
+])
+'''
+        with self.assertRaises(SandboxExecutionError) as se:
+            sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        e = se.exception
+        self.assertIsInstance(e.exc_value, KeyError)
+
+        e = se.exception.exc_value
+        self.assertEqual(e.args[0], 'global_ns')
+        self.assertEqual(e.args[1], 'set_unknown')
+
+        # TemplateGlobalVariable tries to access 'illegal' but that is expected
+        # to throw.
+        source = '''
+illegal = True
+TemplateGlobalVariable()
+'''
+        with self.assertRaises(SandboxExecutionError) as se:
+            sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        e = se.exception
+        self.assertIsInstance(e.exc_value, NameError)
+
+        # TemplateGlobalUPPERVariable sets SOURCES with DIRS, but the context
+        # used when running the template is not expected to access variables
+        # from the global context.
+        sandbox2 = self.sandbox(metadata={'templates': sandbox.templates})
+        source = '''
+DIRS += ['foo']
+TemplateGlobalUPPERVariable()
+'''
+        sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+        self.assertEqual(sandbox2._context, {
+            'SOURCES': [],
+            'DIRS': ['foo'],
+        })
+
+        # However, the result of the template is mixed with the global
+        # context.
+        sandbox2 = self.sandbox(metadata={'templates': sandbox.templates})
+        source = '''
+SOURCES += ['qux.cpp']
+TemplateInherit([
+    'bar.cpp',
+    'foo.cpp',
+])
+SOURCES += ['hoge.cpp']
+'''
+        sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        self.assertEqual(sandbox2._context, {
+            'SOURCES': ['qux.cpp', 'bar.cpp', 'foo.cpp', 'hoge.cpp'],
+            'USE_LIBS': ['foo'],
+            'DIRS': [],
+        })
+
+        # Template names must be CamelCase. Here, we can define the template
+        # inline because the error happens before inspect.getsourcelines.
+        source = '''
+@template
+def foo():
+    pass
+'''
+
+        with self.assertRaises(SandboxExecutionError) as se:
+            sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        e = se.exception
+        self.assertIsInstance(e.exc_value, NameError)
+
+        e = se.exception.exc_value
+        self.assertEqual(e.message,
+            'Template function names must be CamelCase.')
+
+        # Template names must not already be registered.
+        source = '''
+@template
+def Template():
+    pass
+'''
+        with self.assertRaises(SandboxExecutionError) as se:
+            sandbox2.exec_source(source, sandbox.normalize_path('foo.mozbuild'))
+
+        e = se.exception
+        self.assertIsInstance(e.exc_value, KeyError)
+
+        e = se.exception.exc_value
+        self.assertEqual(e.message,
+            'A template named "Template" was already declared in %s.' %
+            sandbox.normalize_path('templates.mozbuild'))
+
+
 if __name__ == '__main__':
     main()