Bug 1231315 - Build CONFIGURE_DEFINE_FILES at build time instead of during configure. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 08 Dec 2015 20:21:09 +0900
changeset 276370 75e62a17dbba803107221515cbe3fab7f540fcdf
parent 276369 3033ec8d8b3aadcef610dbdf6f5d5110dd9027c0
child 276371 08dca8061836a3ef43ba18ea0b2853b2945952b6
push id69144
push usermh@glandium.org
push dateMon, 14 Dec 2015 23:06:53 +0000
treeherdermozilla-inbound@75e62a17dbba [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1231315
milestone45.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1231315 - Build CONFIGURE_DEFINE_FILES at build time instead of during configure. r=gps
python/mozbuild/mozbuild/action/process_define_files.py
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/data/test_config/file.h.in
python/mozbuild/mozbuild/test/backend/data/test_config/moz.build
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/process_define_files.py
@@ -0,0 +1,94 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import os
+import re
+import sys
+from buildconfig import topobjdir
+from mozbuild.backend.configenvironment import ConfigEnvironment
+from mozbuild.util import FileAvoidWrite
+import mozpack.path as mozpath
+
+
+def process_define_file(output, input):
+    '''Creates the given config header. A config header is generated by
+    taking the corresponding source file and replacing some #define/#undef
+    occurences:
+        "#undef NAME" is turned into "#define NAME VALUE"
+        "#define NAME" is unchanged
+        "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE"
+        "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */"
+        Whitespaces are preserved.
+
+    As a special rule, "#undef ALLDEFINES" is turned into "#define NAME
+    VALUE" for all the defined variables.
+    '''
+
+    path = os.path.abspath(input)
+
+    config = ConfigEnvironment.from_config_status(
+        mozpath.join(topobjdir, 'config.status'))
+
+    if mozpath.basedir(path,
+                       [mozpath.join(config.topsrcdir, 'js/src')]) and \
+            not config.substs.get('JS_STANDALONE'):
+        config = ConfigEnvironment.from_config_status(
+            mozpath.join(topobjdir, 'js', 'src', 'config.status'))
+
+    with open(path, 'rU') as input:
+        r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U)
+        for l in input:
+            m = r.match(l)
+            if m:
+                cmd = m.group('cmd')
+                name = m.group('name')
+                value = m.group('value')
+                if name:
+                    if name == 'ALLDEFINES':
+                        if cmd == 'define':
+                            raise Exception(
+                                '`#define ALLDEFINES` is not allowed in a '
+                                'CONFIGURE_DEFINE_FILE')
+                        defines = '\n'.join(sorted(
+                            '#define %s %s' % (name, val)
+                            for name, val in config.defines.iteritems()
+                            if name not in config.non_global_defines))
+                        l = l[:m.start('cmd') - 1] \
+                            + defines + l[m.end('name'):]
+                    elif name in config.defines:
+                        if cmd == 'define' and value:
+                            l = l[:m.start('value')] \
+                                + str(config.defines[name]) \
+                                + l[m.end('value'):]
+                        elif cmd == 'undef':
+                            l = l[:m.start('cmd')] \
+                                + 'define' \
+                                + l[m.end('cmd'):m.end('name')] \
+                                + ' ' \
+                                + str(config.defines[name]) \
+                                + l[m.end('name'):]
+                    elif cmd == 'undef':
+                       l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):]
+
+            output.write(l)
+
+    return {config.source}
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Process define files.')
+
+    parser.add_argument('input', help='Input define file.')
+
+    args = parser.parse_args(argv)
+
+    return process_define_file(sys.stdout, args.input)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -12,17 +12,16 @@ import re
 import mozpack.path as mozpath
 import mozwebidlcodegen
 
 from .base import BuildBackend
 
 from ..frontend.data import (
     ConfigFileSubstitution,
     ExampleWebIDLInterface,
-    HeaderFileSubstitution,
     IPDLFile,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     TestManifest,
     TestWebIDLFile,
     UnifiedSources,
@@ -206,20 +205,16 @@ class CommonBackend(BuildBackend):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
             # to other
             if mozpath.basename(obj.output_path) == 'Makefile':
                 return False
             with self._get_preprocessor(obj) as pp:
                 pp.do_include(obj.input_path)
             self.backend_input_files.add(obj.input_path)
 
-        elif isinstance(obj, HeaderFileSubstitution):
-            self._create_config_header(obj)
-            self.backend_input_files.add(obj.input_path)
-
         # We should consider aggregating WebIDL types in emitter.py.
         elif isinstance(obj, WebIDLFile):
             self._webidls.sources.add(mozpath.join(obj.srcdir, obj.basename))
 
         elif isinstance(obj, GeneratedEventWebIDLFile):
             self._webidls.generated_events_sources.add(mozpath.join(
                 obj.srcdir, obj.basename))
 
@@ -367,59 +362,8 @@ class CommonBackend(BuildBackend):
             f.write('\n'.join(includeTemplate % { "cppfile": s } for
                               s in source_filenames))
 
     def _write_unified_files(self, unified_source_mapping, output_directory,
                              poison_windows_h=False):
         for unified_file, source_filenames in unified_source_mapping:
             self._write_unified_file(unified_file, source_filenames,
                                      output_directory, poison_windows_h)
-
-    def _create_config_header(self, obj):
-        '''Creates the given config header. A config header is generated by
-        taking the corresponding source file and replacing some #define/#undef
-        occurences:
-            "#undef NAME" is turned into "#define NAME VALUE"
-            "#define NAME" is unchanged
-            "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE"
-            "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */"
-            Whitespaces are preserved.
-
-        As a special rule, "#undef ALLDEFINES" is turned into "#define NAME
-        VALUE" for all the defined variables.
-        '''
-        with self._write_file(obj.output_path) as fh, \
-             open(obj.input_path, 'rU') as input:
-            r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U)
-            for l in input:
-                m = r.match(l)
-                if m:
-                    cmd = m.group('cmd')
-                    name = m.group('name')
-                    value = m.group('value')
-                    if name:
-                        if name == 'ALLDEFINES':
-                            if cmd == 'define':
-                                raise Exception(
-                                    '`#define ALLDEFINES` is not allowed in a '
-                                    'CONFIGURE_DEFINE_FILE')
-                            defines = '\n'.join(sorted(
-                                '#define %s %s' % (name, val)
-                                for name, val in obj.config.defines.iteritems()
-                                if name not in obj.config.non_global_defines))
-                            l = l[:m.start('cmd') - 1] \
-                                + defines + l[m.end('name'):]
-                        elif name in obj.config.defines:
-                            if cmd == 'define' and value:
-                                l = l[:m.start('value')] \
-                                    + str(obj.config.defines[name]) \
-                                    + l[m.end('value'):]
-                            elif cmd == 'undef':
-                                l = l[:m.start('cmd')] \
-                                    + 'define' \
-                                    + l[m.end('cmd'):m.end('name')] \
-                                    + ' ' \
-                                    + str(obj.config.defines[name]) \
-                                    + l[m.end('name'):]
-                        elif cmd == 'undef':
-                           l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):]
-
-                fh.write(l)
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -122,20 +122,16 @@ class BaseConfigSubstitution(ContextDeri
         self.output_path = None
         self.relpath = None
 
 
 class ConfigFileSubstitution(BaseConfigSubstitution):
     """Describes a config file that will be generated using substitutions."""
 
 
-class HeaderFileSubstitution(BaseConfigSubstitution):
-    """Describes a header file that will be generated using substitutions."""
-
-
 class VariablePassthru(ContextDerived):
     """A dict of variables to pass through to backend.mk unaltered.
 
     The purpose of this object is to facilitate rapid transitioning of
     variables from Makefile.in to moz.build. In the ideal world, this class
     does not exist and every variable has a richer class representing it.
     As long as we rely on this class, we lose the ability to have flexibility
     in our build backends since we will continue to be tied to our rules.mk.
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -39,17 +39,16 @@ from .data import (
     FinalTargetPreprocessedFiles,
     GeneratedEventWebIDLFile,
     GeneratedFile,
     GeneratedSources,
     GeneratedWebIDLFile,
     ExampleWebIDLInterface,
     ExternalStaticLibrary,
     ExternalSharedLibrary,
-    HeaderFileSubstitution,
     HostDefines,
     HostLibrary,
     HostProgram,
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     IPDLFile,
     JARManifest,
@@ -544,18 +543,20 @@ class TreeMetadataEmitter(LoggingMixin):
         # the recursive make backend.
         for o in self._emit_directory_traversal_from_context(context): yield o
 
         for path in context['CONFIGURE_SUBST_FILES']:
             yield self._create_substitution(ConfigFileSubstitution, context,
                 path)
 
         for path in context['CONFIGURE_DEFINE_FILES']:
-            yield self._create_substitution(HeaderFileSubstitution, context,
-                path)
+            script = mozpath.join(mozpath.dirname(mozpath.dirname(__file__)),
+                                  'action', 'process_define_files.py')
+            yield GeneratedFile(context, script, 'process_define_file', path,
+                                [mozpath.join(context.srcdir, path + '.in')])
 
         for obj in self._process_xpidl(context):
             yield obj
 
         # Proxy some variables as-is until we have richer classes to represent
         # them. We should aim to keep this set small because it violates the
         # desired abstraction of the build definition away from makefiles.
         passthru = VariablePassthru(context)
deleted file mode 100644
--- a/python/mozbuild/mozbuild/test/backend/data/test_config/file.h.in
+++ /dev/null
@@ -1,15 +0,0 @@
-/* Comment */
-#define foo
-#define foo 42
-#undef foo
-#define bar
-#define bar 42
-#undef bar
-
-# undef baz
-
-#ifdef foo
-#   undef   foo
-#  define foo    42
-  #     define   foo   42   
-#endif
--- a/python/mozbuild/mozbuild/test/backend/data/test_config/moz.build
+++ b/python/mozbuild/mozbuild/test/backend/data/test_config/moz.build
@@ -1,6 +1,3 @@
 CONFIGURE_SUBST_FILES = [
     'file',
 ]
-CONFIGURE_DEFINE_FILES = [
-    'file.h',
-]
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -693,46 +693,26 @@ class TestRecursiveMakeBackend(BackendTe
             'DIST_FILES_0_PATH := $(DEPTH)/dist/bin/',
             'PP_TARGETS += DIST_FILES_0',
         ]
 
         found = [str for str in lines if 'DIST_FILES' in str]
         self.assertEqual(found, expected)
 
     def test_config(self):
-        """Test that CONFIGURE_SUBST_FILES and CONFIGURE_DEFINE_FILES are
-        properly handled."""
+        """Test that CONFIGURE_SUBST_FILES are properly handled."""
         env = self._consume('test_config', RecursiveMakeBackend)
 
         self.assertEqual(
             open(os.path.join(env.topobjdir, 'file'), 'r').readlines(), [
                 '#ifdef foo\n',
                 'bar baz\n',
                 '@bar@\n',
             ])
 
-        self.assertEqual(
-            open(os.path.join(env.topobjdir, 'file.h'), 'r').readlines(), [
-                '/* Comment */\n',
-                '#define foo\n',
-                '#define foo baz qux\n',
-                '#define foo baz qux\n',
-                '#define bar\n',
-                '#define bar 42\n',
-                '/* #undef bar */\n',
-                '\n',
-                '# define baz 1\n',
-                '\n',
-                '#ifdef foo\n',
-                '#   define   foo baz qux\n',
-                '#  define foo    baz qux\n',
-                '  #     define   foo   baz qux   \n',
-                '#endif\n',
-            ])
-
     def test_jar_manifests(self):
         env = self._consume('jar-manifests', RecursiveMakeBackend)
 
         with open(os.path.join(env.topobjdir, 'backend.mk'), 'rb') as fh:
             lines = fh.readlines()
 
         lines = [line.rstrip() for line in lines]