bug 1198226 - Add HOST_{CFLAGS,CXXFLAGS,DEFINES} support to mozbuild frontend+recursive make backend. r=mshal
authorTed Mielczarek <ted@mielczarek.org>
Fri, 04 Sep 2015 13:33:04 -0400
changeset 261554 444ea60893cde03b26b6eb2d7af33aef5d968a06
parent 261553 53ac5605d91a66c3faa63ce21d7caab41170af6a
child 261555 3492202f465e1572cdbc4cf930b31f10e7735ab2
push id64766
push usertmielczarek@mozilla.com
push dateWed, 09 Sep 2015 12:37:45 +0000
treeherdermozilla-inbound@a18a6dca709b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1198226
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1198226 - Add HOST_{CFLAGS,CXXFLAGS,DEFINES} support to mozbuild frontend+recursive make backend. r=mshal
config/config.mk
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build
python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build
python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/config/config.mk
+++ b/config/config.mk
@@ -423,16 +423,19 @@ COMPILE_CXXFLAGS = $(if $(DISABLE_STL_WR
 COMPILE_CMFLAGS = $(OS_COMPILE_CMFLAGS) $(MOZBUILD_CMFLAGS)
 COMPILE_CMMFLAGS = $(OS_COMPILE_CMMFLAGS) $(MOZBUILD_CMMFLAGS)
 ASFLAGS += $(MOZBUILD_ASFLAGS)
 
 ifndef CROSS_COMPILE
 HOST_CFLAGS += $(RTL_FLAGS)
 endif
 
+HOST_CFLAGS += $(HOST_DEFINES) $(MOZBUILD_HOST_CFLAGS)
+HOST_CXXFLAGS += $(HOST_DEFINES) $(MOZBUILD_HOST_CXXFLAGS)
+
 #
 # Name of the binary code directories
 #
 # Override defaults
 
 # Default location of include files
 ifndef LIBXUL_SDK
 IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -39,16 +39,17 @@ from ..frontend.data import (
     DistFiles,
     DirectoryTraversal,
     Exports,
     ExternalLibrary,
     FinalTargetFiles,
     GeneratedFile,
     GeneratedInclude,
     GeneratedSources,
+    HostDefines,
     HostLibrary,
     HostProgram,
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     JARManifest,
     JavaJarData,
     JavaScriptModules,
@@ -499,17 +500,18 @@ class RecursiveMakeBackend(CommonBackend
                 if isinstance(v, list):
                     for item in v:
                         backend_file.write('%s += %s\n' % (k, item))
                 elif isinstance(v, bool):
                     if v:
                         backend_file.write('%s := 1\n' % k)
                 else:
                     backend_file.write('%s := %s\n' % (k, v))
-
+        elif isinstance(obj, HostDefines):
+            self._process_defines(obj, backend_file, which='HOST_DEFINES')
         elif isinstance(obj, Defines):
             self._process_defines(obj, backend_file)
 
         elif isinstance(obj, Exports):
             self._process_exports(obj, obj.exports, backend_file)
 
         elif isinstance(obj, GeneratedFile):
             backend_file.write('GENERATED_FILES += %s\n' % obj.output)
@@ -940,21 +942,21 @@ class RecursiveMakeBackend(CommonBackend
                        or None if there is no such dictionary.
         """
         for path, strings in element.walk():
             for s in strings:
                 source = mozpath.normpath(mozpath.join(obj.srcdir, s))
                 dest = mozpath.join(namespace, path, mozpath.basename(s))
                 yield source, dest, strings.flags_for(s)
 
-    def _process_defines(self, obj, backend_file):
+    def _process_defines(self, obj, backend_file, which='DEFINES'):
         """Output the DEFINES rules to the given backend file."""
         defines = list(obj.get_defines())
         if defines:
-            backend_file.write('DEFINES +=')
+            backend_file.write(which + ' +=')
             for define in defines:
                 backend_file.write(' %s' % define)
             backend_file.write('\n')
 
     def _process_exports(self, obj, exports, backend_file):
         # This may not be needed, but is present for backwards compatibility
         # with the old make rules, just in case.
         if not obj.dist_install:
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -1514,16 +1514,21 @@ VARIABLES = {
         """Flags passed to the C++ compiler for all of the C++ source files
            declared in this directory.
 
            Note that the ordering of flags matters here; these flags will be
            added to the compiler's command line in the same order as they
            appear in the moz.build file.
         """, None),
 
+    'HOST_DEFINES': (OrderedDict, dict,
+        """Dictionary of compiler defines to declare for host compilation.
+        See ``DEFINES`` for specifics.
+        """, None),
+
     'CMFLAGS': (List, list,
         """Flags passed to the Objective-C compiler for all of the Objective-C
            source files declared in this directory.
 
            Note that the ordering of flags matters here; these flags will be
            added to the compiler's command line in the same order as they
            appear in the moz.build file.
         """, None),
@@ -1541,16 +1546,34 @@ VARIABLES = {
         """Flags passed to the assembler for all of the assembly source files
            declared in this directory.
 
            Note that the ordering of flags matters here; these flags will be
            added to the assembler's command line in the same order as they
            appear in the moz.build file.
         """, None),
 
+    'HOST_CFLAGS': (List, list,
+        """Flags passed to the host C compiler for all of the C source files
+           declared in this directory.
+
+           Note that the ordering of flags matters here, these flags will be
+           added to the compiler's command line in the same order as they
+           appear in the moz.build file.
+        """, None),
+
+    'HOST_CXXFLAGS': (List, list,
+        """Flags passed to the host C++ compiler for all of the C++ source files
+           declared in this directory.
+
+           Note that the ordering of flags matters here; these flags will be
+           added to the compiler's command line in the same order as they
+           appear in the moz.build file.
+        """, None),
+
     'LDFLAGS': (List, list,
         """Flags passed to the linker when linking all of the libraries and
            executables declared in this directory.
 
            Note that the ordering of flags matters here; these flags will be
            added to the linker's command line in the same order as they
            appear in the moz.build file.
         """, None),
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -168,18 +168,19 @@ class XPIDLFile(ContextDerived):
 
         self.source_path = source
         self.basename = mozpath.basename(source)
         self.module = module
         self.add_to_manifest = add_to_manifest
 
         self.install_target = context['FINAL_TARGET']
 
-class Defines(ContextDerived):
-    """Context derived container object for DEFINES, which is an OrderedDict.
+class BaseDefines(ContextDerived):
+    """Context derived container object for DEFINES/HOST_DEFINES,
+    which are OrderedDicts.
     """
     __slots__ = ('defines')
 
     def __init__(self, context, defines):
         ContextDerived.__init__(self, context)
         self.defines = defines
 
     def get_defines(self):
@@ -192,16 +193,22 @@ class Defines(ContextDerived):
                 yield('-D%s=%s' % (define, shell_quote(value)))
 
     def update(self, more_defines):
         if isinstance(more_defines, Defines):
             self.defines.update(more_defines.defines)
         else:
             self.defines.update(more_defines)
 
+class Defines(BaseDefines):
+    pass
+
+class HostDefines(BaseDefines):
+    pass
+
 class Exports(ContextDerived):
     """Context derived container object for EXPORTS, which is a
     HierarchicalStringList.
 
     We need an object derived from ContextDerived for use in the backend, so
     this object fills that role. It just has a reference to the underlying
     HierarchicalStringList, which is created when parsing EXPORTS.
     """
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -40,16 +40,17 @@ from .data import (
     GeneratedFile,
     GeneratedInclude,
     GeneratedSources,
     GeneratedWebIDLFile,
     ExampleWebIDLInterface,
     ExternalStaticLibrary,
     ExternalSharedLibrary,
     HeaderFileSubstitution,
+    HostDefines,
     HostLibrary,
     HostProgram,
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     IPDLFile,
     JARManifest,
     JavaScriptModules,
@@ -583,17 +584,17 @@ class TreeMetadataEmitter(LoggingMixin):
 
         if context.config.substs.get('OS_TARGET') == 'WINNT' and \
                 context['DELAYLOAD_DLLS']:
             context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll)
                 for dll in context['DELAYLOAD_DLLS']])
             context['OS_LIBS'].append('delayimp')
 
         for v in ['CFLAGS', 'CXXFLAGS', 'CMFLAGS', 'CMMFLAGS', 'ASFLAGS',
-                  'LDFLAGS']:
+                  'LDFLAGS', 'HOST_CFLAGS', 'HOST_CXXFLAGS']:
             if v in context and context[v]:
                 passthru.variables['MOZBUILD_' + v] = context[v]
 
         # NO_VISIBILITY_FLAGS is slightly different
         if context['NO_VISIBILITY_FLAGS']:
             passthru.variables['VISIBILITY_FLAGS'] = ''
 
         if isinstance(context, TemplateContext) and context.template == 'Gyp':
@@ -618,16 +619,20 @@ class TreeMetadataEmitter(LoggingMixin):
 
         for obj in self._process_test_harness_files(context):
             yield obj
 
         defines = context.get('DEFINES')
         if defines:
             yield Defines(context, defines)
 
+        host_defines = context.get('HOST_DEFINES')
+        if host_defines:
+            yield HostDefines(context, host_defines)
+
         resources = context.get('RESOURCE_FILES')
         if resources:
             yield Resources(context, resources, defines)
 
         for pref in sorted(context['JS_PREFERENCE_FILES']):
             yield JsPreferenceFile(context, pref)
 
         self._handle_programs(context)
copy from python/mozbuild/mozbuild/test/backend/data/defines/moz.build
copy to python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build
--- a/python/mozbuild/mozbuild/test/backend/data/defines/moz.build
+++ b/python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build
@@ -1,14 +1,14 @@
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 value = 'xyz'
-DEFINES = {
+HOST_DEFINES = {
     'FOO': True,
 }
 
-DEFINES['BAZ'] = '"ab\'cd"'
-DEFINES.update({
+HOST_DEFINES['BAZ'] = '"ab\'cd"'
+HOST_DEFINES.update({
     'BAR': 7,
     'VALUE': value,
     'QUX': False,
 })
--- a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build
+++ b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build
@@ -14,13 +14,15 @@ RESFILE = 'bar.res'
 RCINCLUDE = 'bar.rc'
 DEFFILE = 'baz.def'
 
 USE_STATIC_LIBS = True
 
 CFLAGS += ['-fno-exceptions', '-w']
 CXXFLAGS += ['-fcxx-exceptions', '-include foo.h']
 LDFLAGS += ['-framework Foo', '-x']
+HOST_CFLAGS += ['-funroll-loops', '-wall']
+HOST_CXXFLAGS += ['-funroll-loops-harder', '-wall-day-everyday']
 WIN32_EXE_LDFLAGS += ['-subsystem:console']
 
 DISABLE_STL_WRAPPING = True
 
 ALLOW_COMPILER_WARNINGS = True
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -302,16 +302,24 @@ class TestRecursiveMakeBackend(BackendTe
                 'MOZBUILD_CXXFLAGS += -include foo.h',
             ],
             'MOZBUILD_LDFLAGS': [
                 'MOZBUILD_LDFLAGS += -framework Foo',
                 'MOZBUILD_LDFLAGS += -x',
                 'MOZBUILD_LDFLAGS += -DELAYLOAD:foo.dll',
                 'MOZBUILD_LDFLAGS += -DELAYLOAD:bar.dll',
             ],
+            'MOZBUILD_HOST_CFLAGS': [
+                'MOZBUILD_HOST_CFLAGS += -funroll-loops',
+                'MOZBUILD_HOST_CFLAGS += -wall',
+            ],
+            'MOZBUILD_HOST_CXXFLAGS': [
+                'MOZBUILD_HOST_CXXFLAGS += -funroll-loops-harder',
+                'MOZBUILD_HOST_CXXFLAGS += -wall-day-everyday',
+            ],
             'WIN32_EXE_LDFLAGS': [
                 'WIN32_EXE_LDFLAGS += -subsystem:console',
             ],
         }
 
         for var, val in expected.items():
             # print("test_variable_passthru[%s]" % (var))
             found = [str for str in lines if str.startswith(var)]
@@ -573,16 +581,29 @@ class TestRecursiveMakeBackend(BackendTe
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         var = 'DEFINES'
         defines = [val for val in lines if val.startswith(var)]
 
         expected = ['DEFINES += -DFOO -DBAZ=\'"ab\'\\\'\'cd"\' -UQUX -DBAR=7 -DVALUE=\'xyz\'']
         self.assertEqual(defines, expected)
 
+    def test_host_defines(self):
+        """Test that HOST_DEFINES are written to backend.mk correctly."""
+        env = self._consume('host-defines', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        var = 'HOST_DEFINES'
+        defines = [val for val in lines if val.startswith(var)]
+
+        expected = ['HOST_DEFINES += -DFOO -DBAZ=\'"ab\'\\\'\'cd"\' -UQUX -DBAR=7 -DVALUE=\'xyz\'']
+        self.assertEqual(defines, expected)
+
     def test_local_includes(self):
         """Test that LOCAL_INCLUDES are written to backend.mk correctly."""
         env = self._consume('local_includes', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
copy from python/mozbuild/mozbuild/test/frontend/data/defines/moz.build
copy to python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build
--- a/python/mozbuild/mozbuild/test/frontend/data/defines/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build
@@ -1,14 +1,14 @@
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 value = 'xyz'
-DEFINES = {
+HOST_DEFINES = {
     'FOO': True,
 }
 
-DEFINES['BAZ'] = '"abcd"'
-DEFINES.update({
+HOST_DEFINES['BAZ'] = '"abcd"'
+HOST_DEFINES.update({
     'BAR': 7,
     'VALUE': value,
     'QUX': False,
 })
--- a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build
@@ -16,13 +16,15 @@ RESFILE = 'bar.res'
 RCINCLUDE = 'bar.rc'
 DEFFILE = 'baz.def'
 
 USE_STATIC_LIBS = True
 
 CFLAGS += ['-fno-exceptions', '-w']
 CXXFLAGS += ['-fcxx-exceptions', '-include foo.h']
 LDFLAGS += ['-framework Foo', '-x']
+HOST_CFLAGS += ['-funroll-loops', '-wall']
+HOST_CXXFLAGS += ['-funroll-loops-harder', '-wall-day-everyday']
 WIN32_EXE_LDFLAGS += ['-subsystem:console']
 
 DISABLE_STL_WRAPPING = True
 
 ALLOW_COMPILER_WARNINGS = True
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -15,16 +15,17 @@ from mozbuild.frontend.data import (
     ConfigFileSubstitution,
     Defines,
     DistFiles,
     DirectoryTraversal,
     Exports,
     GeneratedFile,
     GeneratedInclude,
     GeneratedSources,
+    HostDefines,
     HostSources,
     IPDLFile,
     JARManifest,
     JsPreferenceFile,
     LocalInclude,
     Program,
     ReaderSummary,
     Resources,
@@ -171,16 +172,19 @@ class TestEmitterBasic(unittest.TestCase
             'RESFILE': 'bar.res',
             'RCINCLUDE': 'bar.rc',
             'DEFFILE': 'baz.def',
             'USE_STATIC_LIBS': True,
             'MOZBUILD_CFLAGS': ['-fno-exceptions', '-w'],
             'MOZBUILD_CXXFLAGS': ['-fcxx-exceptions', '-include foo.h'],
             'MOZBUILD_LDFLAGS': ['-framework Foo', '-x', '-DELAYLOAD:foo.dll',
                                  '-DELAYLOAD:bar.dll'],
+            'MOZBUILD_HOST_CFLAGS': ['-funroll-loops', '-wall'],
+            'MOZBUILD_HOST_CXXFLAGS': ['-funroll-loops-harder',
+                                       '-wall-day-everyday'],
             'WIN32_EXE_LDFLAGS': ['-subsystem:console'],
         }
 
         variables = objs[0].variables
         maxDiff = self.maxDiff
         self.maxDiff = None
         self.assertEqual(wanted, variables)
         self.maxDiff = maxDiff
@@ -680,16 +684,35 @@ class TestEmitterBasic(unittest.TestCase
             'BAZ': '"abcd"',
             'FOO': True,
             'VALUE': 'xyz',
             'QUX': False,
         }
 
         self.assertEqual(defines, expected)
 
+    def test_host_defines(self):
+        reader = self.reader('host-defines')
+        objs = self.read_topsrcdir(reader)
+
+        defines = {}
+        for o in objs:
+            if isinstance(o, HostDefines):
+                defines = o.defines
+
+        expected = {
+            'BAR': 7,
+            'BAZ': '"abcd"',
+            'FOO': True,
+            'VALUE': 'xyz',
+            'QUX': False,
+        }
+
+        self.assertEqual(defines, expected)
+
     def test_jar_manifests(self):
         reader = self.reader('jar-manifests')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 1)
         for obj in objs:
             self.assertIsInstance(obj, JARManifest)
             self.assertTrue(os.path.isabs(obj.path))