Bug 1416062 - Start compiling things in the tup backend, limited to xpcom/*; r=chmanchester
authorMike Shal <mshal@mozilla.com>
Tue, 07 Nov 2017 18:50:21 -0500
changeset 446471 32467b9838efbd3ea202238a55165ac8ab7d96fb
parent 446470 0806fc45779f5c88c0686f62cdf0ef0360f50b3c
child 446472 b0876dc6eb63582ef413ba69db672d1c4ae05748
push id1648
push usermtabara@mozilla.com
push dateThu, 01 Mar 2018 12:45:47 +0000
treeherdermozilla-release@cbb9688c2eeb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1416062
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1416062 - Start compiling things in the tup backend, limited to xpcom/*; r=chmanchester Compiling C/C++/asm just requires the list of sources from _process_unified_sources() and the Sources/GeneratedSources objects, along with the ComputedFlags and PerSourceFlags. The assembler invocation will need to be tweaked to support yasm as well, but this works so far for the xpcom directory. MozReview-Commit-ID: 91BSKV9XPLU
python/mozbuild/mozbuild/backend/tup.py
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -8,35 +8,41 @@ import os
 import sys
 
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozbuild.backend.base import PartialBackend, HybridBackend
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.shellutil import quote as shell_quote
 from mozbuild.util import OrderedDefaultDict
+from collections import defaultdict
 
 from mozpack.files import (
     FileFinder,
 )
 
 from .common import CommonBackend
 from ..frontend.data import (
     ChromeManifestEntry,
+    ComputedFlags,
     ContextDerived,
     Defines,
     FinalTargetFiles,
     FinalTargetPreprocessedFiles,
     GeneratedFile,
+    GeneratedSources,
     HostDefines,
     JARManifest,
     ObjdirFiles,
+    PerSourceFlag,
+    Sources,
 )
 from ..util import (
     FileAvoidWrite,
+    expand_variables,
 )
 from ..frontend.context import (
     AbsolutePath,
     ObjDirPath,
 )
 
 
 class BackendTupfile(object):
@@ -50,48 +56,53 @@ class BackendTupfile(object):
         self.relobjdir = mozpath.relpath(objdir, topobjdir)
         self.environment = environment
         self.name = mozpath.join(objdir, 'Tupfile')
         self.rules_included = False
         self.shell_exported = False
         self.defines = []
         self.host_defines = []
         self.delayed_generated_files = []
+        self.per_source_flags = defaultdict(list)
+        self.local_flags = defaultdict(list)
+        self.sources = defaultdict(list)
 
         self.fh = FileAvoidWrite(self.name, capture_diff=True)
         self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
         self.fh.write('\n')
 
     def write(self, buf):
         self.fh.write(buf)
 
     def include_rules(self):
         if not self.rules_included:
             self.write('include_rules\n')
             self.rules_included = True
 
-    def rule(self, cmd, inputs=None, outputs=None, display=None, extra_outputs=None, check_unchanged=False):
+    def rule(self, cmd, inputs=None, outputs=None, display=None,
+             extra_inputs=None, extra_outputs=None, check_unchanged=False):
         inputs = inputs or []
         outputs = outputs or []
         display = display or ""
         self.include_rules()
         flags = ""
         if check_unchanged:
             # This flag causes tup to compare the outputs with the previous run
             # of the command, and skip the rest of the DAG for any that are the
             # same.
             flags += "o"
 
         if display:
             caret_text = flags + ' ' + display
         else:
             caret_text = flags
 
-        self.write(': %(inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
+        self.write(': %(inputs)s%(extra_inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
             'inputs': ' '.join(inputs),
+            'extra_inputs': ' | ' + ' '.join(extra_inputs) if extra_inputs else '',
             'display': '^%s^ ' % caret_text if caret_text else '',
             'cmd': ' '.join(cmd),
             'outputs': ' '.join(outputs),
             'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
         })
 
     def symlink_rule(self, source, output=None, output_group=None):
         outputs = [output] if output else [mozpath.basename(source)]
@@ -101,16 +112,39 @@ class BackendTupfile(object):
         # The !tup_ln macro does a symlink or file copy (depending on the
         # platform) without shelling out to a subprocess.
         self.rule(
             cmd=['!tup_ln'],
             inputs=[source],
             outputs=outputs,
         )
 
+    def gen_sources_rules(self, extra_inputs):
+        compilers = [
+            ('.S', 'AS', 'ASFLAGS'),
+            ('.cpp', 'CXX', 'CXXFLAGS'),
+            ('.c', 'CC', 'CFLAGS'),
+        ]
+        for extension, compiler, flags in compilers:
+            srcs = sorted(self.sources[extension])
+            for src in srcs:
+                # AS can be set to $(CC), so we need to call expand_variables on
+                # the compiler to get the real value.
+                cmd = [expand_variables(self.environment.substs[compiler], self.environment.substs)]
+                cmd.extend(self.local_flags[flags])
+                cmd.extend(self.per_source_flags[src])
+                cmd.extend(['-c', '%f', '-o', '%o'])
+                self.rule(
+                    cmd=cmd,
+                    inputs=[src],
+                    extra_inputs=extra_inputs,
+                    outputs=['%B.o'],
+                    display='%s %%f' % compiler,
+                )
+
     def export_shell(self):
         if not self.shell_exported:
             # These are used by mach/mixin/process.py to determine the current
             # shell.
             for var in ('SHELL', 'MOZILLABUILD', 'COMSPEC'):
                 self.write('export %s\n' % var)
             self.shell_exported = True
 
@@ -213,32 +247,40 @@ class TupOnly(CommonBackend, PartialBack
         elif isinstance(obj, HostDefines):
             self._process_defines(backend_file, obj, host=True)
         elif isinstance(obj, FinalTargetFiles):
             self._process_final_target_files(obj)
         elif isinstance(obj, FinalTargetPreprocessedFiles):
             self._process_final_target_pp_files(obj, backend_file)
         elif isinstance(obj, JARManifest):
             self._consume_jar_manifest(obj)
+        elif isinstance(obj, PerSourceFlag):
+            backend_file.per_source_flags[obj.file_name].extend(obj.flags)
+        elif isinstance(obj, ComputedFlags):
+            self._process_computed_flags(obj, backend_file)
+        elif isinstance(obj, (Sources, GeneratedSources)):
+            if obj.relobjdir.startswith('xpcom'):
+                backend_file.sources[obj.canonical_suffix].extend(obj.files)
 
         return True
 
     def consume_finished(self):
         CommonBackend.consume_finished(self)
 
         # The approach here is similar to fastermake.py, but we
         # simply write out the resulting files here.
         for target, entries in self._manifest_entries.iteritems():
             with self._write_file(mozpath.join(self.environment.topobjdir,
                                                target)) as fh:
                 fh.write(''.join('%s\n' % e for e in sorted(entries)))
 
         for objdir, backend_file in sorted(self._backend_files.items()):
             for obj in backend_file.delayed_generated_files:
                 self._process_generated_file(backend_file, obj)
+            backend_file.gen_sources_rules([self._installed_files])
             with self._write_file(fh=backend_file):
                 pass
 
         with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh:
             acdefines_flags = ' '.join(['-D%s=%s' % (name, shell_quote(value))
                 for (name, value) in sorted(self.environment.acdefines.iteritems())])
             # TODO: AB_CD only exists in Makefiles at the moment.
             acdefines_flags += ' -DAB_CD=en-US'
@@ -380,16 +422,26 @@ class TupOnly(CommonBackend, PartialBack
                                                       output_group=self._installed_files)
 
     def _process_final_target_pp_files(self, obj, backend_file):
         for i, (path, files) in enumerate(obj.files.walk()):
             for f in files:
                 self._preprocess(backend_file, f.full_path,
                                  destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path))
 
+    def _process_computed_flags(self, obj, backend_file):
+        for var, flags in obj.get_flags():
+            backend_file.local_flags[var] = flags
+
+    def _process_unified_sources(self, obj):
+        backend_file = self._get_backend_file_for(obj)
+        if obj.relobjdir.startswith('xpcom'):
+            files = [f[0] for f in obj.unified_source_mapping]
+            backend_file.sources[obj.canonical_suffix].extend(files)
+
     def _handle_idl_manager(self, manager):
         if self.environment.is_artifact_build:
             return
 
         dist_idl_backend_file = self._get_backend_file('dist/idl')
         for idl in manager.idls.values():
             dist_idl_backend_file.symlink_rule(idl['source'], output_group=self._installed_idls)