Bug 1372381 - Compile host libraries, host programs, and simple programs in the Tup backend. r=mshal
authorChris Manchester <cmanchester@mozilla.com>
Mon, 30 Apr 2018 11:10:44 -0700
changeset 472534 539a4a607ef98975ae0cde61c793723a6b1e19c5
parent 472533 0a1897a68d63001f12a4dd9620ab8cefff207176
child 472535 c2420955ce61a3e5c650b309e96564f9a494bdf8
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1372381
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1372381 - Compile host libraries, host programs, and simple programs in the Tup backend. r=mshal MozReview-Commit-ID: 2AcpqiOqSSf
python/mozbuild/mozbuild/backend/tup.py
python/mozbuild/mozbuild/frontend/data.py
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -35,17 +35,20 @@ from ..frontend.data import (
     GeneratedFile,
     GeneratedSources,
     HostDefines,
     HostSources,
     JARManifest,
     ObjdirFiles,
     PerSourceFlag,
     Program,
+    SimpleProgram,
+    HostLibrary,
     HostProgram,
+    HostSimpleProgram,
     SharedLibrary,
     Sources,
     StaticLibrary,
     VariablePassthru,
 )
 from ..util import (
     FileAvoidWrite,
     expand_variables,
@@ -74,17 +77,19 @@ class BackendTupfile(object):
         self.delayed_installed_files = []
         self.per_source_flags = defaultdict(list)
         self.local_flags = defaultdict(list)
         self.sources = defaultdict(list)
         self.host_sources = defaultdict(list)
         self.variables = {}
         self.static_lib = None
         self.shared_lib = None
-        self.program = None
+        self.programs = []
+        self.host_programs = []
+        self.host_library = None
         self.exports = set()
 
         # These files are special, ignore anything that generates them or
         # depends on them.
         self._skip_files = [
             'signmar',
             'libxul.so',
             'libtestcrasher.so',
@@ -214,21 +219,27 @@ class TupBackend(CommonBackend):
     """
 
     def _init(self):
         CommonBackend._init(self)
 
         self._backend_files = {}
         self._cmd = MozbuildObject.from_environment()
         self._manifest_entries = OrderedDefaultDict(set)
-        self._compile_env_gen_files = (
+
+        # These are a hack to approximate things that are needed for the
+        # compile phase.
+        self._compile_env_files = (
+            '*.api',
             '*.c',
+            '*.cfg',
             '*.cpp',
             '*.h',
             '*.inc',
+            '*.msg',
             '*.py',
             '*.rs',
         )
 
         # These are 'group' dependencies - All rules that list these as an output
         # will be built before any rules that list this as an input.
         self._installed_idls = '$(MOZ_OBJ_ROOT)/<installed-idls>'
         self._installed_files = '$(MOZ_OBJ_ROOT)/<installed-files>'
@@ -347,30 +358,37 @@ class TupBackend(CommonBackend):
             display='LINK %o'
         )
         backend_file.symlink_rule(mozpath.join(backend_file.objdir,
                                                shlib.lib_name),
                                   output=mozpath.join(self.environment.topobjdir,
                                                       shlib.install_target,
                                                       shlib.lib_name))
 
+    def _gen_programs(self, backend_file):
+        for p in backend_file.programs:
+            self._gen_program(backend_file, p)
 
-    def _gen_program(self, backend_file):
-        cc_or_cxx = 'CXX' if backend_file.program.cxx_link else 'CC'
-        objs, _, shared_libs, os_libs, static_libs = self._expand_libs(backend_file.program)
+    def _gen_program(self, backend_file, prog):
+        cc_or_cxx = 'CXX' if prog.cxx_link else 'CC'
+        objs, _, shared_libs, os_libs, static_libs = self._expand_libs(prog)
         static_libs = self._lib_paths(backend_file.objdir, static_libs)
         shared_libs = self._lib_paths(backend_file.objdir, shared_libs)
 
         inputs = objs + static_libs + shared_libs
 
-        list_file_name = '%s.list' % backend_file.program.name.replace('.', '_')
+        list_file_name = '%s.list' % prog.name.replace('.', '_')
         list_file = self._make_list_file(backend_file.objdir, objs, list_file_name)
 
-        outputs = [mozpath.relpath(backend_file.program.output_path.full_path,
-                                   backend_file.objdir)]
+        if isinstance(prog, SimpleProgram):
+            outputs = [prog.name]
+        else:
+            outputs = [mozpath.relpath(prog.output_path.full_path,
+                                       backend_file.objdir)]
+
         cmd = (
             [backend_file.environment.substs[cc_or_cxx], '-o', '%o'] +
             backend_file.local_flags['CXX_LDFLAGS'] +
             [list_file] +
             backend_file.local_flags['LDFLAGS'] +
             static_libs +
             [backend_file.environment.substs['MOZ_PROGRAM_LDFLAGS']] +
             shared_libs +
@@ -380,16 +398,67 @@ class TupBackend(CommonBackend):
         backend_file.rule(
             cmd=cmd,
             inputs=inputs,
             outputs=outputs,
             display='LINK %o'
         )
 
 
+    def _gen_host_library(self, backend_file):
+        objs = backend_file.host_library.objs
+        inputs = objs
+        outputs = [backend_file.host_library.name]
+        cmd = (
+            [backend_file.environment.substs['HOST_AR']] +
+            [backend_file.environment.substs['HOST_AR_FLAGS'].replace('$@', '%o')] +
+            objs
+        )
+        backend_file.rule(
+            cmd=cmd,
+            inputs=inputs,
+            outputs=outputs,
+            display='AR %o'
+        )
+
+
+    def _gen_host_programs(self, backend_file):
+        for p in backend_file.host_programs:
+            self._gen_host_program(backend_file, p)
+
+
+    def _gen_host_program(self, backend_file, prog):
+        _, _, _, extra_libs, _ = self._expand_libs(prog)
+        objs = prog.objs
+        outputs = [prog.program]
+        host_libs = []
+        for lib in prog.linked_libraries:
+            if isinstance(lib, HostLibrary):
+                host_libs.append(lib)
+        host_libs = self._lib_paths(backend_file.objdir, host_libs)
+
+        inputs = objs + host_libs
+        use_cxx = any(f.endswith(('.cc', '.cpp')) for f in prog.source_files())
+        cc_or_cxx = 'HOST_CXX' if use_cxx else 'HOST_CC'
+        cmd = (
+            [backend_file.environment.substs[cc_or_cxx], '-o', '%o'] +
+            backend_file.local_flags['HOST_CXX_LDFLAGS'] +
+            backend_file.local_flags['HOST_LDFLAGS'] +
+            objs +
+            host_libs +
+            extra_libs
+        )
+        backend_file.rule(
+            cmd=cmd,
+            inputs=inputs,
+            outputs=outputs,
+            display='LINK %o'
+        )
+
+
     def _gen_static_library(self, backend_file):
         ar = [
             backend_file.environment.substs['AR'],
             backend_file.environment.substs['AR_FLAGS'].replace('$@', '%o')
         ]
 
         objs, _, shared_libs, _, static_libs = self._expand_libs(backend_file.static_lib)
         static_libs = self._lib_paths(backend_file.objdir, static_libs)
@@ -421,17 +490,17 @@ class TupBackend(CommonBackend):
             return True
 
         backend_file = self._get_backend_file_for(obj)
 
         if isinstance(obj, GeneratedFile):
             skip_files = []
 
             if self.environment.is_artifact_build:
-                skip_files = self._compile_env_gen_files
+                skip_files = self._compile_env_gen
 
             for f in obj.outputs:
                 if any(mozpath.match(f, p) for p in skip_files):
                     return False
 
             if backend_file.requires_delay(obj.inputs):
                 backend_file.delayed_generated_files.append(obj)
             else:
@@ -463,20 +532,22 @@ class TupBackend(CommonBackend):
         elif isinstance(obj, HostSources):
             backend_file.host_sources[obj.canonical_suffix].extend(obj.files)
         elif isinstance(obj, VariablePassthru):
             backend_file.variables = obj.variables
         elif isinstance(obj, StaticLibrary):
             backend_file.static_lib = obj
         elif isinstance(obj, SharedLibrary):
             backend_file.shared_lib = obj
-        elif isinstance(obj, HostProgram):
-            pass
-        elif isinstance(obj, Program):
-            backend_file.program = obj
+        elif isinstance(obj, (HostProgram, HostSimpleProgram)):
+            backend_file.host_programs.append(obj)
+        elif isinstance(obj, HostLibrary):
+            backend_file.host_library = obj
+        elif isinstance(obj, (Program, SimpleProgram)):
+            backend_file.programs.append(obj)
         elif isinstance(obj, DirectoryTraversal):
             pass
 
         return True
 
     def consume_finished(self):
         CommonBackend.consume_finished(self)
 
@@ -489,21 +560,23 @@ class TupBackend(CommonBackend):
 
         if self._built_in_addons:
             with self._write_file(mozpath.join(self.environment.topobjdir,
                                                self._built_in_addons_file)) as fh:
                 json.dump({'system': sorted(list(self._built_in_addons))}, fh)
 
         for objdir, backend_file in sorted(self._backend_files.items()):
             backend_file.gen_sources_rules([self._installed_files])
-            for condition, gen_method in ((backend_file.shared_lib, self._gen_shared_library),
-                                          (backend_file.static_lib and backend_file.static_lib.no_expand_lib,
-                                           self._gen_static_library),
-                                          (backend_file.program, self._gen_program)):
-                if condition:
+            for var, gen_method in ((backend_file.shared_lib, self._gen_shared_library),
+                                    (backend_file.static_lib and backend_file.static_lib.no_expand_lib,
+                                     self._gen_static_library),
+                                    (backend_file.programs, self._gen_programs),
+                                    (backend_file.host_programs, self._gen_host_programs),
+                                    (backend_file.host_library, self._gen_host_library)):
+                if var:
                     backend_file.export_shell()
                     gen_method(backend_file)
             for obj in backend_file.delayed_generated_files:
                 self._process_generated_file(backend_file, obj)
             for path, output, output_group in backend_file.delayed_installed_files:
                 backend_file.symlink_rule(path, output=output, output_group=output_group)
             with self._write_file(fh=backend_file):
                 pass
@@ -538,17 +611,16 @@ class TupBackend(CommonBackend):
         if not os.path.exists(mozpath.join(self.environment.topsrcdir, ".tup")):
             tup = self.environment.substs.get('TUP', 'tup')
             self._cmd.run_process(cwd=self.environment.topsrcdir, log_name='tup', args=[tup, 'init'])
 
     def _process_generated_file(self, backend_file, obj):
         # TODO: These are directories that don't work in the tup backend
         # yet, because things they depend on aren't built yet.
         skip_directories = (
-            'layout/style/test', # HostSimplePrograms
             'toolkit/library', # libxul.so
         )
         if obj.script and obj.method and obj.relobjdir not in skip_directories:
             backend_file.export_shell()
             cmd = self._py_action('file_generate')
             if obj.localized:
                 cmd.append('--locale=en-US')
             cmd.extend([
@@ -618,16 +690,21 @@ class TupBackend(CommonBackend):
         if target.startswith('_tests'):
             # TODO: TEST_HARNESS_FILES present a few challenges for the tup
             # backend (bug 1372381).
             return
 
         for path, files in obj.files.walk():
             self._add_features(target, path)
             for f in files:
+                output_group = None
+                if any(mozpath.match(mozpath.basename(f), p)
+                       for p in self._compile_env_files):
+                    output_group = self._installed_files
+
                 if not isinstance(f, ObjDirPath):
                     backend_file = self._get_backend_file(mozpath.join(target, path))
                     if '*' in f:
                         if f.startswith('/') or isinstance(f, AbsolutePath):
                             basepath, wild = os.path.split(f.full_path)
                             if '*' in basepath:
                                 raise Exception("Wildcards are only supported in the filename part of "
                                                 "srcdir-relative or absolute paths.")
@@ -641,40 +718,38 @@ class TupBackend(CommonBackend):
                                     if '*' not in p:
                                         yield p + '/'
                             prefix = ''.join(_prefix(f.full_path))
                             self.backend_input_files.add(prefix)
                             finder = FileFinder(prefix)
                             for p, _ in finder.find(f.full_path[len(prefix):]):
                                 backend_file.symlink_rule(mozpath.join(prefix, p),
                                                           output=mozpath.join(f.target_basename, p),
-                                                          output_group=self._installed_files)
+                                                          output_group=output_group)
                     else:
-                        backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=self._installed_files)
+                        backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=output_group)
                 else:
                     if (self.environment.is_artifact_build and
                         any(mozpath.match(f.target_basename, p) for p in self._compile_env_gen_files)):
                         # If we have an artifact build we never would have generated this file,
                         # so do not attempt to install it.
                         continue
 
                     # We're not generating files in these directories yet, so
                     # don't attempt to install files generated from them.
-                    if f.context.relobjdir not in ('layout/style/test',
-                                                   'toolkit/library',
+                    if f.context.relobjdir not in ('toolkit/library',
                                                    'js/src/shell'):
                         output = mozpath.join('$(MOZ_OBJ_ROOT)', target, path,
                                               f.target_basename)
                         gen_backend_file = self._get_backend_file(f.context.relobjdir)
                         if gen_backend_file.requires_delay([f]):
-                            output_group = self._installed_files if f.target_basename.endswith('.h') else None
                             gen_backend_file.delayed_installed_files.append((f.full_path, output, output_group))
                         else:
                             gen_backend_file.symlink_rule(f.full_path, output=output,
-                                                          output_group=self._installed_files)
+                                                          output_group=output_group)
 
 
     def _process_final_target_pp_files(self, obj, backend_file):
         for i, (path, files) in enumerate(obj.files.walk()):
             self._add_features(obj.install_target, path)
             for f in files:
                 self._preprocess(backend_file, f.full_path,
                                  destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path),
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -527,16 +527,24 @@ class SimpleProgram(BaseProgram):
 
 
 class HostSimpleProgram(HostMixin, BaseProgram):
     """Context derived container object for each program in
     HOST_SIMPLE_PROGRAMS"""
     SUFFIX_VAR = 'HOST_BIN_SUFFIX'
     KIND = 'host'
 
+    def source_files(self):
+        for srcs in self.sources.values():
+            for f in srcs:
+                if ('host_%s' % mozpath.basename(mozpath.splitext(f)[0]) ==
+                    mozpath.splitext(self.program)[0]):
+                    return [f]
+        return []
+
 
 def cargo_output_directory(context, target_var):
     # cargo creates several directories and places its build artifacts
     # in those directories.  The directory structure depends not only
     # on the target, but also what sort of build we are doing.
     rust_build_kind = 'release'
     if context.config.substs.get('MOZ_DEBUG_RUST'):
         rust_build_kind = 'debug'