Bug 1319228 - Generate rules for rust in the Tup backend via cargo --build-plan. r=ted
☠☠ backed out by f513c7af73e6 ☠ ☠
authorChris Manchester <cmanchester@mozilla.com>
Wed, 13 Jun 2018 13:00:13 -0700
changeset 476684 badf116dde30da2fded47fd9fbe5669358247410
parent 476683 a218f97e1b486ed37bba6c1cf05c448005f4c7fd
child 476685 6ba05238789ffd1a0e7cdeef9028d822e18cad1f
push id9374
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:43:20 +0000
treeherdermozilla-beta@160e085dfb0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1319228
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1319228 - Generate rules for rust in the Tup backend via cargo --build-plan. r=ted MozReview-Commit-ID: 9m0Prz3AvmC
python/mozbuild/mozbuild/backend/cargo_build_defs.py
python/mozbuild/mozbuild/backend/tup.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/backend/cargo_build_defs.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+cargo_extra_outputs = {
+    'bindgen': [
+        'tests.rs',
+        'host-target.txt',
+    ],
+    'cssparser': [
+        'tokenizer.rs',
+    ],
+    'gleam': [
+        'gl_and_gles_bindings.rs',
+        'gl_bindings.rs',
+        'gles_bindings.rs',
+    ],
+    'khronos_api': [
+        'webgl_exts.rs',
+    ],
+    'libloading': [
+        'libglobal_static.a',
+        'src/os/unix/global_static.o',
+    ],
+    'selectors': [
+        'ascii_case_insensitive_html_attributes.rs',
+    ],
+    'style': [
+        'gecko/atom_macro.rs',
+        'gecko/pseudo_element_definition.rs',
+        'gecko_properties.rs',
+        'properties.rs',
+        'gecko/bindings.rs',
+        'gecko/structs.rs',
+    ],
+    'webrender': [
+        'shaders.rs',
+    ],
+}
+
+cargo_extra_flags = {
+    'style': [
+        '-l', 'static=global_static',
+        '-L', 'native=%(libloading_outdir)s',
+    ]
+}
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
 import os
+import itertools
 import json
 import sys
 import shutil
 
 import mozpack.path as mozpath
 from mozbuild import shellutil
 from mozbuild.base import MozbuildObject
 from mozbuild.backend.base import PartialBackend, HybridBackend
@@ -40,30 +41,35 @@ from ..frontend.data import (
     JARManifest,
     ObjdirFiles,
     PerSourceFlag,
     Program,
     SimpleProgram,
     HostLibrary,
     HostProgram,
     HostSimpleProgram,
+    RustLibrary,
     SharedLibrary,
     Sources,
     StaticLibrary,
     VariablePassthru,
 )
 from ..util import (
     FileAvoidWrite,
     expand_variables,
 )
 from ..frontend.context import (
     AbsolutePath,
     ObjDirPath,
     RenamedSourcePath,
 )
+from .cargo_build_defs import (
+    cargo_extra_outputs,
+    cargo_extra_flags,
+)
 
 
 class BackendTupfile(object):
     """Represents a generated Tupfile.
     """
 
     def __init__(self, objdir, environment, topsrcdir, topobjdir, dry_run):
         self.topsrcdir = topsrcdir
@@ -77,16 +83,17 @@ class BackendTupfile(object):
         self.outputs = set()
         self.delayed_generated_files = []
         self.delayed_installed_files = []
         self.per_source_flags = defaultdict(list)
         self.local_flags = defaultdict(list)
         self.sources = defaultdict(list)
         self.host_sources = defaultdict(list)
         self.variables = {}
+        self.rust_library = None
         self.static_lib = None
         self.shared_lib = None
         self.programs = []
         self.host_programs = []
         self.host_library = None
         self.exports = set()
 
         # These files are special, ignore anything that generates them or
@@ -240,16 +247,17 @@ class TupBackend(CommonBackend):
             '*.py',
             '*.rs',
         )
 
         # These are 'group' dependencies - All rules that list these as an output
         # will be built before any rules that list this as an input.
         self._installed_idls = '$(MOZ_OBJ_ROOT)/<installed-idls>'
         self._installed_files = '$(MOZ_OBJ_ROOT)/<installed-files>'
+        self._rust_libs = '$(MOZ_OBJ_ROOT)/<rust-libs>'
         # The preprocessor including source-repo.h and buildid.h creates
         # dependencies that aren't specified by moz.build and cause errors
         # in Tup. Express these as a group dependency.
         self._early_generated_files = '$(MOZ_OBJ_ROOT)/<early-generated-files>'
 
         self._built_in_addons = set()
         self._built_in_addons_file = 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json'
 
@@ -541,16 +549,18 @@ class TupBackend(CommonBackend):
         elif isinstance(obj, ComputedFlags):
             self._process_computed_flags(obj, backend_file)
         elif isinstance(obj, (Sources, GeneratedSources)):
             backend_file.sources[obj.canonical_suffix].extend(obj.files)
         elif isinstance(obj, HostSources):
             backend_file.host_sources[obj.canonical_suffix].extend(obj.files)
         elif isinstance(obj, VariablePassthru):
             backend_file.variables = obj.variables
+        elif isinstance(obj, RustLibrary):
+            backend_file.rust_library = obj
         elif isinstance(obj, StaticLibrary):
             backend_file.static_lib = obj
         elif isinstance(obj, SharedLibrary):
             backend_file.shared_lib = obj
         elif isinstance(obj, (HostProgram, HostSimpleProgram)):
             backend_file.host_programs.append(obj)
         elif isinstance(obj, HostLibrary):
             backend_file.host_library = obj
@@ -578,17 +588,18 @@ class TupBackend(CommonBackend):
 
         for objdir, backend_file in sorted(self._backend_files.items()):
             backend_file.gen_sources_rules([self._installed_files])
             for var, gen_method in ((backend_file.shared_lib, self._gen_shared_library),
                                     (backend_file.static_lib and backend_file.static_lib.no_expand_lib,
                                      self._gen_static_library),
                                     (backend_file.programs, self._gen_programs),
                                     (backend_file.host_programs, self._gen_host_programs),
-                                    (backend_file.host_library, self._gen_host_library)):
+                                    (backend_file.host_library, self._gen_host_library),
+                                    (backend_file.rust_library, self._gen_rust)):
                 if var:
                     backend_file.export_shell()
                     gen_method(backend_file)
             for obj in backend_file.delayed_generated_files:
                 self._process_generated_file(backend_file, obj)
             for path, output, output_group in backend_file.delayed_installed_files:
                 backend_file.symlink_rule(path, output=output, output_group=output_group)
             with self._write_file(fh=backend_file):
@@ -616,16 +627,184 @@ class TupBackend(CommonBackend):
             fh.write('IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser\n')
             fh.write('IDL_PARSER_CACHE_DIR = $(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl\n')
 
         # Run 'tup init' if necessary.
         if not os.path.exists(mozpath.join(self.environment.topsrcdir, ".tup")):
             tup = self.environment.substs.get('TUP', 'tup')
             self._cmd.run_process(cwd=self.environment.topsrcdir, log_name='tup', args=[tup, 'init'])
 
+
+    def _get_cargo_flags(self, obj):
+        cargo_flags = ['--build-plan', '-Z', 'unstable-options']
+        if not self.environment.substs.get('MOZ_DEBUG_RUST'):
+            cargo_flags += ['--release']
+        cargo_flags += [
+            '--frozen',
+            '--manifest-path', mozpath.join(obj.srcdir, 'Cargo.toml'),
+            '--lib',
+            '--target=%s' % self.environment.substs['RUST_TARGET'],
+        ]
+        if obj.features:
+            cargo_flags += [
+                '--features', ' '.join(obj.features)
+            ]
+        return cargo_flags
+
+    def _get_cargo_env(self, backend_file):
+        lib = backend_file.rust_library
+        env = {
+            'CARGO_TARGET_DIR': mozpath.normpath(mozpath.join(lib.objdir,
+                                                              lib.target_dir)),
+            'RUSTC': self.environment.substs['RUSTC'],
+            'MOZ_SRC': self.environment.topsrcdir,
+            'MOZ_DIST': self.environment.substs['DIST'],
+            'LIBCLANG_PATH': self.environment.substs['MOZ_LIBCLANG_PATH'],
+            'CLANG_PATH': self.environment.substs['MOZ_CLANG_PATH'],
+            'PKG_CONFIG_ALLOW_CROSS': '1',
+            'RUST_BACKTRACE': 'full',
+            'MOZ_TOPOBJDIR': self.environment.topobjdir,
+            'PYTHON': self.environment.substs['PYTHON'],
+            'PYTHONDONTWRITEBYTECODE': '1',
+        }
+        cargo_incremental = self.environment.substs.get('CARGO_INCREMENTAL')
+        if cargo_incremental is not None:
+            # TODO (bug 1468527): CARGO_INCREMENTAL produces outputs that Tup
+            # doesn't know about, disable it unconditionally for now.
+            pass # env['CARGO_INCREMENTAL'] = cargo_incremental
+
+        rust_simd = self.environment.substs.get('MOZ_RUST_SIMD')
+        if rust_simd is not None:
+            env['RUSTC_BOOTSTRAP'] = '1'
+
+        linker_env_var = ('CARGO_TARGET_%s_LINKER' %
+                          self.environment.substs['RUST_TARGET_ENV_NAME'])
+
+        env.update({
+            'MOZ_CARGO_WRAP_LDFLAGS': ' '.join(backend_file.local_flags['LDFLAGS']),
+            'MOZ_CARGO_WRAP_LD': backend_file.environment.substs['CC'],
+            linker_env_var: mozpath.join(self.environment.topsrcdir,
+                                         'build', 'cargo-linker'),
+            'RUSTFLAGS': '%s %s' % (' '.join(self.environment.substs['MOZ_RUST_DEFAULT_FLAGS']),
+                                    ' '.join(self.environment.substs['RUSTFLAGS'])),
+        })
+        return env
+
+    def _gen_cargo_rules(self, backend_file, build_plan, cargo_env):
+        invocations = build_plan['invocations']
+        processed = set()
+
+        def get_libloading_outdir():
+            for invocation in invocations:
+                if (invocation['package_name'] == 'libloading' and
+                    invocation['outputs'][0].endswith('.rlib')):
+                    return invocation['env']['OUT_DIR']
+
+        def display_name(invocation):
+            output_str = ''
+            if invocation['outputs']:
+                output_str = ' -> %s' % ' '.join([os.path.basename(f)
+                                                  for f in invocation['outputs']])
+            return '{name} v{version} {kind}{output}'.format(
+                name=invocation['package_name'],
+                version=invocation['package_version'],
+                kind=invocation['kind'],
+                output=output_str
+            )
+
+        def cargo_quote(s):
+            return shell_quote(s.replace('\n', '\\n'))
+
+        def _process(key, invocation):
+            if key in processed:
+                return
+            processed.add(key)
+            inputs = set()
+            shortname = invocation['package_name']
+            for dep in invocation['deps']:
+                # We'd expect to just handle dependencies transitively (so use
+                # invocations[dep]['outputs'] here, but because the weird host dependencies
+                # sometimes get used in the final library and not intermediate
+                # libraries, tup doesn't work well with them. So build up the full set
+                # of intermediate dependencies with 'full-deps'
+                depmod = invocations[dep]
+                _process(dep, depmod)
+                inputs.update(depmod['full-deps'])
+
+            command = [
+                'cd %s &&' % invocation['cwd'],
+                'env',
+            ]
+            envvars = invocation.get('env')
+            for k, v in itertools.chain(cargo_env.iteritems(),
+                                        envvars.iteritems()):
+                command.append("%s=%s" % (k, cargo_quote(v)))
+            command.append(invocation['program'])
+            command.extend(cargo_quote(a.replace('dep-info,', ''))
+                           for a in invocation['args'])
+            outputs = invocation['outputs']
+            if os.path.basename(invocation['program']) == 'build-script-build':
+                for output in cargo_extra_outputs.get(shortname, []):
+                    outputs.append(os.path.join(invocation['env']['OUT_DIR'], output))
+
+            if (invocation['target_kind'][0] == 'custom-build' and
+                os.path.basename(invocation['program']) == 'rustc'):
+                flags = cargo_extra_flags.get(shortname, [])
+                for flag in flags:
+                    command.append(flag % {'libloading_outdir': get_libloading_outdir()})
+
+            if 'rustc' in invocation['program']:
+                header = 'RUSTC'
+            else:
+                inputs.add(invocation['program'])
+                header = 'RUN'
+
+            invocation['full-deps'] = set(inputs)
+            invocation['full-deps'].update(invocation['outputs'])
+
+            backend_file.rule(
+                command,
+                inputs=sorted(inputs),
+                outputs=outputs,
+                extra_outputs=[self._rust_libs],
+                extra_inputs=[self._installed_files],
+                display='%s %s' % (header, display_name(invocation)),
+            )
+
+            for dst, link in invocation['links'].iteritems():
+                backend_file.symlink_rule(link, dst, self._rust_libs)
+
+        for val in enumerate(invocations):
+            _process(*val)
+
+
+    def _gen_rust(self, backend_file):
+        # TODO (bug 1468547): The gtest rust library depends on many of the same
+        # libraries as the main rust library, so we'll need to handle these all
+        # at once in order to build the gtest rust library.
+        if 'toolkit/library/gtest' in backend_file.objdir:
+            return
+
+        cargo_flags = self._get_cargo_flags(backend_file.rust_library)
+        cargo_env = self._get_cargo_env(backend_file)
+
+        output_lines = []
+        def accumulate_output(line):
+            output_lines.append(line)
+
+        cargo_status = self._cmd.run_process(
+            [self.environment.substs['CARGO'], 'build'] + cargo_flags,
+            line_handler=accumulate_output,
+            explicit_env=cargo_env)
+
+        cargo_plan = json.loads(''.join(output_lines))
+        self._gen_cargo_rules(backend_file, cargo_plan, cargo_env)
+        self.backend_input_files |= set(cargo_plan['inputs'])
+
+
     def _process_generated_file(self, backend_file, obj):
         # TODO: These are directories that don't work in the tup backend
         # yet, because things they depend on aren't built yet.
         skip_directories = (
             'toolkit/library', # libxul.so
         )
         if obj.script and obj.method and obj.relobjdir not in skip_directories:
             backend_file.export_shell()