Bug 1300208 - Allow specifying multiple rust crates to mozbuild within the same FINAL_LIBRARY, r=froydnj
authorMichael Layzell <michael@thelayzells.com>
Fri, 02 Sep 2016 18:05:09 -0400
changeset 314637 362852b89f4e5a2254f26e8d601279331ff6bc2a
parent 314636 496081cb3214f37199064dc709b5fd3dd13d6a72
child 314638 a222c243ea01c03cf8b8b666b642b412e82cc704
push id30732
push usercbook@mozilla.com
push dateWed, 21 Sep 2016 10:04:03 +0000
treeherdermozilla-central@560b2c805bf7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1300208
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1300208 - Allow specifying multiple rust crates to mozbuild within the same FINAL_LIBRARY, r=froydnj MozReview-Commit-ID: IIjV4Kg7wOv
config/expandlibs_gen.py
config/rules.mk
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml
python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml
toolkit/library/gtest/rust/Cargo.toml
toolkit/library/rust/Cargo.toml
--- a/config/expandlibs_gen.py
+++ b/config/expandlibs_gen.py
@@ -16,16 +16,21 @@ def generate(args):
     desc = LibDescriptor()
     for arg in args:
         if isObject(arg):
             if os.path.exists(arg):
                 desc['OBJS'].append(os.path.abspath(arg))
             else:
                 raise Exception("File not found: %s" % arg)
         elif os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
+            # We want to skip static libraries with the name foo-rs-prelink
+            # as they are individually linked for every final library, and
+            # thus should not be included in the descriptor file
+            if '-rs-prelink' in os.path.basename(arg):
+                continue
             if os.path.exists(arg) or os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
                 desc['LIBS'].append(os.path.abspath(arg))
             else:
                 raise Exception("File not found: %s" % arg)
     return desc
 
 if __name__ == '__main__':
     parser = OptionParser()
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -935,16 +935,40 @@ cargo_build_flags += --verbose
 # has full visibility into how changes in Rust sources might affect the final
 # build.
 force-cargo-build:
 	$(REPORT_BUILD)
 	env CARGO_TARGET_DIR=. RUSTC=$(RUSTC) $(CARGO) build $(cargo_build_flags) --
 
 $(RUST_LIBRARY_FILE): force-cargo-build
 endif # CARGO_FILE
+
+ifdef RUST_PRELINK
+# Make target for building a prelinked rust library. This merges rust .rlibs
+# together into a single .a file which is used within the FINAL_LIBRARY.
+#
+# RUST_PRELINK_FLAGS, RUST_PRELINK_SRC, and RUST_PRELINK_DEPS are set in
+# recursivemake.py, and together tell rustc how to find the libraries to link
+# together, but we compute the optimization flags below
+
+RUST_PRELINK_FLAGS += -g
+RUST_PRELINK_FLAGS += -C panic=abort
+
+ifdef MOZ_DEBUG
+RUST_PRELINK_FLAGS += -C opt-level=1
+RUST_PRELINK_FLAGS += -C debug-assertions
+else
+RUST_PRELINK_FLAGS += -C opt-level=2
+RUST_PRELINK_FLAGS += -C lto
+endif
+
+$(RUST_PRELINK): $(RUST_PRELINK_DEPS) $(RUST_PRELINK_SRC)
+	$(REPORT_BUILD)
+	$(RUSTC) -o $@ --crate-type staticlib --target $(RUST_TARGET) $(RUST_PRELINK_FLAGS) $(RUST_PRELINK_SRC)
+endif # RUST_PRELINK
 endif # MOZ_RUST
 
 $(SOBJS):
 	$(REPORT_BUILD)
 	$(AS) -o $@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(LOCAL_INCLUDES) -c $<
 
 $(CPPOBJS):
 	$(REPORT_BUILD_VERBOSE)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1231,35 +1231,59 @@ class RecursiveMakeBackend(CommonBackend
             elif isinstance(obj, (HostLibrary, HostProgram, HostSimpleProgram)):
                 assert isinstance(lib, HostLibrary)
                 backend_file.write_once('HOST_LIBS += %s/%s\n'
                                    % (relpath, lib.import_name))
 
         # We have to link any Rust libraries after all intermediate static
         # libraries have been listed to ensure that the Rust libraries are
         # searched after the C/C++ objects that might reference Rust symbols.
-        # Building Rust crates normally takes care of Rust->Rust linkage, but
-        # we have to be careful: a savvy user might have specified that there
-        # is a staticlib (that contains the Rust runtime) and several other
-        # rlibs (which are plain archive files) to be linked into a given
-        # library.  We need to ensure that the staticlib comes after the
-        # rlibs to ensure symbols are found correctly.
-        if isinstance(obj, SharedLibrary) and any(isinstance(o, RustLibrary)
-                                                  for o in obj.linked_libraries):
-            libs = [l for l in obj.linked_libraries if isinstance(l, RustLibrary)]
-            def name_cmp(l1, l2):
-                if l1.crate_type == 'staticlib':
-                    return 1
-                if l2.crate_type == 'staticlib':
-                    return -1
-                return cmp(l1.basename, l2.basename)
-            libs.sort(cmp=name_cmp)
-            for l in libs:
-                relpath = pretty_relpath(l)
-                backend_file.write('STATIC_LIBS += %s/%s\n' % (relpath, l.import_name))
+
+        def find_rlibs(obj):
+            if isinstance(obj, RustLibrary):
+                yield obj
+            elif isinstance(obj, StaticLibrary) and not obj.no_expand_lib:
+                for l in obj.linked_libraries:
+                    for rlib in find_rlibs(l):
+                        yield rlib
+
+        # Check if we have any rust libraries to prelink and include in our
+        # final library. If we do, write out the RUST_PRELINK information
+        rlibs = []
+        if isinstance(obj, (SharedLibrary, StaticLibrary)):
+            for l in obj.linked_libraries:
+                rlibs += find_rlibs(l)
+        if rlibs:
+            prelink_libname = '%s/%s%s-rs-prelink%s' \
+                              % (relpath,
+                                 obj.config.lib_prefix,
+                                 obj.basename,
+                                 obj.config.lib_suffix)
+            backend_file.write('RUST_PRELINK := %s\n' % prelink_libname)
+            backend_file.write_once('STATIC_LIBS += %s\n' % prelink_libname)
+
+            extern_crate_file = mozpath.join(
+                obj.objdir, '%s-rs-prelink.rs' % obj.basename)
+            with self._write_file(extern_crate_file) as f:
+                f.write('// AUTOMATICALLY GENERATED.  DO NOT EDIT.\n\n')
+                for rlib in rlibs:
+                    f.write('extern crate %s;\n'
+                            % rlib.basename.replace('-', '_'))
+            backend_file.write('RUST_PRELINK_SRC := %s\n' % extern_crate_file)
+
+            backend_file.write('RUST_PRELINK_FLAGS :=\n')
+            backend_file.write('RUST_PRELINK_DEPS :=\n')
+            for rlib in rlibs:
+                rlib_relpath = pretty_relpath(rlib)
+                backend_file.write('RUST_PRELINK_FLAGS += --extern %s=%s/%s\n'
+                                   % (rlib.basename, rlib_relpath, rlib.import_name))
+                backend_file.write('RUST_PRELINK_FLAGS += -L %s/%s\n'
+                                   % (rlib_relpath, rlib.deps_path))
+                backend_file.write('RUST_PRELINK_DEPS += %s/%s\n'
+                                   % (rlib_relpath, rlib.import_name))
 
         for lib in obj.linked_system_libs:
             if obj.KIND == 'target':
                 backend_file.write_once('OS_LIBS += %s\n' % lib)
             else:
                 backend_file.write_once('HOST_EXTRA_LIBS += %s\n' % lib)
 
         # Process library-based defines
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -459,43 +459,39 @@ class StaticLibrary(Library):
         self.no_expand_lib = no_expand_lib
 
 
 class RustLibrary(StaticLibrary):
     """Context derived container object for a static library"""
     __slots__ = (
         'cargo_file',
         'crate_type',
+        'deps_path',
     )
 
     def __init__(self, context, basename, cargo_file, crate_type, **args):
         StaticLibrary.__init__(self, context, basename, **args)
         self.cargo_file = cargo_file
         self.crate_type = crate_type
         # We need to adjust our naming here because cargo replaces '-' in
         # package names defined in Cargo.toml with underscores in actual
         # filenames. But we need to keep the basename consistent because
         # many other things in the build system depend on that.
-        assert self.crate_type == 'staticlib'
-        self.lib_name = '%s%s%s' % (
-            context.config.lib_prefix,
-            basename.replace('-', '_'),
-            context.config.lib_suffix
-        )
+        assert self.crate_type == 'rlib'
+        self.lib_name = 'lib%s.rlib' % basename.replace('-', '_')
         # cargo creates several directories and places its build artifacts
         # in those directories.  The directory structure depends not only
         # on the target, but also what sort of build we are doing.
         rust_build_kind = 'release'
         if context.config.substs.get('MOZ_DEBUG'):
             rust_build_kind = 'debug'
-        self.import_name = '%s/%s/%s' % (
-            context.config.substs['RUST_TARGET'],
-            rust_build_kind,
-            self.lib_name,
-        )
+        build_dir = mozpath.join(context.config.substs['RUST_TARGET'],
+                                 rust_build_kind)
+        self.import_name = mozpath.join(build_dir, self.lib_name)
+        self.deps_path = mozpath.join(build_dir, 'deps')
 
 
 class SharedLibrary(Library):
     """Context derived container object for a shared library"""
     __slots__ = (
         'soname',
         'variant',
         'symbols_file',
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -431,17 +431,17 @@ class TreeMetadataEmitter(LoggingMixin):
 
         crate_type = lib_section.get('crate-type', None)
         if not crate_type:
             raise SandboxValidationError(
                 'Can\'t determine a crate-type for %s from Cargo.toml' % libname,
                 context)
 
         crate_type = crate_type[0]
-        if crate_type != 'staticlib':
+        if crate_type != 'rlib':
             raise SandboxValidationError(
                 'crate-type %s is not permitted for %s' % (crate_type, libname),
                 context)
 
 
         return RustLibrary(context, libname, cargo_file, crate_type, **static_args)
 
     def _handle_linkables(self, context, passthru, generated_files):
@@ -654,27 +654,16 @@ class TreeMetadataEmitter(LoggingMixin):
                 is_rust_library = context.get('IS_RUST_LIBRARY')
                 if is_rust_library:
                     lib = self._rust_library(context, libname, static_args)
                 else:
                     lib = StaticLibrary(context, libname, **static_args)
                 self._libs[libname].append(lib)
                 self._linkage.append((context, lib, 'USE_LIBS'))
 
-                # Multiple staticlibs for a library means multiple copies
-                # of the Rust runtime, which will result in linking errors
-                # later on.
-                if is_rust_library:
-                    staticlibs = [l for l in self._libs[libname]
-                                  if isinstance(l, RustLibrary) and l.crate_type == 'staticlib']
-                    if len(staticlibs) > 1:
-                        raise SandboxValidationError(
-                            'Cannot have multiple Rust staticlibs in %s: %s' % (libname, ', '.join(l.basename for l in staticlibs)),
-                            context)
-
                 has_linkables = True
 
             if lib_defines:
                 if not libname:
                     raise SandboxValidationError('LIBRARY_DEFINES needs a '
                         'LIBRARY_NAME to take effect', context)
                 lib.lib_defines.update(lib_defines)
 
--- a/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml
+++ b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml
@@ -1,12 +1,12 @@
 [package]
 name = "random-crate"
 version = "0.1.0"
 authors = [
   "Nobody <nobody@mozilla.org>",
 ]
 
 [lib]
-crate-type = ["staticlib"]
+crate-type = ["rlib"]
 
 [dependencies]
 deep-crate = { version = "0.1.0", path = "the/depths" }
--- a/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml
+++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml
@@ -1,9 +1,9 @@
 [package]
 name = "random-crate"
 version = "0.1.0"
 authors = [
   "Nobody <nobody@mozilla.org>",
 ]
 
 [lib]
-crate-type = ["staticlib"]
+crate-type = ["rlib"]
--- a/toolkit/library/gtest/rust/Cargo.toml
+++ b/toolkit/library/gtest/rust/Cargo.toml
@@ -5,17 +5,17 @@ authors = ["nobody@mozilla.org"]
 license = "MPL-2.0"
 description = "Testing code for libgkrust"
 
 [dependencies]
 mp4parse-gtest = { path = "../../../../dom/media/gtest" }
 
 [lib]
 path = "lib.rs"
-crate-type = ["staticlib"]
+crate-type = ["rlib"]
 test = false
 doctest = false
 bench = false
 doc = false
 plugin = false
 harness = false
 
 # Explicitly specify what our profiles use.
--- a/toolkit/library/rust/Cargo.toml
+++ b/toolkit/library/rust/Cargo.toml
@@ -5,17 +5,17 @@ authors = ["nobody@mozilla.org"]
 license = "MPL-2.0"
 description = "Rust code for libxul"
 
 [dependencies]
 mp4parse_capi = { path = "../../../media/libstagefright/binding/mp4parse_capi" }
 
 [lib]
 path = "lib.rs"
-crate-type = ["staticlib"]
+crate-type = ["rlib"]
 test = false
 doctest = false
 bench = false
 doc = false
 plugin = false
 harness = false
 
 # Explicitly specify what our profiles use.