Bug 1163224 - add build system support for multiple Rust crates; r=glandium
authorNathan Froyd <froydnj.com>
Thu, 21 Apr 2016 09:54:01 -0400
changeset 332160 841c2247f57d86fa30025f9d0c8d3f461f3b2be5
parent 332159 173a441d7bb1be25728c5348135d3935ab20195a
child 332161 8c852c602947803ab731dc0dbb45994b60781360
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1163224
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1163224 - add build system support for multiple Rust crates; r=glandium Our current build system support for Rust compiles any Rust crate into a so-called staticlib, which is a static library (.a file) that includes the Rust runtime. That staticlib is then linked into libxul. For supporting multiple crates, this approach breaks down, as linking multiple copies of the Rust runtime is going to fail. For supporting multiple crates, the approach taken here is to compile each crate into a so-called rlib, which is essentially a staticlib without the Rust runtime linked in. The build system takes note of every crate destined for linking with libxul (treating them like static libraries generated from C/C++ files), and generates a super-crate, whimsically named "rul", that is compiled as a staticlib (so as to include the Rust runtime) and then linked into libxul. Thus only one copy of the Rust runtime is included, and the Rust compiler can take care of any inter-crate dependencies. This patch currently only supports Rust code in shared libraries, not in binaries.
config/rules.mk
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -235,19 +235,20 @@ endif
 COBJS = $(notdir $(CSRCS:.c=.$(OBJ_SUFFIX)))
 SOBJS = $(notdir $(SSRCS:.S=.$(OBJ_SUFFIX)))
 # CPPSRCS can have different extensions (eg: .cpp, .cc)
 CPPOBJS = $(notdir $(addsuffix .$(OBJ_SUFFIX),$(basename $(CPPSRCS))))
 CMOBJS = $(notdir $(CMSRCS:.m=.$(OBJ_SUFFIX)))
 CMMOBJS = $(notdir $(CMMSRCS:.mm=.$(OBJ_SUFFIX)))
 # ASFILES can have different extensions (.s, .asm)
 ASOBJS = $(notdir $(addsuffix .$(OBJ_SUFFIX),$(basename $(ASFILES))))
-RSOBJS = $(addprefix lib,$(notdir $(RSSRCS:.rs=.$(LIB_SUFFIX))))
+RSOBJS = $(addprefix lib,$(notdir $(RSSRCS:.rs=.rlib)))
+RS_STATICLIB_CRATE_OBJ = $(addprefix lib,$(notdir $(RS_STATICLIB_CRATE_SRC:.rs=.$(LIB_SUFFIX))))
 ifndef OBJS
-_OBJS = $(COBJS) $(SOBJS) $(CPPOBJS) $(CMOBJS) $(CMMOBJS) $(ASOBJS) $(RSOBJS)
+_OBJS = $(COBJS) $(SOBJS) $(CPPOBJS) $(CMOBJS) $(CMMOBJS) $(ASOBJS) $(RSOBJS) $(RS_STATICLIB_CRATE_OBJ)
 OBJS = $(strip $(_OBJS))
 endif
 
 HOST_COBJS = $(addprefix host_,$(notdir $(HOST_CSRCS:.c=.$(OBJ_SUFFIX))))
 # HOST_CPPOBJS can have different extensions (eg: .cpp, .cc)
 HOST_CPPOBJS = $(addprefix host_,$(notdir $(addsuffix .$(OBJ_SUFFIX),$(basename $(HOST_CPPSRCS)))))
 HOST_CMOBJS = $(addprefix host_,$(notdir $(HOST_CMSRCS:.m=.$(OBJ_SUFFIX))))
 HOST_CMMOBJS = $(addprefix host_,$(notdir $(HOST_CMMSRCS:.mm=.$(OBJ_SUFFIX))))
@@ -854,19 +855,22 @@ endif
 define src_objdep
 $(basename $2$(notdir $1)).$(OBJ_SUFFIX): $1 $$(call mkdir_deps,$$(MDDEPDIR))
 endef
 $(foreach f,$(CSRCS) $(SSRCS) $(CPPSRCS) $(CMSRCS) $(CMMSRCS) $(ASFILES),$(eval $(call src_objdep,$(f))))
 $(foreach f,$(HOST_CSRCS) $(HOST_CPPSRCS) $(HOST_CMSRCS) $(HOST_CMMSRCS),$(eval $(call src_objdep,$(f),host_)))
 
 # The Rust compiler only outputs library objects, and so we need different
 # mangling to generate dependency rules for it.
-mk_libname = $(basename lib$(notdir $1)).$(LIB_SUFFIX)
+mk_libname = $(basename lib$(notdir $1)).rlib
 src_libdep = $(call mk_libname,$1): $1 $$(call mkdir_deps,$$(MDDEPDIR))
+mk_global_crate_libname = $(basename lib$(notdir $1)).$(LIB_SUFFIX)
+crate_src_libdep = $(call mk_global_crate_libname,$1): $1 $$(call mkdir_deps,$$(MDDEPDIR))
 $(foreach f,$(RSSRCS),$(eval $(call src_libdep,$(f))))
+$(foreach f,$(RS_STATICLIB_CRATE_SRC),$(eval $(call crate_src_libdep,$(f))))
 
 $(OBJS) $(HOST_OBJS) $(PROGOBJS) $(HOST_PROGOBJS): $(GLOBAL_DEPS)
 
 # Rules for building native targets must come first because of the host_ prefix
 $(HOST_COBJS):
 	$(REPORT_BUILD_VERBOSE)
 	$(ELOG) $(HOST_CC) $(HOST_OUTOPTION)$@ -c $(HOST_CPPFLAGS) $(HOST_CFLAGS) $(INCLUDES) $(NSPR_CFLAGS) $(_VPATH_SRCS)
 
@@ -909,18 +913,22 @@ ifdef ASFILES
 	$(REPORT_BUILD_VERBOSE)
 	$(AS) $(ASOUTOPTION)$@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(AS_DASH_C_FLAG) $(_VPATH_SRCS)
 endif
 
 ifdef MOZ_RUST
 # Assume any system libraries rustc links against are already
 # in the target's LIBS.
 $(RSOBJS):
-	$(REPORT_BUILD_VERBOSE)
-	$(RUSTC) $(RUSTFLAGS) --crate-type staticlib --emit dep-info=$(MDDEPDIR)/$(call mk_libname,$<).pp,link=$(call mk_libname,$<) $(_VPATH_SRCS)
+	$(REPORT_BUILD)
+	$(RUSTC) $(RUSTFLAGS) --crate-type rlib --emit dep-info=$(MDDEPDIR)/$(call mk_libname,$<).pp,link=$(call mk_libname,$<) $(_VPATH_SRCS)
+
+$(RS_STATICLIB_CRATE_OBJ):
+	$(REPORT_BUILD)
+	$(RUSTC) $(RUSTFLAGS) --crate-type staticlib $(RLIB_EXTERN_CRATE_OPTIONS) --emit dep-info=$(MDDEPDIR)/$(call mk_global_crate_libname,$(RS_STATICLIB_CRATE_SRC)).pp,link=$@ $(RS_STATICLIB_CRATE_SRC)
 endif
 
 $(SOBJS):
 	$(REPORT_BUILD)
 	$(AS) -o $@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(LOCAL_INCLUDES) $(TARGET_LOCAL_INCLUDES) -c $<
 
 $(CPPOBJS):
 	$(REPORT_BUILD_VERBOSE)
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -26,16 +26,17 @@ from mozbuild.frontend.data import (
     ExampleWebIDLInterface,
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
+    RustRlibLibrary,
     SharedLibrary,
     TestManifest,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
 )
 from mozbuild.jar import (
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -56,16 +56,17 @@ from ..frontend.data import (
     JARManifest,
     JavaJarData,
     Library,
     LocalInclude,
     ObjdirFiles,
     ObjdirPreprocessedFiles,
     PerSourceFlag,
     Program,
+    RustRlibLibrary,
     SharedLibrary,
     SimpleProgram,
     Sources,
     StaticLibrary,
     TestManifest,
     VariablePassthru,
     XPIDLFile,
 )
@@ -568,16 +569,20 @@ class RecursiveMakeBackend(CommonBackend
             # automated.
             if isinstance(obj.wrapped, JavaJarData):
                 self._process_java_jar_data(obj.wrapped, backend_file)
             elif isinstance(obj.wrapped, AndroidEclipseProjectData):
                 self._process_android_eclipse_project_data(obj.wrapped, backend_file)
             else:
                 return False
 
+        elif isinstance(obj, RustRlibLibrary):
+            # Nothing to do because |Sources| has done the work for us.
+            pass
+
         elif isinstance(obj, SharedLibrary):
             self._process_shared_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
         elif isinstance(obj, StaticLibrary):
             self._process_static_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
@@ -1142,16 +1147,27 @@ class RecursiveMakeBackend(CommonBackend
             backend_file.write('IS_COMPONENT := 1\n')
         if libdef.soname:
             backend_file.write('DSO_SONAME := %s\n' % libdef.soname)
         if libdef.is_sdk:
             backend_file.write('SDK_LIBRARY := %s\n' % libdef.import_name)
         if libdef.symbols_file:
             backend_file.write('SYMBOLS_FILE := %s\n' % libdef.symbols_file)
 
+        rust_rlibs = [o for o in libdef.linked_libraries if isinstance(o, RustRlibLibrary)]
+        if rust_rlibs:
+            # write out Rust file with extern crate declarations.
+            extern_crate_file = mozpath.join(libdef.objdir, 'rul.rs')
+            with self._write_file(extern_crate_file) as f:
+                f.write('// AUTOMATICALLY GENERATED.  DO NOT EDIT.\n\n')
+                for rlib in rust_rlibs:
+                    f.write('extern crate %s;\n' % rlib.crate_name)
+
+            backend_file.write('RS_STATICLIB_CRATE_SRC := %s\n' % extern_crate_file)
+
     def _process_static_library(self, libdef, backend_file):
         backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename)
         backend_file.write('FORCE_STATIC_LIB := 1\n')
         backend_file.write('REAL_LIBRARY := %s\n' % libdef.lib_name)
         if libdef.is_sdk:
             backend_file.write('SDK_LIBRARY := %s\n' % libdef.import_name)
         if libdef.no_expand_lib:
             backend_file.write('NO_EXPAND_LIBS := 1\n')
@@ -1188,16 +1204,19 @@ class RecursiveMakeBackend(CommonBackend
                     self._build_target_for_obj(lib))
             relpath = pretty_relpath(lib)
             if isinstance(obj, Library):
                 if isinstance(lib, StaticLibrary):
                     backend_file.write_once('STATIC_LIBS += %s/%s\n'
                                         % (relpath, lib.import_name))
                     if isinstance(obj, SharedLibrary):
                         write_shared_and_system_libs(lib)
+                elif isinstance(lib, RustRlibLibrary):
+                    backend_file.write_once('RLIB_EXTERN_CRATE_OPTIONS += --extern %s=%s/%s\n'
+                                            % (lib.crate_name, relpath, lib.rlib_filename))
                 elif isinstance(obj, SharedLibrary):
                     assert lib.variant != lib.COMPONENT
                     backend_file.write_once('SHARED_LIBS += %s/%s\n'
                                         % (relpath, lib.import_name))
             elif isinstance(obj, (Program, SimpleProgram)):
                 if isinstance(lib, StaticLibrary):
                     backend_file.write_once('STATIC_LIBS += %s/%s\n'
                                         % (relpath, lib.import_name))
@@ -1206,16 +1225,24 @@ class RecursiveMakeBackend(CommonBackend
                     assert lib.variant != lib.COMPONENT
                     backend_file.write_once('SHARED_LIBS += %s/%s\n'
                                         % (relpath, lib.import_name))
             elif isinstance(obj, (HostLibrary, HostProgram, HostSimpleProgram)):
                 assert isinstance(lib, HostLibrary)
                 backend_file.write_once('HOST_LIBS += %s/%s\n'
                                    % (relpath, lib.import_name))
 
+        # We have to link the Rust super-crate after all intermediate static
+        # libraries have been listed to ensure that the Rust objects are
+        # searched after the C/C++ objects that might reference Rust symbols.
+        # Building the Rust super-crate will take care of Rust->Rust linkage.
+        if isinstance(obj, SharedLibrary) and any(isinstance(o, RustRlibLibrary)
+                                                  for o in obj.linked_libraries):
+            backend_file.write('STATIC_LIBS += librul.$(LIB_SUFFIX)\n')
+
         for lib in obj.linked_system_libs:
             if obj.KIND == 'target':
                 backend_file.write_once('OS_LIBS += %s\n' % lib)
             else:
                 backend_file.write_once('HOST_EXTRA_LIBS += %s\n' % lib)
 
         # Process library-based defines
         self._process_defines(obj.lib_defines, backend_file)
@@ -1393,16 +1420,22 @@ class RecursiveMakeBackend(CommonBackend
             # that if the Makefile.in disappears, this will force
             # moz.build traversal. This means that when we remove empty
             # Makefile.in files, the old file will get replaced with
             # the autogenerated one automatically.
             self.backend_input_files.add(obj.input_path)
 
         self._makefile_out_count += 1
 
+    def _handle_linked_rust_crates(self, obj, extern_crate_file):
+        backend_file = self._get_backend_file_for(obj)
+
+        backend_file.write('RS_STATICLIB_CRATE_SRC := %s\n' % extern_crate_file)
+        backend_file.write('STATIC_LIBS += librul.$(LIB_SUFFIX)\n')
+
     def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
                              unified_ipdl_cppsrcs_mapping):
         # Write out a master list of all IPDL source files.
         mk = Makefile()
 
         mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources))
 
         self._add_unified_build_rules(mk, unified_ipdl_cppsrcs_mapping,
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -440,16 +440,26 @@ class Library(BaseLibrary):
     )
 
     def __init__(self, context, basename, real_name=None, is_sdk=False):
         BaseLibrary.__init__(self, context, real_name or basename)
         self.basename = basename
         self.is_sdk = is_sdk
 
 
+class RustRlibLibrary(Library):
+    """Context derived container object for a Rust rlib"""
+
+    def __init__(self, context, basename, crate_name, rlib_filename, link_into):
+        Library.__init__(self, context, basename)
+        self.crate_name = crate_name
+        self.rlib_filename = rlib_filename
+        self.link_into = link_into
+
+
 class StaticLibrary(Library):
     """Context derived container object for a static library"""
     __slots__ = (
         'link_into',
         'no_expand_lib',
     )
 
     def __init__(self, context, basename, real_name=None, is_sdk=False,
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -21,16 +21,17 @@ from mozbuild.util import (
 import mozpack.path as mozpath
 import mozinfo
 
 from .data import (
     AndroidAssetsDirs,
     AndroidExtraPackages,
     AndroidExtraResDirs,
     AndroidResDirs,
+    BaseSources,
     BrandingFiles,
     ChromeManifestEntry,
     ConfigFileSubstitution,
     ContextWrapped,
     Defines,
     DirectoryTraversal,
     Exports,
     FinalTargetFiles,
@@ -54,16 +55,17 @@ from .data import (
     Linkable,
     LocalInclude,
     ObjdirFiles,
     ObjdirPreprocessedFiles,
     PerSourceFlag,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     Program,
+    RustRlibLibrary,
     SdkFiles,
     SharedLibrary,
     SimpleProgram,
     Sources,
     StaticLibrary,
     TestHarnessFiles,
     TestWebIDLFile,
     TestManifest,
@@ -188,17 +190,17 @@ class TreeMetadataEmitter(LoggingMixin):
             objs = list(self._emit_libs_derived(contexts))
             self._emitter_time += time.time() - start
 
             for o in emit_objs(objs): yield o
 
     def _emit_libs_derived(self, contexts):
         # First do FINAL_LIBRARY linkage.
         for lib in (l for libs in self._libs.values() for l in libs):
-            if not isinstance(lib, StaticLibrary) or not lib.link_into:
+            if not isinstance(lib, (StaticLibrary, RustRlibLibrary)) or not lib.link_into:
                 continue
             if lib.link_into not in self._libs:
                 raise SandboxValidationError(
                     'FINAL_LIBRARY ("%s") does not match any LIBRARY_NAME'
                     % lib.link_into, contexts[lib.objdir])
             candidates = self._libs[lib.link_into]
 
             # When there are multiple candidates, but all are in the same
@@ -590,16 +592,17 @@ class TreeMetadataEmitter(LoggingMixin):
                 lib.lib_defines.update(lib_defines)
 
         # Only emit sources if we have linkables defined in the same context.
         # Note the linkables are not emitted in this function, but much later,
         # after aggregation (because of e.g. USE_LIBS processing).
         if not has_linkables:
             return
 
+        rust_sources = []
         sources = defaultdict(list)
         gen_sources = defaultdict(list)
         all_flags = {}
         for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
             srcs = sources[symbol]
             gen_srcs = gen_sources[symbol]
             context_srcs = context.get(symbol, [])
             for f in context_srcs:
@@ -685,17 +688,51 @@ class TreeMetadataEmitter(LoggingMixin):
                 # Now sort the files to let groupby work.
                 sorted_files = sorted(srcs, key=canonical_suffix_for_file)
                 for canonical_suffix, files in itertools.groupby(
                         sorted_files, canonical_suffix_for_file):
                     arglist = [context, list(files), canonical_suffix]
                     if (variable.startswith('UNIFIED_') and
                             'FILES_PER_UNIFIED_FILE' in context):
                         arglist.append(context['FILES_PER_UNIFIED_FILE'])
-                    yield cls(*arglist)
+                    obj = cls(*arglist)
+
+                    # Rust is special.  Each Rust file that we compile
+                    # is a separate crate and gets compiled into its own
+                    # rlib file (Rust's equivalent of a .a file).  When
+                    # we go to link Rust sources into a library or
+                    # executable, we have a separate, single crate that
+                    # gets compiled as a staticlib (like a rlib, but
+                    # includes the actual Rust runtime).
+                    if canonical_suffix == '.rs':
+                        if not final_lib:
+                            raise SandboxValidationError(
+                                'Rust sources must be destined for a FINAL_LIBRARY')
+                        # If we're building a shared library, we're going to
+                        # auto-generate a module that includes all of the rlib
+                        # files.  This means we can't permit Rust files as
+                        # immediate inputs of the shared library.
+                        if libname and shared_lib:
+                            raise SandboxValidationError(
+                                'No Rust sources permitted as an immediate input of %s: %s' % (shlib, files))
+                        rust_sources.append(obj)
+                    yield obj
+
+        # Rust sources get translated into rlibs, which are essentially static
+        # libraries.  We need to note all of them for linking, too.
+        if libname and static_lib:
+            for s in rust_sources:
+                for f in s.files:
+                    (base, _) = mozpath.splitext(mozpath.basename(f))
+                    crate_name = context.relsrcdir.replace('/', '_') + '_' + base
+                    rlib_filename = 'lib' + base + '.rlib'
+                    lib = RustRlibLibrary(context, libname, crate_name,
+                                          rlib_filename, final_lib)
+                    self._libs[libname].append(lib)
+                    self._linkage.append((context, lib, 'USE_LIBS'))
 
         for f, flags in all_flags.iteritems():
             if flags.flags:
                 ext = mozpath.splitext(f)[1]
                 yield PerSourceFlag(context, f, flags.flags)
 
 
     def emit_from_context(self, context):