Bug 1575760 - Make `mach vendor rust` create a .cargo/config and check it in the tree. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Mon, 26 Aug 2019 22:20:32 +0000
changeset 553714 80fa83b977a0da53a967e32b152ddf09584ba433
parent 553713 9e0c871919f09f693a7f148bc23ac066c6446a2e
child 553715 c8660505bc7e65f20c5959fb4e940df17a1c3d9e
push id2165
push userffxbld-merge
push dateMon, 14 Oct 2019 16:30:58 +0000
treeherdermozilla-release@0eae18af659f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1575760
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1575760 - Make `mach vendor rust` create a .cargo/config and check it in the tree. r=nalexander Maybe back when .cargo/config.in was added, the directory indicated for vendored crates needed to be absolute. That is at least not the case with the current supported versions of rust. The current setup has a few caveats: - .cargo/config.in has shown to become stale (it currently contains multiple unused entries) - non-gecko build tasks have to generate a .cargo/config on their own if they want to use vendored crates - in turn, non-gecko build tasks that don't, may unknowingly get their dependencies from crates.io (see the recent attempt at moving geckodriver builds to a separate task). By checking in a .cargo/config file, we can alleviate the last two, but that comes at the price of `cargo update` not wanting to act when .cargo/config exists, because of the source replacement configuration. But rust vendor gently generates a suitable configuration on its own, so we can use that to generate a .cargo/config automatically. Which addresses the first caveat of the current setup. That leaves us with `cargo update` not working out of the box, but that just requires people running it to manually remove .cargo/config first. Which is arguably what rust wants you to do in the first place. It's kind of incidental that we started with a .cargo/config.in rather than .cargo/config. Now, while a simple .cargo/config works, that's not enough for the case where the objdir doesn't live inside the source directory. In that case cargo looks for the configuration from the objdir, and fails to find it. So we still need a .cargo/config.in, which we generate with a little trick. Differential Revision: https://phabricator.services.mozilla.com/D43012
.cargo/.gitignore
.cargo/config
.cargo/config.in
js/src/make-source-package.sh
js/src/wasm/cranelift/Cargo.toml
python/mozbuild/mozbuild/vendor_rust.py
taskcluster/scripts/builder/build-sm-mozjs-crate.sh
taskcluster/scripts/builder/build-sm-rust-bindings.sh
deleted file mode 100644
--- a/.cargo/.gitignore
+++ /dev/null
@@ -1,1 +0,0 @@
-config
copy from .cargo/config.in
copy to .cargo/config
--- a/.cargo/config.in
+++ b/.cargo/config
@@ -1,45 +1,41 @@
-# Note: if you add more configure substitutions here with required values
-# you will also need to fix the sed commands in:
-# taskcluster/scripts/builder/build-sm-mozjs-crate.sh
-# taskcluster/scripts/builder/build-sm-rust-bindings.sh
+# This file contains vendoring instructions for cargo.
+# It was generated by `mach vendor rust`.
+# Please do not edit.
 
-[source.crates-io]
-registry = 'https://github.com/rust-lang/crates.io-index'
-replace-with = 'vendored-sources'
-
-[source."https://github.com/servo/serde"]
-git = "https://github.com/servo/serde"
-branch = "deserialize_from_enums10"
+[source."https://github.com/hsivonen/packed_simd"]
+branch = "rust_1_32"
+git = "https://github.com/hsivonen/packed_simd"
 replace-with = "vendored-sources"
 
-[source."https://github.com/retep998/winapi-rs"]
+[source."https://github.com/froydnj/winapi-rs"]
+branch = "aarch64"
 git = "https://github.com/froydnj/winapi-rs"
-branch = "aarch64"
 replace-with = "vendored-sources"
 
-[source."https://github.com/rust-lang-nursery/packed_simd"]
-git = "https://github.com/hsivonen/packed_simd"
-branch = "rust_1_32"
+[source."https://github.com/alexcrichton/mio-named-pipes"]
+branch = "master"
+git = "https://github.com/alexcrichton/mio-named-pipes"
+replace-with = "vendored-sources"
+
+[source."https://github.com/NikVolf/tokio-named-pipes"]
+branch = "stable"
+git = "https://github.com/NikVolf/tokio-named-pipes"
 replace-with = "vendored-sources"
 
 [source."https://github.com/CraneStation/Cranelift"]
 git = "https://github.com/CraneStation/Cranelift"
+replace-with = "vendored-sources"
 rev = "164f91a1f473e582e18e48d056c51787d9a1c24d"
-replace-with = "vendored-sources"
 
-[source."https://github.com/ChunMinChang/coreaudio-sys"]
-git = "https://github.com/ChunMinChang/coreaudio-sys"
-branch = "gecko-build"
+[source.crates-io]
 replace-with = "vendored-sources"
 
-[source."https://github.com/alexcrichton/mio-named-pipes"]
-git = "https://github.com/alexcrichton/mio-named-pipes"
-replace-with = "vendored-sources"
-
-[source."https://github.com/NikVolf/tokio-named-pipes"]
-git = "https://github.com/NikVolf/tokio-named-pipes"
-branch = "stable"
-replace-with = "vendored-sources"
-
+# Take advantage of the fact that cargo will treat lines starting with #
+# as comments to add preprocessing directives for when this file is included
+# from .cargo/config.in.
+#define REPLACE_NAME vendored-sources
+#define VENDORED_DIRECTORY third_party/rust
+#ifndef top_srcdir
 [source.vendored-sources]
-directory = '@top_srcdir@/third_party/rust'
+directory = "third_party/rust"
+#endif
--- a/.cargo/config.in
+++ b/.cargo/config.in
@@ -1,45 +1,10 @@
-# Note: if you add more configure substitutions here with required values
-# you will also need to fix the sed commands in:
-# taskcluster/scripts/builder/build-sm-mozjs-crate.sh
-# taskcluster/scripts/builder/build-sm-rust-bindings.sh
-
-[source.crates-io]
-registry = 'https://github.com/rust-lang/crates.io-index'
-replace-with = 'vendored-sources'
+# Please do not edit this file.
 
-[source."https://github.com/servo/serde"]
-git = "https://github.com/servo/serde"
-branch = "deserialize_from_enums10"
-replace-with = "vendored-sources"
-
-[source."https://github.com/retep998/winapi-rs"]
-git = "https://github.com/froydnj/winapi-rs"
-branch = "aarch64"
-replace-with = "vendored-sources"
+# Note: this file is only really needed when objdir is not a subdirectory of
+# the top source directory.
 
-[source."https://github.com/rust-lang-nursery/packed_simd"]
-git = "https://github.com/hsivonen/packed_simd"
-branch = "rust_1_32"
-replace-with = "vendored-sources"
-
-[source."https://github.com/CraneStation/Cranelift"]
-git = "https://github.com/CraneStation/Cranelift"
-rev = "164f91a1f473e582e18e48d056c51787d9a1c24d"
-replace-with = "vendored-sources"
+#include config
+#filter substitution
 
-[source."https://github.com/ChunMinChang/coreaudio-sys"]
-git = "https://github.com/ChunMinChang/coreaudio-sys"
-branch = "gecko-build"
-replace-with = "vendored-sources"
-
-[source."https://github.com/alexcrichton/mio-named-pipes"]
-git = "https://github.com/alexcrichton/mio-named-pipes"
-replace-with = "vendored-sources"
-
-[source."https://github.com/NikVolf/tokio-named-pipes"]
-git = "https://github.com/NikVolf/tokio-named-pipes"
-branch = "stable"
-replace-with = "vendored-sources"
-
-[source.vendored-sources]
-directory = '@top_srcdir@/third_party/rust'
+[source.@REPLACE_NAME@]
+directory = "@top_srcdir@/@VENDORED_DIRECTORY@"
--- a/js/src/make-source-package.sh
+++ b/js/src/make-source-package.sh
@@ -124,16 +124,17 @@ case $cmd in
     cp -pPR \
         ${TOPSRCDIR}/build \
         ${TOPSRCDIR}/config \
         ${TOPSRCDIR}/python \
         ${tgtpath}/
 
     ${MKDIR} -p ${tgtpath}/.cargo
     cp -pPR \
+        ${TOPSRCDIR}/.cargo/config \
         ${TOPSRCDIR}/.cargo/config.in \
         ${tgtpath}/.cargo/
 
     ${MKDIR} -p ${tgtpath}/third_party
     cp -pPR \
         ${TOPSRCDIR}/third_party/python \
         ${TOPSRCDIR}/third_party/rust \
         ${tgtpath}/third_party/
--- a/js/src/wasm/cranelift/Cargo.toml
+++ b/js/src/wasm/cranelift/Cargo.toml
@@ -9,17 +9,17 @@ crate-type = ["rlib"]
 name = "baldrdash"
 
 [dependencies]
 # The build system redirects the versions of cranelift-codegen and
 # cranelift-wasm to pinned commits. If you want to update Cranelift in Gecko,
 # you should update the following files:
 # - $TOP_LEVEL/Cargo.toml, look for the revision (rev) hashes of both cranelift
 # dependencies (codegen and wasm).
-# - $TOP_LEVEL/.cargo/config.in, look for the revision (rev) field of the
+# - $TOP_LEVEL/.cargo/config, look for the revision (rev) field of the
 # Cranelift source.
 cranelift-codegen = { version = "0.40", default-features = false }
 cranelift-wasm = "0.40"
 target-lexicon = "0.4.0"
 log = { version = "0.4.6", default-features = false, features = ["release_max_level_info"] }
 env_logger = "0.5.6"
 
 [build-dependencies]
--- a/python/mozbuild/mozbuild/vendor_rust.py
+++ b/python/mozbuild/mozbuild/vendor_rust.py
@@ -6,25 +6,46 @@ from __future__ import absolute_import, 
 
 import errno
 import hashlib
 import logging
 import os
 import re
 import subprocess
 import sys
+from collections import OrderedDict
 from distutils.version import LooseVersion
+from itertools import dropwhile
 
+import pytoml
 import mozpack.path as mozpath
 from mozbuild.base import (
     BuildEnvironmentNotFoundException,
     MozbuildObject,
 )
 
 
+CARGO_CONFIG_TEMPLATE = '''\
+# This file contains vendoring instructions for cargo.
+# It was generated by `mach vendor rust`.
+# Please do not edit.
+
+{config}
+
+# Take advantage of the fact that cargo will treat lines starting with #
+# as comments to add preprocessing directives for when this file is included
+# from .cargo/config.in.
+#define REPLACE_NAME {replace_name}
+#define VENDORED_DIRECTORY {directory}
+#ifndef top_srcdir
+{replace}
+#endif
+'''
+
+
 class VendorRust(MozbuildObject):
     def get_cargo_path(self):
         try:
             return self.substs['CARGO']
         except (BuildEnvironmentNotFoundException, KeyError):
             # Default if this tree isn't configured.
             from mozfile import which
             cargo = which('cargo')
@@ -312,22 +333,93 @@ license file's hash.
 
         cargo = self._ensure_cargo()
         if not cargo:
             return
 
         relative_vendor_dir = 'third_party/rust'
         vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir)
 
+        cargo_config = os.path.join(self.topsrcdir, '.cargo', 'config')
+        # First, remove .cargo/config
+        try:
+            os.remove(cargo_config)
+        except Exception:
+            pass
+
         # We use check_call instead of mozprocess to ensure errors are displayed.
         # We do an |update -p| here to regenerate the Cargo.lock file with minimal
         # changes. See bug 1324462
         subprocess.check_call([cargo, 'update', '-p', 'gkrust'], cwd=self.topsrcdir)
 
-        subprocess.check_call([cargo, 'vendor', '--quiet', vendor_dir], cwd=self.topsrcdir)
+        output = subprocess.check_output([cargo, 'vendor', vendor_dir],
+                                         stderr=subprocess.STDOUT,
+                                         cwd=self.topsrcdir)
+
+        # Get the snippet of configuration that cargo vendor outputs, and
+        # update .cargo/config with it.
+        # XXX(bug 1576765): Hopefully do something better after
+        # https://github.com/rust-lang/cargo/issues/7280 is addressed.
+        config = '\n'.join(dropwhile(lambda l: not l.startswith('['),
+                                     output.splitlines()))
+
+        # The config is toml, parse it as such.
+        config = pytoml.loads(config)
+
+        # For each replace-with, extract their configuration and update the
+        # corresponding directory to be relative to topsrcdir.
+        replaces = {
+            v['replace-with']
+            for v in config['source'].values()
+            if 'replace-with' in v
+        }
+
+        # We only really expect one replace-with
+        if len(replaces) != 1:
+            self.log(
+                logging.ERROR, 'vendor_failed', {},
+                '''cargo vendor didn't output a unique replace-with. Found: %s.''' % replaces)
+            sys.exit(1)
+
+        replace_name = replaces.pop()
+        replace = config['source'].pop(replace_name)
+        replace['directory'] = mozpath.relpath(
+            mozpath.normsep(os.path.normcase(replace['directory'])),
+            mozpath.normsep(os.path.normcase(self.topsrcdir)),
+        )
+
+        # Introduce some determinism for the output.
+        def recursive_sort(obj):
+            if isinstance(obj, dict):
+                return OrderedDict(sorted(
+                    (k, recursive_sort(v)) for k, v in obj.items()))
+            if isinstance(obj, list):
+                return [recursive_sort(o) for o in obj]
+            return obj
+
+        config = recursive_sort(config)
+
+        # Normalize pytoml output:
+        # - removing empty lines
+        # - remove empty [section]
+        def toml_dump(data):
+            dump = pytoml.dumps(data)
+            if isinstance(data, dict):
+                for k, v in data.items():
+                    if all(isinstance(v2, dict) for v2 in v.values()):
+                        dump = dump.replace('[%s]' % k, '')
+            return dump.strip()
+
+        with open(cargo_config, 'w') as fh:
+            fh.write(CARGO_CONFIG_TEMPLATE.format(
+                config=toml_dump(config),
+                replace_name=replace_name,
+                directory=replace['directory'],
+                replace=toml_dump({'source': {replace_name: replace}}),
+            ))
 
         if not self._check_licenses(vendor_dir):
             self.log(
                 logging.ERROR, 'license_check_failed', {},
                 '''The changes from `mach vendor rust` will NOT be added to version control.''')
             sys.exit(1)
 
         self.repository.add_remove_files(vendor_dir)
--- a/taskcluster/scripts/builder/build-sm-mozjs-crate.sh
+++ b/taskcluster/scripts/builder/build-sm-mozjs-crate.sh
@@ -1,19 +1,14 @@
 #!/usr/bin/env bash
 
 set -xe
 
 source $(dirname $0)/sm-tooltool-config.sh
 
-# Ensure that we have a .config/cargo that points us to our vendored crates
-# rather than to crates.io.
-cd "$SRCDIR/.cargo"
-sed -e "s|@top_srcdir@|$SRCDIR|" -e 's|@[^@]*@||g' < config.in > config
-
 cd "$SRCDIR/js/src"
 
 export PATH="$PATH:$MOZ_FETCHES_DIR/cargo/bin:$MOZ_FETCHES_DIR/rustc/bin"
 export RUST_BACKTRACE=1
 export AUTOMATION=1
 
 cargo build --verbose --frozen --features debugmozjs
 cargo build --verbose --frozen
--- a/taskcluster/scripts/builder/build-sm-rust-bindings.sh
+++ b/taskcluster/scripts/builder/build-sm-rust-bindings.sh
@@ -1,19 +1,14 @@
 #!/usr/bin/env bash
 
 set -xe
 
 source $(dirname $0)/sm-tooltool-config.sh
 
-# Ensure that we have a .config/cargo that points us to our vendored crates
-# rather than to crates.io.
-cd "$SRCDIR/.cargo"
-sed -e "s|@top_srcdir@|$SRCDIR|" -e 's|@[^@]*@||g' < config.in > config
-
 cd "$SRCDIR/js/rust"
 
 export LD_LIBRARY_PATH="$MOZ_FETCHES_DIR/gcc/lib64"
 # Enable backtraces if we panic.
 export RUST_BACKTRACE=1
 
 cargo test --verbose --frozen --features debugmozjs
 cargo test --verbose --frozen