servo: Merge #18325 - Switch back to pinning Rust by Nightly date instead of commit hash… (from servo:rustup-toolchain); r=nox
authorSimon Sapin <simon.sapin@exyr.org>
Thu, 31 Aug 2017 07:47:34 -0500
changeset 378058 5df6f8fe6c50631fe00f47b69d16b8d555d90503
parent 378057 2b3fcd296b8b98cc39b001daf89177998a215321
child 378059 67b9e1d526da542719353ff4275f3f9fbafb326a
push id94412
push userarchaeopteryx@coole-files.de
push dateFri, 01 Sep 2017 08:46:09 +0000
treeherdermozilla-inbound@d56571d7f1be [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnox
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
servo: Merge #18325 - Switch back to pinning Rust by Nightly date instead of commit hash… (from servo:rustup-toolchain); r=nox … this time using a `rust-toolchain` file compatible with rustup: https://github.com/rust-lang-nursery/rustup.rs/#the-toolchain-file And upgrade to rustc 1.21.0-nightly (c11f689d2 2017-08-29) ---- Now if both `system-rust` and `system-cargo` are set to `true` in `.servobuild`’s `[tools]` section, and the corresponding `rustc` and `cargo` binaries are in fact rustup’s wrappers, then rustup will use the correct version based on `rust-toolchain`. CC https://github.com/servo/servo/issues/11361 Unlike https://github.com/servo/servo/pull/17927, this does not make mach use rustup directly. That should wait until https://github.com/rust-lang-nursery/rustup.rs/issues/1099 is fixed. Source-Repo: https://github.com/servo/servo Source-Revision: c4800a6c83e6fdabaf7c4eff70a24487d16f18ff
servo/README.md
servo/appveyor.yml
servo/python/servo/bootstrap_commands.py
servo/python/servo/command_base.py
servo/python/servo/devenv_commands.py
servo/rust-commit-hash
servo/rust-toolchain
--- a/servo/README.md
+++ b/servo/README.md
@@ -119,17 +119,17 @@ Pre-installed Android tools are needed. 
 [details](https://github.com/servo/servo/wiki/Building-for-Android)
 
 ## The Rust compiler
 
 Servo's build system automatically downloads a Rust compiler to build itself.
 This is normally a specific revision of Rust upstream, but sometimes has a
 backported patch or two.
 If you'd like to know which nightly build of Rust we use, see
-[`rust-commit-hash`](https://github.com/servo/servo/blob/master/rust-commit-hash).
+[`rust-toolchain`](https://github.com/servo/servo/blob/master/rust-toolchain).
 
 ## Building
 
 Servo is built with [Cargo](https://crates.io/), the Rust package manager. We also use Mozilla's
 Mach tools to orchestrate the build and other tasks.
 
 ### Normal build
 
--- a/servo/appveyor.yml
+++ b/servo/appveyor.yml
@@ -31,18 +31,18 @@ environment:
   matrix:
   - TARGET: nightly-x86_64-pc-windows-msvc
 
 branches:
   only:
     - master
 
 cache:
-  - .servo -> rust-commit-hash
-  - .cargo -> rust-commit-hash
+  - .servo -> rust-toolchain
+  - .cargo -> rust-toolchain
   - .ccache
 
 # Uncomment these lines to expose RDP access information to the build machine in the build log.
 #init:
 #  - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 #
 #on_finish:
 #  - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
--- a/servo/python/servo/bootstrap_commands.py
+++ b/servo/python/servo/bootstrap_commands.py
@@ -63,98 +63,99 @@ class MachCommands(CommandBase):
                      action='append',
                      default=[],
                      help='Download rust stdlib for specified target')
     @CommandArgument('--stable',
                      action='store_true',
                      help='Use stable rustc version')
     def bootstrap_rustc(self, force=False, target=[], stable=False):
         self.set_use_stable_rust(stable)
-        version = self.rust_version()
-        rust_path = self.rust_path()
-        rust_dir = path.join(self.context.sharedir, "rust", rust_path)
-        install_dir = path.join(self.context.sharedir, "rust", version)
-        if not self.config["build"]["llvm-assertions"]:
-            if not self.use_stable_rust():
-                install_dir += "-alt"
+        rust_dir = path.join(self.context.sharedir, "rust", self.rust_path())
+        install_dir = path.join(self.context.sharedir, "rust", self.rust_install_dir())
+        version = self.rust_stable_version() if stable else "nightly"
+        static_s3 = "https://static-rust-lang-org.s3.amazonaws.com/dist"
 
         if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
             print("Rust compiler already downloaded.", end=" ")
             print("Use |bootstrap-rust --force| to download again.")
         else:
             if path.isdir(rust_dir):
                 shutil.rmtree(rust_dir)
             os.makedirs(rust_dir)
 
             # The nightly Rust compiler is hosted on the nightly server under the date with a name
             # rustc-nightly-HOST-TRIPLE.tar.gz, whereas the stable compiler is named
             # rustc-VERSION-HOST-TRIPLE.tar.gz. We just need to pull down and extract it,
             # giving a directory name that will be the same as the tarball name (rustc is
             # in that directory).
             if stable:
-                tarball = "rustc-%s-%s.tar.gz" % (version, host_triple())
-                rustc_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/" + tarball
+                base_url = static_s3
             else:
-                tarball = "%s/rustc-nightly-%s.tar.gz" % (version, host_triple())
+                import toml
+                import re
+                channel = "%s/%s/channel-rust-nightly.toml" % (static_s3, self.rust_nightly_date())
+                version_string = toml.load(urllib2.urlopen(channel))["pkg"]["rustc"]["version"]
+                short_commit = re.search("\(([0-9a-f]+) ", version_string).group(1)
+                commit_api = "https://api.github.com/repos/rust-lang/rust/commits/" + short_commit
+                nightly_commit_hash = json.load(urllib2.urlopen(commit_api))["sha"]
+
                 base_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds"
                 if not self.config["build"]["llvm-assertions"]:
                     base_url += "-alt"
-                rustc_url = base_url + "/" + tarball
+                base_url += "/" + nightly_commit_hash
+
+            rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple())
             tgz_file = rust_dir + '-rustc.tar.gz'
-
             download_file("Rust compiler", rustc_url, tgz_file)
 
             print("Extracting Rust compiler...")
             extract(tgz_file, install_dir)
             print("Rust compiler ready.")
 
         # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz` for the nightly
         # releases, or rust-std-VERSION-TRIPLE.tar.gz for stable releases, with
         # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory.
         # This `lib` directory needs to be extracted and merged with the `rustc/lib`
         # directory from the host compiler above.
-        nightly_suffix = "" if stable else "-nightly"
-        stable_version = "-{}".format(version) if stable else ""
         lib_dir = path.join(install_dir,
-                            "rustc{}{}-{}".format(nightly_suffix, stable_version, host_triple()),
+                            "rustc-%s-%s" % (version, host_triple()),
                             "rustc", "lib", "rustlib")
 
         # ensure that the libs for the host's target is downloaded
         host_target = host_triple()
         if host_target not in target:
             target.append(host_target)
 
         for target_triple in target:
             target_lib_dir = path.join(lib_dir, target_triple)
             if path.exists(target_lib_dir):
                 # No need to check for force. If --force the directory is already deleted
                 print("Rust lib for target {} already downloaded.".format(target_triple), end=" ")
                 print("Use |bootstrap-rust --force| to download again.")
                 continue
 
+            tarball = "rust-std-%s-%s.tar.gz" % (version, target_triple)
+            tgz_file = path.join(install_dir, tarball)
             if self.use_stable_rust():
-                std_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-std-%s-%s.tar.gz"
-                           % (version, target_triple))
-                tgz_file = install_dir + ('rust-std-%s-%s.tar.gz' % (version, target_triple))
+                std_url = static_s3 + "/" + tarball
             else:
-                std_url = ("https://s3.amazonaws.com/rust-lang-ci/rustc-builds/%s/rust-std-nightly-%s.tar.gz"
-                           % (version, target_triple))
-                tgz_file = install_dir + ('rust-std-nightly-%s.tar.gz' % target_triple)
+                std_url = static_s3 + "/" + self.rust_nightly_date() + "/" + tarball
 
             download_file("Host rust library for target %s" % target_triple, std_url, tgz_file)
             print("Extracting Rust stdlib for target %s..." % target_triple)
             extract(tgz_file, install_dir)
             shutil.copytree(path.join(install_dir,
-                                      "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple),
-                                      "rust-std-%s" % target_triple, "lib", "rustlib", target_triple),
+                                      "rust-std-%s-%s" % (version, target_triple),
+                                      "rust-std-%s" % target_triple,
+                                      "lib", "rustlib", target_triple),
                             path.join(install_dir,
-                                      "rustc%s%s-%s" % (nightly_suffix, stable_version, host_triple()),
-                                      "rustc", "lib", "rustlib", target_triple))
-            shutil.rmtree(path.join(install_dir,
-                          "rust-std%s%s-%s" % (nightly_suffix, stable_version, target_triple)))
+                                      "rustc-%s-%s" % (version, host_triple()),
+                                      "rustc",
+                                      "lib", "rustlib", target_triple))
+            shutil.rmtree(path.join(install_dir, "rust-std-%s-%s" % (version, target_triple)))
 
             print("Rust {} libs ready.".format(target_triple))
 
     @Command('bootstrap-rust-docs',
              description='Download the Rust documentation',
              category='bootstrap')
     @CommandArgument('--force', '-f',
                      action='store_true',
@@ -166,18 +167,18 @@ class MachCommands(CommandBase):
         if not force and path.exists(docs_dir):
             print("Rust docs already downloaded.", end=" ")
             print("Use |bootstrap-rust-docs --force| to download again.")
             return
 
         if path.isdir(docs_dir):
             shutil.rmtree(docs_dir)
         docs_name = self.rust_path().replace("rustc-", "rust-docs-")
-        docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-docs-nightly-%s.tar.gz"
-                    % host_triple())
+        docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/%s/rust-docs-nightly-%s.tar.gz"
+                    % (self.rust_nightly_date(), host_triple()))
         tgz_file = path.join(rust_root, 'doc.tar.gz')
 
         download_file("Rust docs", docs_url, tgz_file)
 
         print("Extracting Rust docs...")
         temp_dir = path.join(rust_root, "temp_docs")
         if path.isdir(temp_dir):
             shutil.rmtree(temp_dir)
@@ -190,30 +191,29 @@ class MachCommands(CommandBase):
 
     @Command('bootstrap-cargo',
              description='Download the Cargo build tool',
              category='bootstrap')
     @CommandArgument('--force', '-f',
                      action='store_true',
                      help='Force download even if cargo already exists')
     def bootstrap_cargo(self, force=False):
-        cargo_dir = path.join(self.context.sharedir, "cargo",
-                              self.cargo_build_id())
+        cargo_dir = path.join(self.context.sharedir, "cargo", self.rust_nightly_date())
         if not force and path.exists(path.join(cargo_dir, "cargo", "bin", "cargo" + BIN_SUFFIX)):
             print("Cargo already downloaded.", end=" ")
             print("Use |bootstrap-cargo --force| to download again.")
             return
 
         if path.isdir(cargo_dir):
             shutil.rmtree(cargo_dir)
         os.makedirs(cargo_dir)
 
         tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
-        nightly_url = "https://s3.amazonaws.com/rust-lang-ci/rustc-builds/%s/%s" % \
-            (self.cargo_build_id()[len("rust-"):], tgz_file)
+        nightly_url = "https://static-rust-lang-org.s3.amazonaws.com/dist/%s/%s" % \
+            (self.rust_nightly_date(), tgz_file)
 
         download_file("Cargo nightly", nightly_url, tgz_file)
 
         print("Extracting Cargo nightly...")
         nightly_dir = path.join(cargo_dir,
                                 path.basename(tgz_file).replace(".tar.gz", ""))
         extract(tgz_file, cargo_dir, movedir=nightly_dir)
         print("Cargo ready.")
@@ -286,29 +286,27 @@ class MachCommands(CommandBase):
              category='bootstrap')
     @CommandArgument('--force', '-f',
                      action='store_true',
                      help='Actually remove stuff')
     @CommandArgument('--keep',
                      default='1',
                      help='Keep up to this many most recent nightlies')
     def clean_nightlies(self, force=False, keep=None):
-        self.set_use_stable_rust(False)
-        rust_current_nightly = self.rust_version()
-        self.set_use_stable_rust(True)
-        rust_current_stable = self.rust_version()
+        rust_current_nightly = self.rust_nightly_date()
+        rust_current_stable = self.rust_stable_version()
         print("Current Rust nightly version: {}".format(rust_current_nightly))
         print("Current Rust stable version: {}".format(rust_current_stable))
         to_keep = set()
         if int(keep) == 1:
             # Optimize keep=1 case to not invoke git
             to_keep.add(rust_current_nightly)
             to_keep.add(rust_current_stable)
         else:
-            for version_file in ['rust-commit-hash', 'rust-stable-version']:
+            for version_file in ['rust-toolchain', 'rust-stable-version']:
                 cmd = subprocess.Popen(
                     ['git', 'log', '--oneline', '--no-color', '-n', keep, '--patch', version_file],
                     stdout=subprocess.PIPE,
                     universal_newlines=True
                 )
                 stdout, _ = cmd.communicate()
                 for line in stdout.splitlines():
                     if line.startswith(b"+") and not line.startswith(b"+++"):
--- a/servo/python/servo/command_base.py
+++ b/servo/python/servo/command_base.py
@@ -280,61 +280,71 @@ class CommandBase(object):
         self.config["android"].setdefault("toolchain", "")
         # Set default android target
         self.handle_android_target("armv7-linux-androideabi")
 
         self.set_cargo_root()
         self.set_use_stable_rust(False)
 
     _use_stable_rust = False
-    _rust_version = None
-    _rust_version_is_stable = False
-    _cargo_build_id = None
+    _rust_stable_version = None
+    _rust_nightly_date = None
 
     def set_cargo_root(self):
         if not self.config["tools"]["system-cargo"]:
             self.config["tools"]["cargo-root"] = path.join(
-                self.context.sharedir, "cargo", self.cargo_build_id())
+                self.context.sharedir, "cargo", self.rust_nightly_date())
 
     def set_use_stable_rust(self, use_stable_rust=True):
         self._use_stable_rust = use_stable_rust
         if not self.config["tools"]["system-rust"]:
             self.config["tools"]["rust-root"] = path.join(
                 self.context.sharedir, "rust", self.rust_path())
         if use_stable_rust:
             # Cargo maintainer's position is that CARGO_INCREMENTAL is a nightly-only feature
             # and should not be used on the stable channel.
             # https://github.com/rust-lang/cargo/issues/3835
             self.config["build"]["incremental"] = False
 
     def use_stable_rust(self):
         return self._use_stable_rust
 
-    def rust_path(self):
-        version = self.rust_version()
+    def rust_install_dir(self):
         if self._use_stable_rust:
-            return os.path.join(version, "rustc-%s-%s" % (version, host_triple()))
-        if not self.config["build"]["llvm-assertions"]:
-            version += "-alt"
-        return os.path.join(version, "rustc-nightly-%s" % (host_triple()))
+            return self.rust_stable_version()
+        elif not self.config["build"]["llvm-assertions"]:
+            return self.rust_nightly_date() + "-alt"
+        else:
+            return self.rust_nightly_date()
+
+    def rust_path(self):
+        if self._use_stable_rust:
+            version = self.rust_stable_version()
+        else:
+            version = "nightly"
+
+        subdir = "rustc-%s-%s" % (version, host_triple())
+        return os.path.join(self.rust_install_dir(), subdir)
 
-    def rust_version(self):
-        if self._rust_version is None or self._use_stable_rust != self._rust_version_is_stable:
-            filename = path.join(self.context.topdir,
-                                 "rust-stable-version" if self._use_stable_rust else "rust-commit-hash")
+    def rust_stable_version(self):
+        if self._rust_stable_version is None:
+            filename = path.join("rust-stable-version")
             with open(filename) as f:
-                self._rust_version = f.read().strip()
-        return self._rust_version
+                self._rust_stable_version = f.read().strip()
+        return self._rust_stable_version
 
-    def cargo_build_id(self):
-        if self._cargo_build_id is None:
-            filename = path.join(self.context.topdir, "rust-commit-hash")
+    def rust_nightly_date(self):
+        if self._rust_nightly_date is None:
+            filename = path.join(self.context.topdir, "rust-toolchain")
             with open(filename) as f:
-                self._cargo_build_id = "rust-" + f.read().strip()
-        return self._cargo_build_id
+                toolchain = f.read().strip()
+                prefix = "nightly-"
+                assert toolchain.startswith(prefix)
+                self._rust_nightly_date = toolchain[len(prefix):]
+        return self._rust_nightly_date
 
     def get_top_dir(self):
         return self.context.topdir
 
     def get_target_dir(self):
         if "CARGO_TARGET_DIR" in os.environ:
             return os.environ["CARGO_TARGET_DIR"]
         else:
@@ -582,17 +592,17 @@ class CommandBase(object):
         target_path = path.join(base_target_path, target_platform)
         target_exists = path.exists(target_path)
 
         # Always check if all needed MSVC dependencies are installed
         if "msvc" in target_platform:
             Registrar.dispatch("bootstrap", context=self.context)
 
         if not (self.config['tools']['system-rust'] or (rustc_binary_exists and target_exists)):
-            print("looking for rustc at %s" % (rustc_path))
+            print("Looking for rustc at %s" % (rustc_path))
             Registrar.dispatch("bootstrap-rust", context=self.context, target=filter(None, [target]),
                                stable=self._use_stable_rust)
 
         cargo_path = path.join(self.config["tools"]["cargo-root"], "cargo", "bin",
                                "cargo" + BIN_SUFFIX)
         cargo_binary_exists = path.exists(cargo_path)
 
         if not self.config["tools"]["system-cargo"] and not cargo_binary_exists:
--- a/servo/python/servo/devenv_commands.py
+++ b/servo/python/servo/devenv_commands.py
@@ -131,17 +131,16 @@ class MachCommands(CommandBase):
         '--dry-run', '-d', action='store_true',
         help='Show outdated packages.')
     def update_cargo(self, params=None, package=None, all_packages=None, dry_run=None):
         if not params:
             params = []
 
         if dry_run:
             import toml
-            import json
             import httplib
             import colorama
 
             cargo_file = open(path.join(self.context.topdir, "Cargo.lock"))
             content = toml.load(cargo_file)
 
             packages = {}
             outdated_packages = 0
@@ -257,41 +256,26 @@ class MachCommands(CommandBase):
         grep_paths = root_dirs_abs + tests_dirs_abs
         return call(
             ["git"] + ["grep"] + params + ['--'] + grep_paths + [':(exclude)*.min.js', ':(exclude)*.min.css'],
             env=self.build_env())
 
     @Command('rustup',
              description='Update the Rust version to latest Nightly',
              category='devenv')
-    @CommandArgument('--master',
-                     action='store_true',
-                     help='Use the latest commit of the "master" branch')
-    def rustup(self, master=False):
-        if master:
-            url = "https://api.github.com/repos/rust-lang/rust/git/refs/heads/master"
-            commit = json.load(urllib2.urlopen(url))["object"]["sha"]
-        else:
-            import toml
-            import re
-            url = "https://static.rust-lang.org/dist/channel-rust-nightly.toml"
-            version = toml.load(urllib2.urlopen(url))["pkg"]["rustc"]["version"]
-            short_commit = re.search("\(([0-9a-f]+) ", version).group(1)
-            url = "https://api.github.com/repos/rust-lang/rust/commits/" + short_commit
-            commit = json.load(urllib2.urlopen(url))["sha"]
-        filename = path.join(self.context.topdir, "rust-commit-hash")
+    def rustup(self):
+        url = "https://static.rust-lang.org/dist/channel-rust-nightly-date.txt"
+        nightly_date = urllib2.urlopen(url).read()
+        filename = path.join(self.context.topdir, "rust-toolchain")
         with open(filename, "w") as f:
-            f.write(commit + "\n")
+            f.write("nightly-%s\n" % nightly_date)
 
-        # Reset self.config["tools"]["rust-root"]
-        self._rust_version = None
+        # Reset self.config["tools"]["rust-root"] and self.config["tools"]["cargo-root"]
+        self._rust_nightly_date = None
         self.set_use_stable_rust(False)
-
-        # Reset self.config["tools"]["cargo-root"]
-        self._cargo_build_id = None
         self.set_cargo_root()
 
         self.fetch()
 
     @Command('fetch',
              description='Fetch Rust, Cargo and Cargo dependencies',
              category='devenv')
     def fetch(self):
deleted file mode 100644
--- a/servo/rust-commit-hash
+++ /dev/null
@@ -1,1 +0,0 @@
-13d94d5fa8129a34f5c77a1bcd76983f5aed2434
new file mode 100644
--- /dev/null
+++ b/servo/rust-toolchain
@@ -0,0 +1,1 @@
+nightly-2017-08-30