Bug 1573435 - Use toolchain fetches for spidermonkey builds. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 13 Aug 2019 18:03:40 +0900
changeset 488147 ab4d3a77070a11190b1ebf09e4594ef0954f3e1d
parent 488146 84afd114aae980e01e7835779f26bfa78df151ea
child 488148 3103e4be4bb4d63926e309af68ed23074207b064
push id36435
push usercbrindusan@mozilla.com
push dateThu, 15 Aug 2019 09:46:49 +0000
treeherdermozilla-central@0db07ff50ab5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1573435
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573435 - Use toolchain fetches for spidermonkey builds. r=nalexander Differential Revision: https://phabricator.services.mozilla.com/D41739
js/src/devtools/automation/autospider.py
js/src/devtools/automation/variants/asan
js/src/devtools/automation/variants/fuzzing
js/src/devtools/automation/variants/msan
js/src/devtools/automation/variants/tsan
taskcluster/ci/spidermonkey/kind.yml
taskcluster/ci/spidermonkey/linux.yml
taskcluster/ci/spidermonkey/windows.yml
taskcluster/scripts/builder/build-sm-mozjs-crate.sh
taskcluster/scripts/builder/build-sm-rust-bindings.sh
taskcluster/scripts/builder/build-sm.sh
taskcluster/scripts/builder/sm-tooltool-config.sh
taskcluster/taskgraph/transforms/spidermonkey.py
--- a/js/src/devtools/automation/autospider.py
+++ b/js/src/devtools/automation/autospider.py
@@ -16,26 +16,28 @@ import subprocess
 import sys
 
 from collections import Counter, namedtuple
 from logging import info
 from os import environ as env
 from subprocess import Popen
 from threading import Timer
 
-Dirs = namedtuple('Dirs', ['scripts', 'js_src', 'source', 'tooltool'])
+Dirs = namedtuple('Dirs', ['scripts', 'js_src', 'source', 'tooltool', 'fetches'])
 
 
 def directories(pathmodule, cwd, fixup=lambda s: s):
     scripts = pathmodule.join(fixup(cwd), fixup(pathmodule.dirname(__file__)))
     js_src = pathmodule.abspath(pathmodule.join(scripts, "..", ".."))
     source = pathmodule.abspath(pathmodule.join(js_src, "..", ".."))
     tooltool = pathmodule.abspath(env.get('TOOLTOOL_CHECKOUT',
                                           pathmodule.join(source, "..", "..")))
-    return Dirs(scripts, js_src, source, tooltool)
+    fetches = pathmodule.abspath(env.get('MOZ_FETCHES_DIR',
+                                         pathmodule.join(source, "..", "..")))
+    return Dirs(scripts, js_src, source, tooltool, fetches)
 
 
 # Some scripts will be called with sh, which cannot use backslashed
 # paths. So for direct subprocess.* invocation, use normal paths from
 # DIR, but when running under the shell, use POSIX style paths.
 DIR = directories(os.path, os.getcwd())
 PDIR = directories(posixpath, os.environ["PWD"],
                    fixup=lambda s: re.sub(r'^(\w):', r'/\1', s))
@@ -104,17 +106,17 @@ args = parser.parse_args()
 logging.basicConfig(level=logging.INFO, format='%(message)s')
 
 OBJDIR = args.objdir
 OUTDIR = os.path.join(OBJDIR, "out")
 POBJDIR = posixpath.join(PDIR.source, args.objdir)
 MAKE = env.get('MAKE', 'make')
 MAKEFLAGS = env.get('MAKEFLAGS', '-j6' + ('' if AUTOMATION else ' -s'))
 
-for d in ('scripts', 'js_src', 'source', 'tooltool'):
+for d in ('scripts', 'js_src', 'source', 'tooltool', 'fetches'):
     info("DIR.{name} = {dir}".format(name=d, dir=getattr(DIR, d)))
 
 
 def set_vars_from_script(script, vars):
     '''Run a shell script, then dump out chosen environment variables. The build
        system uses shell scripts to do some configuration that we need to
        borrow. On Windows, the script itself must output the variable settings
        (in the form "export FOO=<value>"), since otherwise there will be
@@ -223,17 +225,17 @@ elif platform.system() == 'Windows':
     compiler = 'cl'
 else:
     compiler = 'gcc'
 
 info("using compiler '{}'".format(compiler))
 
 cxx = {'clang': 'clang++', 'gcc': 'g++', 'cl': 'cl'}.get(compiler)
 
-compiler_dir = env.get('GCCDIR', os.path.join(DIR.tooltool, compiler))
+compiler_dir = env.get('GCCDIR', os.path.join(DIR.fetches, compiler))
 info("looking for compiler under {}/".format(compiler_dir))
 if os.path.exists(os.path.join(compiler_dir, 'bin', compiler)):
     env.setdefault('CC', os.path.join(compiler_dir, 'bin', compiler))
     env.setdefault('CXX', os.path.join(compiler_dir, 'bin', cxx))
     if compiler == 'clang':
         platlib = 'lib'
     else:
         platlib = 'lib64' if word_bits == 64 else 'lib'
@@ -244,17 +246,17 @@ else:
 
 bindir = os.path.join(OBJDIR, 'dist', 'bin')
 env['LD_LIBRARY_PATH'] = ':'.join(
     p for p in (bindir, env.get('LD_LIBRARY_PATH')) if p)
 
 for v in ('CC', 'CXX', 'LD_LIBRARY_PATH'):
     info("default {name} = {value}".format(name=v, value=env[v]))
 
-rust_dir = os.path.join(DIR.tooltool, 'rustc')
+rust_dir = os.path.join(DIR.fetches, 'rustc')
 if os.path.exists(os.path.join(rust_dir, 'bin', 'rustc')):
     env.setdefault('RUSTC', os.path.join(rust_dir, 'bin', 'rustc'))
     env.setdefault('CARGO', os.path.join(rust_dir, 'bin', 'cargo'))
 else:
     env.setdefault('RUSTC', 'rustc')
     env.setdefault('CARGO', 'cargo')
 
 if platform.system() == 'Darwin':
@@ -332,16 +334,17 @@ def run_command(command, check=False, **
         raise subprocess.CalledProcessError(status, command, output=stderr)
     return stdout, stderr, status
 
 
 # Replacement strings in environment variables.
 REPLACEMENTS = {
     'DIR': DIR.scripts,
     'TOOLTOOL_CHECKOUT': DIR.tooltool,
+    'MOZ_FETCHES_DIR': DIR.fetches,
     'MOZ_UPLOAD_DIR': env['MOZ_UPLOAD_DIR'],
     'OUTDIR': OUTDIR,
 }
 
 # Add in environment variable settings for this variant. Normally used to
 # modify the flags passed to the shell or to set the GC zeal mode.
 for k, v in variant.get('env', {}).items():
     env[k.encode('ascii')] = v.encode('ascii').format(**REPLACEMENTS)
--- a/js/src/devtools/automation/variants/asan
+++ b/js/src/devtools/automation/variants/asan
@@ -1,10 +1,10 @@
 {
     "configure-args": "--enable-debug-symbols='-gline-tables-only' --enable-gczeal --disable-jemalloc --enable-address-sanitizer",
     "optimize": true,
     "debug": false,
     "compiler": "clang",
     "env": {
-        "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer"
+        "LLVM_SYMBOLIZER": "{MOZ_FETCHES_DIR}/clang/bin/llvm-symbolizer"
     },
     "use_minidump": false
 }
--- a/js/src/devtools/automation/variants/fuzzing
+++ b/js/src/devtools/automation/variants/fuzzing
@@ -1,13 +1,13 @@
 {
     "configure-args": "--enable-fuzzing --enable-gczeal --enable-debug-symbols='-gline-tables-only -gdwarf-2' --disable-jemalloc --disable-stdcxx-compat --enable-address-sanitizer --enable-ctypes --enable-nspr-build",
     "optimize": true,
     "debug": false,
     "compiler": "clang",
     "env": {
         "JITTEST_EXTRA_ARGS": "--jitflags=none",
         "JSTESTS_EXTRA_ARGS": "--jitflags=none",
-        "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer",
-        "ASAN_SYMBOLIZER_PATH": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer"
+        "LLVM_SYMBOLIZER": "{MOZ_FETCHES_DIR}/clang/bin/llvm-symbolizer",
+        "ASAN_SYMBOLIZER_PATH": "{MOZ_FETCHES_DIR}/clang/bin/llvm-symbolizer"
     },
     "use_minidump": false
 }
--- a/js/src/devtools/automation/variants/msan
+++ b/js/src/devtools/automation/variants/msan
@@ -1,14 +1,14 @@
 {
     "configure-args": "--enable-debug-symbols='-gline-tables-only' --disable-jemalloc --enable-memory-sanitizer --without-system-zlib",
     "optimize": true,
     "debug": false,
     "compiler": "clang",
     "env": {
         "JITTEST_EXTRA_ARGS": "--jitflags=interp --ignore-timeouts={DIR}/cgc-jittest-timeouts.txt",
         "JSTESTS_EXTRA_ARGS": "--jitflags=interp --exclude-file={DIR}/cgc-jstests-slow.txt",
-        "MSAN_OPTIONS": "external_symbolizer_path={TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer:log_path={OUTDIR}/sanitize_log"
+        "MSAN_OPTIONS": "external_symbolizer_path={MOZ_FETCHES_DIR}/clang/bin/llvm-symbolizer:log_path={OUTDIR}/sanitize_log"
     },
     "ignore-test-failures": "true",
     "max-errors": 7,
     "use_minidump": false
 }
--- a/js/src/devtools/automation/variants/tsan
+++ b/js/src/devtools/automation/variants/tsan
@@ -1,12 +1,12 @@
 {
     "configure-args": "--enable-debug-symbols='-gline-tables-only' --disable-jemalloc --enable-thread-sanitizer",
     "optimize": true,
     "debug": false,
     "compiler": "clang",
     "env": {
-        "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer",
+        "LLVM_SYMBOLIZER": "{MOZ_FETCHES_DIR}/clang/bin/llvm-symbolizer",
         "JITTEST_EXTRA_ARGS": "--jitflags=tsan --ignore-timeouts={DIR}/cgc-jittest-timeouts.txt --unusable-error-status --exclude-from={DIR}/tsan-slow.txt",
         "JSTESTS_EXTRA_ARGS": "--exclude-file={DIR}/cgc-jstests-slow.txt"
     },
     "use_minidump": false
 }
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -6,17 +6,16 @@ loader: taskgraph.loader.transform:loade
 
 kind-dependencies:
     - toolchain
 
 transforms:
     - taskgraph.transforms.spidermonkey:transforms
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
-    - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
     attributes:
         retrigger: true
     treeherder:
         kind: build
@@ -53,21 +52,22 @@ job-defaults:
             - python/**
             - taskcluster/moz.build
             - taskcluster/ci/spidermonkey/kind.yml
             - testing/mozbase/**
             - testing/web-platform/**
             - test.mozbuild
             - toolkit/mozapps/installer/package-name.mk
             - toolkit/mozapps/installer/upload-files.mk
-    toolchains:
-        by-worker-type:
-            .*-b-win2012:
-                - win64-clang-cl
-                - win64-rust
-            default:
-                - linux64-clang
-                - linux64-gcc
-                - linux64-rust
+    fetches:
+        toolchain:
+            by-worker-type:
+                .*-b-win2012:
+                    - win64-clang-cl
+                    - win64-rust
+                default:
+                    - linux64-clang
+                    - linux64-gcc
+                    - linux64-rust
 
 jobs-from:
     - linux.yml
     - windows.yml
--- a/taskcluster/ci/spidermonkey/linux.yml
+++ b/taskcluster/ci/spidermonkey/linux.yml
@@ -136,20 +136,21 @@ sm-tsan-linux64/opt:
     description: "Spidermonkey Thread Sanitizer"
     index:
         job-name: sm-tsan-linux64-opt
     treeherder:
         symbol: SM(tsan)
         platform: linux64/opt
     run:
         spidermonkey-variant: tsan
-    toolchains:
-        - linux64-binutils
-        - linux64-clang
-        - linux64-rust
+    fetches:
+        toolchain:
+            - linux64-binutils
+            - linux64-clang
+            - linux64-rust
 
 sm-rootanalysis-linux64/debug:
     description: "Spidermonkey Root Analysis"
     index:
         job-name: sm-rootanalysis-linux64-debug
     treeherder:
         platform: linux64/debug
         symbol: SM(r)
--- a/taskcluster/ci/spidermonkey/windows.yml
+++ b/taskcluster/ci/spidermonkey/windows.yml
@@ -14,106 +14,113 @@ sm-plain-win64/debug:
     index:
         job-name: sm-plain-win64-debug
     treeherder:
         platform: windows2012-64/debug
         symbol: SM(p)
     run:
         spidermonkey-variant: plaindebug
         spidermonkey-platform: win64
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
 
 sm-plain-win32/debug:
     description: "Spidermonkey Plain win32 debug"
     index:
         job-name: sm-plain-win32-debug
     treeherder:
         platform: windows2012-32/debug
         symbol: SM(p)
     run:
         spidermonkey-variant: plaindebug
         spidermonkey-platform: win32
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
 
 sm-plain-win64/opt:
     description: "Spidermonkey Plain win64 opt"
     index:
         job-name: sm-plain-win64-opt
     treeherder:
         platform: windows2012-64/opt
         symbol: SM(p)
     run:
         spidermonkey-variant: plain
         spidermonkey-platform: win64
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
 
 sm-plain-win32/opt:
     description: "Spidermonkey Plain win32 opt"
     index:
         job-name: sm-plain-win32-opt
     treeherder:
         platform: windows2012-32/opt
         symbol: SM(p)
     run:
         spidermonkey-variant: plain
         spidermonkey-platform: win32
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
 
 sm-compacting-win64/debug:
     description: "Spidermonkey Compacting win64 debug"
     index:
         job-name: sm-compacting-win64-debug
     treeherder:
         platform: windows2012-64/debug
         symbol: SM(cgc)
     run:
         spidermonkey-variant: compacting
         spidermonkey-platform: win64
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
 
 sm-compacting-win32/debug:
     description: "Spidermonkey Compacting win32 debug"
     index:
         job-name: sm-compacting-win32-debug
     treeherder:
         platform: windows2012-32/debug
         symbol: SM(cgc)
     run:
         spidermonkey-variant: compacting
         spidermonkey-platform: win32
     run-on-projects: []
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
 
 sm-plain-win64-aarch64/opt:
     description: "Spidermonkey Plain win64-aarch64 opt"
     index:
         job-name: sm-plain-win64-aarch64-opt
     treeherder:
         platform: windows2012-aarch64/opt
         symbol: SM(p)
         tier: 2
     run:
         spidermonkey-variant: plain
         spidermonkey-platform: win64
-    toolchains:
-        - win64-clang-cl
-        - win64-rust
+    fetches:
+        toolchain:
+            - win64-clang-cl
+            - win64-rust
--- a/taskcluster/scripts/builder/build-sm-mozjs-crate.sh
+++ b/taskcluster/scripts/builder/build-sm-mozjs-crate.sh
@@ -6,14 +6,14 @@ source $(dirname $0)/sm-tooltool-config.
 
 # Ensure that we have a .config/cargo that points us to our vendored crates
 # rather than to crates.io.
 cd "$SRCDIR/.cargo"
 sed -e "s|@top_srcdir@|$SRCDIR|" -e 's|@[^@]*@||g' < config.in > config
 
 cd "$SRCDIR/js/src"
 
-export PATH="$PATH:$TOOLTOOL_CHECKOUT/cargo/bin:$TOOLTOOL_CHECKOUT/rustc/bin"
+export PATH="$PATH:$MOZ_FETCHES_DIR/cargo/bin:$MOZ_FETCHES_DIR/rustc/bin"
 export RUST_BACKTRACE=1
 export AUTOMATION=1
 
 cargo build --verbose --frozen --features debugmozjs
 cargo build --verbose --frozen
--- a/taskcluster/scripts/builder/build-sm-rust-bindings.sh
+++ b/taskcluster/scripts/builder/build-sm-rust-bindings.sh
@@ -6,14 +6,14 @@ source $(dirname $0)/sm-tooltool-config.
 
 # Ensure that we have a .config/cargo that points us to our vendored crates
 # rather than to crates.io.
 cd "$SRCDIR/.cargo"
 sed -e "s|@top_srcdir@|$SRCDIR|" -e 's|@[^@]*@||g' < config.in > config
 
 cd "$SRCDIR/js/rust"
 
-export LD_LIBRARY_PATH="$TOOLTOOL_CHECKOUT/gcc/lib64"
+export LD_LIBRARY_PATH="$MOZ_FETCHES_DIR/gcc/lib64"
 # Enable backtraces if we panic.
 export RUST_BACKTRACE=1
 
 cargo test --verbose --frozen --features debugmozjs
 cargo test --verbose --frozen
--- a/taskcluster/scripts/builder/build-sm.sh
+++ b/taskcluster/scripts/builder/build-sm.sh
@@ -14,11 +14,11 @@ BUILD_STATUS=$?
 # Ensure upload dir exists
 mkdir -p $UPLOAD_DIR
 
 # Copy artifacts for upload by TaskCluster
 cp -rL $SRCDIR/obj-spider/dist/bin/{js,jsapi-tests,js-gdb.py} $UPLOAD_DIR
 
 # Fuzzing users would really like to have llvm-symbolizer available in the same
 # directory as the built output.
-gzip -c $TOOLTOOL_CHECKOUT/clang/bin/llvm-symbolizer > $UPLOAD_DIR/llvm-symbolizer.gz || true
+gzip -c $MOZ_FETCHES_DIR/clang/bin/llvm-symbolizer > $UPLOAD_DIR/llvm-symbolizer.gz || true
 
 exit $BUILD_STATUS
--- a/taskcluster/scripts/builder/sm-tooltool-config.sh
+++ b/taskcluster/scripts/builder/sm-tooltool-config.sh
@@ -56,16 +56,16 @@ fi
 # necessary for the JS shell, but it's less duplication to share tooltool
 # manifests.
 BROWSER_PLATFORM=$PLATFORM_OS$BITS
 
 (cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain${TOOLTOOL_MANIFEST:+ -v $TOOLTOOL_AUTH_FLAGS --tooltool-url $TOOLTOOL_SERVER --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST}${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
 
 ) || exit 1 # end of set -e scope
 
-# Add all the tooltool binaries to our $PATH.
-for bin in $TOOLTOOL_CHECKOUT/*/bin $TOOLTOOL_CHECKOUT/VC/bin/Hostx64/x86; do
+# Add all the fetches and tooltool binaries to our $PATH.
+for bin in $MOZ_FETCHES_DIR/*/bin $TOOLTOOL_CHECKOUT/VC/bin/Hostx64/x86; do
     if [ ! -d "$bin" ]; then
         continue
     fi
     absbin=$(cd "$bin" && pwd)
     export PATH="$absbin:$PATH"
 done
--- a/taskcluster/taskgraph/transforms/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/spidermonkey.py
@@ -9,16 +9,16 @@ import copy
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def handle_keyed_by(config, jobs):
     """Resolve fields that can be keyed by platform, etc."""
     fields = [
-        "toolchains"
+        "fetches.toolchain"
     ]
     for job in jobs:
         job = copy.deepcopy(job)  # don't overwrite dict values here
         for field in fields:
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
 
         yield job