Bug 1573435 - Use toolchain fetches for hazard builds. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 13 Aug 2019 17:43:56 +0900
changeset 488146 84afd114aae980e01e7835779f26bfa78df151ea
parent 488145 874a06471c7ae2d9847ac3a1a9f562a16b903683
child 488147 ab4d3a77070a11190b1ebf09e4594ef0954f3e1d
push id36435
push usercbrindusan@mozilla.com
push dateThu, 15 Aug 2019 09:46:49 +0000
treeherdermozilla-central@0db07ff50ab5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1573435
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573435 - Use toolchain fetches for hazard builds. r=nalexander Differential Revision: https://phabricator.services.mozilla.com/D41738
js/src/devtools/rootAnalysis/mozconfig.haz
taskcluster/ci/hazard/kind.yml
taskcluster/scripts/builder/build-haz-linux.sh
taskcluster/scripts/builder/hazard-analysis.sh
--- a/js/src/devtools/rootAnalysis/mozconfig.haz
+++ b/js/src/devtools/rootAnalysis/mozconfig.haz
@@ -17,31 +17,31 @@ MOZ_AUTOMATION_PACKAGE_TESTS=0
 MOZ_AUTOMATION_UPLOAD=0
 
 ac_add_options --enable-js-shell
 
 # The objdir must be at a known location so its path can be stripped from the
 # filenames stored by the analysis
 mk_add_options MOZ_OBJDIR=obj-analyzed
 
-export LLVM_CONFIG=$TOOLTOOL_DIR/clang/bin/llvm-config
-export CBINDGEN="${TOOLTOOL_DIR}/cbindgen/cbindgen"
+export LLVM_CONFIG=$MOZ_FETCHES_DIR/clang/bin/llvm-config
+export CBINDGEN="${MOZ_FETCHES_DIR}/cbindgen/cbindgen"
 
 # The configuration options are chosen to compile the most code
 # (--enable-debug, --enable-tests) in the trickiest way possible
 # (--enable-optimize) to maximize the chance of seeing tricky static orderings.
 ac_add_options --enable-debug
 ac_add_options --enable-tests
 ac_add_options --enable-optimize
-ac_add_options --with-compiler-wrapper=$TOOLTOOL_DIR/sixgill/usr/libexec/sixgill/scripts/wrap_gcc/basecc
+ac_add_options --with-compiler-wrapper=$MOZ_FETCHES_DIR/sixgill/usr/libexec/sixgill/scripts/wrap_gcc/basecc
 ac_add_options --without-ccache
 
 ac_add_options --disable-replace-malloc
 
 # -Wattributes is very verbose due to attributes being ignored on template
 # instantiations. -Wignored-attributes is very verbose due to attributes being
 # ignored on template parameters.
 CFLAGS="$CFLAGS -Wno-attributes -Wno-ignored-attributes"
 CPPFLAGS="$CPPFLAGS -Wno-attributes -Wno-ignored-attributes"
 CXXFLAGS="$CXXFLAGS -Wno-attributes -Wno-ignored-attributes"
 
-NODEJS="$TOOLTOOL_DIR/node/bin/node"
-NASM="$TOOLTOOL_DIR/nasm/nasm"
+NODEJS="$MOZ_FETCHES_DIR/node/bin/node"
+NASM="$MOZ_FETCHES_DIR/nasm/nasm"
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -5,17 +5,16 @@
 loader: taskgraph.loader.transform:loader
 
 kind-dependencies:
     - toolchain
 
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
-    - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
     attributes:
         retrigger: true
     treeherder:
         kind: build
@@ -38,38 +37,40 @@ jobs:
             using: hazard
             command: >
                 cd /builds/worker/checkouts/gecko/taskcluster/scripts/builder
                 && ./build-haz-linux.sh --project shell $HOME/workspace
         when:
             files-changed:
                 - js/public/**
                 - js/src/**
-        toolchains:
-            - linux64-clang
-            - linux64-gcc-6
-            - linux64-gcc-sixgill
-            - linux64-rust
-            - linux64-cbindgen
-            - linux64-node
+        fetches:
+            toolchain:
+                - linux64-clang
+                - linux64-gcc-6
+                - linux64-gcc-sixgill
+                - linux64-rust
+                - linux64-cbindgen
+                - linux64-node
 
     linux64-haz/debug:
         description: "Browser Hazard Analysis Linux"
         index:
             product: firefox
             job-name: browser-haz-debug
         treeherder:
             platform: linux64/debug
             symbol: H
         run:
             using: hazard
             mozconfig: "browser/config/mozconfigs/linux64/hazards"
             command: >
                 cd /builds/worker/checkouts/gecko/taskcluster/scripts/builder
                 && ./build-haz-linux.sh --project browser $HOME/workspace
-        toolchains:
-            - linux64-clang
-            - linux64-gcc-6
-            - linux64-gcc-sixgill
-            - linux64-rust
-            - linux64-cbindgen
-            - linux64-nasm
-            - linux64-node
+        fetches:
+            toolchain:
+                - linux64-clang
+                - linux64-gcc-6
+                - linux64-gcc-sixgill
+                - linux64-rust
+                - linux64-cbindgen
+                - linux64-nasm
+                - linux64-node
--- a/taskcluster/scripts/builder/build-haz-linux.sh
+++ b/taskcluster/scripts/builder/build-haz-linux.sh
@@ -3,28 +3,26 @@
 function usage() {
     echo "Usage: $0 [--project <shell|browser>] <workspace-dir> flags..."
     echo "flags are treated the same way as a commit message would be"
     echo "(as in, they are scanned for directives just like a try: ... line)"
 }
 
 PROJECT=shell
 WORKSPACE=
-DO_TOOLTOOL=1
 while [[ $# -gt 0 ]]; do
     if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then
         usage
         exit 0
     elif [[ "$1" == "--project" ]]; then
         shift
         PROJECT="$1"
         shift
     elif [[ "$1" == "--no-tooltool" ]]; then
         shift
-        DO_TOOLTOOL=
     elif [[ -z "$WORKSPACE" ]]; then
         WORKSPACE=$( cd "$1" && pwd )
         shift
         break
     fi
 done
 
 SCRIPT_FLAGS=$*
@@ -32,38 +30,26 @@ SCRIPT_FLAGS=$*
 # Ensure all the scripts in this dir are on the path....
 DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 PATH=$DIRNAME:$PATH
 
 # Use GECKO_BASE_REPOSITORY as a signal for whether we are running in automation.
 export AUTOMATION=${GECKO_BASE_REPOSITORY:+1}
 
 : "${GECKO_PATH:="$DIRNAME"/../../..}"
-: "${TOOLTOOL_CACHE:=$WORKSPACE/tt-cache}"
 
 if ! [ -d "$GECKO_PATH" ]; then
     echo "GECKO_PATH must be set to a directory containing a gecko source checkout" >&2
     exit 1
 fi
 
-# Directory to populate with tooltool-installed tools
-export TOOLTOOL_DIR="$WORKSPACE"
-
 # Directory to hold the (useless) object files generated by the analysis.
 export MOZ_OBJDIR="$WORKSPACE/obj-analyzed"
 mkdir -p "$MOZ_OBJDIR"
 
-if [ -n "$DO_TOOLTOOL" ]; then (
-    cd "$TOOLTOOL_DIR"
-    "$GECKO_PATH"/mach artifact toolchain -v\
-                ${TOOLTOOL_MANIFEST:+ --tooltool-url https://tooltool.mozilla-releng.net/ \
-                                      --tooltool-manifest "$GECKO_PATH/$TOOLTOOL_MANIFEST"} \
-                --cache-dir "$TOOLTOOL_CACHE"${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}}
-) fi
-
 export NO_MERCURIAL_SETUP_CHECK=1
 
 if [[ "$PROJECT" = "browser" ]]; then (
     cd "$WORKSPACE"
     set "$WORKSPACE"
     # Mozbuild config:
     export MOZBUILD_STATE_PATH=$WORKSPACE/mozbuild/
     # Create .mozbuild so mach doesn't complain about this
--- a/taskcluster/scripts/builder/hazard-analysis.sh
+++ b/taskcluster/scripts/builder/hazard-analysis.sh
@@ -3,24 +3,24 @@
 [ -n "$WORKSPACE" ]
 [ -n "$MOZ_OBJDIR" ]
 [ -n "$GECKO_PATH" ]
 
 HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell
 JSBIN="$HAZARD_SHELL_OBJDIR/dist/bin/js"
 JS_SRCDIR=$GECKO_PATH/js/src
 ANALYSIS_SRCDIR=$JS_SRCDIR/devtools/rootAnalysis
-GCCDIR="$TOOLTOOL_DIR/gcc"
+GCCDIR="$MOZ_FETCHES_DIR/gcc"
 
 export CC="$GCCDIR/bin/gcc"
 export CXX="$GCCDIR/bin/g++"
-export PATH="$GCCDIR/bin:$TOOLTOOL_DIR/clang/bin:$PATH"
+export PATH="$GCCDIR/bin:$MOZ_FETCHES_DIR/clang/bin:$PATH"
 export LD_LIBRARY_PATH="$GCCDIR/lib64"
-export RUSTC="$TOOLTOOL_DIR/rustc/bin/rustc"
-export CARGO="$TOOLTOOL_DIR/rustc/bin/cargo"
+export RUSTC="$MOZ_FETCHES_DIR/rustc/bin/rustc"
+export CARGO="$MOZ_FETCHES_DIR/rustc/bin/cargo"
 
 PYTHON=python2.7
 if ! which $PYTHON; then
     PYTHON=python
 fi
 
 function check_commit_msg () {
     ( set +e;
@@ -67,18 +67,18 @@ function configure_analysis () {
     mkdir -p "$analysis_dir" || true
     (
         cd "$analysis_dir"
         cat > defaults.py <<EOF
 js = "$JSBIN"
 analysis_scriptdir = "$ANALYSIS_SRCDIR"
 objdir = "$MOZ_OBJDIR"
 source = "$GECKO_PATH"
-sixgill = "$TOOLTOOL_DIR/sixgill/usr/libexec/sixgill"
-sixgill_bin = "$TOOLTOOL_DIR/sixgill/usr/bin"
+sixgill = "$MOZ_FETCHES_DIR/sixgill/usr/libexec/sixgill"
+sixgill_bin = "$MOZ_FETCHES_DIR/sixgill/usr/bin"
 EOF
 
         local rev
         rev=$(cd $GECKO_PATH && hg log -r . -T '{node|short}')
         cat > run-analysis.sh <<EOF
 #!/bin/sh
 if [ \$# -eq 0 ]; then
   set gcTypes
@@ -103,17 +103,17 @@ function run_analysis () {
 
     (
         cd "$analysis_dir"
         $PYTHON "$ANALYSIS_SRCDIR/analyze.py" -v --buildcommand="$GECKO_PATH/taskcluster/scripts/builder/hazard-${build_type}.sh"
     )
 }
 
 function analysis_self_test () {
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname "$JSBIN")" $PYTHON "$ANALYSIS_SRCDIR/run-test.py" -v --js "$JSBIN" --sixgill "$TOOLTOOL_DIR/sixgill" --gccdir "$GCCDIR"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname "$JSBIN")" $PYTHON "$ANALYSIS_SRCDIR/run-test.py" -v --js "$JSBIN" --sixgill "$MOZ_FETCHES_DIR/sixgill" --gccdir "$GCCDIR"
 }
 
 function grab_artifacts () {
     local analysis_dir
     analysis_dir="$1"
     local artifacts
     artifacts="$2"