Bug 1573435 - Use toolchain fetches for source-test builds. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 14 Aug 2019 11:06:46 +0900
changeset 488148 3103e4be4bb4d63926e309af68ed23074207b064
parent 488147 ab4d3a77070a11190b1ebf09e4594ef0954f3e1d
child 488149 73576d4b23de5f35476f4e3d439adf498b9ab5d1
push id36435
push usercbrindusan@mozilla.com
push dateThu, 15 Aug 2019 09:46:49 +0000
treeherdermozilla-central@0db07ff50ab5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1573435
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573435 - Use toolchain fetches for source-test builds. r=nalexander While here, use GECKO_PATH instead of $HOME/checkouts/gecko. Differential Revision: https://phabricator.services.mozilla.com/D41889
python/mozbuild/mozbuild/code-analysis/mach_commands.py
taskcluster/ci/source-test/clang.yml
taskcluster/ci/source-test/coverity.yml
taskcluster/ci/source-test/infer.yml
taskcluster/ci/source-test/kind.yml
taskcluster/scripts/misc/source-test-clang-setup.sh
taskcluster/scripts/misc/source-test-common.sh
taskcluster/scripts/misc/source-test-infer-setup.sh
--- a/python/mozbuild/mozbuild/code-analysis/mach_commands.py
+++ b/python/mozbuild/mozbuild/code-analysis/mach_commands.py
@@ -925,17 +925,17 @@ class StaticAnalysis(MachCommandBase):
         self.TOOLS_CHECKER_NOT_FOUND = 7
         self.TOOLS_CHECKER_FAILED_FILE = 8
         self.TOOLS_CHECKER_LIST_EMPTY = 9
         self.TOOLS_GRADLE_FAILED = 10
 
         # Configure the tree or download clang-tidy package, depending on the option that we choose
         if intree_tool:
             _, config, _ = self._get_config_environment()
-            clang_tools_path = self.topsrcdir
+            clang_tools_path = os.environ['MOZ_FETCHES_DIR']
             self._clang_tidy_path = mozpath.join(
                 clang_tools_path, "clang-tidy", "bin",
                 "clang-tidy" + config.substs.get('BIN_SUFFIX', ''))
             self._clang_format_path = mozpath.join(
                 clang_tools_path, "clang-tidy", "bin",
                 "clang-format" + config.substs.get('BIN_SUFFIX', ''))
             self._clang_apply_replacements = mozpath.join(
                 clang_tools_path, "clang-tidy", "bin",
@@ -1808,17 +1808,17 @@ class StaticAnalysis(MachCommandBase):
                     args += [':({0}){1}'.format(','.join(magics), pattern)]
         return args
 
     def _get_infer(self, force=False, skip_cache=False, download_if_needed=True,
                    verbose=False, intree_tool=False):
         rc, config, _ = self._get_config_environment()
         if rc != 0:
             return rc
-        infer_path = self.topsrcdir if intree_tool else \
+        infer_path = os.environ['MOZ_FETCHES_DIR'] if intree_tool else \
             mozpath.join(self._mach_context.state_dir, 'infer')
         self._infer_path = mozpath.join(infer_path, 'infer', 'bin', 'infer' +
                                         config.substs.get('BIN_SUFFIX', ''))
         if intree_tool:
             return not os.path.exists(self._infer_path)
         if os.path.exists(self._infer_path) and not force:
             return 0
 
--- a/taskcluster/ci/source-test/clang.yml
+++ b/taskcluster/ci/source-test/clang.yml
@@ -18,23 +18,24 @@ job-defaults:
         docker-image: {in-tree: debian7-amd64-build}
         max-run-time: 5400
     treeherder:
         kind: other
         tier: 2
     run:
         using: run-task
         tooltool-downloads: public
-    toolchains:
-        - linux64-clang
-        - linux64-clang-tidy
-        - linux64-rust
-        - linux64-cbindgen
-        - linux64-nasm
-        - linux64-node
+    fetches:
+        toolchain:
+            - linux64-clang
+            - linux64-clang-tidy
+            - linux64-rust
+            - linux64-cbindgen
+            - linux64-nasm
+            - linux64-node
     when:
         # Extension list from https://hg.mozilla.org/mozilla-central/file/tip/python/mozbuild/mozbuild/mach_commands.py#l1664
         files-changed:
             - '**/*.c'
             - '**/*.cpp'
             - '**/*.cc'
             - '**/*.cxx'
             - '**/*.m'
--- a/taskcluster/ci/source-test/coverity.yml
+++ b/taskcluster/ci/source-test/coverity.yml
@@ -17,22 +17,23 @@ job-defaults:
         docker-image: {in-tree: debian7-amd64-build}
         max-run-time: 5400
     treeherder:
         kind: other
         tier: 2
     run:
         using: run-task
         tooltool-downloads: public
-    toolchains:
-        - linux64-clang-7
-        - linux64-rust
-        - linux64-cbindgen
-        - linux64-nasm
-        - linux64-node
+    fetches:
+        toolchain:
+            - linux64-clang-7
+            - linux64-rust
+            - linux64-cbindgen
+            - linux64-nasm
+            - linux64-node
     when:
         # Extension list from https://hg.mozilla.org/mozilla-central/file/tip/python/mozbuild/mozbuild/mach_commands.py#l1664
         files-changed:
             - '**/*.c'
             - '**/*.cpp'
             - '**/*.cc'
             - '**/*.cxx'
             - '**/*.m'
--- a/taskcluster/ci/source-test/infer.yml
+++ b/taskcluster/ci/source-test/infer.yml
@@ -18,25 +18,26 @@ job-defaults:
         docker-image: {in-tree: android-build}
         max-run-time: 5400
     treeherder:
         kind: other
         tier: 2
     run:
         using: run-task
         tooltool-downloads: public
-    toolchains:
-        - linux64-infer
-        - linux64-android-sdk-linux-repack
-        - linux64-android-ndk-linux-repack
-        - linux64-rust-android
-        - linux64-clang
-        - linux64-cbindgen
-        - linux64-nasm
-        - linux64-node
+    fetches:
+        toolchain:
+            - linux64-infer
+            - linux64-android-sdk-linux-repack
+            - linux64-android-ndk-linux-repack
+            - linux64-rust-android
+            - linux64-clang
+            - linux64-cbindgen
+            - linux64-nasm
+            - linux64-node
     when:
         files-changed:
             - 'mobile/**/*.java'
 
 infer:
     description: Run static-analysis (infer) on Java patches
     treeherder:
         symbol: infer
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -2,17 +2,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.try_job:transforms
     - taskgraph.transforms.source_test:transforms
-    - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
     - fetch
     - toolchain
 
 jobs-from:
--- a/taskcluster/scripts/misc/source-test-clang-setup.sh
+++ b/taskcluster/scripts/misc/source-test-clang-setup.sh
@@ -1,19 +1,19 @@
 #!/bin/bash
 source $HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-common.sh
 
 # Add clang-tidy to PATH
-export PATH=$MOZBUILD_STATE_PATH/clang-tidy/bin:$PATH
+export PATH=$MOZ_FETCHES_DIR/clang-tidy/bin:$PATH
 
 # Use toolchain clang
-export LD_LIBRARY_PATH=$MOZBUILD_STATE_PATH/clang/lib
+export LD_LIBRARY_PATH=$MOZ_FETCHES_DIR/clang/lib
 
 # Write custom mozconfig
-export MOZCONFIG=$HOME/checkouts/gecko/mozconfig
+export MOZCONFIG=$GECKO_PATH/mozconfig
 # Enable debug mode
 echo "ac_add_options --enable-debug"  > $MOZCONFIG
 # Enable GC zeal, a testing and debugging feature that helps find GC-related bugs in JSAPI applications.
 echo "ac_add_options --enable-gczeal" > $MOZCONFIG
 
 # Mach lookup clang-tidy in clang-tools
 mkdir -p $MOZBUILD_STATE_PATH/clang-tools
-ln -s $MOZBUILD_STATE_PATH/clang-tidy $MOZBUILD_STATE_PATH/clang-tools/clang-tidy
+ln -s $MOZ_FETCHES_DIR/clang-tidy $MOZBUILD_STATE_PATH/clang-tools/clang-tidy
--- a/taskcluster/scripts/misc/source-test-common.sh
+++ b/taskcluster/scripts/misc/source-test-common.sh
@@ -1,22 +1,16 @@
 #! /bin/bash -vex
 
 set -x -e
 
 export MOZBUILD_STATE_PATH=$HOME/workspace
 
-# Setup toolchains
-pushd $MOZBUILD_STATE_PATH
-$HOME/checkouts/gecko/mach artifact toolchain -v $MOZ_TOOLCHAINS
-
 # Add toolchain binaries to PATH to run ./mach configure
-export PATH=$MOZBUILD_STATE_PATH/clang/bin:$PATH
-export PATH=$MOZBUILD_STATE_PATH/rustc/bin:$PATH
-export PATH=$MOZBUILD_STATE_PATH/cbindgen:$PATH
-export PATH=$MOZBUILD_STATE_PATH/nasm:$PATH
-export PATH=$MOZBUILD_STATE_PATH/node/bin:$PATH
+export PATH=$MOZ_FETCHES_DIR/clang/bin:$PATH
+export PATH=$MOZ_FETCHES_DIR/rustc/bin:$PATH
+export PATH=$MOZ_FETCHES_DIR/cbindgen:$PATH
+export PATH=$MOZ_FETCHES_DIR/nasm:$PATH
+export PATH=$MOZ_FETCHES_DIR/node/bin:$PATH
 
 # Use clang as host compiler
-export CC=$MOZBUILD_STATE_PATH/clang/bin/clang
-export CXX=$MOZBUILD_STATE_PATH/clang/bin/clang++
-
-popd
+export CC=$MOZ_FETCHES_DIR/clang/bin/clang
+export CXX=$MOZ_FETCHES_DIR/clang/bin/clang++
--- a/taskcluster/scripts/misc/source-test-infer-setup.sh
+++ b/taskcluster/scripts/misc/source-test-infer-setup.sh
@@ -1,18 +1,18 @@
 #!/bin/bash
-source $HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-common.sh
+source $GECKO_PATH/taskcluster/scripts/misc/source-test-common.sh
 
 # Write custom mozconfig
-MOZCONFIG=$HOME/checkouts/gecko/mozconfig
+MOZCONFIG=$GECKO_PATH/mozconfig
 echo "ac_add_options --enable-application=mobile/android" > $MOZCONFIG
 echo "ac_add_options --target=arm-linux-androideabi" >> $MOZCONFIG
-echo "ac_add_options --with-android-sdk=${MOZBUILD_STATE_PATH}/android-sdk-linux" >> $MOZCONFIG
-echo "ac_add_options --with-android-ndk=${MOZBUILD_STATE_PATH}/android-ndk" >> $MOZCONFIG
+echo "ac_add_options --with-android-sdk=${MOZ_FETCHES_DIR}/android-sdk-linux" >> $MOZCONFIG
+echo "ac_add_options --with-android-ndk=${MOZ_FETCHES_DIR}/android-ndk" >> $MOZCONFIG
 
 # Write custom grade properties
 export GRADLE_USER_HOME=$HOME/workspace/gradle
 mkdir -p $GRADLE_USER_HOME
 echo "org.gradle.daemon=false" >> ${GRADLE_USER_HOME}/gradle.properties
 
 # Mach lookup infer in infer...
 mkdir -p $MOZBUILD_STATE_PATH/infer/infer
-mv $MOZBUILD_STATE_PATH/infer/{bin,lib} $MOZBUILD_STATE_PATH/infer/infer
+mv $MOZ_FETCHES_DIR/infer/{bin,lib} $MOZBUILD_STATE_PATH/infer/infer