Bug 1382860 - Use docker_worker_add_public_artifacts for toolchain docker-worker jobs. r=dustin
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 21 Jul 2017 07:30:14 +0900
changeset 419011 4ed067a3c2941aada4f32eb63b68d5574a2a2c80
parent 419010 5b8225dac5532f0726c9a5f9bf81e97ce8367331
child 419012 394b3d22db1988839462c9832f4ef309aef556a1
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1382860
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1382860 - Use docker_worker_add_public_artifacts for toolchain docker-worker jobs. r=dustin This affects the location of the artifacts directory, so adjust the scripts and artifact definitions as a consequence.
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/scripts/misc/build-binutils-linux.sh
taskcluster/scripts/misc/build-cctools-port-macosx.sh
taskcluster/scripts/misc/build-cctools-port.sh
taskcluster/scripts/misc/build-clang-linux.sh
taskcluster/scripts/misc/build-clang-macosx.sh
taskcluster/scripts/misc/build-clang-tidy-linux.sh
taskcluster/scripts/misc/build-clang-tidy-macosx.sh
taskcluster/scripts/misc/build-gcc-linux.sh
taskcluster/scripts/misc/build-hfsplus-linux.sh
taskcluster/scripts/misc/build-libdmg-hfsplus.sh
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -16,17 +16,17 @@ linux64-clang:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: toolchain-script
         script: build-clang-linux.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/clang.tar.xz
+        toolchain-artifact: public/build/clang.tar.xz
 
 linux64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: linux64-clang-tidy
     treeherder:
         kind: build
@@ -41,17 +41,17 @@ linux64-clang-tidy:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy-linux.sh
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/clang-tidy.tar.xz
+        toolchain-artifact: public/build/clang-tidy.tar.xz
 
 linux64-gcc:
     description: "GCC toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(gcc)
         tier: 1
@@ -59,17 +59,17 @@ linux64-gcc:
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-gcc-linux.sh
         resources:
             - 'build/unix/build-gcc/**'
-        toolchain-artifact: public/gcc.tar.xz
+        toolchain-artifact: public/build/gcc.tar.xz
 
 linux64-binutils:
     description: "Binutils toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(binutil)
         tier: 1
@@ -77,17 +77,17 @@ linux64-binutils:
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-binutils-linux.sh
         resources:
             - 'build/unix/build-binutils/**'
-        toolchain-artifact: public/binutils.tar.xz
+        toolchain-artifact: public/build/binutils.tar.xz
 
 linux64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(cctools)
         tier: 1
@@ -97,17 +97,17 @@ linux64-cctools-port:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
     run:
         using: toolchain-script
         script: build-cctools-port.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/cctools.tar.xz
+        toolchain-artifact: public/build/cctools.tar.xz
 
 linux64-hfsplus:
     description: "hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(hfs+)
         tier: 1
@@ -118,25 +118,25 @@ linux64-hfsplus:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
     run:
         using: toolchain-script
         script: build-hfsplus-linux.sh
         resources:
             - 'build/unix/build-hfsplus/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/hfsplus-tools.tar.xz
+        toolchain-artifact: public/build/hfsplus-tools.tar.xz
 
 linux64-libdmg:
     description: "libdmg-hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(libdmg-hfs+)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-libdmg-hfsplus.sh
-        toolchain-artifact: public/dmg.tar.xz
+        toolchain-artifact: public/build/dmg.tar.xz
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -17,17 +17,17 @@ macosx64-clang:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-clang-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/clang.tar.xz
+        toolchain-artifact: public/build/clang.tar.xz
 
 macosx64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: macosx64-clang-tidy
     treeherder:
         kind: build
@@ -43,17 +43,17 @@ macosx64-clang-tidy:
     run:
         using: toolchain-script
         script: build-clang-tidy-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/clang-tidy.tar.xz
+        toolchain-artifact: public/build/clang-tidy.tar.xz
 
 macosx64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(cctools)
         tier: 1
@@ -64,9 +64,9 @@ macosx64-cctools-port:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-cctools-port-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-artifact: public/cctools.tar.bz2
+        toolchain-artifact: public/build/cctools.tar.bz2
--- a/taskcluster/scripts/misc/build-binutils-linux.sh
+++ b/taskcluster/scripts/misc/build-binutils-linux.sh
@@ -1,16 +1,16 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building binutils for Linux.
 
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 build/unix/build-binutils/build-binutils.sh $HOME_DIR
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
 cp $HOME_DIR/binutils.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-cctools-port-macosx.sh
+++ b/taskcluster/scripts/misc/build-cctools-port-macosx.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building cctools (Apple's binutils) for Mac OS X on
 # Linux using ctools-port (https://github.com/tpoechtrager/cctools-port).
 WORKSPACE=$HOME/workspace
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 # Repository info
 : CROSSTOOL_PORT_REPOSITORY    ${CROSSTOOL_PORT_REPOSITORY:=https://github.com/tpoechtrager/cctools-port}
 : CROSSTOOL_PORT_REV           ${CROSSTOOL_PORT_REV:=8e9c3f2506b51cf56725eaa60b6e90e240e249ca}
 
 # Set some crosstools-port directories
 CROSSTOOLS_SOURCE_DIR=$WORKSPACE/crosstools-port
 CROSSTOOLS_CCTOOLS_DIR=$CROSSTOOLS_SOURCE_DIR/cctools
--- a/taskcluster/scripts/misc/build-cctools-port.sh
+++ b/taskcluster/scripts/misc/build-cctools-port.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building cctools (Apple's binutils) for Linux using
 # cctools-port (https://github.com/tpoechtrager/cctools-port).
 WORKSPACE=$HOME/workspace
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 # Repository info
 : CROSSTOOL_PORT_REPOSITORY    ${CROSSTOOL_PORT_REPOSITORY:=https://github.com/tpoechtrager/cctools-port}
 : CROSSTOOL_PORT_REV           ${CROSSTOOL_PORT_REV:=8e9c3f2506b51cf56725eaa60b6e90e240e249ca}
 
 # Set some crosstools-port directories
 CROSSTOOLS_SOURCE_DIR=$WORKSPACE/crosstools-port
 CROSSTOOLS_CCTOOLS_DIR=$CROSSTOOLS_SOURCE_DIR/cctools
--- a/taskcluster/scripts/misc/build-clang-linux.sh
+++ b/taskcluster/scripts/misc/build-clang-linux.sh
@@ -1,16 +1,16 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building clang for Linux.
 
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
 # gets a bit too verbose here
 set +x
 
--- a/taskcluster/scripts/misc/build-clang-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-macosx.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building clang for Mac OS X on Linux.
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
 # ld needs libLTO.so from llvm
 export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
 # these variables are used in build-clang.py
--- a/taskcluster/scripts/misc/build-clang-tidy-linux.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-linux.sh
@@ -1,16 +1,16 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building clang for Linux.
 
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
 # gets a bit too verbose here
 set +x
 
--- a/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building clang for Mac OS X on Linux.
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
 # ld needs libLTO.so from llvm
 export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
 # these variables are used in build-clang.py
--- a/taskcluster/scripts/misc/build-gcc-linux.sh
+++ b/taskcluster/scripts/misc/build-gcc-linux.sh
@@ -1,16 +1,16 @@
 #!/bin/bash
 set -e
 
 # This script is for building GCC for Linux.
 
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 build/unix/build-gcc/build-gcc.sh $HOME_DIR
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
 cp $HOME_DIR/gcc.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-hfsplus-linux.sh
+++ b/taskcluster/scripts/misc/build-hfsplus-linux.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building hfsplus for Linux.
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
 export PATH=$PATH:$HOME_DIR/src/clang/bin
 
 build/unix/build-hfsplus/build-hfsplus.sh $HOME_DIR
--- a/taskcluster/scripts/misc/build-libdmg-hfsplus.sh
+++ b/taskcluster/scripts/misc/build-libdmg-hfsplus.sh
@@ -1,17 +1,17 @@
 #!/bin/bash
 set -x -e -v
 
 # This script is for building libdmg-hfsplus to get the `dmg` and `hfsplus`
 # tools for producing DMG archives on Linux.
 
 WORKSPACE=$HOME/workspace
 STAGE=$WORKSPACE/dmg
-UPLOAD_DIR=$WORKSPACE/artifacts
+UPLOAD_DIR=$HOME/artifacts
 
 # There's no single well-maintained fork of libdmg-hfsplus, so we forked
 # https://github.com/andreas56/libdmg-hfsplus/ to get a specific version and
 # backport some patches.
 : LIBDMG_REPOSITORY    ${LIBDMG_REPOSITORY:=https://github.com/mozilla/libdmg-hfsplus}
 # The `mozilla` branch contains our fork.
 : LIBDMG_REV           ${LIBDMG_REV:=ba04b00435a0853f1499d751617177828ee8ec00}
 
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 from taskgraph.util.schema import Schema
 from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_tc_vcs_cache,
     docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_add_public_artifacts,
     support_vcs_checkout,
 )
 from taskgraph.util.hash import hash_paths
 from taskgraph import GECKO
 
 
 TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}'
 
@@ -75,22 +76,17 @@ def add_optimizations(config, run, taskd
 def docker_worker_toolchain(config, job, taskdesc):
     run = job['run']
     taskdesc['run-on-projects'] = ['trunk', 'try']
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
     worker['caches'] = []
 
-    worker['artifacts'].append({
-        'name': 'public',
-        'path': '/home/worker/workspace/artifacts/',
-        'type': 'directory',
-    })
-
+    docker_worker_add_public_artifacts(config, job, taskdesc)
     docker_worker_add_tc_vcs_cache(config, job, taskdesc)
     docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
     support_vcs_checkout(config, job, taskdesc)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],