Bug 1374940 - Add artifact paths to toolchain jobs definitions. r=dustin
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 21 Jun 2017 15:13:35 +0900
changeset 418756 2023ee951ea672f47cb01f5df84bf72a88fcfb1b
parent 418755 eac3bc21f3708cf404a1115c57addedcd0104bb1
child 418757 ba64040f39ad2656ea5e10d4aced392dce5b75f1
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1374940
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1374940 - Add artifact paths to toolchain jobs definitions. r=dustin The toolchain jobs produce artifacts that are going to be used by other jobs, but there is no reliable way for the decision task to know the name of those artifacts. So we make their definition required in the toolchain job definitions.
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/ci/toolchain/windows.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/job/toolchain.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -16,16 +16,17 @@ linux64-clang:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: toolchain-script
         script: build-clang-linux.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang.tar.xz
 
 linux64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: linux64-clang-tidy
     treeherder:
         kind: build
@@ -40,16 +41,17 @@ linux64-clang-tidy:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy-linux.sh
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang-tidy.tar.xz
 
 linux64-gcc:
     description: "GCC toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(gcc)
         tier: 1
@@ -57,16 +59,17 @@ linux64-gcc:
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-gcc-linux.sh
         resources:
             - 'build/unix/build-gcc/**'
+        toolchain-artifact: public/gcc.tar.xz
 
 linux64-binutils:
     description: "Binutils toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(binutil)
         tier: 1
@@ -74,16 +77,17 @@ linux64-binutils:
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-binutils-linux.sh
         resources:
             - 'build/unix/build-binutils/**'
+        toolchain-artifact: public/binutils.tar.xz
 
 linux64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(cctools)
         tier: 1
@@ -93,16 +97,17 @@ linux64-cctools-port:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
     run:
         using: toolchain-script
         script: build-cctools-port.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/cctools.tar.xz
 
 linux64-hfsplus:
     description: "hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(hfs+)
         tier: 1
@@ -113,23 +118,25 @@ linux64-hfsplus:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
     run:
         using: toolchain-script
         script: build-hfsplus-linux.sh
         resources:
             - 'build/unix/build-hfsplus/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/hfsplus-tools.tar.xz
 
 linux64-libdmg:
     description: "libdmg-hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(libdmg-hfs+)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-libdmg-hfsplus.sh
+        toolchain-artifact: public/dmg.tar.xz
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -17,16 +17,17 @@ macosx64-clang:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-clang-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang.tar.xz
 
 macosx64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: macosx64-clang-tidy
     treeherder:
         kind: build
@@ -42,16 +43,17 @@ macosx64-clang-tidy:
     run:
         using: toolchain-script
         script: build-clang-tidy-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang-tidy.tar.xz
 
 macosx64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(cctools)
         tier: 1
@@ -62,8 +64,9 @@ macosx64-cctools-port:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-cctools-port-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/cctools.tar.bz2
--- a/taskcluster/ci/toolchain/windows.yml
+++ b/taskcluster/ci/toolchain/windows.yml
@@ -15,16 +15,17 @@ win32-clang-cl:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang32-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+        toolchain-artifact: public/build/clang.tar.bz2
 
 win64-clang-cl:
     description: "Clang-cl toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TW64(clang-cl)
         tier: 2
@@ -34,16 +35,17 @@ win64-clang-cl:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang64-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+        toolchain-artifact: public/build/clang.tar.bz2
 
 win32-clang-tidy:
     description: "Clang-tidy toolchain build"
     index:
         product: static-analysis
         job-name: win32-clang-tidy
     treeherder:
         kind: build
@@ -56,16 +58,17 @@ win32-clang-tidy:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy32-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+        toolchain-artifact: public/build/clang-tidy.tar.bz2
 
 win64-clang-tidy:
     description: "Clang-tidy toolchain build"
     index:
         product: static-analysis
         job-name: win64-clang-tidy
     treeherder:
         kind: build
@@ -78,8 +81,9 @@ win64-clang-tidy:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy64-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+        toolchain-artifact: public/build/clang-tidy.tar.bz2
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -159,9 +159,13 @@ specific locale involved. Currently this
 
 signed
 ======
 Signals that the output of this task contains signed artifacts.
 
 repackage_type
 ==============
 This is the type of repackage. Can be ``repackage`` or 
-``repacakge_signing``.
+``repackage_signing``.
+
+toolchain-artifact
+==================
+For toolchain jobs, this is the path to the artifact for that toolchain.
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -34,16 +34,19 @@ toolchain_run_schema = Schema({
         False,
         'public',
         'internal',
     ),
 
     # Paths/patterns pointing to files that influence the outcome of a
     # toolchain build.
     Optional('resources'): [basestring],
+
+    # Path to the artifact produced by the toolchain job
+    Required('toolchain-artifact'): basestring,
 })
 
 
 def add_optimizations(config, run, taskdesc):
     files = list(run.get('resources', []))
     # This file
     files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
     # The script
@@ -125,16 +128,19 @@ def docker_worker_toolchain(config, job,
         '--',
         'bash',
         '-c',
         'cd /home/worker && '
         './workspace/build/src/taskcluster/scripts/misc/{}'.format(
             run['script'])
     ]
 
+    attributes = taskdesc.setdefault('attributes', {})
+    attributes['toolchain-artifact'] = run['toolchain-artifact']
+
     add_optimizations(config, run, taskdesc)
 
 
 @run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
 def windows_toolchain(config, job, taskdesc):
     run = job['run']
     taskdesc['run-on-projects'] = ['trunk', 'try']
 
@@ -176,9 +182,12 @@ def windows_toolchain(config, job, taskd
 
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
         ' '.join(hg_command),
         # do something intelligent.
         r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
     ]
 
+    attributes = taskdesc.setdefault('attributes', {})
+    attributes['toolchain-artifact'] = run['toolchain-artifact']
+
     add_optimizations(config, run, taskdesc)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -210,17 +210,16 @@ task_description_schema = Schema({
         # command in the docker image
         Optional('command'): [taskref_or_string],
 
         # the maximum time to run, in seconds
         Required('max-run-time'): int,
 
         # the exit status code that indicates the task should be retried
         Optional('retry-exit-status'): int,
-
     }, {
         Required('implementation'): 'generic-worker',
         Required('os'): Any('windows', 'macosx'),
         # see http://schemas.taskcluster.net/generic-worker/v1/payload.json
         # and https://docs.taskcluster.net/reference/workers/generic-worker/payload
 
         # command is a list of commands to run, sequentially
         # on Windows, each command is a string, on OS X and Linux, each command is