Bug 1374940 - Allow to define a list of toolchains to use for a given TC job. r=dustin
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 21 Jun 2017 15:38:21 +0900
changeset 418757 ba64040f39ad2656ea5e10d4aced392dce5b75f1
parent 418756 2023ee951ea672f47cb01f5df84bf72a88fcfb1b
child 418758 9c7f4cb53632c82b8edc2271c9303dc5929f7109
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1374940
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1374940 - Allow to define a list of toolchains to use for a given TC job. r=dustin Such a definition automatically sets up the corresponding dependencies in the taskgraph, and adds the necessary artifact definitions for use in the corresponding jobs. The jobs end up with a MOZ_TOOLCHAINS environment variable with a list of path@task-id strings, where task-id is corresponding to the (possibly optimized) toolchain job, and path corresponding to the toolchain-artifact defined for that toolchain job.
taskcluster/ci/android-stuff/kind.yml
taskcluster/ci/artifact-build/kind.yml
taskcluster/ci/build/kind.yml
taskcluster/ci/hazard/kind.yml
taskcluster/ci/spidermonkey/kind.yml
taskcluster/ci/static-analysis/kind.yml
taskcluster/ci/toolchain/kind.yml
taskcluster/ci/valgrind/kind.yml
taskcluster/taskgraph/transforms/toolchain.py
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -3,17 +3,21 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # The name of this kind should suggest it's not meant to be permanent.  This is
 # a temporary place to generate these tasks in Bug 1286075 until they are
 # rewritten in a better way.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.try_job:transforms
    - taskgraph.transforms.android_stuff:transforms
    - taskgraph.transforms.task:transforms
 
 jobs:
     android-api-15-gradle-dependencies:
         description: "Android armv7 API 15+ gradle dependencies"
         index:
--- a/taskcluster/ci/artifact-build/kind.yml
+++ b/taskcluster/ci/artifact-build/kind.yml
@@ -1,16 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
    - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs:
     linux64-artifact/opt:
         description: "Linux64 Opt Artifact Build"
         index:
             product: firefox
--- a/taskcluster/ci/build/kind.yml
+++ b/taskcluster/ci/build/kind.yml
@@ -1,18 +1,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
    - taskgraph.transforms.build:transforms
    - taskgraph.transforms.build_attrs:transforms
    - taskgraph.transforms.build_lints:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
     - android.yml
     - linux.yml
     - macosx.yml
     - windows.yml
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -1,16 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
    - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
     treeherder:
         kind: build
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -1,16 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
    - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
     treeherder:
         platform: linux64/opt
         kind: build
         tier: 1
--- a/taskcluster/ci/static-analysis/kind.yml
+++ b/taskcluster/ci/static-analysis/kind.yml
@@ -1,16 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
    - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
     index:
         product: firefox
     treeherder:
         symbol: S
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.try_job:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
    - linux.yml
    - macosx.yml
    - windows.yml
--- a/taskcluster/ci/valgrind/kind.yml
+++ b/taskcluster/ci/valgrind/kind.yml
@@ -1,16 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
+kind-dependencies:
+   - toolchain
+
 transforms:
    - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs:
     linux64-valgrind/opt:
         description: "Linux64 Valgrind Opt"
         index:
             product: firefox
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/toolchain.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+from taskgraph.transforms.base import TransformSequence
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def use_toolchains(config, jobs):
+    """Add dependencies corresponding to toolchains to use, and pass a list
+    of corresponding artifacts to jobs using toolchains.
+    """
+    artifacts = {}
+    # Toolchain jobs can depend on other toolchain jobs, but we don't have full
+    # tasks for them, since they're being transformed. So scan the jobs list in
+    # that case, otherwise, use the list of tasks for the kind dependencies.
+    if config.kind == 'toolchain':
+        jobs = list(jobs)
+        for job in jobs:
+            artifact = job.get('run', {}).get('toolchain-artifact')
+            if artifact:
+                artifacts[job['name']] = artifact
+    else:
+        for task in config.kind_dependencies_tasks:
+            if task.kind != 'toolchain':
+                continue
+            artifact = task.attributes.get('toolchain-artifact')
+            if artifact:
+                artifacts[task.label.replace('%s-' % task.kind, '')] = artifact
+
+    for job in jobs:
+        env = job.setdefault('worker', {}).setdefault('env', {})
+
+        toolchains = job.pop('toolchains', [])
+
+        if config.kind == 'toolchain' and job['name'] in toolchains:
+            raise Exception("Toolchain job %s can't use itself as toolchain"
+                            % job['name'])
+
+        filenames = {}
+        for t in toolchains:
+            if t not in artifacts:
+                raise Exception('Missing toolchain job for %s-%s: %s'
+                                % (config.kind, job['name'], t))
+
+            f = os.path.basename(artifacts[t])
+            if f in filenames:
+                # Build jobs don't support toolchain artifacts with the same
+                # name: they would overwrite one with the other.
+                raise Exception('%s-%s cannot use both %s and %s toolchains: '
+                                'they both have the same artifact name %s'
+                                % (config.kind, job['name'], filenames[f],
+                                   t, f))
+            filenames[f] = t
+
+        if toolchains:
+            job.setdefault('dependencies', {}).update(
+                ('toolchain-%s' % t, 'toolchain-%s' % t)
+                for t in toolchains
+            )
+            # Pass a list of artifact-path@task-id to the job for all the
+            # toolchain artifacts it's going to need, where task-id is
+            # corresponding to the (possibly optimized) toolchain job, and
+            # artifact-path to the toolchain-artifact defined for that
+            # toolchain job.
+            env['MOZ_TOOLCHAINS'] = {'task-reference': ' '.join(
+                '%s@<toolchain-%s>' % (artifacts[t], t)
+                for t in toolchains
+            )}
+
+        yield job