Bug 1413976: Support defining toolchain and docker-image tasks by reference to mozilla-centrals definitions; r=dustin
authorTom Prince <mozilla@hocat.ca>
Fri, 17 Nov 2017 16:35:34 -0700
changeset 29477 502d6658a41df319188af3ff9c2348328f23ab4b
parent 29476 306fff0f8a9d1fe8b7eeafd86c68166d78288767
child 29478 a76a4d378bf9b4e212e38c964d79ecd18e23e9a2
push id2108
push userclokep@gmail.com
push dateMon, 22 Jan 2018 17:53:55 +0000
treeherdercomm-beta@c44930d8ad9b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1413976
Bug 1413976: Support defining toolchain and docker-image tasks by reference to mozilla-centrals definitions; r=dustin MozReview-Commit-ID: 9Lqq3UMbLxe
build/virtualenv_packages.txt
taskcluster/ci/config.yml
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/source-test/mozlint.yml
taskcluster/ci/toolchain/kind.yml
taskcluster/comm_taskgraph/__init__.py
new file mode 100644
--- /dev/null
+++ b/build/virtualenv_packages.txt
@@ -0,0 +1,1 @@
+comm.pth:comm/taskcluster
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -1,8 +1,9 @@
+trust-domain: comm
 treeherder:
     group-names:
         'tc': 'Executed by TaskCluster'
         'tc-X': 'Xpcshell tests executed by TaskCluster'
         'tc-Z': 'MozMill tests executed by TaskCluster'
         'I': 'Docker Image Builds'
         'TL': 'Toolchain builds for Linux 64-bits'
         'TM': 'Toolchain builds for OSX'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: comm_taskgraph:reference_loader
+base-path: taskcluster/ci
+
+jobs:
+  - desktop1604-test
+  - desktop-build
+  - lint
--- a/taskcluster/ci/source-test/mozlint.yml
+++ b/taskcluster/ci/source-test/mozlint.yml
@@ -3,22 +3,18 @@ eslint:
     platform: lint/opt
     treeherder:
         symbol: ES
         kind: test
         tier: 1
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
         docker-image:
-          indexed: "docker.images.v2.level-3.lint.latest"
+          in-tree: "lint"
         max-run-time: 1800
-        # FIXME: https://bugzilla.mozilla.org/show_bug.cgi?id=1408574
-        volumes:
-        - /builds/worker/checkouts
-        - /builds/worker/.cache
     run:
         using: run-task
         comm-checkout: true
         command: >
             cd /builds/worker/checkouts/gecko/ &&
             cp -r /build/node_modules_eslint node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-mozilla node_modules &&
             ln -s ../tools/lint/eslint/eslint-plugin-spidermonkey-js node_modules &&
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: comm_taskgraph:reference_loader
+base-path: taskcluster/ci
+
+jobs:
+    - linux64-clang-3.9
+    - linux64-gcc
+    - linux64-rust
+    - linux64-sccache
+    - linux64-gcc-4.9
+    - linux64-rust-1.19
+    - win32-clang-cl
+    - win32-rust
+    - win64-sccache
+    - win64-rust-1.19
+    - win33-clang-cl
new file mode 100644
--- /dev/null
+++ b/taskcluster/comm_taskgraph/__init__.py
@@ -0,0 +1,44 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+from taskgraph.util.yaml import load_yaml
+from taskgraph.util.python_path import find_object
+
+
+def _get_aliases(kind, job):
+    aliases = {job['name']}
+
+    if kind == 'toolchain':
+        if job['run'].get('toolchain-alias'):
+            aliases.add(job['run'].get('toolchain-alias'))
+
+    return aliases
+
+
+def _get_loader(path, config):
+    try:
+        loader = config['loader']
+    except KeyError:
+        raise KeyError("{!r} does not define `loader`".format(path))
+    return find_object(loader)
+
+
+def reference_loader(kind, path, config, params, loaded_tasks):
+    """
+    Loads selected jobs from a different taskgraph hierarchy.
+
+    This loads jobs of the given kind from the taskgraph rooted at `base-path`,
+    and includes all the jobs with names or aliaes matching the names in the
+    `jobs` key.
+    """
+    base_path = config.pop('base-path')
+    sub_path = os.path.join(base_path, kind)
+    sub_config = load_yaml(sub_path, "kind.yml")
+    loader = _get_loader(sub_path, sub_config)
+    inputs = loader(kind, sub_path, sub_config, params, loaded_tasks)
+
+    jobs = set(config.pop('jobs'))
+
+    config.update(sub_config)
+
+    return (job for job in inputs if (_get_aliases(kind, job) & jobs))