Bug 1286075: add a toolchain kind; r=gps
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:04:31 +0000
changeset 313715 ae09f23c30124fcfb4cab32d355a96487280c0c1
parent 313714 882f3c9c1bc061e793a26da57970041f75193e9f
child 313716 a84691d694b7ebb9344ce5a784943976e7ae5cc7
push id20528
push userryanvm@gmail.com
push dateWed, 14 Sep 2016 00:48:16 +0000
treeherderfx-team@8a996d7c71a2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1286075
milestone51.0a1
Bug 1286075: add a toolchain kind; r=gps MozReview-Commit-ID: EEeQdA9aPPq
taskcluster/ci/toolchain/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/job/toolchain.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: toolchain build
+    treeherder:
+        kind: build
+        tier: 1
+        platform: linux64/opt
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+
+jobs:
+    linux64-clang/opt:
+        index:
+            product: firefox
+            job-name:
+                gecko-v2: linux64-clang-opt
+        treeherder:
+            symbol: Cc(Clang)
+        run:
+            using: toolchain-script
+            script: build-clang-linux.sh
+        when:
+            files-changed:
+                - 'build/build-clang/**'
+                - 'taskcluster/scripts/misc/build-clang-linux.sh'
+
+    linux64-gcc/opt:
+        index:
+            product: firefox
+            job-name:
+                gecko-v2: linux64-gcc-opt
+        treeherder:
+            symbol: Cc(GCC)
+        run:
+            using: toolchain-script
+            script: build-gcc-linux.sh
+        when:
+            files-changed:
+                - 'build/unix/build-gcc/**'
+                - 'taskcluster/scripts/misc/build-gcc-linux.sh'
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -52,16 +52,23 @@ Valgrind tasks produce builds instrument
 static-analysis
 ---------------
 
 Static analysis builds use the compiler to perform some detailed analysis of
 the source code while building.  The useful output from these tasks are their
 build logs, and while they produce a binary, they do not upload it as an
 artifact.
 
+toolchain
+---------
+
+Toolchain builds create the compiler toolchains used to build Firefox.  These
+will eventually be dependencies of the builds themselves, but for the moment
+are run manually via try pushes and the results uploaded to tooltool.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running toolchain-building jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_gecko_vcs_env_vars
+)
+
+toolchain_run_schema = Schema({
+    Required('using'): 'toolchain-script',
+
+    # the script (in taskcluster/scripts/misc) to run
+    Required('script'): basestring,
+})
+
+
+@run_job_using("docker-worker", "toolchain-script", schema=toolchain_run_schema)
+def docker_worker_toolchain(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    worker['artifacts'].append({
+        'name': 'public',
+        'path': '/home/worker/workspace/artifacts/',
+        'type': 'directory',
+    })
+
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'TOOLS_DISABLE': 'true',
+    })
+
+    # tooltool downloads; note that this downloads using the API endpoint directly,
+    # rather than via relengapi-proxy
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+
+    command = ' && '.join([
+        "cd /home/worker/",
+        "./bin/checkout-sources.sh",
+        "./workspace/build/src/taskcluster/scripts/misc/" + run['script'],
+    ])
+    worker['command'] = ["/bin/bash", "-c", command]
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -30,16 +30,17 @@ BUILD_KINDS = set([
     'upload-symbols',
     'valgrind',
     'static-analysis',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-check',
+    'toolchain',
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)