Bug 1286075: add a build kind, modify tests to use it; r=jlund
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:34:06 +0000
changeset 354989 a1911eb6ed8b024fdad878de7bef5568c95e11c0
parent 354988 40e59df093a7332f90a34de57a1d08766fc9a93c
child 354990 40d5bfd3a633dfba55c9764db7bc934304da3f8b
push id6570
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:26:13 +0000
treeherdermozilla-beta@f455459b2ae5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund
bugs1286075
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1286075: add a build kind, modify tests to use it; r=jlund MozReview-Commit-ID: DkpkkSRxVB9
taskcluster/ci/android-test/kind.yml
taskcluster/ci/build/android-partner.yml
taskcluster/ci/build/android.yml
taskcluster/ci/build/kind.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/build/macosx.yml
taskcluster/ci/build/mulet.yml
taskcluster/ci/build/windows.yml
taskcluster/ci/desktop-test/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/test.py
taskcluster/taskgraph/transforms/build.py
taskcluster/taskgraph/transforms/build_attrs.py
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/mulet.py
taskcluster/taskgraph/try_option_syntax.py
--- a/taskcluster/ci/android-test/kind.yml
+++ b/taskcluster/ci/android-test/kind.yml
@@ -1,12 +1,12 @@
 implementation: taskgraph.task.test:TestTask
 
 kind-dependencies:
-    - legacy
+    - build
 
 transforms:
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.android_test:transforms
    - taskgraph.transforms.tests.all_kinds:transforms
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.make_task_description:transforms
    - taskgraph.transforms.task:transforms
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/android-partner.yml
@@ -0,0 +1,32 @@
+android-partner-sample1/opt:
+    description: "Android armv7 API 15+ partner sample 1"
+    index:
+        product: mobile
+        job-name: android-api-15-partner-sample1-opt
+    treeherder:
+        platform: android-4-0-armv7-api15-partner1/opt
+        symbol: tc(B)
+        tier: 2
+    run-on-projects:
+        - try
+    worker-type: aws-provisioner-v1/android-api-15
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+        env:
+            EXTRA_CHECKOUT_REPOSITORIES: "PARTNER\n"
+            PARTNER_BASE_REPOSITORY: "https://github.com/mozilla/fennec-distribution-sample"
+            PARTNER_DEST_DIR: "/home/worker/workspace/build/partner"
+            PARTNER_HEAD_REPOSITORY: "https://github.com/mozilla/fennec-distribution-sample"
+            PARTNER_HEAD_REV: "master"
+    run:
+        using: mozharness
+        actions: [get-secrets build multi-l10n update]
+        config:
+            - builds/releng_base_android_64_builds.py
+            - disable_signing.py
+            - platform_supports_post_upload_to_latest.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: api-15-partner-sample1
+        tooltool-downloads: internal
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/android.yml
@@ -0,0 +1,102 @@
+android-api-15/debug:
+    description: "Android 4.0 API15+ Debug"
+    index:
+        product: mobile
+        job-name:
+            buildbot: android-api-15-debug
+            gecko-v1: android-api-15.debug
+            gecko-v2: android-api-15-debug
+    treeherder:
+        platform: android-4-0-armv7-api15/debug
+        symbol: tc(B)
+    worker-type: aws-provisioner-v1/android-api-15
+    worker:
+        implementation: docker-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        actions: [get-secrets build multi-l10n update]
+        config:
+            - builds/releng_base_android_64_builds.py
+            - disable_signing.py
+            - platform_supports_post_upload_to_latest.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: api-15-debug
+        tooltool-downloads: internal
+
+android-x86/opt:
+    description: "Android 4.2 x86 Opt"
+    index:
+        product: mobile
+        job-name: android-x86-opt
+    treeherder:
+        platform: android-4-2-x86/opt
+        symbol: tc(B)
+        tier: 1
+    worker-type: aws-provisioner-v1/android-api-15
+    worker:
+        implementation: docker-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        actions: [get-secrets build multi-l10n update]
+        config:
+            - builds/releng_base_android_64_builds.py
+            - disable_signing.py
+            - platform_supports_post_upload_to_latest.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: x86
+        tooltool-downloads: internal
+
+android-api-15/opt:
+    description: "Android 4.0 API15+ Opt"
+    index:
+        product: mobile
+        job-name: android-api-15-opt
+    treeherder:
+        platform: android-4-0-armv7-api15/opt
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/android-api-15
+    worker:
+        implementation: docker-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        actions: [get-secrets build multi-l10n update]
+        config:
+            - builds/releng_base_android_64_builds.py
+            - disable_signing.py
+            - platform_supports_post_upload_to_latest.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: api-15
+        tooltool-downloads: internal
+
+android-api-15-gradle/opt:
+    description: "Android 4.0 API15+ (Gradle) Opt"
+    index:
+        product: mobile
+        job-name: android-api-15-gradle-opt
+    treeherder:
+        platform: android-4-0-armv7-api15/opt
+        symbol: tc(Bg)
+        tier: 2
+    worker-type: aws-provisioner-v1/android-api-15
+    worker:
+        implementation: docker-worker
+        max-run-time: 7200
+        env:
+            # Bug 1292762 - Set GRADLE_USER_HOME to avoid sdk-manager-plugin intermittent
+            GRADLE_USER_HOME: /home/worker/workspace/build/src/dotgradle
+    run:
+        using: mozharness
+        actions: [get-secrets build multi-l10n update]
+        config:
+            - builds/releng_base_android_64_builds.py
+            - disable_signing.py
+            - platform_supports_post_upload_to_latest.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: api-15-gradle
+        tooltool-downloads: internal
+
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/kind.yml
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build:transforms
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs-from:
+    - android-partner.yml
+    - android.yml
+    - linux.yml
+    - macosx.yml
+    - mulet.yml
+    - windows.yml
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/linux.yml
@@ -0,0 +1,176 @@
+linux64/opt:
+    description: "Linux64 Opt"
+    index:
+        product: firefox
+        job-name: linux64-opt
+    treeherder:
+        platform: linux64/opt
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux64/pgo:
+    description: "Linux64 Opt PGO"
+    index:
+        product: firefox
+        job-name: linux64-pgo-opt
+    treeherder:
+        platform: linux64/pgo
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    coalesce-name: linux64-pgo
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        options: [enable-pgo]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux64/debug:
+    description: "Linux64 Debug"
+    index:
+        product: firefox
+        job-name:
+            buildbot: linux64-debug
+            gecko-v1: linux64.debug
+            gecko-v2: linux64-debug
+    treeherder:
+        platform: linux64/debug
+        symbol: tc(B)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: debug
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux/opt:
+    description: "Linux32 Opt"
+    index:
+        product: firefox
+        job-name: linux-opt
+    treeherder:
+        platform: linux32/opt
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    coalesce-name: opt_linux32
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_32_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux/debug:
+    description: "Linux32 Debug"
+    index:
+        product: firefox
+        job-name: linux-debug
+    treeherder:
+        platform: linux32/debug
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    coalesce-name: dbg_linux32
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_32_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: debug
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux64-asan/opt:
+    description: "Linux64 Opt ASAN"
+    index:
+        product: firefox
+        job-name: linux64-asan-opt
+    treeherder:
+        platform: linux64/asan
+        symbol: tc(Bo)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: asan-tc
+        tooltool-downloads: public
+        need-xvfb: true
+
+linux64-asan/debug:
+    description: "Linux64 Debug ASAN"
+    index:
+        product: firefox
+        job-name: linux64-asan-debug
+    treeherder:
+        platform: linux64/asan
+        symbol: tc(Bd)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test generate-build-stats update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: asan-tc-and-debug
+        tooltool-downloads: public
+        need-xvfb: true
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/macosx.yml
@@ -0,0 +1,46 @@
+macosx64/debug:
+    description: "MacOS X x64 Cross-compile"
+    index:
+        product: firefox
+        job-name: macosx64-debug
+    treeherder:
+        platform: osx-10-7/debug
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/dbg-macosx64
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build generate-build-stats update]
+        config:
+            - builds/releng_base_mac_64_cross_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: cross-debug
+        tooltool-downloads: internal
+
+macosx64/opt:
+    description: "MacOS X x64 Cross-compile"
+    index:
+        product: firefox
+        job-name: macosx64-opt
+    treeherder:
+        platform: osx-10-7/opt
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/opt-macosx64
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build generate-build-stats update]
+        config:
+            - builds/releng_base_mac_64_cross_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        tooltool-downloads: internal
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/mulet.yml
@@ -0,0 +1,43 @@
+linux64-mulet/debug:
+    description: "Linux64 Mulet Debug"
+    index:
+        product: b2g
+        job-name:
+            buildbot: linux64-mulet
+            gecko-v1: mulet.dbg
+            gecko-v2: mulet-dbg
+    treeherder:
+        platform: mulet-linux64/debug  # See bug 1286086
+        symbol: B
+        tier: 3
+    worker-type: aws-provisioner-v1/mulet-debug
+    worker:
+        implementation: docker-worker
+        max-run-time: 3600
+    run:
+        using: mach-via-build-mulet-linux.sh
+        mozconfig: b2g/dev/config/mozconfigs/linux64/mulet_dbg
+        tooltool-manifest: b2g/dev/config/tooltool-manifests/linux64/releng.manifest
+
+linux64-mulet/opt:
+    description: "Linux64 Mulet Opt"
+    index:
+        product: b2g
+        job-name:
+            buildbot: linux64-mulet
+            gecko-v1: mulet.opt
+            gecko-v2: mulet-opt
+    treeherder:
+        platform: mulet-linux64/opt # ?!?%
+        symbol: B
+        tier: 3
+    worker-type: aws-provisioner-v1/mulet-opt
+    worker:
+        implementation: docker-worker
+        max-run-time: 3600
+    run:
+        using: mach-via-build-mulet-linux.sh
+        mozconfig: b2g/dev/config/mozconfigs/linux64/mulet
+        tooltool-manifest: b2g/dev/config/tooltool-manifests/linux64/releng.manifest
+
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build/windows.yml
@@ -0,0 +1,80 @@
+win32/debug:
+    description: "Win32 Debug"
+    index:
+        product: firefox
+        job-name:
+            gecko-v2: win32-debug
+    treeherder:
+        platform: windowsxp/debug
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        implementation: generic-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_win32_debug.py
+
+win32/opt:
+    description: "Win32 Opt"
+    index:
+        product: firefox
+        job-name:
+            gecko-v2: win32-opt
+    treeherder:
+        platform: windowsxp/opt
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        implementation: generic-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_win32_opt.py
+
+win64/debug:
+    description: "Win64 Debug"
+    index:
+        product: firefox
+        job-name:
+            gecko-v2: win64-debug
+    treeherder:
+        platform: windows8-64/debug
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        implementation: generic-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_win64_debug.py
+
+win64/opt:
+    description: "Win64 Opt"
+    index:
+        product: firefox
+        job-name:
+            gecko-v2: win64-opt
+    treeherder:
+        platform: windows8-64/opt
+        symbol: tc(B)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        implementation: generic-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_win64_opt.py
+
--- a/taskcluster/ci/desktop-test/kind.yml
+++ b/taskcluster/ci/desktop-test/kind.yml
@@ -1,12 +1,12 @@
 implementation: taskgraph.task.test:TestTask
 
 kind-dependencies:
-    - legacy
+    - build
 
 transforms:
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.desktop_test:transforms
    - taskgraph.transforms.tests.all_kinds:transforms
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.make_task_description:transforms
    - taskgraph.transforms.task:transforms
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -1,17 +1,20 @@
 Task Kinds
 ==========
 
 This section lists and documents the available task kinds.
 
-Builds
+build
 ------
 
-Builds are currently implemented by the ``legacy`` kind.
+Builds are tasks that produce an installer or other output that can be run by
+users or automated tests.  This is more restrictive than most definitions of
+"build" in a Mozilla context: it does not include tasks that run build-like
+actions for static analysis or to produce instrumented artifacts.
 
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -57,19 +57,16 @@ class TestTask(transform.TransformTask):
     def get_builds_by_platform(cls, dep_kind, loaded_tasks):
         """Find the build tasks on which tests will depend, keyed by
         platform/type.  Returns a dictionary mapping build platform to task
         label."""
         builds_by_platform = {}
         for task in loaded_tasks:
             if task.kind != dep_kind:
                 continue
-            # remove this check when builds are no longer legacy
-            if task.attributes['legacy_kind'] != 'build':
-                continue
 
             build_platform = task.attributes.get('build_platform')
             build_type = task.attributes.get('build_type')
             if not build_platform or not build_type:
                 continue
             platform = "{}/{}".format(build_platform, build_type)
             if platform in builds_by_platform:
                 raise Exception("multiple build jobs for " + platform)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the build
+kind.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_defaults(config, jobs):
+    """Set defaults, including those that differ per worker implementation"""
+    for job in jobs:
+        job['treeherder'].setdefault('kind', 'build')
+        job['treeherder'].setdefault('tier', 1)
+        if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
+            job['worker'].setdefault('docker-image', {'in-tree': 'desktop-build'})
+        yield job
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/build_attrs.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_build_attributes(config, jobs):
+    """
+    Set the build_platform and build_type attributes based on the job name.
+    Although not all jobs using this transform are actual "builds", the try
+    option syntax treats them as such, and this arranges the attributes
+    appropriately for that purpose.
+    """
+    for job in jobs:
+        build_platform, build_type = job['name'].split('/')
+
+        # pgo builds are represented as a different platform, type opt
+        if build_type == 'pgo':
+            build_platform = build_platform + '-pgo'
+            build_type = 'opt'
+
+        attributes = job.setdefault('attributes', {})
+        attributes.update({
+            'build_platform': build_platform,
+            'build_type': build_type,
+        })
+
+        yield job
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common support for various job types.  These functions are all named after the
+worker implementation they operate on, and take the same three parameters, for
+consistency.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
+
+
+def docker_worker_add_workspace_cache(config, job, taskdesc):
+    """Add the workspace cache based on the build platform/type and level,
+    except on try where workspace caches are not used."""
+    if config.params['project'] == 'try':
+        return
+
+    taskdesc['worker'].setdefault('caches', []).append({
+        'type': 'persistent',
+        'name': 'level-{}-{}-build-{}-{}-workspace'.format(
+            config.params['level'], config.params['project'],
+            taskdesc['attributes']['build_platform'],
+            taskdesc['attributes']['build_type'],
+        ),
+        'mount-point': "/home/worker/workspace",
+    })
+
+
+def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
+    taskdesc['worker'].setdefault('caches', []).append({
+        'type': 'persistent',
+        'name': 'level-{}-{}-tc-vcs'.format(
+            config.params['level'], config.params['project']),
+        'mount-point': "/home/worker/.tc-vcs",
+    })
+
+
+def docker_worker_add_public_artifacts(config, job, taskdesc):
+    taskdesc['worker'].setdefault('artifacts', []).append({
+        'name': 'public/build',
+        'path': '/home/worker/artifacts/',
+        'type': 'directory',
+    })
+
+
+def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
+    """Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker."""
+    env = taskdesc['worker'].setdefault('env', {})
+    env.update({
+        'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+        'GECKO_HEAD_REF': config.params['head_rev'],
+        'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+        'GECKO_HEAD_REV': config.params['head_rev'],
+    })
+
+
+def docker_worker_setup_secrets(config, job, taskdesc):
+    """Set up access to secrets via taskcluster-proxy.  The value of
+    run['secrets'] should be a boolean or a list of secret names that
+    can be accessed."""
+    if not job['run'].get('secrets'):
+        return
+
+    taskdesc['worker']['taskcluster-proxy'] = True
+    secrets = job['run']['secrets']
+    if secrets is True:
+        secrets = ['*']
+    for sec in secrets:
+        taskdesc['scopes'].append(SECRET_SCOPE.format(
+            job['treeherder']['kind'], config.params['level'], sec))
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -0,0 +1,195 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+Support for running jobs via mozharness.  Ideally, most stuff gets run this
+way, and certainly anything using mozharness should use this approach.
+
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_workspace_cache,
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_setup_secrets,
+    docker_worker_add_public_artifacts
+)
+
+COALESCE_KEY = 'builds.{project}.{name}'
+
+mozharness_run_schema = Schema({
+    Required('using'): 'mozharness',
+
+    # the mozharness script used to run this task, relative to the testing/
+    # directory and using forward slashes even on Windows
+    Required('script'): basestring,
+
+    # the config files required for the task, relative to
+    # testing/mozharness/configs and using forward slashes even on Windows
+    Required('config'): [basestring],
+
+    # any additional actions to pass to the mozharness command; not supported
+    # on Windows
+    Optional('actions'): [basestring],
+
+    # any additional options (without leading --) to be passed to mozharness;
+    # not supported on Windows
+    Optional('options'): [basestring],
+
+    # --custom-build-variant-cfg value (not supported on Windows)
+    Optional('custom-build-variant-cfg'): basestring,
+
+    # If not false, tooltool downloads will be enabled via relengAPIProxy
+    # for either just public files, or all files.  Not supported on Windows
+    Required('tooltool-downloads', default=False): Any(
+        False,
+        'public',
+        'internal',
+    ),
+
+    # The set of secret names to which the task has access; these are prefixed
+    # with `project/releng/gecko/{treeherder.kind}/level-{level}/`.  Setting
+    # this will enable any worker features required and set the task's scopes
+    # appropriately.  `true` here means ['*'], all secrets.  Not supported on
+    # Windows
+    Required('secrets', default=False): Any(bool, [basestring]),
+
+    # If true, taskcluster proxy will be enabled; note that it may also be enabled
+    # automatically e.g., for secrets support.  Not supported on Windows.
+    Required('taskcluster-proxy', default=False): bool,
+
+    # If true, the build scripts will start Xvfb.  Not supported on Windows.
+    Required('need-xvfb', default=False): bool,
+
+    # If false, indicate that builds should skip producing artifacts.  Not
+    # supported on Windows.
+    Required('keep-artifacts', default=True): bool,
+})
+
+
+@run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema)
+def mozharness_on_docker_worker_setup(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['implementation'] = job['worker']['implementation']
+
+    # running via mozharness assumes desktop-build (which contains build.sh)
+    taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
+
+    worker['relengapi-proxy'] = False  # but maybe enabled for tooltool below
+    worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
+
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_workspace_cache(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker.setdefault('env', {})
+    env.update({
+        'MOZHARNESS_CONFIG': ' '.join(run['config']),
+        'MOZHARNESS_SCRIPT': run['script'],
+        'MH_BRANCH': config.params['project'],
+        'MH_BUILD_POOL': 'taskcluster',
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    if 'actions' in run:
+        env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
+
+    if 'options' in run:
+        env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
+
+    if 'custom-build-variant-cfg' in run:
+        env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
+
+    # if we're not keeping artifacts, set some env variables to empty values
+    # that will cause the build process to skip copying the results to the
+    # artifacts directory.  This will have no effect for operations that are
+    # not builds.
+    if not run['keep-artifacts']:
+        env['DIST_TARGET_UPLOADS'] = ''
+        env['DIST_UPLOADS'] = ''
+
+    # Xvfb
+    if run['need-xvfb']:
+        env['NEED_XVFB'] = 'true'
+
+    # tooltool downloads
+    if run['tooltool-downloads']:
+        worker['relengapi-proxy'] = True
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'tooltool-cache',
+            'mount-point': '/home/worker/tooltool-cache',
+        })
+        taskdesc['scopes'].extend([
+            'docker-worker:relengapi-proxy:tooltool.download.public',
+        ])
+        if run['tooltool-downloads'] == 'internal':
+            taskdesc['scopes'].append(
+                'docker-worker:relengapi-proxy:tooltool.download.internal')
+        env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+        env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+        env['TOOLTOOL_REV'] = 'master'
+
+    docker_worker_setup_secrets(config, job, taskdesc)
+
+    worker['command'] = ["/bin/bash", "bin/build.sh"]
+
+
+# We use the generic worker to run tasks on Windows
+@run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema)
+def mozharness_on_windows(config, job, taskdesc):
+    run = job['run']
+
+    # fail if invalid run options are included
+    invalid = []
+    for prop in ['actions', 'options', 'custom-build-variant-cfg',
+                 'tooltool-downloads', 'secrets', 'taskcluster-proxy',
+                 'need-xvfb']:
+        if prop in run and run[prop]:
+            invalid.append(prop)
+    if not run.get('keep-artifacts', True):
+        invalid.append('keep-artifacts')
+    if invalid:
+        raise Exception("Jobs run using mozharness on Windows do not support properties " +
+                        ', '.join(invalid))
+
+    worker = taskdesc['worker']
+
+    worker['artifacts'] = [{
+        'path': r'public\build',
+        'type': 'directory',
+    }]
+
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool',
+        'TOOLTOOL_REV': 'master',
+    })
+
+    mh_command = [r'c:\mozilla-build\python\python.exe']
+    mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
+    for cfg in run['config']:
+        mh_command.append('--config ' + cfg.replace('/', '\\'))
+    mh_command.append('--branch ' + config.params['project'])
+    mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build')
+    worker['command'] = [
+        r'mkdir .\build\src',
+        r'hg share c:\builds\hg-shared\mozilla-central .\build\src',
+        r'hg pull -u -R .\build\src --rev %GECKO_HEAD_REV% %GECKO_HEAD_REPOSITORY%',
+        ' '.join(mh_command),
+    ]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mulet.py
@@ -0,0 +1,119 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running mulet tasks via build-mulet-linux.sh
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_workspace_cache,
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_add_public_artifacts
+)
+
+COALESCE_KEY = 'builds.{project}.{name}'
+
+build_mulet_linux_schema = Schema({
+    Required('using'): 'mach-via-build-mulet-linux.sh',
+
+    # The pathname of the mozconfig to use
+    Required('mozconfig'): basestring,
+
+    # The tooltool manifest to use
+    Required('tooltool-manifest'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mach-via-build-mulet-linux.sh", schema=build_mulet_linux_schema)
+def docker_worker_make_via_build_mulet_linux_sh(config, job, taskdesc):
+    run = job['run']
+    worker = taskdesc.get('worker')
+
+    # assumes the builder image (which contains the gecko checkout command)
+    taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
+
+    worker['taskcluster-proxy'] = False
+
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_workspace_cache(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker.setdefault('env', {})
+    env.update({
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    env['MOZCONFIG'] = run['mozconfig']
+    env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+
+    # tooltool downloads
+    worker['relengapi-proxy'] = True
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        # N.B. different from build.sh
+        # TODO(taskdiff): grepping suggests this isn't used..
+        'mount-point': '/home/worker/tools/tooltool-cache',
+    })
+    taskdesc['scopes'].extend([
+        'docker-worker:relengapi-proxy:tooltool.download.public',
+    ])
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+
+    worker['command'] = [
+        "/bin/bash",
+        "-c",
+        "checkout-gecko workspace"
+        " && cd ./workspace/gecko/taskcluster/scripts/builder"
+        " && buildbot_step 'Build' ./build-mulet-linux.sh $HOME/workspace",
+    ]
+
+mulet_simulator_schema = Schema({
+    Required('using'): 'mulet-simulator',
+
+    # The shell command to run with `bash -exc`.  This will have parameters
+    # substituted for {..} and will be enclosed in a {task-reference: ..} block
+    # so it can refer to the parent task as <build>
+    Required('shell-command'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mulet-simulator", schema=mulet_simulator_schema)
+def docker_worker_mulet_simulator(config, job, taskdesc):
+    run = job['run']
+    worker = taskdesc.get('worker')
+
+    # assumes the builder image (which contains the gecko checkout command)
+    taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
+
+    worker['taskcluster-proxy'] = False
+
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    taskdesc.setdefault('routes', []).extend([
+        'index.gecko.v1.{project}.latest.simulator.opt'.format(**config.params),
+    ])
+
+    # TODO(taskdiff): has the scope for this cache, but not the cache
+    taskdesc.setdefault('scopes', []).extend([
+        'docker-worker:cache:level-{level}-{project}-tc-vcs'.format(**config.params),
+    ])
+
+    shell_command = run['shell-command'].format(**config.params)
+
+    worker['command'] = [
+        "/bin/bash",
+        "-exc",
+        {'task-reference': shell_command},
+    ]
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -18,16 +18,17 @@ TRY_DELIMITER = 'try:'
 # mappings from the single char alias to a longer more recognizable form.
 BUILD_TYPE_ALIASES = {
     'o': 'opt',
     'd': 'debug'
 }
 
 # consider anything in this whitelist of kinds to be governed by -b/-p
 BUILD_KINDS = set([
+    'build',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying