Bug 1510465: [taskgraph] Use `mach` job type more consistently; r=dustin
authorTom Prince <mozilla@hocat.ca>
Wed, 28 Nov 2018 18:35:19 +0000
changeset 507775 0146ddab71767245ac84916ff0e038567e0e57b1
parent 507774 7235cbf1093860707e4094dd2cb1fe4c3f229596
child 507776 95ae4f717250683c7f338ab8d8b7f751d1d438c5
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1510465
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1510465: [taskgraph] Use `mach` job type more consistently; r=dustin Differential Revision: https://phabricator.services.mozilla.com/D13158
taskcluster/ci/addon/kind.yml
taskcluster/ci/source-test/doc.yml
taskcluster/ci/source-test/file-metadata.yml
taskcluster/ci/source-test/jsshell.yml
taskcluster/ci/upload-generated-sources/kind.yml
taskcluster/ci/upload-symbols/kind.yml
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/python_test.py
taskcluster/taskgraph/transforms/source_test.py
--- a/taskcluster/ci/addon/kind.yml
+++ b/taskcluster/ci/addon/kind.yml
@@ -24,16 +24,14 @@ jobs:
         worker:
             docker-image: {in-tree: debian7-amd64-build}
             max-run-time: 1800
             artifacts:
                 - type: file
                   name: public/tps.xpi
                   path: /builds/worker/checkouts/gecko/tps-out/tps.xpi
         run:
-            using: run-task
-            command: >
-                cd /builds/worker/checkouts/gecko &&
-                ./mach tps-build --dest tps-out
+            using: mach
+            mach: tps-build --dest tps-out
             sparse-profile: tps
         when:
             files-changed:
                 - 'services/sync/tps/extensions/tps/**'
--- a/taskcluster/ci/source-test/doc.yml
+++ b/taskcluster/ci/source-test/doc.yml
@@ -10,20 +10,18 @@ generate:
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 1800
         artifacts:
             - type: file
               name: public/docs.tar.gz
               path: /builds/worker/checkouts/gecko/docs-out/main.tar.gz
     run:
-        using: run-task
-        command: >
-            cd /builds/worker/checkouts/gecko &&
-            ./mach doc --outdir docs-out --no-open --no-serve --archive
+        using: mach
+        mach: doc --outdir docs-out --no-open --no-serve --archive
         sparse-profile: sphinx-docs
     optimization:
         skip-unless-schedules: [docs]
 
 upload:
     description: Generate and upload the Sphinx documentation
     platform: lint/opt
     treeherder:
@@ -32,15 +30,15 @@ upload:
         tier: 3
     run-on-projects: [mozilla-central]
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 1800
         taskcluster-proxy: true
     run:
-        using: run-task
-        command: cd /builds/worker/checkouts/gecko && ./mach doc --upload --no-open --no-serve
+        using: mach
+        mach: doc --upload --no-open --no-serve
         sparse-profile: sphinx-docs
     scopes:
         - secrets:get:project/releng/gecko/build/level-{level}/gecko-docs-upload
     optimization:
         skip-unless-schedules: [docs]
--- a/taskcluster/ci/source-test/file-metadata.yml
+++ b/taskcluster/ci/source-test/file-metadata.yml
@@ -11,20 +11,18 @@ job-defaults:
 bugzilla-components:
     description: Generate metadata about source files and Bugzilla
     treeherder:
         symbol: Bugzilla
     index:
         product: source
         job-name: source-bugzilla-info
     run:
-        using: run-task
-        command: >
-            cd /builds/worker/checkouts/gecko
-            && ./mach file-info bugzilla-automation /builds/worker/artifacts
+        using: mach
+        mach: file-info bugzilla-automation /builds/worker/artifacts
     worker:
         artifacts:
             - type: directory
               path: /builds/worker/artifacts
               name: public
         max-run-time: 2700
     when:
         files-changed:
--- a/taskcluster/ci/source-test/jsshell.yml
+++ b/taskcluster/ci/source-test/jsshell.yml
@@ -15,21 +15,19 @@ job-defaults:
                         v8:
                             SHELL: /bin/bash
                             JSSHELL: /home/cltbld/fetches/d8/d8
                 max-run-time: 1800
     treeherder:
         kind: test
         tier: 2
     run:
-        using: run-task
+        using: mach
         workdir: /home/cltbld
-        command: >
-            cd $GECKO_PATH &&
-            ./mach jsshell-bench --perfherder={shell} --binary=$JSSHELL {test}
+        mach: jsshell-bench --perfherder={shell} --binary=$JSSHELL {test}
     run-on-projects:
         by-shell:
             sm: ['mozilla-central', 'try', 'integration']
             default: ['mozilla-central', 'try']
     fetches:
         build:
             - target.jsshell.zip
         fetch:
--- a/taskcluster/ci/upload-generated-sources/kind.yml
+++ b/taskcluster/ci/upload-generated-sources/kind.yml
@@ -22,20 +22,18 @@ job-template:
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     treeherder:
         symbol: Ugs
         kind: build
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 600
     run:
-        using: run-task
-        command: >
-            cd /builds/worker/checkouts/gecko &&
-            ./mach python build/upload_generated_sources.py ${ARTIFACT_URL}
+        using: mach
+        mach: python build/upload_generated_sources.py ${ARTIFACT_URL}
         sparse-profile: upload-generated-sources
     optimization:
         only-if-dependencies-run: null
     scopes:
         - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload
     run-on-projects:
         by-build-platform:
             .*devedition.*: ['mozilla-beta', 'maple']
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -23,20 +23,18 @@ job-template:
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 900
         env:
             ARTIFACT_TASKID: {"task-reference": "<build>"}
             # {level} gets replaced in the upload_symbols transform
             SYMBOL_SECRET: "project/releng/gecko/build/level-{level}/gecko-symbol-upload"
     run:
-        using: run-task
-        command: >
-            cd /builds/worker/checkouts/gecko &&
-            ./mach python toolkit/crashreporter/tools/upload_symbols.py https://queue.taskcluster.net/v1/task/${ARTIFACT_TASKID}/artifacts/public/build/target.crashreporter-symbols-full.zip
+        using: mach
+        mach: python toolkit/crashreporter/tools/upload_symbols.py https://queue.taskcluster.net/v1/task/${ARTIFACT_TASKID}/artifacts/public/build/target.crashreporter-symbols-full.zip
         sparse-profile: upload-symbols
     optimization:
         only-if-dependencies-run: null
     scopes:
         - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload
     run-on-projects:
         by-build-platform:
             .*devedition.*: ['mozilla-beta', 'maple']
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -27,16 +27,17 @@ mach_schema = Schema({
 
     # Base work directory used to set up the task.
     Required('workdir'): basestring,
 })
 
 
 @run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
 @run_job_using("native-engine", "mach", schema=mach_schema, defaults={'comm-checkout': False})
+@run_job_using("generic-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
 def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
     run['command'] = 'cd {workdir}/checkouts/gecko && ./mach {mach}'.format(**run)
     run['using'] = 'run-task'
     del run['mach']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/job/python_test.py
+++ b/taskcluster/taskgraph/transforms/job/python_test.py
@@ -28,15 +28,14 @@ python_test_schema = Schema({
 @run_job_using(
     'docker-worker',
     'python-test',
     schema=python_test_schema,
     defaults={'python-version': 2, 'subsuite': 'default'})
 def docker_worker_python_test(config, job, taskdesc):
     run = job['run']
 
-    # defer to the run_task implementation
-    run['command'] = 'cd {workdir}/checkouts/gecko && ' \
-        './mach python-test --python {python-version} --subsuite {subsuite}'.format(**run)
-    run['using'] = 'run-task'
+    # defer to the mach implementation
+    run['mach'] = 'python-test --python {python-version} --subsuite {subsuite}'.format(**run)
+    run['using'] = 'mach'
     del run['python-version']
     del run['subsuite']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -142,17 +142,17 @@ def split_jsshell(config, jobs):
             new_job['description'] = '{} on {}'.format(new_job['description'], all_shells[shell])
             new_job['shell'] = shell
 
             group = 'js-bench-{}'.format(shell)
             symbol = split_symbol(new_job['treeherder']['symbol'])[1]
             new_job['treeherder']['symbol'] = join_symbol(group, symbol)
 
             run = new_job['run']
-            run['command'] = run['command'].format(shell=shell, SHELL=shell.upper(), test=test)
+            run['mach'] = run['mach'].format(shell=shell, SHELL=shell.upper(), test=test)
             yield new_job
 
 
 def add_build_dependency(config, job):
     """
     Add build dependency to the job and installer_url to env.
     """
     key = job['platform']