Bug 1510465: [taskgraph] Use mach job type more consistently in release tasks; r=Callek
authorTom Prince <mozilla@hocat.ca>
Wed, 28 Nov 2018 19:19:42 +0000
changeset 504996 95ae4f717250683c7f338ab8d8b7f751d1d438c5
parent 504995 0146ddab71767245ac84916ff0e038567e0e57b1
child 504997 4e1a901cc0a5b0ca50f00cc929a967e0e6c0063e
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1510465
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1510465: [taskgraph] Use mach job type more consistently in release tasks; r=Callek Differential Revision: https://phabricator.services.mozilla.com/D13159
taskcluster/ci/cron-bouncer-check/kind.yml
taskcluster/ci/release-bouncer-check/kind.yml
taskcluster/ci/release-secondary-update-verify-config/kind.yml
taskcluster/ci/release-update-verify-config/kind.yml
taskcluster/taskgraph/transforms/bouncer_check.py
taskcluster/taskgraph/transforms/update_verify_config.py
--- a/taskcluster/ci/cron-bouncer-check/kind.yml
+++ b/taskcluster/ci/cron-bouncer-check/kind.yml
@@ -12,17 +12,16 @@ transforms:
 job-defaults:
     name: bouncer-check
     description: bouncer check
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 1200
         docker-image: {in-tree: "update-verify"}
     run:
-        using: run-task
         sparse-profile: mozharness
     attributes:
         build_platform: linux64
         build_type: opt
         cron: true
     treeherder:
         symbol: Rel(ckbouncer)
         kind: test
--- a/taskcluster/ci/release-bouncer-check/kind.yml
+++ b/taskcluster/ci/release-bouncer-check/kind.yml
@@ -18,17 +18,16 @@ job-defaults:
     description: release bouncer check
     run-on-projects: []  # to make sure this never runs as part of CI
     shipping-phase: push
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 1200
         docker-image: {in-tree: "update-verify"}
     run:
-        using: run-task
         sparse-profile: mozharness
     attributes:
         build_platform: linux64
         build_type: opt
     treeherder:
         symbol: Rel(BncChk)
         kind: test
         tier: 1
--- a/taskcluster/ci/release-secondary-update-verify-config/kind.yml
+++ b/taskcluster/ci/release-secondary-update-verify-config/kind.yml
@@ -19,17 +19,16 @@ job-defaults:
       docker-image:
          in-tree: "update-verify"
       max-run-time: 3600
       artifacts:
          - name: public/build/update-verify.cfg
            path: /builds/worker/checkouts/gecko/update-verify.cfg
            type: file
    run:
-      using: run-task
       sparse-profile: mozharness
    extra:
       app-name: browser
       branch-prefix: mozilla
       product: firefox
       archive-prefix:
          by-release-level:
             staging: "http://ftp.stage.mozaws.net/pub"
--- a/taskcluster/ci/release-update-verify-config/kind.yml
+++ b/taskcluster/ci/release-update-verify-config/kind.yml
@@ -18,17 +18,16 @@ job-defaults:
       docker-image:
          in-tree: "update-verify"
       max-run-time: 3600
       artifacts:
          - name: public/build/update-verify.cfg
            path: /builds/worker/checkouts/gecko/update-verify.cfg
            type: file
    run:
-      using: run-task
       sparse-profile: mozharness
    extra:
       app-name: browser
       branch-prefix: mozilla
       archive-prefix:
          by-release-level:
             staging: "http://ftp.stage.mozaws.net/pub"
             production: "https://archive.mozilla.org/pub"
--- a/taskcluster/taskgraph/transforms/bouncer_check.py
+++ b/taskcluster/taskgraph/transforms/bouncer_check.py
@@ -1,99 +1,95 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
-import copy
 import json
-import subprocess
+from pipes import quote as shell_quote
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.scriptworker import get_release_config
 from taskgraph.util.schema import (
     resolve_keyed_by,
 )
 
 import logging
 logger = logging.getLogger(__name__)
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def add_command(config, jobs):
     for job in jobs:
-        job = copy.deepcopy(job)  # don't overwrite dict values here
         command = [
-            "cd", "/builds/worker/checkouts/gecko", "&&",
-            "./mach", "python",
-            "testing/mozharness/scripts/release/bouncer_check.py",
+            "python", "testing/mozharness/scripts/release/bouncer_check.py",
         ]
-        job["run"]["command"] = command
+        job['run'].update({
+            'using': 'mach',
+            'mach': command,
+        })
         yield job
 
 
 @transforms.add
 def add_previous_versions(config, jobs):
     release_config = get_release_config(config)
     if not release_config.get("partial_versions"):
         for job in jobs:
             yield job
     else:
         extra_params = []
         for partial in release_config["partial_versions"].split(","):
             extra_params.append("--previous-version={}".format(partial.split("build")[0].strip()))
 
         for job in jobs:
-            job = copy.deepcopy(job)  # don't overwrite dict values here
-            job["run"]["command"].extend(extra_params)
+            job["run"]["mach"].extend(extra_params)
             yield job
 
 
 @transforms.add
 def handle_keyed_by(config, jobs):
     """Resolve fields that can be keyed by project, etc."""
     fields = [
         "run.config",
         "run.product-field",
         "run.extra-config",
     ]
 
     release_config = get_release_config(config)
     version = release_config["version"]
 
     for job in jobs:
-        job = copy.deepcopy(job)  # don't overwrite dict values here
         for field in fields:
             resolve_keyed_by(item=job, field=field, item_name=job['name'],
                              project=config.params['project'])
 
         for cfg in job["run"]["config"]:
-            job["run"]["command"].extend(["--config", cfg])
+            job["run"]["mach"].extend(["--config", cfg])
 
         if config.kind == "cron-bouncer-check":
-            job["run"]["command"].extend([
+            job["run"]["mach"].extend([
                 "--product-field={}".format(job["run"]["product-field"]),
                 "--products-url={}".format(job["run"]["products-url"]),
             ])
             del job["run"]["product-field"]
             del job["run"]["products-url"]
         elif config.kind == "release-bouncer-check":
-            job["run"]["command"].append("--version={}".format(version))
+            job["run"]["mach"].append("--version={}".format(version))
 
         del job["run"]["config"]
 
         if 'extra-config' in job['run']:
             env = job['worker'].setdefault('env', {})
             env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(job['run']['extra-config'])
             del job["run"]["extra-config"]
 
         yield job
 
 
 @transforms.add
 def command_to_string(config, jobs):
     """Convert command to string to make it work properly with run-task"""
     for job in jobs:
-        job = copy.deepcopy(job)  # don't overwrite dict values here
-        job["run"]["command"] = subprocess.list2cmdline(job["run"]["command"])
+        job["run"]["mach"] = ' '.join(map(shell_quote, job["run"]["mach"]))
         yield job
--- a/taskcluster/taskgraph/transforms/update_verify_config.py
+++ b/taskcluster/taskgraph/transforms/update_verify_config.py
@@ -59,18 +59,17 @@ def add_command(config, tasks):
     release_config = get_release_config(config)
 
     for task in tasks:
         task["description"] = "generate update verify config for {}".format(
             task["attributes"]["build_platform"]
         )
 
         command = [
-            "cd", "/builds/worker/checkouts/gecko", "&&"
-            "./mach", "python",
+            "python",
             "testing/mozharness/scripts/release/update-verify-config-creator.py",
             "--product", task["extra"]["product"],
             "--stage-product", task["shipping-product"],
             "--app-name", task["extra"]["app-name"],
             "--branch-prefix", task["extra"]["branch-prefix"],
             "--platform", task["extra"]["platform"],
             "--to-version", release_config["version"],
             "--to-app-version", release_config["appVersion"],
@@ -109,11 +108,14 @@ def add_command(config, tasks):
             if arg == "include-version":
                 task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]]
             if arg == "mar-channel-id-override":
                 task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]]
 
             command.append("--{}".format(arg))
             command.append(task["extra"][arg])
 
-        task["run"]["command"] = " ".join(command)
+        task['run'].update({
+            'using': 'mach',
+            'mach': " ".join(command),
+        })
 
         yield task