Bug 1509962: [update-verify] Use in-tree update-verify code; r=mtabara
authorTom Prince <mozilla@hocat.ca>
Mon, 03 Dec 2018 06:44:28 +0000
changeset 508409 4c4352291d629aaaaf39718c3cea57870db048d1
parent 508408 3ed6bf5ca77217e5884de76be8e66d8cf880b251
child 508410 7eb869c7fd0a51a5809065ecaca31dfcc3b45e78
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1509962
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509962: [update-verify] Use in-tree update-verify code; r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D12977
build/sparse-profiles/update-verify
taskcluster/ci/release-final-verify/kind.yml
taskcluster/ci/release-secondary-final-verify/kind.yml
taskcluster/ci/release-secondary-update-verify/kind.yml
taskcluster/ci/release-update-verify/kind.yml
taskcluster/taskgraph/transforms/final_verify.py
taskcluster/taskgraph/transforms/job/run_task.py
taskcluster/taskgraph/transforms/update_verify.py
tools/update-verify/release/common/download_builds.sh
tools/update-verify/release/updates/verify.sh
tools/update-verify/scripts/chunked-verify.sh
new file mode 100644
--- /dev/null
+++ b/build/sparse-profiles/update-verify
@@ -0,0 +1,4 @@
+%include build/sparse-profiles/mach
+
+[include]
+path:tools/update-verify/
--- a/taskcluster/ci/release-final-verify/kind.yml
+++ b/taskcluster/ci/release-final-verify/kind.yml
@@ -6,31 +6,29 @@ loader: taskgraph.loader.transform:loade
 
 kind-dependencies:
    - release-bouncer-check
    - release-update-verify-config
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.final_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    name: final-verify
    run-on-projects: []  # to make sure this never runs as part of CI
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 7200
       retry-exit-status: [1]
-      env:
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
+
 
 jobs:
    firefox:
       description: final verify
       shipping-phase: push
       shipping-product: firefox
       treeherder:
          platform: firefox-release/opt
--- a/taskcluster/ci/release-secondary-final-verify/kind.yml
+++ b/taskcluster/ci/release-secondary-final-verify/kind.yml
@@ -8,31 +8,28 @@ kind-dependencies:
    - post-balrog-dummy
    - post-beetmover-dummy
    - release-secondary-balrog-submit-toplevel
    - release-secondary-update-verify-config
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.final_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    name: secondary-final-verify
    run-on-projects: []  # to make sure this never runs as part of CI
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 7200
       retry-exit-status: [1]
-      env:
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
 
 jobs:
    firefox:
       description: secondary final verify
       shipping-phase: promote
       shipping-product: firefox
       treeherder:
          platform: linux64/opt
--- a/taskcluster/ci/release-secondary-update-verify/kind.yml
+++ b/taskcluster/ci/release-secondary-update-verify/kind.yml
@@ -8,38 +8,35 @@ kind-dependencies:
    - post-balrog-dummy
    - post-beetmover-dummy
    - release-secondary-balrog-submit-toplevel
    - release-secondary-update-verify-config
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.update_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    name: secondary-update-verify
    run-on-projects: []  # to make sure this never runs as part of CI
    shipping-phase: promote
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
       artifacts:
          - name: 'public/build/diff-summary.log'
            path: '/builds/worker/tools/release/updates/diff-summary.log'
            type: file
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 7200
       retry-exit-status:
          - 255
       env:
-         NO_BBCONFIG: "1"
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
          CHANNEL: "beta-localtest"
    extra:
       chunks: 12
 
 jobs:
    firefox-secondary-linux64:
       description: linux64 secondary channel update verify
       shipping-product: firefox
--- a/taskcluster/ci/release-update-verify/kind.yml
+++ b/taskcluster/ci/release-update-verify/kind.yml
@@ -8,38 +8,34 @@ kind-dependencies:
    - post-balrog-dummy
    - post-beetmover-dummy
    - release-balrog-submit-toplevel
    - release-update-verify-config
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.update_verify:transforms
+   - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    name: update-verify
    run-on-projects: []  # to make sure this never runs as part of CI
    shipping-phase: promote
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
       artifacts:
          - name: 'public/build/diff-summary.log'
            path: '/builds/worker/tools/release/updates/diff-summary.log'
            type: file
-      implementation: docker-worker
-      os: linux
       docker-image:
          in-tree: "update-verify"
       max-run-time: 5400
       retry-exit-status:
          - 255
-      env:
-         NO_BBCONFIG: "1"
-         BUILD_TOOLS_REPO: https://hg.mozilla.org/build/tools
    extra:
       chunks: 12
 
 jobs:
    firefox-linux64:
       description: linux64 update verify
       shipping-product: firefox
       worker:
--- a/taskcluster/taskgraph/transforms/final_verify.py
+++ b/taskcluster/taskgraph/transforms/final_verify.py
@@ -23,21 +23,21 @@ def add_command(config, tasks):
         final_verify_configs = []
         for upstream in task.get("dependencies", {}).keys():
             if 'update-verify-config' in upstream:
                 final_verify_configs.append(
                     "{}update-verify.cfg".format(
                         get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
                     )
                 )
-        task["worker"]["command"] = [
-            "/bin/bash",
-            "-c",
-            {
-                "task-reference": "hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " +
-                                  "./final-verification.sh " +
-                                  " ".join(final_verify_configs)
-            }
-        ]
+        task['run'] = {
+            'using': 'run-task',
+            'command': {
+                'task-reference': 'cd /builds/worker/checkouts/gecko && '
+                                  'tools/update-verify/release/final-verification.sh '
+                                  + ' '.join(final_verify_configs),
+            },
+            'sparse-profile': 'update-verify',
+        }
         for thing in ("BUILD_TOOLS_REPO",):
             thing = "worker.env.{}".format(thing)
             resolve_keyed_by(task, thing, thing, **config.params)
         yield task
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Support for running jobs that are invoked via the `run-task` script.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from taskgraph.transforms.task import taskref_or_string
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
 from taskgraph.transforms.job.common import support_vcs_checkout
 from voluptuous import Required, Any
 
 run_task_schema = Schema({
     Required('using'): 'run-task',
 
@@ -28,17 +29,17 @@ run_task_schema = Schema({
 
     # if true, perform a checkout of a comm-central based branch inside the
     # gecko checkout
     Required('comm-checkout'): bool,
 
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
-    Required('command'): Any([basestring], basestring),
+    Required('command'): Any([taskref_or_string], taskref_or_string),
 
     # Base work directory used to set up the task.
     Required('workdir'): basestring,
 })
 
 
 def common_setup(config, job, taskdesc, command, checkoutdir):
     run = job['run']
@@ -78,17 +79,18 @@ def docker_worker_run_task(config, job, 
         worker['caches'].append({
             'type': 'persistent',
             'name': 'level-{level}-{project}-dotcache'.format(**config.params),
             'mount-point': '{workdir}/.cache'.format(**run),
             'skip-untrusted': True,
         })
 
     run_command = run['command']
-    if isinstance(run_command, basestring):
+    # dict is for the case of `{'task-reference': basestring}`.
+    if isinstance(run_command, (basestring, dict)):
         run_command = ['bash', '-cx', run_command]
     if run['comm-checkout']:
         command.append('--comm-checkout={workdir}/checkouts/gecko/comm'.format(**run))
     command.append('--fetch-hgfingerprint')
     command.append('--')
     command.extend(run_command)
     worker['command'] = command
 
--- a/taskcluster/taskgraph/transforms/update_verify.py
+++ b/taskcluster/taskgraph/transforms/update_verify.py
@@ -26,26 +26,26 @@ def add_command(config, tasks):
             chunked = deepcopy(task)
             chunked["treeherder"]["symbol"] = add_suffix(
                 chunked["treeherder"]["symbol"], this_chunk)
             chunked["label"] = "release-update-verify-{}-{}/{}".format(
                 chunked["name"], this_chunk, total_chunks
             )
             if not chunked["worker"].get("env"):
                 chunked["worker"]["env"] = {}
-            chunked["worker"]["command"] = [
-                "/bin/bash",
-                "-c",
-                "hg clone $BUILD_TOOLS_REPO tools && " +
-                "tools/scripts/release/updates/chunked-verify.sh " +
-                "UNUSED UNUSED {} {}".format(
-                    total_chunks,
-                    this_chunk,
-                )
-            ]
+            chunked["run"] = {
+                'using': 'run-task',
+                'command': 'cd /builds/worker/checkouts/gecko && '
+                           'tools/update-verify/scripts/chunked-verify.sh '
+                           '{} {}'.format(
+                               total_chunks,
+                               this_chunk,
+                           ),
+                'sparse-profile': 'update-verify',
+            }
             for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
                 thing = "worker.env.{}".format(thing)
                 resolve_keyed_by(
                     chunked, thing, thing,
                     **{
                         'project': config.params['project'],
                         'release-type': config.params['release_type'],
                     }
--- a/tools/update-verify/release/common/download_builds.sh
+++ b/tools/update-verify/release/common/download_builds.sh
@@ -1,12 +1,12 @@
 pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
-retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
+retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
 
 download_builds() {
   # cleanup
   mkdir -p downloads/
   rm -rf downloads/*
 
   source_url="$1"
   target_url="$2"
--- a/tools/update-verify/release/updates/verify.sh
+++ b/tools/update-verify/release/updates/verify.sh
@@ -22,17 +22,17 @@ diff_summary_log="$PWD/diff-summary.log"
 if [ -e ${diff_summary_log} ]; then
   rm ${diff_summary_log}
 fi
 touch ${diff_summary_log}
 
 pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
-retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
+retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"
 cert_replacer="$MY_DIR/../replace-updater-certs.py"
 
 dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der"
 nightly_overrides="dep1.der nightly_aurora_level3_primary.der dep2.der nightly_aurora_level3_secondary.der release_primary.der nightly_aurora_level3_primary.der release_secondary.der nightly_aurora_level3_secondary.der"
 release_overrides="dep1.der release_primary.der dep2.der release_secondary.der nightly_aurora_level3_primary.der release_primary.der nightly_aurora_level3_secondary.der release_secondary.der"
 
 runmode=0
 config_file="updates.cfg"
--- a/tools/update-verify/scripts/chunked-verify.sh
+++ b/tools/update-verify/scripts/chunked-verify.sh
@@ -2,88 +2,38 @@
 set -ex
 set -o pipefail
 # This ugly hack is a cross-platform (Linux/Mac/Windows+MSYS) way to get the
 # absolute path to the directory containing this script
 pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
 SCRIPTS_DIR="$MY_DIR/.."
-PYTHON="/tools/python/bin/python"
-if [ ! -x $PYTHON ]; then
-    PYTHON=python
-fi
-JSONTOOL="$PYTHON $SCRIPTS_DIR/buildfarm/utils/jsontool.py"
-workdir=`pwd`
-
-platform=$1
-configDict=$2
-chunks=$3
-thisChunk=$4
-channel=$5
+PYTHON='./mach python'
 
-if [ -n "$PROPERTIES_FILE" -a -f "$PROPERTIES_FILE" ]; then
-    # Buildbot only
-    if $JSONTOOL -k properties.NO_BBCONFIG $PROPERTIES_FILE; then
-       NO_BBCONFIG=$($JSONTOOL -k properties.NO_BBCONFIG $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.CHANNEL $PROPERTIES_FILE; then
-       CHANNEL=$($JSONTOOL -k properties.CHANNEL $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.VERIFY_CONFIG $PROPERTIES_FILE; then
-       VERIFY_CONFIG=$($JSONTOOL -k properties.VERIFY_CONFIG $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.TOTAL_CHUNKS $PROPERTIES_FILE; then
-       chunks=$($JSONTOOL -k properties.TOTAL_CHUNKS $PROPERTIES_FILE);
-    fi
-    if $JSONTOOL -k properties.THIS_CHUNK $PROPERTIES_FILE; then
-       thisChunk=$($JSONTOOL -k properties.THIS_CHUNK $PROPERTIES_FILE);
-    fi
-    if [ -z "$NO_BBCONFIG" -a -z "$BUILDBOT_CONFIGS" ]; then
-        export BUILDBOT_CONFIGS="https://hg.mozilla.org/build/buildbot-configs"
-    fi
-    # Get the assumed slavebuilddir, and read in from buildbot if this is not
-    # Release promotion
-    SLAVEBUILDDIR=$(basename $(cd "$SCRIPTS_DIR/.."; pwd))
-    if [ -z "$NO_BBCONFIG" ]; then
-        RELEASE_CONFIG=$($JSONTOOL -k properties.release_config $PROPERTIES_FILE)
-        TAG=$($JSONTOOL -k properties.release_tag $PROPERTIES_FILE)
-        SLAVEBUILDDIR=$($JSONTOOL -k properties.slavebuilddir $PROPERTIES_FILE)
-    fi
-
-    $PYTHON -u $SCRIPTS_DIR/buildfarm/maintenance/purge_builds.py \
-        -s 16 -n info -n 'rel-*' -n 'tb-rel-*' -n $SLAVEBUILDDIR
-fi
+chunks=$1
+thisChunk=$2
 
 if [ -n "$TASKCLUSTER_VERIFY_CONFIG" ]; then
     wget -O "$SCRIPTS_DIR/release/updates/update-verify.cfg" "$TASKCLUSTER_VERIFY_CONFIG"
     VERIFY_CONFIG="update-verify.cfg"
-fi
-
-if [ -z "$VERIFY_CONFIG" -a -n "$NO_BBCONFIG" ]; then
-    echo "Unable to run without VERIFY_CONFIG specified when using NO_BBCONFIG"
+else
+    echo "Unable to run without TASKCLUSTER_VERIFY_CONFIG."
     exit 1
 fi
 
-if [ -z "$NO_BBCONFIG" ]; then
-  $PYTHON $MY_DIR/chunked-verify.py -t $TAG -r $RELEASE_CONFIG \
-  -b $BUILDBOT_CONFIGS -p $platform --chunks $chunks --this-chunk $thisChunk \
-  --config-dict $configDict --release-channel $channel \
-    2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
+# release promotion
+if [ -n "$CHANNEL" ]; then
+  EXTRA_PARAMS="--verify-channel $CHANNEL"
 else
-  # release promotion
-  if [ -n "$CHANNEL" ]; then
-    EXTRA_PARAMS="--verify-channel $CHANNEL"
-  else
-    EXTRA_PARAMS=""
-  fi
-  $PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \
-  --verify-config $VERIFY_CONFIG $EXTRA_PARAMS \
-  2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
+  EXTRA_PARAMS=""
 fi
+$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \
+--verify-config $VERIFY_CONFIG $EXTRA_PARAMS \
+2>&1 | tee $SCRIPTS_DIR/../verify_log.txt
 
 print_failed_msg()
 {
   echo "-------------------------"
   echo "This run has failed, see the above log"
   echo
   return 1
 }