Bug 1290620 - Use run-task from decision task; r=dustin
authorGregory Szorc <gps@mozilla.com>
Fri, 29 Jul 2016 21:03:48 -0700
changeset 307724 b18ad418cba80522c888d329ae4eb37ab07fb436
parent 307723 9b080fde55c442af41360afa7f10defd9f1b9ab1
child 307725 b2a6aaf59783cecf8cde63fe51fbd65e14b026df
push id80163
push usercbook@mozilla.com
push dateTue, 02 Aug 2016 15:13:03 +0000
treeherdermozilla-inbound@9f3c3f219840 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1290620, 1289643
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1290620 - Use run-task from decision task; r=dustin This commit does a lot. But it's really not too difficult to comprehend once you focus on the final state, which is basically the same as the "lint" image and derived tasks. Before, the "decision" image contained a "checkout-gecko" script and "run-action" and "run-decision" scripts. The latter 2 invoked the first script. The "run-task" script basically does what the combination of these scripts were doing before. So we switch to it. While we're here, we also replaced the custom Mercurial installation in this image with the shared install-mercurial.sh script. The system-setup.sh script for the decision image is now short and sweet. The YAML files for tasks using this image have been updated to use "run-task." We no longer have to pass an environment variable to hold command arguments. So we revert to putting these arguments inline in the task's command. Dustin never liked passing the arguments as environment variables, so it should make him happy ;) We add --log-no-times because "run-task" prefixes its own timestamps on output lines. The path to the Gecko checkout has been changed from /home/worker/workspace to /home/worker/checkouts to match changes made in bug 1289643. Finally, since "checkout-gecko-and-run" is no longer used, we delete it. The Docker image version has been bumped accordingly. MozReview-Commit-ID: HAGuCz6668Q
.taskcluster.yml
taskcluster/taskgraph/action.yml
testing/docker/decision/Dockerfile
testing/docker/decision/VERSION
testing/docker/decision/bin/checkout-gecko
testing/docker/decision/bin/run-action
testing/docker/decision/bin/run-decision
testing/docker/decision/system-setup.sh
testing/docker/recipes/checkout-gecko-and-run
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -65,46 +65,52 @@ tasks:
       payload:
         env:
           # checkout-gecko uses these to check out the source; the inputs
           # to `mach taskgraph decision` are all on the command line.
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
-          # Arguments passed into `mach taskgraph decision`
-          # TODO use mozilla-unified for the base repository once the tc-vcs
-          # tar.gz archives are created or tc-vcs isn't being used.
-          DECISION_ARGS: >
-            --pushlog-id='{{pushlog_id}}'
-            --project='{{project}}'
-            --message='{{comment}}'
-            --owner='{{owner}}'
-            --level='{{level}}'
-            --base-repository='https://hg.mozilla.org/mozilla-central'
-            --head-repository='{{{url}}}'
-            --head-ref='{{revision}}'
-            --head-rev='{{revision}}'
-            --revision-hash='{{revision_hash}}'
 
         cache:
           level-{{level}}-hg-shared: /home/worker/hg-shared
-          level-{{level}}-gecko-decision: /home/worker/workspace
+          level-{{level}}-checkouts: /home/worker/checkouts
 
         features:
           taskclusterProxy: true
 
         # Note: This task is built server side without the context or tooling that
         # exist in tree so we must hard code the version
-        image: 'taskcluster/decision:0.1.2'
+        image: 'taskcluster/decision:0.1.3'
 
         maxRunTime: 1800
 
+        # TODO use mozilla-unified for the base repository once the tc-vcs
+        # tar.gz archives are created or tc-vcs isn't being used.
         command:
-          - /home/worker/bin/run-decision
+          - /home/worker/bin/run-task
+          - '--vcs-checkout=/home/worker/checkouts/gecko'
+          - '--'
+          - bash
+          - -cx
+          - >
+              cd /home/worker/checkouts/gecko &&
+              ln -s /home/worker/artifacts artifacts &&
+              ./mach --log-no-times taskgraph decision
+              --pushlog-id='{{pushlog_id}}'
+              --project='{{project}}'
+              --message='{{comment}}'
+              --owner='{{owner}}'
+              --level='{{level}}'
+              --base-repository='https://hg.mozilla.org/mozilla-central'
+              --head-repository='{{{url}}}'
+              --head-ref='{{revision}}'
+              --head-rev='{{revision}}'
+              --revision-hash='{{revision_hash}}'
 
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             expires: '{{#from_now}}7 days{{/from_now}}'
 
       extra:
--- a/taskcluster/taskgraph/action.yml
+++ b/taskcluster/taskgraph/action.yml
@@ -26,39 +26,46 @@ routes:
   - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
 
 payload:
   env:
     GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
     GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
     GECKO_HEAD_REF: '{{head_ref}}'
     GECKO_HEAD_REV: '{{head_rev}}'
-    ACTION_ARGS: >
-      --decision-id='{{decision_task_id}}'
-      --task-labels='{{task_labels}}'
 
   cache:
     level-{{level}}-hg-shared: /home/worker/hg-shared
-    level-{{level}}-gecko-decision: /home/worker/workspace
+    level-{{level}}-checkouts: /home/worker/checkouts
 
   features:
     taskclusterProxy: true
 
   # Note: This task is built server side without the context or tooling that
   # exist in tree so we must hard code the version
-  image: 'taskcluster/decision:0.1.2'
+  image: 'taskcluster/decision:0.1.3'
 
   # Virtually no network or other potentially risky operations happen as part
   # of the task timeout aside from the initial clone. We intentionally have
   # set this to a lower value _all_ decision tasks should use a root
   # repository which is cached.
   maxRunTime: 1800
 
   command:
-    - /home/worker/bin/run-action
+    - /home/worker/bin/run-task
+    - '--vcs-checkout=/home/worker/checkouts/gecko'
+    - '--'
+    - bash
+    - -cx
+    - >
+        cd /home/worker/checkouts/gecko &&
+        ln -s /home/worker/artifacts artifacts &&
+        ./mach --log-no-times taskgraph action-task
+        --decision-id='{{decision_task_id}}'
+        --task-label='{{task_labels}}'
 
   artifacts:
     'public':
       type: 'directory'
       path: '/home/worker/artifacts'
       expires: '{{#from_now}}7 days{{/from_now}}'
 
 extra:
--- a/testing/docker/decision/Dockerfile
+++ b/testing/docker/decision/Dockerfile
@@ -3,19 +3,23 @@ MAINTAINER    Greg Arndt <garndt@mozilla
 
 # Add worker user
 RUN useradd -d /home/worker -s /bin/bash -m worker
 RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts
 
 # %include testing/docker/recipes/tooltool.py
 ADD topsrcdir/testing/docker/recipes/tooltool.py /tmp/tooltool.py
 
+# %include testing/docker/recipes/install-mercurial.sh
+ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
+
 ADD system-setup.sh /tmp/system-setup.sh
 RUN bash /tmp/system-setup.sh
 
-ADD bin /home/worker/bin
+# %include testing/docker/recipes/run-task
+ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
 
 ENV PATH /home/worker/bin:$PATH
 ENV SHELL /bin/bash
 ENV HOME /home/worker
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
--- a/testing/docker/decision/VERSION
+++ b/testing/docker/decision/VERSION
@@ -1,1 +1,1 @@
-0.1.2
+0.1.3
deleted file mode 100755
--- a/testing/docker/decision/bin/checkout-gecko
+++ /dev/null
@@ -1,31 +0,0 @@
-#! /bin/bash -e
-
-set -ex
-
-# Ensure we have at least enough to check gecko out...
-test $GECKO_BASE_REPOSITORY
-
-DESTDIR=$1
-
-res=`curl --fail --retry 5 http://taskcluster/secrets/v1/secret/project/taskcluster/gecko/hgfingerprint`
-FP=`echo $res | jq -r .secret.content`
-
-if [[ ! "$FP" =~ ^[a-f0-9:]+$ ]]; then
-    echo
-    echo "[taskcluster:error] HG host fingerprint not properly loaded"
-    exit 1
-fi
-
-if [ -n "${GECKO_HEAD_REV}" ]; then
-  args="--revision ${GECKO_HEAD_REV}"
-else
-  args="--branch ${GECKO_HEAD_REF}"
-fi
-
-hg --config hostfingerprints.hg.mozilla.org=${FP} robustcheckout \
-  --sharebase /home/worker/hg-shared \
-  --purge \
-  ${args} \
-  --upstream ${GECKO_BASE_REPOSITORY} \
-  ${GECKO_HEAD_REPOSITORY} \
-  ${DESTDIR}
deleted file mode 100755
--- a/testing/docker/decision/bin/run-action
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root.
-if [ $(id -u) = 0 ]; then
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
-
-    exec sudo -E -u worker /home/worker/bin/run-action "${@}"
-fi
-
-/home/worker/bin/checkout-gecko /home/worker/workspace/gecko
-cd /home/worker/workspace/gecko
-ln -s /home/worker/artifacts artifacts &&
-eval "./mach taskgraph action-task ${ACTION_ARGS}"
deleted file mode 100755
--- a/testing/docker/decision/bin/run-decision
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root.
-if [ $(id -u) = 0 ]; then
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
-
-    exec sudo -E -u worker /home/worker/bin/run-decision "${@}"
-fi
-
-/home/worker/bin/checkout-gecko /home/worker/workspace/gecko
-cd /home/worker/workspace/gecko
-ln -s /home/worker/artifacts artifacts
-eval "./mach taskgraph decision ${DECISION_ARGS}"
--- a/testing/docker/decision/system-setup.sh
+++ b/testing/docker/decision/system-setup.sh
@@ -16,68 +16,16 @@ BUILD=/root/build
 mkdir $BUILD
 
 tooltool_fetch() {
     cat >manifest.tt
     python2.7 /tmp/tooltool.py fetch
     rm manifest.tt
 }
 
-# Install Mercurial from custom debs since distro packages tend to lag behind.
 cd $BUILD
-tooltool_fetch <<EOF
-[
-{
-    "size": 44878,
-    "digest": "7b1fc1217e0dcaeea852b0af2dc559b1aafb704fbee7e29cbec75af57bacb84910a7ec92b5c33f04ee98f23b3a57f1fa451173fe7c8a96f58faefe319dc7dde1",
-    "algorithm": "sha512",
-    "filename": "mercurial_3.8.4_amd64.deb"
-},
-{
-    "size": 1818422,
-    "digest": "b476e2612e7495a1c7c5adfd84511aa7479e26cc9070289513ec705fbfc4c61806ce2dbcceca0e63f2e80669be416f3467a3cebb522dcb8a6aeb62cdd3df82f2",
-    "algorithm": "sha512",
-    "filename": "mercurial-common_3.8.4_all.deb"
-}
-]
-EOF
-
-dpkg -i mercurial-common_3.8.4_all.deb mercurial_3.8.4_amd64.deb
-
-mkdir -p /usr/local/mercurial
-chown 755 /usr/local/mercurial
-cd /usr/local/mercurial
-tooltool_fetch <<'EOF'
-[
-{
-    "size": 11849,
-    "digest": "c88d9b8afd6649bd28bbacfa654ebefec8087a01d1662004aae088d485edeb03a92df1193d1310c0369d7721f475b974fcd4a911428ec65936f7e40cf1609c49",
-    "algorithm": "sha512",
-    "filename": "robustcheckout.py"
-}
-]
-EOF
-
-chmod 644 /usr/local/mercurial/robustcheckout.py
-
-# Install a global hgrc file with reasonable defaults.
-mkdir -p /etc/mercurial
-cat >/etc/mercurial/hgrc <<EOF
-# By default the progress bar starts after 3s and updates every 0.1s. We
-# change this so it shows and updates every 1.0s.
-[progress]
-delay = 1.0
-refresh = 1.0
-
-[web]
-cacerts = /etc/ssl/certs/ca-certificates.crt
-
-[extensions]
-robustcheckout = /usr/local/mercurial/robustcheckout.py
-EOF
-
-chmod 644 /etc/mercurial/hgrc
+. /tmp/install-mercurial.sh
 
 cd /
 rm -rf $BUILD
 apt-get clean
 apt-get autoclean
 rm $0
deleted file mode 100755
--- a/testing/docker/recipes/checkout-gecko-and-run
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-DEST=$1
-CHECKOUTS=`dirname ${DEST}`
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root. There
-# may not be a cache mount/directory on some instances. So create the
-# directory if missing.
-if [ $(id -u) = 0 ]; then
-    mkdir -p ${CHECKOUTS}
-    chown worker:worker /home/worker/hg-shared ${CHECKOUTS}
-
-    exec sudo -E -u worker ${0} "${@}"
-fi
-
-shift
-
-# We set the base repository to mozilla-central so tc-vcs doesn't get
-# confused. Switch to mozilla-unified because robustcheckout works best
-# with it.
-if [ "${GECKO_BASE_REPOSITORY}" = "https://hg.mozilla.org/mozilla-central" ]; then
-    GECKO_BASE_REPOSITORY=https://hg.mozilla.org/mozilla-unified
-fi
-
-/usr/bin/hg robustcheckout \
-    --sharebase /home/worker/hg-shared \
-    --purge \
-    --upstream ${GECKO_BASE_REPOSITORY} \
-    --revision ${GECKO_HEAD_REV} \
-    ${GECKO_HEAD_REPOSITORY} \
-    ${DEST}
-
-exec "${@}"