Bug 1290531 - Use checkout-gecko-and-run from decision task; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Fri, 29 Jul 2016 15:45:53 -0700
changeset 394548 6690559ecbcfbbb70e34b4b38cbf4c711c0edfa6
parent 394506 046bf0827ebd37f929d34d36fc78e2e2eca2c5a2
child 526838 58b58daa2fc8533b6cf68cc5182e35dba81b483a
push id24601
push userbmo:gps@mozilla.com
push dateFri, 29 Jul 2016 23:22:35 +0000
reviewersdustin
bugs1290531, 1289643
milestone50.0a1
Bug 1290531 - Use checkout-gecko-and-run from decision task; r?dustin This commit does a lot. But it's really not too difficult to comprehend once you focus on the final state, which is basically the same as the "lint" image and derived tasks. Before, the "decision" image contained a "checkout-gecko" script and "run-action" and "run-decision" scripts. The latter 2 invoked the first script. The "checkout-gecko-and-run" script basically does what the combination of these scripts were doing before. So we switch to it. While we're here, we also replaced the custom Mercurial installation in this image with the shared install-mercurial.sh script. The system-setup.sh script for the decision image is now short and sweet. The YAML files for tasks using this image have been updated to use "checkout-gecko-and-run." We no longer have to pass an environment variable to hold command arguments. So we revert to putting these arguments inline in the task's command. Yay. Finally, the path to the Gecko checkout has been changed from /home/worker/workspace to /home/worker/checkouts to match changes made in bug 1289643. The Docker image has been bumped accordingly. MozReview-Commit-ID: HAGuCz6668Q
.taskcluster.yml
taskcluster/taskgraph/action.yml
testing/docker/decision/Dockerfile
testing/docker/decision/VERSION
testing/docker/decision/bin/checkout-gecko
testing/docker/decision/bin/run-action
testing/docker/decision/bin/run-decision
testing/docker/decision/system-setup.sh
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -65,46 +65,50 @@ tasks:
       payload:
         env:
           # checkout-gecko uses these to check out the source; the inputs
           # to `mach taskgraph decision` are all on the command line.
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
-          # Arguments passed into `mach taskgraph decision`
-          # TODO use mozilla-unified for the base repository once the tc-vcs
-          # tar.gz archives are created or tc-vcs isn't being used.
-          DECISION_ARGS: >
-            --pushlog-id='{{pushlog_id}}'
-            --project='{{project}}'
-            --message='{{comment}}'
-            --owner='{{owner}}'
-            --level='{{level}}'
-            --base-repository='https://hg.mozilla.org/mozilla-central'
-            --head-repository='{{{url}}}'
-            --head-ref='{{revision}}'
-            --head-rev='{{revision}}'
-            --revision-hash='{{revision_hash}}'
 
         cache:
           level-{{level}}-hg-shared: /home/worker/hg-shared
-          level-{{level}}-gecko-decision: /home/worker/workspace
+          level-{{level}}-checkouts: /home/worker/checkouts
 
         features:
           taskclusterProxy: true
 
         # Note: This task is built server side without the context or tooling that
         # exist in tree so we must hard code the version
-        image: 'taskcluster/decision:0.1.2'
+        image: 'taskcluster/decision:0.1.3'
 
         maxRunTime: 1800
 
+        # TODO use mozilla-unified for the base repository once the tc-vcs
+        # tar.gz archives are created or tc-vcs isn't being used.
         command:
-          - /home/worker/bin/run-decision
+          - /home/worker/bin/checkout-gecko-and-run
+          - /home/worker/checkouts/gecko
+          - bash
+          - -cx
+          - >
+              cd /home/worker/checkouts/gecko &&
+              ln -s /home/worker/artifacts artifacts &&
+              ./mach taskgraph decision --pushlog-id='{{pushlog_id}}'
+              --project='{{project}}'
+              --message='{{comment}}'
+              --owner='{{owner}}'
+              --level='{{level}}'
+              --base-repository='https://hg.mozilla.org/mozilla-central'
+              --head-repository='{{{url}}}'
+              --head-ref='{{revision}}'
+              --head-rev='{{revision}}'
+              --revision-hash='{{revision_hash}}'
 
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             expires: '{{#from_now}}7 days{{/from_now}}'
 
       extra:
--- a/taskcluster/taskgraph/action.yml
+++ b/taskcluster/taskgraph/action.yml
@@ -26,39 +26,43 @@ routes:
   - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
 
 payload:
   env:
     GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
     GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
     GECKO_HEAD_REF: '{{head_ref}}'
     GECKO_HEAD_REV: '{{head_rev}}'
-    ACTION_ARGS: >
-      --decision-id='{{decision_task_id}}'
-      --task-labels='{{task_labels}}'
 
   cache:
     level-{{level}}-hg-shared: /home/worker/hg-shared
-    level-{{level}}-gecko-decision: /home/worker/workspace
+    level-{{level}}-checkouts: /home/worker/checkouts
 
   features:
     taskclusterProxy: true
 
   # Note: This task is built server side without the context or tooling that
   # exist in tree so we must hard code the version
-  image: 'taskcluster/decision:0.1.2'
+  image: 'taskcluster/decision:0.1.3'
 
   # Virtually no network or other potentially risky operations happen as part
   # of the task timeout aside from the initial clone. We intentionally have
   # set this to a lower value _all_ decision tasks should use a root
   # repository which is cached.
   maxRunTime: 1800
 
   command:
-    - /home/worker/bin/run-action
+    - /home/worker/bin/checkout-gecko-and-run
+    - /home/worker/checkouts/gecko
+    - bash
+    - -cx
+    - >
+        cd /home/worker/checkouts/gecko &&
+        ln -s /home/worker/artifacts artifacts &&
+        ./mach taskgraph action-task --decision-id='{{decision_task_id}}' --task-label='{{task_labels}}'
 
   artifacts:
     'public':
       type: 'directory'
       path: '/home/worker/artifacts'
       expires: '{{#from_now}}7 days{{/from_now}}'
 
 extra:
--- a/testing/docker/decision/Dockerfile
+++ b/testing/docker/decision/Dockerfile
@@ -3,19 +3,23 @@ MAINTAINER    Greg Arndt <garndt@mozilla
 
 # Add worker user
 RUN useradd -d /home/worker -s /bin/bash -m worker
 RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts
 
 # %include testing/docker/recipes/tooltool.py
 ADD topsrcdir/testing/docker/recipes/tooltool.py /tmp/tooltool.py
 
+# %include testing/docker/recipes/install-mercurial.sh
+ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
+
 ADD system-setup.sh /tmp/system-setup.sh
 RUN bash /tmp/system-setup.sh
 
-ADD bin /home/worker/bin
+# %include testing/docker/recipes/checkout-gecko-and-run
+ADD topsrcdir/testing/docker/recipes/checkout-gecko-and-run /home/worker/bin/checkout-gecko-and-run
 
 ENV PATH /home/worker/bin:$PATH
 ENV SHELL /bin/bash
 ENV HOME /home/worker
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
--- a/testing/docker/decision/VERSION
+++ b/testing/docker/decision/VERSION
@@ -1,1 +1,1 @@
-0.1.2
+0.1.3
deleted file mode 100755
--- a/testing/docker/decision/bin/checkout-gecko
+++ /dev/null
@@ -1,31 +0,0 @@
-#! /bin/bash -e
-
-set -ex
-
-# Ensure we have at least enough to check gecko out...
-test $GECKO_BASE_REPOSITORY
-
-DESTDIR=$1
-
-res=`curl --fail --retry 5 http://taskcluster/secrets/v1/secret/project/taskcluster/gecko/hgfingerprint`
-FP=`echo $res | jq -r .secret.content`
-
-if [[ ! "$FP" =~ ^[a-f0-9:]+$ ]]; then
-    echo
-    echo "[taskcluster:error] HG host fingerprint not properly loaded"
-    exit 1
-fi
-
-if [ -n "${GECKO_HEAD_REV}" ]; then
-  args="--revision ${GECKO_HEAD_REV}"
-else
-  args="--branch ${GECKO_HEAD_REF}"
-fi
-
-hg --config hostfingerprints.hg.mozilla.org=${FP} robustcheckout \
-  --sharebase /home/worker/hg-shared \
-  --purge \
-  ${args} \
-  --upstream ${GECKO_BASE_REPOSITORY} \
-  ${GECKO_HEAD_REPOSITORY} \
-  ${DESTDIR}
deleted file mode 100755
--- a/testing/docker/decision/bin/run-action
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root.
-if [ $(id -u) = 0 ]; then
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
-
-    exec sudo -E -u worker /home/worker/bin/run-action "${@}"
-fi
-
-/home/worker/bin/checkout-gecko /home/worker/workspace/gecko
-cd /home/worker/workspace/gecko
-ln -s /home/worker/artifacts artifacts &&
-eval "./mach taskgraph action-task ${ACTION_ARGS}"
deleted file mode 100755
--- a/testing/docker/decision/bin/run-decision
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -ex
-
-# The script starts executing as root. We need to change ownership
-# of the caches because they are initially owned by root:root.
-if [ $(id -u) = 0 ]; then
-    chown worker:worker /home/worker/hg-shared /home/worker/workspace
-
-    exec sudo -E -u worker /home/worker/bin/run-decision "${@}"
-fi
-
-/home/worker/bin/checkout-gecko /home/worker/workspace/gecko
-cd /home/worker/workspace/gecko
-ln -s /home/worker/artifacts artifacts
-eval "./mach taskgraph decision ${DECISION_ARGS}"
--- a/testing/docker/decision/system-setup.sh
+++ b/testing/docker/decision/system-setup.sh
@@ -16,68 +16,16 @@ BUILD=/root/build
 mkdir $BUILD
 
 tooltool_fetch() {
     cat >manifest.tt
     python2.7 /tmp/tooltool.py fetch
     rm manifest.tt
 }
 
-# Install Mercurial from custom debs since distro packages tend to lag behind.
 cd $BUILD
-tooltool_fetch <<EOF
-[
-{
-    "size": 44878,
-    "digest": "7b1fc1217e0dcaeea852b0af2dc559b1aafb704fbee7e29cbec75af57bacb84910a7ec92b5c33f04ee98f23b3a57f1fa451173fe7c8a96f58faefe319dc7dde1",
-    "algorithm": "sha512",
-    "filename": "mercurial_3.8.4_amd64.deb"
-},
-{
-    "size": 1818422,
-    "digest": "b476e2612e7495a1c7c5adfd84511aa7479e26cc9070289513ec705fbfc4c61806ce2dbcceca0e63f2e80669be416f3467a3cebb522dcb8a6aeb62cdd3df82f2",
-    "algorithm": "sha512",
-    "filename": "mercurial-common_3.8.4_all.deb"
-}
-]
-EOF
-
-dpkg -i mercurial-common_3.8.4_all.deb mercurial_3.8.4_amd64.deb
-
-mkdir -p /usr/local/mercurial
-chown 755 /usr/local/mercurial
-cd /usr/local/mercurial
-tooltool_fetch <<'EOF'
-[
-{
-    "size": 11849,
-    "digest": "c88d9b8afd6649bd28bbacfa654ebefec8087a01d1662004aae088d485edeb03a92df1193d1310c0369d7721f475b974fcd4a911428ec65936f7e40cf1609c49",
-    "algorithm": "sha512",
-    "filename": "robustcheckout.py"
-}
-]
-EOF
-
-chmod 644 /usr/local/mercurial/robustcheckout.py
-
-# Install a global hgrc file with reasonable defaults.
-mkdir -p /etc/mercurial
-cat >/etc/mercurial/hgrc <<EOF
-# By default the progress bar starts after 3s and updates every 0.1s. We
-# change this so it shows and updates every 1.0s.
-[progress]
-delay = 1.0
-refresh = 1.0
-
-[web]
-cacerts = /etc/ssl/certs/ca-certificates.crt
-
-[extensions]
-robustcheckout = /usr/local/mercurial/robustcheckout.py
-EOF
-
-chmod 644 /etc/mercurial/hgrc
+. /tmp/install-mercurial.sh
 
 cd /
 rm -rf $BUILD
 apt-get clean
 apt-get autoclean
 rm $0