Bug 1324767 - Simplify decision task definition; r?dustin
authorJonas Finnemann Jensen <jopsen@gmail.com>
Wed, 21 Dec 2016 10:19:20 +0100
changeset 918932 39900201a5ae19da6b8ce3d040d5ce7aaf37b6f5
parent 918735 c36fbe84042debef0a5d58b7fc88185b401762ce
child 918933 01925d86209d93eea93ae431ffdc3dc373060b66
push id160414
push userjojensen@mozilla.com
push dateWed, 21 Dec 2016 09:30:39 +0000
treeherdertry@01925d86209d [default view] [failures only]
reviewersdustin
bugs1324767
milestone53.0a1
Bug 1324767 - Simplify decision task definition; r?dustin Setting more environment variables, ENTRYPOINT and CMD in the Dockerfile for the decision image as well as taking inputs from environment variables in ./mach taskgraph decision, allows us to greatly simplify the decision task definition in .taskcluster.yml. This is useful as a general cleanup, making it easier to see what is input and what is template. But also simplifies validation of task definitions, which is required when validating the Chain-Of-Trust artifacts before signing.
.taskcluster.yml
taskcluster/mach_commands.py
testing/docker/decision/Dockerfile
testing/docker/decision/VERSION
testing/docker/decision/system-setup.sh
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -64,61 +64,44 @@ tasks:
 
       routes:
         - "index.gecko.v2.{{project}}.latest.firefox.decision"
         - "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"
         - "tc-treeherder-stage.v2.{{project}}.{{revision}}.{{pushlog_id}}"
 
       payload:
         env:
-          # checkout-gecko uses these to check out the source; the inputs
-          # to `mach taskgraph decision` are all on the command line.
-          GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
-          GECKO_HEAD_REPOSITORY: '{{{url}}}'
-          GECKO_HEAD_REF: '{{revision}}'
-          GECKO_HEAD_REV: '{{revision}}'
-          HG_STORE_PATH: /home/worker/checkouts/hg-store
+          # Variables used by both run-task to checkout the source, the
+          # './mach taskgraph decision' command.
+          GECKO_BASE_REPOSITORY:  'https://hg.mozilla.org/mozilla-unified'
+          GECKO_HEAD_REPOSITORY:  '{{{url}}}'
+          GECKO_HEAD_REF:         '{{revision}}'
+          GECKO_HEAD_REV:         '{{revision}}'
+          # Variables only used by './mach taskgraph decision'
+          COMMIT_MESSAGE:         '{{{comment}}}'
+          REVISION_HASH:          '{{revision}}'
+          GECKO_PROJECT:          '{{project}}'
+          PUSHLOG_ID:             '{{pushlog_id}}'
+          PUSH_DATE:              '{{pushdate}}'
+          OWNER_EMAIL:            '{{owner}}'
+          SCM_LEVEL:              '{{level}}'
 
         cache:
           level-{{level}}-checkouts: /home/worker/checkouts
 
         features:
           taskclusterProxy: true
           chainOfTrust: true
 
         # Note: This task is built server side without the context or tooling that
         # exist in tree so we must hard code the hash
-        image: 'taskcluster/decision@sha256:0f59f922d86c471e208b7ea08ab077fc68c3920ed5e6895d69a23e8f3457dc24'
+        image: 'jonasfj/decision@sha256:b72fd20e1077e9552b5271aea0eefda7f23622bac91e150fccb7884333f166fb'
 
         maxRunTime: 1800
 
-        # TODO use mozilla-unified for the base repository once the tc-vcs
-        # tar.gz archives are created or tc-vcs isn't being used.
-        command:
-          - /home/worker/bin/run-task
-          - '--vcs-checkout=/home/worker/checkouts/gecko'
-          - '--'
-          - bash
-          - -cx
-          - >
-              cd /home/worker/checkouts/gecko &&
-              ln -s /home/worker/artifacts artifacts &&
-              ./mach --log-no-times taskgraph decision
-              --pushlog-id='{{pushlog_id}}'
-              --pushdate='{{pushdate}}'
-              --project='{{project}}'
-              --message={{#shellquote}}{{{comment}}}{{/shellquote}}
-              --owner='{{owner}}'
-              --level='{{level}}'
-              --base-repository='https://hg.mozilla.org/mozilla-central'
-              --head-repository='{{{url}}}'
-              --head-ref='{{revision}}'
-              --head-rev='{{revision}}'
-              --revision-hash='{{revision_hash}}'
-
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             expires: '{{#from_now}}364 days{{/from_now}}'
 
       extra:
         treeherder:
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -7,29 +7,50 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import logging
 import sys
 import traceback
 import re
+import os
+import argparse
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
     SubCommand,
 )
 
 from mozbuild.base import MachCommandBase
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 
 
+class EnvDefault(argparse.Action):
+    """
+        Action for use with argparse that supports loading option from
+        environment variable if not specified.
+
+        Credits: http://stackoverflow.com/a/10551190/68333
+    """
+
+    def __init__(self, envvar, required=True, default=None, **kwargs):
+        default = os.environ.get(envvar, default)
+        if required and default:
+            required = False
+        super(EnvDefault, self).__init__(default=default, required=required,
+                                         **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
 class ShowTaskGraphSubCommand(SubCommand):
     """A SubCommand with TaskGraph-specific arguments"""
 
     def __call__(self, func):
         after = SubCommand.__call__(self, func)
         args = [
             CommandArgument('--root', '-r', default='taskcluster/ci',
                             help="root of the taskgraph definition relative to topsrcdir"),
@@ -109,50 +130,50 @@ class MachCommands(MachCommandBase):
         return self.show_taskgraph('optimized_task_graph', options)
 
     @SubCommand('taskgraph', 'decision',
                 description="Run the decision task")
     @CommandArgument('--root', '-r',
                      default='taskcluster/ci',
                      help="root of the taskgraph definition relative to topsrcdir")
     @CommandArgument('--base-repository',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='GECKO_BASE_REPOSITORY',
                      help='URL for "base" repository to clone')
     @CommandArgument('--head-repository',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='GECKO_HEAD_REPOSITORY',
                      help='URL for "head" repository to fetch revision from')
     @CommandArgument('--head-ref',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='GECKO_HEAD_REF',
                      help='Reference (this is same as rev usually for hg)')
     @CommandArgument('--head-rev',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='GECKO_HEAD_REV',
                      help='Commit revision to use from head repository')
     @CommandArgument('--message',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='COMMIT_MESSAGE',
                      help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
     @CommandArgument('--revision-hash',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='REVISION_HASH',
                      help='Treeherder revision hash (long revision id) to attach results to')
     @CommandArgument('--project',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='GECKO_PROJECT',
                      help='Project to use for creating task graph. Example: --project=try')
     @CommandArgument('--pushlog-id',
                      dest='pushlog_id',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='PUSHLOG_ID',
                      default=0)
     @CommandArgument('--pushdate',
                      dest='pushdate',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='PUSH_DATE',
                      type=int,
                      default=0)
     @CommandArgument('--owner',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='OWNER_EMAIL',
                      help='email address of who owns this graph')
     @CommandArgument('--level',
-                     required=True,
+                     required=True, action=EnvDefault, envvar='SCM_LEVEL',
                      help='SCM level of this repository')
     @CommandArgument('--triggered-by',
                      choices=['nightly', 'push'],
                      default='push',
                      help='Source of execution of the decision graph')
     @CommandArgument('--target-tasks-method',
                      help='method for selecting the target tasks to generate')
     def taskgraph_decision(self, **options):
--- a/testing/docker/decision/Dockerfile
+++ b/testing/docker/decision/Dockerfile
@@ -1,28 +1,39 @@
 FROM          ubuntu:16.04
 MAINTAINER    Greg Arndt <garndt@mozilla.com>
 
 # Add worker user
-RUN useradd -d /home/worker -s /bin/bash -m worker
-RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts
+RUN useradd -d /home/worker -s /bin/bash -m worker && \
+    mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts
 
 # %include testing/docker/recipes/tooltool.py
 ADD topsrcdir/testing/docker/recipes/tooltool.py /tmp/tooltool.py
 
 # %include testing/mozharness/external_tools/robustcheckout.py
 ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
 
 # %include testing/docker/recipes/install-mercurial.sh
 ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
 
 ADD system-setup.sh /tmp/system-setup.sh
 RUN bash /tmp/system-setup.sh
 
 # %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
+ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task
 
-ENV PATH /home/worker/bin:$PATH
-ENV SHELL /bin/bash
-ENV HOME /home/worker
+# Set variable normally configured at login, by the shells parent process, these
+# are taken from GNU su manual
+ENV           HOME          /home/worker
+ENV           SHELL         /bin/bash
+ENV           USER          worker
+ENV           LOGNAME       worker
+ENV           HOSTNAME      taskcluster-worker
+ENV           LC_ALL        C
 
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
+# Always use a volume for checkouts, this should be declared a cache in tasks
+VOLUME        /home/worker/checkouts
+ENV           HG_STORE_PATH /home/worker/checkouts/hg-store
+
+# Set some sane defaults
+WORKDIR     /home/worker/
+ENTRYPOINT  ["run-task", "--vcs-checkout=/home/worker/checkouts/gecko", "--"]
+CMD         ["/home/worker/checkouts/gecko/mach", "--log-no-times", "taskgraph", "decision"]
--- a/testing/docker/decision/VERSION
+++ b/testing/docker/decision/VERSION
@@ -1,1 +1,1 @@
-0.1.7
+0.2.0-alpha1
--- a/testing/docker/decision/system-setup.sh
+++ b/testing/docker/decision/system-setup.sh
@@ -1,14 +1,16 @@
 #!/usr/bin/env bash
 
 set -v -e
 
 test `whoami` == 'root'
 
+export DEBIAN_FRONTEND=noninteractive
+
 apt-get update
 apt-get install -y --force-yes --no-install-recommends \
     ca-certificates \
     python \
     sudo
 
 BUILD=/root/build
 mkdir $BUILD
@@ -21,9 +23,10 @@ tooltool_fetch() {
 
 cd $BUILD
 . /tmp/install-mercurial.sh
 
 cd /
 rm -rf $BUILD
 apt-get clean
 apt-get autoclean
+rm -rf /var/lib/apt/lists/
 rm $0