Bug 1406209 - Use .taskcluster.yml for action tasks templates r=dustin,jonasfj
authorBrian Stack <bstack@mozilla.com>
Mon, 09 Oct 2017 18:39:00 -0700
changeset 386812 215bc8a3310cde31dd6f09c72f8e5c84a904023f
parent 386811 35f1751b91a9fff2c6f4649ce90aec5d1eb72976
child 386813 b5118c0bafd7580ebe8a8873950de060c446f78e
push id96311
push userarchaeopteryx@coole-files.de
push dateWed, 18 Oct 2017 09:52:02 +0000
treeherdermozilla-inbound@a8a1e8cc1980 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, jonasfj
bugs1406209
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1406209 - Use .taskcluster.yml for action tasks templates r=dustin,jonasfj MozReview-Commit-ID: 8I8lIouV6KF
.taskcluster.yml
build/sparse-profiles/taskgraph
taskcluster/taskgraph/actions/registry.py
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -1,139 +1,193 @@
 # This file is rendered via JSON-e by
 # - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
 # - cron tasks - taskcluster/taskgraph/cron/decision.py
+# - action tasks - taskcluster/taskgraph/actions/registry.py
 version: 1
 tasks:
-  $let:
-    # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
-    ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
-    # ensure there's no trailing `/` on the repo URL
-    repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
-  in:
-  - taskId: '${as_slugid("decision")}'
-    taskGroupId: '${as_slugid("decision")}' # same as tsakId; this is how automation identifies a decision tsak
-    schedulerId: 'gecko-level-${repository.level}'
-
-    created: {$fromNow: ''}
-    deadline: {$fromNow: '1 day'}
-    expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
-    metadata:
-      $merge:
-        - owner: "${ownerEmail}"
-          source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
-        - $if: 'tasks_for == "hg-push"'
-          then:
-            name: "Gecko Decision Task"
-            description: 'The task that creates all of the other tasks in the task graph'
-          else:
-            name: "Decision Task for cron job ${cron.job_name}"
-            description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
-
-    provisionerId: "aws-provisioner-v1"
-    workerType: "gecko-${repository.level}-decision"
-
-    tags:
-      $if: 'tasks_for == "hg-push"'
-      then: {createdForUser: "${ownerEmail}"}
-
-    routes:
-      $if: 'tasks_for == "hg-push"'
-      then:
-        - "index.gecko.v2.${repository.project}.latest.firefox.decision"
-        - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
-        - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-        - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-        - "notify.email.${ownerEmail}.on-failed"
-        - "notify.email.${ownerEmail}.on-exception"
-      else:
-        - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
-        - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-        - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-
-    scopes:
-      $if: 'tasks_for == "hg-push"'
-      then:
-        - 'assume:repo:${repoUrl[8:]}:*'
-        - 'queue:route:notify.email.${ownerEmail}.*'
-      else:
-        - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
-
-    dependencies: []
-    requires: all-completed
-
-    priority: lowest
-    retries: 5
+  - $let:
+      # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
+      ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
+      # ensure there's no trailing `/` on the repo URL
+      repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
+    in:
+      taskId: {$if: 'tasks_for != "action"', then: '${as_slugid("decision")}'}
+      taskGroupId:
+        $if: 'tasks_for == "action"'
+        then:
+          '${action.taskGroupId}'
+        else:
+          '${as_slugid("decision")}' # same as taskId; this is how automation identifies a decision tsak
+      schedulerId: 'gecko-level-${repository.level}'
 
-    payload:
-      env:
-        # checkout-gecko uses these to check out the source; the inputs
-        # to `mach taskgraph decision` are all on the command line.
-        GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
-        GECKO_HEAD_REPOSITORY: '${repoUrl}'
-        GECKO_HEAD_REF: '${push.revision}'
-        GECKO_HEAD_REV: '${push.revision}'
-        GECKO_COMMIT_MSG: '${push.comment}'
-        HG_STORE_PATH: /builds/worker/checkouts/hg-store
-        TASKCLUSTER_CACHES: /builds/worker/checkouts
-
-      cache:
-        level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts
-
-      features:
-        taskclusterProxy: true
-        chainOfTrust: true
-
-      # Note: This task is built server side without the context or tooling that
-      # exist in tree so we must hard code the hash
-      # XXX Changing this will break Chain of Trust without an associated puppet and
-      # scriptworker patch!
-      image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
-
-      maxRunTime: 1800
-
-      # TODO use mozilla-unified for the base repository once the tc-vcs
-      # tar.gz archives are created or tc-vcs isn't being used.
-      command:
-        - /builds/worker/bin/run-task
-        - '--vcs-checkout=/builds/worker/checkouts/gecko'
-        - '--sparse-profile=build/sparse-profiles/taskgraph'
-        - '--'
-        - bash
-        - -cx
-        - $let:
-            extraArgs: {$if: 'tasks_for == "hg-push"', then: '', else: '${cron.quoted_args}'}
-          # NOTE: the explicit reference to mozilla-central below is required because android-stuff
-          # still uses tc-vcs, which does not support mozilla-unified
-          # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
-          in: >
-            cd /builds/worker/checkouts/gecko &&
-            ln -s /builds/worker/artifacts artifacts &&
-            ./mach --log-no-times taskgraph decision
-            --pushlog-id='${push.pushlog_id}'
-            --pushdate='${push.pushdate}'
-            --project='${repository.project}'
-            --message="$GECKO_COMMIT_MSG"
-            --owner='${ownerEmail}'
-            --level='${repository.level}'
-            --base-repository='https://hg.mozilla.org/mozilla-central'
-            --head-repository="$GECKO_HEAD_REPOSITORY"
-            --head-ref="$GECKO_HEAD_REF"
-            --head-rev="$GECKO_HEAD_REV"
-            ${extraArgs}
-
-      artifacts:
-        'public':
-          type: 'directory'
-          path: '/builds/worker/artifacts'
-          expires: {$fromNow: '1 year'}
-
-    extra:
-      treeherder:
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 day'}
+      expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
+      metadata:
         $merge:
-          - machine:
-              platform: gecko-decision
+          - owner: "${ownerEmail}"
+            source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
           - $if: 'tasks_for == "hg-push"'
             then:
-              symbol: D
+              name: "Gecko Decision Task"
+              description: 'The task that creates all of the other tasks in the task graph'
             else:
-              groupSymbol: cron
-              symbol: "${cron.job_symbol}"
+              $if: 'tasks_for == "action"'
+              then:
+                name: "Action: ${action.title}"
+                description: '${action.description}'
+              else:
+                name: "Decision Task for cron job ${cron.job_name}"
+                description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
+
+      provisionerId: "aws-provisioner-v1"
+      workerType: "gecko-${repository.level}-decision"
+
+      tags:
+        $if: 'tasks_for == "hg-push"'
+        then: {createdForUser: "${ownerEmail}"}
+        else:
+          $if: 'tasks_for == "action"'
+          then:
+            createdForUser: '${ownerEmail}'
+            kind: 'action-callback'
+
+      routes:
+        $if: 'tasks_for == "hg-push"'
+        then:
+          - "index.gecko.v2.${repository.project}.latest.firefox.decision"
+          - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
+          - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - "notify.email.${ownerEmail}.on-failed"
+          - "notify.email.${ownerEmail}.on-exception"
+        else:
+          - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - $if: 'tasks_for == "action"'
+            then: "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.$ownTaskId"
+            else: "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
+
+      scopes:
+        $if: 'tasks_for == "hg-push"'
+        then:
+          - 'assume:repo:${repoUrl[8:]}:*'
+          - 'queue:route:notify.email.${ownerEmail}.*'
+        else:
+          $if: 'tasks_for == "action"'
+          then:
+            - '${action.repo_scope}'
+          else:
+            - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
+
+      dependencies: []
+      requires: all-completed
+
+      priority: lowest
+      retries: 5
+
+      payload:
+        env:
+          # checkout-gecko uses these to check out the source; the inputs
+          # to `mach taskgraph decision` are all on the command line.
+          $merge:
+            - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
+              GECKO_HEAD_REPOSITORY: '${repoUrl}'
+              GECKO_HEAD_REF: '${push.revision}'
+              GECKO_HEAD_REV: '${push.revision}'
+              GECKO_COMMIT_MSG: {$if: 'tasks_for != "action"', then: '${push.comment}'}
+              HG_STORE_PATH: /builds/worker/checkouts/hg-store
+              TASKCLUSTER_CACHES: /builds/worker/checkouts
+            - $if: 'tasks_for == "action"'
+              then:
+                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'
+                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}
+                ACTION_TASK: {$json: {$eval: 'task'}}
+                ACTION_INPUT: {$json: {$eval: 'input'}}
+                ACTION_CALLBACK: '${action.cb_name}'
+                ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}
+
+        cache:
+          level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts
+
+        features:
+          taskclusterProxy: true
+          chainOfTrust: true
+
+        # Note: This task is built server side without the context or tooling that
+        # exist in tree so we must hard code the hash
+        # XXX Changing this will break Chain of Trust without an associated puppet and
+        # scriptworker patch!
+        image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
+
+        maxRunTime: 1800
+
+        # TODO use mozilla-unified for the base repository once the tc-vcs
+        # tar.gz archives are created or tc-vcs isn't being used.
+        command:
+          - /builds/worker/bin/run-task
+          - '--vcs-checkout=/builds/worker/checkouts/gecko'
+          - '--sparse-profile=build/sparse-profiles/taskgraph'
+          - '--'
+          - bash
+          - -cx
+          - $let:
+              extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
+            # NOTE: the explicit reference to mozilla-central below is required because android-stuff
+            # still uses tc-vcs, which does not support mozilla-unified
+            # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
+            in:
+              $if: 'tasks_for == "action"'
+              then: >
+                cd /builds/worker/checkouts/gecko &&
+                ln -s /builds/worker/artifacts artifacts &&
+                ./mach --log-no-times taskgraph action-callback
+              else: >
+                cd /builds/worker/checkouts/gecko &&
+                ln -s /builds/worker/artifacts artifacts &&
+                ./mach --log-no-times taskgraph decision
+                --pushlog-id='${push.pushlog_id}'
+                --pushdate='${push.pushdate}'
+                --project='${repository.project}'
+                --message="$GECKO_COMMIT_MSG"
+                --owner='${ownerEmail}'
+                --level='${repository.level}'
+                --base-repository='https://hg.mozilla.org/mozilla-central'
+                --head-repository="$GECKO_HEAD_REPOSITORY"
+                --head-ref="$GECKO_HEAD_REF"
+                --head-rev="$GECKO_HEAD_REV"
+                ${extraArgs}
+
+        artifacts:
+          'public':
+            type: 'directory'
+            path: '/builds/worker/artifacts'
+            expires: {$fromNow: '1 year'}
+
+      extra:
+        $merge:
+          - treeherder:
+              $merge:
+                - machine:
+                    platform: gecko-decision
+                - $if: 'tasks_for == "hg-push"'
+                  then:
+                    symbol: D
+                  else:
+                    $if: 'tasks_for == "action"'
+                    then:
+                      groupName: 'action-callback'
+                      groupSymbol: AC
+                      symbol: "${action.symbol}"
+                    else:
+                      groupSymbol: cron
+                      symbol: "${cron.job_symbol}"
+          - $if: 'tasks_for == "action"'
+            then:
+              parent: '${action.taskGroupId}'
+              action:
+                name: '${action.name}'
+                context:
+                  taskGroupId: '${action.taskGroupId}'
+                  taskId: {$eval: 'taskId'}
+                  input: {$eval: 'input'}
+                  parameters: {$eval: 'parameters'}
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -31,8 +31,11 @@ glob:**/*.mozbuild
 
 # Tooltool manifests also need to be opened. Assume they
 # are all somewhere in "tooltool-manifests" directories.
 glob:**/tooltool-manifests/**
 
 # For scheduling android-gradle-dependencies.
 path:mobile/android/config/
 glob:**/*.gradle
+
+# for action-task building
+path:.taskcluster.yml
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -3,31 +3,27 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
-import inspect
 import re
 import yaml
 from slugid import nice as slugid
 from mozbuild.util import memoize
 from types import FunctionType
 from collections import namedtuple
-from taskgraph import create
+from taskgraph import create, GECKO
 from taskgraph.util import taskcluster
-from taskgraph.util.docker import docker_image
 from taskgraph.parameters import Parameters
 
 
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
-
 actions = []
 callbacks = {}
 
 Action = namedtuple('Action', [
     'name', 'title', 'description', 'order', 'context', 'schema', 'task_template_builder',
 ])
 
 
@@ -168,123 +164,65 @@ def register_callback_action(name, title
     mem = {"registered": False}  # workaround nonlocal missing in 2.x
 
     def register_callback(cb):
         assert isinstance(cb, FunctionType), 'callback must be a function'
         assert isinstance(symbol, basestring), 'symbol must be a string'
         assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters'
         assert not mem['registered'], 'register_callback_action must be used as decorator'
         assert cb.__name__ not in callbacks, 'callback name {} is not unique'.format(cb.__name__)
-        source_path = os.path.relpath(inspect.stack()[1][1], GECKO)
 
         @register_task_action(name, title, description, order, context, schema)
         def build_callback_action_task(parameters):
             if not available(parameters):
                 return None
 
             match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository'])
             if not match:
                 raise Exception('Unrecognized head_repository')
             repo_scope = 'assume:repo:{}/{}:*'.format(
                 match.group(1), match.group(2))
 
             task_group_id = os.environ.get('TASK_ID', slugid())
 
-            return {
-                'created': {'$fromNow': ''},
-                'deadline': {'$fromNow': '12 hours'},
-                'expires': {'$fromNow': '1 year'},
-                'metadata': {
-                    'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
-                    'source': '{}/raw-file/{}/{}'.format(
-                        parameters['head_repository'], parameters['head_rev'], source_path,
-                    ),
-                    'name': 'Action: {}'.format(title),
-                    'description': 'Task executing callback for action.\n\n---\n' + description,
-                },
-                'workerType': 'gecko-{}-decision'.format(parameters['level']),
-                'provisionerId': 'aws-provisioner-v1',
-                'taskGroupId': task_group_id,
-                'schedulerId': 'gecko-level-{}'.format(parameters['level']),
-                'scopes': [
-                    repo_scope,
-                ],
-                'tags': {
-                    'createdForUser': parameters['owner'],
-                    'kind': 'action-callback',
-                },
-                'routes': [
-                    'tc-treeherder.v2.{}.{}.{}'.format(
-                        parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
-                    'tc-treeherder-stage.v2.{}.{}.{}'.format(
-                        parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
-                    'index.gecko.v2.{}.pushlog-id.{}.actions.${{ownTaskId}}'.format(
-                        parameters['project'], parameters['pushlog_id'])
-                ],
-                'payload': {
-                    'env': {
-                        'GECKO_BASE_REPOSITORY': 'https://hg.mozilla.org/mozilla-unified',
-                        'GECKO_HEAD_REPOSITORY': parameters['head_repository'],
-                        'GECKO_HEAD_REF': parameters['head_ref'],
-                        'GECKO_HEAD_REV': parameters['head_rev'],
-                        'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
-                        'ACTION_TASK_GROUP_ID': task_group_id,
-                        'ACTION_TASK_ID': {'$json': {'$eval': 'taskId'}},
-                        'ACTION_TASK': {'$json': {'$eval': 'task'}},
-                        'ACTION_INPUT': {'$json': {'$eval': 'input'}},
-                        'ACTION_CALLBACK': cb.__name__,
-                        'ACTION_PARAMETERS': {'$json': {'$eval': 'parameters'}},
-                        'TASKCLUSTER_CACHES': '/builds/worker/checkouts',
-                    },
-                    'artifacts': {
-                        'public': {
-                            'type': 'directory',
-                            'path': '/builds/worker/artifacts',
-                            'expires': {'$fromNow': '1 year'},
+            template = os.path.join(GECKO, '.taskcluster.yml')
+
+            with open(template, 'r') as f:
+                taskcluster_yml = yaml.safe_load(f)
+                if taskcluster_yml['version'] != 1:
+                    raise Exception('actions.json must be updated to work with .taskcluster.yml')
+                if not isinstance(taskcluster_yml['tasks'], list):
+                    raise Exception('.taskcluster.yml "tasks" must be a list for action tasks')
+
+                return {
+                    '$let': {
+                        'tasks_for': 'action',
+                        'repository': {
+                            'url': parameters['head_repository'],
+                            'project': parameters['project'],
+                            'level': parameters['level'],
+                        },
+                        'push': {
+                            'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
+                            'pushlog_id': parameters['pushlog_id'],
+                            'revision': parameters['head_rev'],
+                        },
+                        'action': {
+                            'name': name,
+                            'title': title,
+                            'description': description,
+                            'taskGroupId': task_group_id,
+                            'repo_scope': repo_scope,
+                            'cb_name': cb.__name__,
+                            'symbol': symbol,
                         },
                     },
-                    'cache': {
-                        'level-{}-checkouts-sparse-v1'.format(parameters['level']):
-                            '/builds/worker/checkouts',
-                    },
-                    'features': {
-                        'taskclusterProxy': True,
-                        'chainOfTrust': True,
-                    },
-                    'image': docker_image('decision'),
-                    'maxRunTime': 1800,
-                    'command': [
-                        '/builds/worker/bin/run-task',
-                        '--vcs-checkout=/builds/worker/checkouts/gecko',
-                        '--sparse-profile=build/sparse-profiles/taskgraph',
-                        '--', 'bash', '-cx',
-                        """\
-cd /builds/worker/checkouts/gecko &&
-ln -s /builds/worker/artifacts artifacts &&
-./mach --log-no-times taskgraph action-callback""",
-                    ],
-                },
-                'extra': {
-                    'treeherder': {
-                        'groupName': 'action-callback',
-                        'groupSymbol': 'AC',
-                        'symbol': symbol,
-                    },
-                    'parent': task_group_id,
-                    'action': {
-                        'name': name,
-                        'context': {
-                            'taskGroupId': task_group_id,
-                            'taskId': {'$eval': 'taskId'},
-                            'input': {'$eval': 'input'},
-                            'parameters': {'$eval': 'parameters'},
-                        },
-                    },
-                },
-            }
+                    'in': taskcluster_yml['tasks'][0]
+                }
+
         mem['registered'] = True
         callbacks[cb.__name__] = cb
     return register_callback
 
 
 def render_actions_json(parameters):
     """
     Render JSON object for the ``public/actions.json`` artifact.