Bug 1486970 - Create revision and pushlog-id index routes for cron decision tasks. r=dustin,aki
authorTom Prince <mozilla@hocat.ca>
Tue, 27 Nov 2018 22:33:09 +0000
changeset 448539 e75b3f5571fe4127b09b468cc8c7dacb6d822863
parent 448538 b59f4aa8d4a32212a3cedeb8779c13fda9863569
child 448540 11828ed488177ec28bfb343c64b466785600fab6
push id35119
push userccoroiu@mozilla.com
push dateThu, 29 Nov 2018 04:26:53 +0000
treeherdermozilla-central@c35dea45f131 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, aki
bugs1486970
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1486970 - Create revision and pushlog-id index routes for cron decision tasks. r=dustin,aki This changeset actually creates the new routes we want for decision tasks, and adds the supporting code for passing push info in to the taskgraph. This must land after scriptworker is updated and deployed. This has the affect of also un-hardcoding the .taskcluster.yml fields applied in the first part of this bug. This changeset in particular will ride trains and not be uplifted (outside of compelling reasons). Differential Revision: https://phabricator.services.mozilla.com/D5195
.taskcluster.yml
taskcluster/taskgraph/cron/decision.py
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -8,22 +8,16 @@ tasks:
   # takes place.
   - $if: 'tasks_for in ["hg-push", "action", "cron"]'
     then:
       $let:
         # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
         ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
         # ensure there's no trailing `/` on the repo URL
         repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
-        # Hardcode cron push info for now, so that we can transition to using real values without breaking callers of Chain of Trust
-        _pushId: {$if: 'tasks_for == "cron"', then: '-1', else: {$eval: 'push.pushlog_id'}}
-        # action tasks can fail because of no pushdate or push comment information in context, so include them in
-        # hardcodes (even though they don't use these variables)
-        _pushDate: {$if: 'tasks_for == "cron" || tasks_for == "action"', then: '0', else: {$eval: 'push.pushdate'}}
-        _pushComment: {$if: 'tasks_for == "cron" || tasks_for == "action"', then: '', else: {$eval: 'push.comment'}}
       in:
         taskId: {$if: 'tasks_for != "action"', then: '${as_slugid("decision")}'}
         taskGroupId:
           $if: 'tasks_for == "action"'
           then:
             '${action.taskGroupId}'
           else:
             '${as_slugid("decision")}' # same as taskId; this is how automation identifies a decision tsak
@@ -64,22 +58,22 @@ tasks:
               kind: 'action-callback'
             else:
               $if: 'tasks_for == "cron"'
               then:
                 kind: cron-task
 
         routes:
           $flatten:
-            - "tc-treeherder.v2.${repository.project}.${push.revision}.${_pushId}"
+            - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
             - $if: 'tasks_for == "hg-push"'
               then:
                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision"
                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
-                - "index.gecko.v2.${repository.project}.pushlog-id.${_pushId}.decision"
+                - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
                 - "notify.email.${ownerEmail}.on-failed"
                 - "notify.email.${ownerEmail}.on-exception"
                 # Send a notification email if the push comes from try
                 - $if: 'repository.project == "try"'
                   then:
                     "notify.email.${ownerEmail}.on-completed"
                 # BUG 1500166 Notify ciduty by email if a nightly hook fails
                 - "notify.email.ciduty+failedcron@mozilla.com.on-failed"
@@ -90,19 +84,21 @@ tasks:
                 # They are still here so external tools that referenced them continue to work.
                 - "index.gecko.v2.${repository.project}.latest.firefox.decision"
                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.firefox.decision"
               else:
                 $if: 'tasks_for == "action"'
                 then:
                 - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-failed"
                 - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-exception"
-                - "index.gecko.v2.${repository.project}.pushlog-id.${_pushId}.actions.${ownTaskId}"
+                - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
                 else:  # cron
                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
+                - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
+                - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
                 # list each cron task on this revision, so actions can find them
                 - 'index.gecko.v2.${repository.project}.revision.${push.revision}.cron.${as_slugid("decision")}'
                 # These are the old index routes for the decision task.
                 - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
 
         scopes:
           $if: 'tasks_for == "hg-push"'
           then:
@@ -142,17 +138,17 @@ tasks:
           env:
             # checkout-gecko uses these to check out the source; the inputs
             # to `mach taskgraph decision` are all on the command line.
             $merge:
               - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
                 GECKO_HEAD_REPOSITORY: '${repoUrl}'
                 GECKO_HEAD_REF: '${push.revision}'
                 GECKO_HEAD_REV: '${push.revision}'
-                GECKO_COMMIT_MSG: {$if: 'tasks_for != "action"', then: '${_pushComment}'}
+                GECKO_COMMIT_MSG: {$if: 'tasks_for != "action"', then: '${push.comment}'}
                 HG_STORE_PATH: /builds/worker/checkouts/hg-store
                 TASKCLUSTER_CACHES: /builds/worker/checkouts
               - $if: 'tasks_for == "action"'
                 then:
                   ACTION_TASK_GROUP_ID: '${action.taskGroupId}'     # taskGroupId of the target task
                   ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
                   ACTION_INPUT: {$json: {$eval: 'input'}}
                   ACTION_CALLBACK: '${action.cb_name}'
@@ -185,18 +181,18 @@ tasks:
                 then: >
                   cd /builds/worker/checkouts/gecko &&
                   ln -s /builds/worker/artifacts artifacts &&
                   ./mach --log-no-times taskgraph action-callback
                 else: >
                   cd /builds/worker/checkouts/gecko &&
                   ln -s /builds/worker/artifacts artifacts &&
                   ./mach --log-no-times taskgraph decision
-                  --pushlog-id='${_pushId}'
-                  --pushdate='${_pushDate}'
+                  --pushlog-id='${push.pushlog_id}'
+                  --pushdate='${push.pushdate}'
                   --project='${repository.project}'
                   --message="$GECKO_COMMIT_MSG"
                   --owner='${ownerEmail}'
                   --level='${repository.level}'
                   --base-repository="$GECKO_BASE_REPOSITORY"
                   --head-repository="$GECKO_HEAD_REPOSITORY"
                   --head-ref="$GECKO_HEAD_REF"
                   --head-rev="$GECKO_HEAD_REV"
--- a/taskcluster/taskgraph/cron/decision.py
+++ b/taskcluster/taskgraph/cron/decision.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 import jsone
 import pipes
 import yaml
 import os
 import slugid
 
 from taskgraph.util.time import current_json_time
+from taskgraph.util.hg import find_hg_revision_push_info
 
 
 def run_decision_task(job, params, root):
     arguments = []
     if 'target-tasks-method' in job:
         arguments.append('--target-tasks-method={}'.format(job['target-tasks-method']))
     return [
         make_decision_task(
@@ -31,16 +32,19 @@ def run_decision_task(job, params, root)
 
 def make_decision_task(params, root, symbol, arguments=[], head_rev=None):
     """Generate a basic decision task, based on the root .taskcluster.yml"""
     with open(os.path.join(root, '.taskcluster.yml'), 'rb') as f:
         taskcluster_yml = yaml.safe_load(f)
 
     if not head_rev:
         head_rev = params['head_rev']
+    push_info = find_hg_revision_push_info(
+        params['repository_url'],
+        params['head_rev'])
 
     slugids = {}
 
     def as_slugid(name):
         # https://github.com/taskcluster/json-e/issues/164
         name = name[0]
         if name not in slugids:
             slugids[name] = slugid.nice()
@@ -54,18 +58,18 @@ def make_decision_task(params, root, sym
         'repository': {
             'url': params['repository_url'],
             'project': params['project'],
             'level': params['level'],
         },
         'push': {
             'revision': params['head_rev'],
             # remainder are fake values, but the decision task expects them anyway
-            'pushlog_id': -1,
-            'pushdate': 0,
+            'pushlog_id': push_info['pushid'],
+            'pushdate': push_info['pushdate'],
             'owner': 'cron',
             'comment': '',
         },
         'cron': {
             'task_id': os.environ.get('TASK_ID', '<cron task id>'),
             'job_name': params['job_name'],
             'job_symbol': symbol,
             # args are shell-quoted since they are given to `bash -c`