Bug 1280129 - Update tasks to use v2 treeherder routes r=dustin
authorGregory Arndt <garndt@mozilla.com>
Tue, 14 Jun 2016 12:56:25 -0500
changeset 301868 099196831d4cf158176631064079088d84abf8db
parent 301867 b850837edec241b11e9922b382d9178b265e55b8
child 301869 f3d5b3e79015645ae443a9841c8914cd9569ff61
push id78486
push usergarndt@mozilla.com
push dateWed, 15 Jun 2016 14:37:33 +0000
treeherdermozilla-inbound@099196831d4c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1280129
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1280129 - Update tasks to use v2 treeherder routes r=dustin Jobs reporting to treeherder should rely on the task route for project, revision, and pushlog ID rather than things stuffed into task.extra.treeherder. This also removes the need for a revision_hash that was calculated by mozilla-taskcluster. MozReview-Commit-ID: EcQM9QRZzgG
.taskcluster.yml
taskcluster/ci/docker-image/image.yml
taskcluster/ci/legacy/tasks/decision/branch.yml
taskcluster/ci/legacy/tasks/decision/try.yml
taskcluster/docs/parameters.rst
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/legacy.py
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -55,18 +55,18 @@ tasks:
         - "docker-worker:cache:level-{{level}}-*"
         - "docker-worker:cache:tooltool-cache"
         - "secrets:get:project/taskcluster/gecko/hgfingerprint"
         # mozilla-taskcluster will append the appropriate assume:repo:<repo>
         # scope here.
 
       routes:
         - "index.gecko.v2.{{project}}.latest.firefox.decision"
-        - "tc-treeherder.{{project}}.{{revision_hash}}"
-        - "tc-treeherder-stage.{{project}}.{{revision_hash}}"
+        - "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"
+        - "tc-treeherder-stage.v2.{{project}}.{{revision}}.{{pushlog_id}}"
 
       payload:
         env:
           # checkout-gecko uses these to check out the source; the inputs
           # to `mach taskgraph decision` are all on the command line.
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
--- a/taskcluster/ci/docker-image/image.yml
+++ b/taskcluster/ci/docker-image/image.yml
@@ -13,18 +13,18 @@ task:
   workerType: taskcluster-images
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   routes:
       - index.docker.images.v1.{{project}}.{{image_name}}.latest
       - index.docker.images.v1.{{project}}.{{image_name}}.pushdate.{{year}}.{{month}}-{{day}}-{{pushtime}}
       - index.docker.images.v1.{{project}}.{{image_name}}.hash.{{context_hash}}
-      - tc-treeherder.{{project}}.{{revision_hash}}
-      - tc-treeherder-stage.{{project}}.{{revision_hash}}
+      - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
+      - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
 
   payload:
     env:
       HASH: '{{context_hash}}'
       PROJECT: '{{project}}'
       CONTEXT_URL: '{{context_url}}'
       CONTEXT_PATH: '{{context_path}}'
       BASE_REPOSITORY: '{{base_repository}}'
@@ -45,14 +45,12 @@ task:
         path: '/artifacts/image.tar'
         expires: '{{#from_now}}1 year{{/from_now}}'
   extra:
     treeherderEnv:
       - staging
       - production
     treeherder:
       jobKind: other
-      revision: {{head_rev}}
-      revision_hash: {{revision_hash}}
       build:
         platform: 'taskcluster-images'
       symbol: 'I'
 
--- a/taskcluster/ci/legacy/tasks/decision/branch.yml
+++ b/taskcluster/ci/legacy/tasks/decision/branch.yml
@@ -12,18 +12,18 @@ scopes:
   - queue:*
   - docker-worker:*
   - scheduler:*
 tasks:
   - taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
     task:
       routes:
         - index.gecko.v2.{{project}}.latest.firefox.decision
-        - tc-treeherder-stage.{{project}}.{{revision_hash}}
-        - tc-treeherder.{{project}}.{{revision_hash}}
+        - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
+        - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
       created: '{{now}}'
       deadline: '{{#from_now}}1 day{{/from_now}}'
       metadata:
         owner: mozilla-taskcluster-maintenance@mozilla.com
         source: {{{source}}}
         name: Initial decision task for {{project}} specific graphs
         description: |
           This is the single most important task as it decides how all other tasks
@@ -74,26 +74,23 @@ tasks:
             checkout-gecko workspace &&
             cd workspace/gecko &&
             ./mach taskcluster-graph
             --pushlog-id='{{pushlog_id}}'
             --project='{{project}}'
             --message='{{comment}}'
             --owner='{{owner}}'
             --level='{{level}}'
-            --revision-hash='{{revision_hash}}'
             --extend-graph > /home/worker/artifacts/graph.json &&
             cp /home/worker/artifacts/graph.json /home/worker/artifacts/all_tasks.json
         graphs:
           - /home/worker/artifacts/graph.json
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             # Arbitrary value for keeping these artifacts around.  They are just the
             # graph.json and context directories for now, so nothing that needs
             # to stay around for long.
             expires: '{{#from_now}}7 days{{/from_now}}'
       extra:
         treeherder:
           symbol: D
-          revision: '{{revision}}'
-          revision_hash: '{{revision_hash}}'
--- a/taskcluster/ci/legacy/tasks/decision/try.yml
+++ b/taskcluster/ci/legacy/tasks/decision/try.yml
@@ -36,18 +36,18 @@ tasks:
         # Bug 1269443: cache scopes, etc. must be listed explicitly
         - "docker-worker:cache:level-1-*"
         - "docker-worker:cache:tooltool-cache"
         - "secrets:get:project/taskcluster/gecko/hgfingerprint"
         - "assume:repo:hg.mozilla.org/try:*"
 
       routes:
         - "index.gecko.v2.{{project}}.latest.firefox.decision"
-        - "tc-treeherder.{{project}}.{{revision_hash}}"
-        - "tc-treeherder-stage.{{project}}.{{revision_hash}}"
+        - "tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
+        - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
 
       payload:
         env:
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
 
@@ -85,21 +85,18 @@ tasks:
             --project='{{project}}'
             --message='{{comment}}'
             --owner='{{owner}}'
             --level='{{level}}'
             --base-repository='https://hg.mozilla.org/mozilla-central'
             --head-repository='{{{url}}}'
             --head-ref='{{revision}}'
             --head-rev='{{revision}}'
-            --revision-hash='{{revision_hash}}'
 
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             expires: '{{#from_now}}7 days{{/from_now}}'
 
       extra:
         treeherder:
           symbol: D
-          revision: '{{revision}}'
-          revision_hash: '{{revision_hash}}'
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -30,19 +30,16 @@ Push Information
    The revision to check out; this can be a short revision string
 
 ``head_ref``
    For Mercurial repositories, this is the same as ``head_rev``.  For
    git repositories, which do not allow pulling explicit revisions, this gives
    the symbolic ref containing ``head_rev`` that should be pulled from
    ``head_repository``.
 
-``revision_hash``
-   The full-length revision string
-
 ``owner``
    Email address indicating the person who made the push.  Note that this
    value may be forged and *must not* be relied on for authentication.
 
 ``message``
    The commit message
 
 ``pushlog_id``
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -85,17 +85,16 @@ def get_decision_parameters(options):
     This also applies per-project parameters, based on the given project.
 
     """
     parameters = {n: options[n] for n in [
         'base_repository',
         'head_repository',
         'head_rev',
         'head_ref',
-        'revision_hash',
         'message',
         'project',
         'pushlog_id',
         'owner',
         'level',
         'target_tasks_method',
     ] if n in options}
 
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -32,32 +32,32 @@ INDEX_URL = 'https://index.taskcluster.n
 
 class DockerImageKind(base.Kind):
 
     def load_tasks(self, params):
         # TODO: make this match the pushdate (get it from a parameter rather than vcs)
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
 
         parameters = {
+            'pushlog_id': params.get('pushlog_id', 0),
             'pushdate': pushdate,
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
             'month': pushdate[4:6],
             'day': pushdate[6:8],
             'project': params['project'],
             'docker_image': docker_image,
             'base_repository': params['base_repository'] or params['head_repository'],
             'head_repository': params['head_repository'],
             'head_ref': params['head_ref'] or params['head_rev'],
             'head_rev': params['head_rev'],
             'owner': params['owner'],
             'level': params['level'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
-            'revision_hash': params['revision_hash'],
             'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
                     .format(repo=params['head_repository'], rev=params['head_rev']),
         }
 
         tasks = []
         templates = Templates(self.path)
         for image_name in self.config['images']:
             context_path = os.path.join('testing', 'docker', image_name)
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -34,17 +34,16 @@ GECKO = os.path.realpath(os.path.join(RO
 TASKID_PLACEHOLDER = 'TaskLabel=={}'
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
 DEFAULT_TRY = 'try: -b do -p all -u all -t all'
 DEFAULT_JOB_PATH = os.path.join(
     'tasks', 'branches', 'base_jobs.yml'
 )
 
-TREEHERDER_ROUTE_PREFIX = 'tc-treeherder-stage'
 TREEHERDER_ROUTES = {
     'staging': 'tc-treeherder-stage',
     'production': 'tc-treeherder'
 }
 
 # time after which a try build's results will expire
 TRY_EXPIRATION = "14 days"
 
@@ -205,45 +204,48 @@ def set_expiration(task, timestamp):
         return
 
     # for docker-worker, artifacts is a dictionary
     # for generic-worker, artifacts is a list
     # for taskcluster-worker, it will depend on what we do in artifacts plugin
     for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
         artifact['expires'] = timestamp
 
-def add_treeherder_revision_info(task, revision, revision_hash):
-    # Only add treeherder information if task.extra.treeherder is present
-    if 'extra' not in task and 'treeherder' not in task.extra:
-        return
+def format_treeherder_route(destination, project, revision, pushlog_id):
+    return "{}.v2.{}.{}.{}".format(destination,
+                                   project,
+                                   revision,
+                                   pushlog_id)
 
-    task['extra']['treeherder']['revision'] = revision
-    task['extra']['treeherder']['revision_hash'] = revision_hash
-
-
-def decorate_task_treeherder_routes(task, suffix):
+def decorate_task_treeherder_routes(task, project, revision, pushlog_id):
     """Decorate the given task with treeherder routes.
 
     Uses task.extra.treeherderEnv if available otherwise defaults to only
     staging.
 
     :param dict task: task definition.
-    :param str suffix: The project/revision_hash portion of the route.
+    :param str project: The project the tasks are running for.
+    :param str revision: The revision for the push
+    :param str pushlog_id: The ID of the push
     """
 
     if 'extra' not in task:
         return
 
     if 'routes' not in task:
         task['routes'] = []
 
     treeheder_env = task['extra'].get('treeherderEnv', ['staging'])
 
     for env in treeheder_env:
-        task['routes'].append('{}.{}'.format(TREEHERDER_ROUTES[env], suffix))
+        route = format_treeherder_route(TREEHERDER_ROUTES[env],
+                                        project,
+                                        revision,
+                                        pushlog_id)
+        task['routes'].append(route)
 
 def decorate_task_json_routes(task, json_routes, parameters):
     """Decorate the given task with routes.json routes.
 
     :param dict task: task definition.
     :param json_routes: the list of routes to use from routes.json
     :param parameters: dictionary of parameters to use in route templates
     """
@@ -337,42 +339,36 @@ class LegacyKind(base.Kind):
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
             'month': pushdate[4:6],
             'day': pushdate[6:8],
             'owner': params['owner'],
             'level': params['level'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
-            'revision_hash': params['revision_hash']
         }.items())
 
-        treeherder_route = '{}.{}'.format(
-            params['project'],
-            params.get('revision_hash', '')
-        )
-
         routes_file = os.path.join(root, 'routes.json')
         with open(routes_file) as f:
             contents = json.load(f)
             json_routes = contents['routes']
             # TODO: Nightly and/or l10n routes
 
         # Task graph we are generating for taskcluster...
         graph = {
             'tasks': [],
             'scopes': set(),
         }
 
-        if params['revision_hash']:
-            for env in TREEHERDER_ROUTES:
-                route = 'queue:route:{}.{}'.format(
-                    TREEHERDER_ROUTES[env],
-                    treeherder_route)
-                graph['scopes'].add(route)
+        for env in TREEHERDER_ROUTES:
+            route = format_treeherder_route(TREEHERDER_ROUTES[env],
+                                            parameters['project'],
+                                            parameters['head_rev'],
+                                            parameters['pushlog_id'])
+            graph['scopes'].add("queue:route:{}".format(route))
 
         graph['metadata'] = {
             'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
             'owner': params['owner'],
             # TODO: Add full mach commands to this example?
             'description': 'Task graph generated via ./mach taskcluster-graph',
             'name': 'task graph local'
         }
@@ -433,25 +429,23 @@ class LegacyKind(base.Kind):
 
             set_interactive_task(build_task, interactive)
 
             # try builds don't use cache
             if project == "try":
                 remove_caches_from_task(build_task)
                 set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
 
-            if params['revision_hash']:
-                add_treeherder_revision_info(build_task['task'],
-                                             params['head_rev'],
-                                             params['revision_hash'])
-                decorate_task_treeherder_routes(build_task['task'],
-                                                treeherder_route)
-                decorate_task_json_routes(build_task['task'],
-                                          json_routes,
-                                          build_parameters)
+            decorate_task_treeherder_routes(build_task['task'],
+                                            build_parameters['project'],
+                                            build_parameters['head_rev'],
+                                            build_parameters['pushlog_id'])
+            decorate_task_json_routes(build_task['task'],
+                                      json_routes,
+                                      build_parameters)
 
             # Ensure each build graph is valid after construction.
             validate_build_task(build_task)
             attributes = build_task['attributes'] = {'kind':'legacy', 'legacy_kind': 'build'}
             if 'build_name' in build:
                 attributes['build_platform'] = build['build_name']
             if 'build_type' in task_extra:
                 attributes['build_type'] = {'dbg': 'debug'}.get(task_extra['build_type'],
@@ -510,19 +504,16 @@ class LegacyKind(base.Kind):
                 post_parameters = merge_dicts(build_parameters,
                                               post_build.get('additional-parameters', {}))
                 post_task = configure_dependent_task(post_build['task'],
                                                      post_parameters,
                                                      mklabel(),
                                                      templates,
                                                      build_treeherder_config)
                 set_interactive_task(post_task, interactive)
-                add_treeherder_revision_info(post_task['task'],
-                                             params['head_rev'],
-                                             params['revision_hash'])
 
                 if project == "try":
                     set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
 
                 post_task['attributes'] = attributes.copy()
                 post_task['attributes']['legacy_kind'] = 'post_build'
                 post_task['attributes']['post_build'] = post_build['job_flag']
                 graph['tasks'].append(post_task)
@@ -560,24 +551,20 @@ class LegacyKind(base.Kind):
                         test_parameters['chunk'] = chunk
                     test_task = configure_dependent_task(test['task'],
                                                          test_parameters,
                                                          mklabel(),
                                                          templates,
                                                          build_treeherder_config)
                     set_interactive_task(test_task, interactive)
 
-                    if params['revision_hash']:
-                        add_treeherder_revision_info(test_task['task'],
-                                                     params['head_rev'],
-                                                     params['revision_hash'])
-                        decorate_task_treeherder_routes(
-                            test_task['task'],
-                            treeherder_route
-                        )
+                    decorate_task_treeherder_routes(test_task['task'],
+                                                    test_parameters['project'],
+                                                    test_parameters['head_rev'],
+                                                    test_parameters['pushlog_id'])
 
                     if project == "try":
                         set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
 
                     test_task['attributes'] = attributes.copy()
                     test_task['attributes']['legacy_kind'] = 'unittest'
                     test_task['attributes']['test_platform'] = attributes['build_platform']
                     test_task['attributes']['unittest_try_name'] = test['unittest_try_name']