Bug 1381613 - Populate a repository-less index route for "trunk" repos; r=dustin
☠☠ backed out by a1ea69a3a0a7 ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Mon, 17 Jul 2017 13:28:48 -0700
changeset 418232 543c503f745c89360398fd3207ddaea5c262a807
parent 418231 f231c7e2d8c82e9158f3fa93db55aa695e55186e
child 418233 e3be5a6ffbcf737097479e5a317e04d60faa5923
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1381613
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1381613 - Populate a repository-less index route for "trunk" repos; r=dustin There are a few places where we walk commit ancestry looking for things attached to a specific revision. Because the repository name is attached to the index path and because a revision can exist in multiple repositories, we often have to perform N index lookups to find a result for a specific revision. This is inefficient. To facilitate faster index lookups by revision, we introduce a new route that doesn't contain the repository name. In theory, we should be able to do this globally - for all repos. However, the configuration of tasks can vary significantly by repo. So e.g. a linux64 build on "central" is sufficiently different from a linux64 build on "beta" or "release." For that reason, this commit takes the conservative approach and only defines a shared route for repositories with a similar configuration: the "trunk" repositories. MozReview-Commit-ID: 8rIgUbzW4eL
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -10,16 +10,17 @@ complexities of worker implementations, 
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
 import time
 from copy import deepcopy
 
+from taskgraph.util.attributes import TRUNK_PROJECTS
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import get_release_config
 from voluptuous import Any, Required, Optional, Extra
 from taskgraph import GECKO
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
@@ -471,16 +472,22 @@ UNKNOWN_GROUP_NAME = "Treeherder group {
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
+# {central, inbound, autoland} write to a "trunk" index prefix. This facilitates
+# walking of tasks with similar configurations.
+V2_TRUNK_ROUTE_TEMPLATES = [
+    "index.gecko.v2.trunk.revision.{head_rev}.{product}.{job-name}",
+]
+
 V2_NIGHTLY_TEMPLATES = [
     "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
 ]
 
 V2_L10N_TEMPLATES = [
@@ -815,19 +822,28 @@ def add_generic_index_routes(config, tas
         raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
 
     subs = config.params.copy()
     subs['job-name'] = job_name
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
+    project = config.params.get('project')
+
     for tpl in V2_ROUTE_TEMPLATES:
         routes.append(tpl.format(**subs))
 
+
+    # Additionally alias all tasks for "trunk" repos into a common
+    # namespace.
+    if project and project in TRUNK_PROJECTS:
+        for tpl in V2_TRUNK_ROUTE_TEMPLATES:
+            routes.append(tpl.format(**subs))
+
     return task
 
 
 @index_builder('nightly')
 def add_nightly_index_routes(config, task):
     index = task.get('index')
     routes = task.setdefault('routes', [])