Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin a=RyanVM
authorJustin Wood <Callek@gmail.com>
Mon, 09 Jan 2017 16:23:04 -0500
changeset 463197 b3774461acc6bee2216c5f57e167f9e5795fb09d
parent 463181 f94f1552ae0a2ef38684663552603595df863606
child 463198 0d84f70bec18e4acee507170509ca234dca2a301
child 463200 ad4fa9eeb1e9b44f52ac6248bed6b2475fc12d5e
child 463201 3eee272cff7f42a81efffdf6a9596700c087b399
child 463205 9ac765fbaf299293bf8723a1ef058b11f21d8fb2
child 463214 77e94e524e134ca65199d589ab80248212bc3e13
child 463218 2c979e6b8c031a47b0770352049243957227d53c
child 463230 53b30c74b62301e0fd68da4ef93758965ec895a1
child 463258 06614ab0872f9198c0b23c7a5fb714baabfc1bf1
child 463259 05e946a1ad2ddb52d2f9e00ec16fd4251c80ca75
child 463261 3938276fd4ec3375257ab7d75b9dcdb052bb8b35
child 463265 969cef63bbba17dac7ece5ae5c15669d9d951d03
child 463321 fd1eee0615a3dde85f2f3e5f7118cc744723c67a
child 463322 e06062e1fda9238379905a96338b6aa23bb4f5a2
child 463338 ad819c7358b6b0e3931753669c495d1609ba5e46
child 463339 7fed485ea9b627fc23b147ee85821980cc5970de
child 463395 ec0d862aa0fd7e54da9b20ddd137acfd0b5fed6f
child 463912 33349153d605ecfc8a6e08ae2066b80ca28c4681
child 464326 c5353485eaf46f84f14268b0dab4a35cd4a88296
child 464956 77aaf6209052fa7482d648e5f362378d822fd7d0
push id41978
push usereglassercamp@mozilla.com
push dateWed, 18 Jan 2017 16:41:50 +0000
reviewersdustin, RyanVM
bugs1301495
milestone53.0a1
Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin a=RyanVM Adds l10n and nightly indexing, matching (better) what Buildbot is currently doing with these types of tasks (This patch is against `date`, will be grafted on review for real landing, using autoland) MozReview-Commit-ID: K0BYwaCm6xL
taskcluster/ci/build/android.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -48,17 +48,18 @@ android-x86/opt:
         tooltool-downloads: internal
 
 android-x86-nightly/opt:
     description: "Android 4.2 x86 Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-x86-nightly-opt
+        job-name: android-x86-opt
+        type: nightly
     treeherder:
         platform: android-4-2-x86/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
@@ -102,17 +103,18 @@ android-api-15/opt:
         tooltool-downloads: internal
 
 android-api-15-nightly/opt:
     description: "Android 4.0 API15+ Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-api-15-nightly-opt
+        job-name: android-api-15-opt
+        type: nightly
     treeherder:
         platform: android-4-0-armv7-api15/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -150,17 +150,18 @@ linux/pgo:
         need-xvfb: true
 
 linux-nightly/opt:
     description: "Linux32 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux32-nightly-opt
+        job-name: linux-opt
+        type: nightly
     treeherder:
         platform: linux32/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
@@ -227,17 +228,18 @@ linux64-asan/debug:
         need-xvfb: true
 
 linux64-nightly/opt:
     description: "Linux64 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux64-nightly-opt
+        job-name: linux64-opt
+        type: nightly
     treeherder:
         platform: linux64/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -32,26 +32,27 @@ job-template:
       by-build-platform:
          default: 36000
          android-api-15-l10n: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-l10n: internal
    index:
+      type: l10n
       product:
          by-build-platform:
             default: firefox
             android-api-15-l10n: mobile
       job-name:
          by-build-platform:
-            linux-l10n: linux32-l10n-opt
-            linux64-l10n: linux64-l10n-opt
-            macosx64: macosx64-l10n-opt
-            android-api-15-l10n: android-l10n-opt
+            linux-l10n: linux-opt
+            linux64-l10n: linux64-opt
+            macosx64: macosx64-opt
+            android-api-15-l10n: android-api-15-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc(L10n)
       tier:
          by-build-platform:
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -31,27 +31,28 @@ job-template:
    run-time:
       by-build-platform:
          default: 36000
          android-api-15-nightly: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-nightly: internal
-#   index:
-#      product:
-#         by-build-platform:
-#            default: firefox
-#            android-api-15-nightly: mobile
-#      job-name:
-#         by-build-platform:
-#            linux-nightly: linux32-nightly-l10n-opt
-#            linux64-nightly: linux64-nightly-l10n-opt
-#            macosx64-nightly: macosx64-nightly-l10n-opt
-#            android-api-15-nightly: android-nightly-l10n-opt
+   index:
+      type: l10n
+      product:
+         by-build-platform:
+            default: firefox
+            android-api-15-nightly: mobile
+      job-name:
+         by-build-platform:
+            linux-nightly: linux-opt
+            linux64-nightly: linux64-opt
+            macosx64-nightly: macosx64-opt
+            android-api-15-nightly: android-api-15-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android-api-15-nightly: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc-L10n(N)
       tier:
          by-build-platform:
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -63,16 +63,19 @@ l10n_description_schema = Schema({
     },
     # Items for the taskcluster index
     Optional('index'): {
         # Product to identify as in the taskcluster index
         Required('product'): _by_platform(basestring),
 
         # Job name to identify as in the taskcluster index
         Required('job-name'): _by_platform(basestring),
+
+        # Type of index
+        Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
     # task object of the dependent task
     Required('dependent-task'): object,
 
     # worker-type to utilize
@@ -350,16 +353,17 @@ def make_job_description(config, jobs):
             },
             'run-on-projects': [],
         }
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
+                'type': job['index'].get('type', 'generic'),
             }
 
         if job.get('dependencies'):
             job_description['dependencies'] = job['dependencies']
         if job.get('env'):
             job_description['worker']['env'] = job['env']
         if job.get('when', {}).get('files-changed'):
             job_description.setdefault('when', {})
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -87,16 +87,19 @@ task_description_schema = Schema({
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any('firefox', 'mobile', 'static-analysis'),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
+        # Type of gecko v2 index to use
+        'type': Any('generic', 'nightly', 'l10n'),
+
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
             # builds below tier-1 in the index.
@@ -369,16 +372,29 @@ GROUP_NAMES = {
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
+V2_NIGHTLY_TEMPLATES = [
+    "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
+]
+
+V2_L10N_TEMPLATES = [
+    "index.gecko.v2.{project}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.latest.{product}-l10n.{job-name}.{locale}",
+]
+
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
 COALESCE_KEY = 'builds.{project}.{name}'
 
@@ -387,16 +403,26 @@ payload_builders = {}
 
 
 def payload_builder(name):
     def wrap(func):
         payload_builders[name] = func
         return func
     return wrap
 
+# define a collection of index builders, depending on the type implementation
+index_builders = {}
+
+
+def index_builder(name):
+    def wrap(func):
+        index_builders[name] = func
+        return func
+    return wrap
+
 
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
 
     image = worker['docker-image']
     if isinstance(image, dict):
         docker_image_task = 'build-docker-image-' + image['in-tree']
@@ -589,38 +615,104 @@ transforms = TransformSequence()
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             task_description_schema, task,
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
+@index_builder('generic')
+def add_generic_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    for tpl in V2_ROUTE_TEMPLATES:
+        routes.append(tpl.format(**subs))
+
+    return task
+
+
+@index_builder('nightly')
+def add_nightly_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['build_date'] = time.strftime("%Y.%m.%d",
+                                       time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    for tpl in V2_NIGHTLY_TEMPLATES:
+        routes.append(tpl.format(**subs))
+
+    return task
+
+
+@index_builder('l10n')
+def add_l10n_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    locales = task['attributes'].get('chunk_locales',
+                                     task['attributes'].get('all_locales'))
+
+    if not locales:
+        raise Exception("Error: Unable to use l10n index for tasks without locales")
+
+    # If there are too many locales, we can't write a route for all of them
+    # See Bug 1323792
+    if len(locales) > 18:  # 18 * 3 = 54, max routes = 64
+        return task
+
+    for locale in locales:
+        for tpl in V2_L10N_TEMPLATES:
+            routes.append(tpl.format(locale=locale, **subs))
+
+    return task
+
+
 @transforms.add
 def add_index_routes(config, tasks):
     for task in tasks:
         index = task.get('index')
-        routes = task.setdefault('routes', [])
 
         if not index:
             yield task
             continue
 
-        job_name = index['job-name']
-        if job_name not in JOB_NAME_WHITELIST:
-            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
-        subs = config.params.copy()
-        subs['job-name'] = job_name
-        subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
-                                                time.gmtime(config.params['build_date']))
-        subs['product'] = index['product']
-
-        for tpl in V2_ROUTE_TEMPLATES:
-            routes.append(tpl.format(**subs))
+        index_type = index.get('type', 'generic')
+        task = index_builders[index_type](config, task)
 
         # The default behavior is to rank tasks according to their tier
         extra_index = task.setdefault('extra', {}).setdefault('index', {})
         rank = index.get('rank', 'by-tier')
 
         if rank == 'by-tier':
             # rank is zero for non-tier-1 tasks and based on pushid for others;
             # this sorts tier-{2,3} builds below tier-1 in the index
@@ -724,24 +816,33 @@ def build_task(config, tasks):
 
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open("testing/mozharness/configs/routes.json", "rb") as f:
         routes_json = json.load(f)
 
-    # we only deal with the 'routes' key here
-    routes = routes_json['routes']
+    for key in ('routes', 'nightly', 'l10n'):
+        if key == 'routes':
+            tc_template = V2_ROUTE_TEMPLATES
+        elif key == 'nightly':
+            tc_template = V2_NIGHTLY_TEMPLATES
+        elif key == 'l10n':
+            tc_template = V2_L10N_TEMPLATES
+
+        routes = routes_json[key]
 
-    # we use different variables than mozharness
-    for mh, tg in [
-            ('{index}', 'index'),
-            ('{build_product}', '{product}'),
-            ('{build_name}-{build_type}', '{job-name}'),
-            ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
-        routes = [r.replace(mh, tg) for r in routes]
+        # we use different variables than mozharness
+        for mh, tg in [
+                ('{index}', 'index'),
+                ('{build_product}', '{product}'),
+                ('{build_name}-{build_type}', '{job-name}'),
+                ('{year}.{month}.{day}.{pushdate}', '{build_date_long}'),
+                ('{year}.{month}.{day}', '{build_date}')]:
+            routes = [r.replace(mh, tg) for r in routes]
 
-    if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
-        raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
-                        "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
+        if sorted(routes) != sorted(tc_template):
+            raise Exception("V2 TEMPLATES do not match Mozharness's routes.json: "
+                            "(tc):%s vs (mh):%s" % (tc_template, routes))
+
 
 check_v2_routes()