Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin a=RyanVM
authorJustin Wood <Callek@gmail.com>
Mon, 09 Jan 2017 16:23:04 -0500
changeset 377163 b3774461acc6bee2216c5f57e167f9e5795fb09d
parent 377162 f94f1552ae0a2ef38684663552603595df863606
child 377164 3eee272cff7f42a81efffdf6a9596700c087b399
child 391110 c5353485eaf46f84f14268b0dab4a35cd4a88296
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, RyanVM
bugs1301495
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin a=RyanVM Adds l10n and nightly indexing, matching (better) what Buildbot is currently doing with these types of tasks (This patch is against `date`, will be grafted on review for real landing, using autoland) MozReview-Commit-ID: K0BYwaCm6xL
taskcluster/ci/build/android.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -48,17 +48,18 @@ android-x86/opt:
         tooltool-downloads: internal
 
 android-x86-nightly/opt:
     description: "Android 4.2 x86 Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-x86-nightly-opt
+        job-name: android-x86-opt
+        type: nightly
     treeherder:
         platform: android-4-2-x86/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
@@ -102,17 +103,18 @@ android-api-15/opt:
         tooltool-downloads: internal
 
 android-api-15-nightly/opt:
     description: "Android 4.0 API15+ Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-api-15-nightly-opt
+        job-name: android-api-15-opt
+        type: nightly
     treeherder:
         platform: android-4-0-armv7-api15/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -150,17 +150,18 @@ linux/pgo:
         need-xvfb: true
 
 linux-nightly/opt:
     description: "Linux32 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux32-nightly-opt
+        job-name: linux-opt
+        type: nightly
     treeherder:
         platform: linux32/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
@@ -227,17 +228,18 @@ linux64-asan/debug:
         need-xvfb: true
 
 linux64-nightly/opt:
     description: "Linux64 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux64-nightly-opt
+        job-name: linux64-opt
+        type: nightly
     treeherder:
         platform: linux64/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -32,26 +32,27 @@ job-template:
       by-build-platform:
          default: 36000
          android-api-15-l10n: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-l10n: internal
    index:
+      type: l10n
       product:
          by-build-platform:
             default: firefox
             android-api-15-l10n: mobile
       job-name:
          by-build-platform:
-            linux-l10n: linux32-l10n-opt
-            linux64-l10n: linux64-l10n-opt
-            macosx64: macosx64-l10n-opt
-            android-api-15-l10n: android-l10n-opt
+            linux-l10n: linux-opt
+            linux64-l10n: linux64-opt
+            macosx64: macosx64-opt
+            android-api-15-l10n: android-api-15-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc(L10n)
       tier:
          by-build-platform:
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -31,27 +31,28 @@ job-template:
    run-time:
       by-build-platform:
          default: 36000
          android-api-15-nightly: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-nightly: internal
-#   index:
-#      product:
-#         by-build-platform:
-#            default: firefox
-#            android-api-15-nightly: mobile
-#      job-name:
-#         by-build-platform:
-#            linux-nightly: linux32-nightly-l10n-opt
-#            linux64-nightly: linux64-nightly-l10n-opt
-#            macosx64-nightly: macosx64-nightly-l10n-opt
-#            android-api-15-nightly: android-nightly-l10n-opt
+   index:
+      type: l10n
+      product:
+         by-build-platform:
+            default: firefox
+            android-api-15-nightly: mobile
+      job-name:
+         by-build-platform:
+            linux-nightly: linux-opt
+            linux64-nightly: linux64-opt
+            macosx64-nightly: macosx64-opt
+            android-api-15-nightly: android-api-15-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android-api-15-nightly: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc-L10n(N)
       tier:
          by-build-platform:
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -63,16 +63,19 @@ l10n_description_schema = Schema({
     },
     # Items for the taskcluster index
     Optional('index'): {
         # Product to identify as in the taskcluster index
         Required('product'): _by_platform(basestring),
 
         # Job name to identify as in the taskcluster index
         Required('job-name'): _by_platform(basestring),
+
+        # Type of index
+        Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
     # task object of the dependent task
     Required('dependent-task'): object,
 
     # worker-type to utilize
@@ -350,16 +353,17 @@ def make_job_description(config, jobs):
             },
             'run-on-projects': [],
         }
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
+                'type': job['index'].get('type', 'generic'),
             }
 
         if job.get('dependencies'):
             job_description['dependencies'] = job['dependencies']
         if job.get('env'):
             job_description['worker']['env'] = job['env']
         if job.get('when', {}).get('files-changed'):
             job_description.setdefault('when', {})
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -87,16 +87,19 @@ task_description_schema = Schema({
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any('firefox', 'mobile', 'static-analysis'),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
+        # Type of gecko v2 index to use
+        'type': Any('generic', 'nightly', 'l10n'),
+
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
             # builds below tier-1 in the index.
@@ -369,16 +372,29 @@ GROUP_NAMES = {
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
+V2_NIGHTLY_TEMPLATES = [
+    "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
+]
+
+V2_L10N_TEMPLATES = [
+    "index.gecko.v2.{project}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.latest.{product}-l10n.{job-name}.{locale}",
+]
+
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
 COALESCE_KEY = 'builds.{project}.{name}'
 
@@ -387,16 +403,26 @@ payload_builders = {}
 
 
 def payload_builder(name):
     def wrap(func):
         payload_builders[name] = func
         return func
     return wrap
 
+# define a collection of index builders, depending on the type implementation
+index_builders = {}
+
+
+def index_builder(name):
+    def wrap(func):
+        index_builders[name] = func
+        return func
+    return wrap
+
 
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
 
     image = worker['docker-image']
     if isinstance(image, dict):
         docker_image_task = 'build-docker-image-' + image['in-tree']
@@ -589,38 +615,104 @@ transforms = TransformSequence()
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             task_description_schema, task,
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
+@index_builder('generic')
+def add_generic_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    for tpl in V2_ROUTE_TEMPLATES:
+        routes.append(tpl.format(**subs))
+
+    return task
+
+
+@index_builder('nightly')
+def add_nightly_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['build_date'] = time.strftime("%Y.%m.%d",
+                                       time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    for tpl in V2_NIGHTLY_TEMPLATES:
+        routes.append(tpl.format(**subs))
+
+    return task
+
+
+@index_builder('l10n')
+def add_l10n_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    locales = task['attributes'].get('chunk_locales',
+                                     task['attributes'].get('all_locales'))
+
+    if not locales:
+        raise Exception("Error: Unable to use l10n index for tasks without locales")
+
+    # If there are too many locales, we can't write a route for all of them
+    # See Bug 1323792
+    if len(locales) > 18:  # 18 * 3 = 54, max routes = 64
+        return task
+
+    for locale in locales:
+        for tpl in V2_L10N_TEMPLATES:
+            routes.append(tpl.format(locale=locale, **subs))
+
+    return task
+
+
 @transforms.add
 def add_index_routes(config, tasks):
     for task in tasks:
         index = task.get('index')
-        routes = task.setdefault('routes', [])
 
         if not index:
             yield task
             continue
 
-        job_name = index['job-name']
-        if job_name not in JOB_NAME_WHITELIST:
-            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
-        subs = config.params.copy()
-        subs['job-name'] = job_name
-        subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
-                                                time.gmtime(config.params['build_date']))
-        subs['product'] = index['product']
-
-        for tpl in V2_ROUTE_TEMPLATES:
-            routes.append(tpl.format(**subs))
+        index_type = index.get('type', 'generic')
+        task = index_builders[index_type](config, task)
 
         # The default behavior is to rank tasks according to their tier
         extra_index = task.setdefault('extra', {}).setdefault('index', {})
         rank = index.get('rank', 'by-tier')
 
         if rank == 'by-tier':
             # rank is zero for non-tier-1 tasks and based on pushid for others;
             # this sorts tier-{2,3} builds below tier-1 in the index
@@ -724,24 +816,33 @@ def build_task(config, tasks):
 
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open("testing/mozharness/configs/routes.json", "rb") as f:
         routes_json = json.load(f)
 
-    # we only deal with the 'routes' key here
-    routes = routes_json['routes']
+    for key in ('routes', 'nightly', 'l10n'):
+        if key == 'routes':
+            tc_template = V2_ROUTE_TEMPLATES
+        elif key == 'nightly':
+            tc_template = V2_NIGHTLY_TEMPLATES
+        elif key == 'l10n':
+            tc_template = V2_L10N_TEMPLATES
+
+        routes = routes_json[key]
 
-    # we use different variables than mozharness
-    for mh, tg in [
-            ('{index}', 'index'),
-            ('{build_product}', '{product}'),
-            ('{build_name}-{build_type}', '{job-name}'),
-            ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
-        routes = [r.replace(mh, tg) for r in routes]
+        # we use different variables than mozharness
+        for mh, tg in [
+                ('{index}', 'index'),
+                ('{build_product}', '{product}'),
+                ('{build_name}-{build_type}', '{job-name}'),
+                ('{year}.{month}.{day}.{pushdate}', '{build_date_long}'),
+                ('{year}.{month}.{day}', '{build_date}')]:
+            routes = [r.replace(mh, tg) for r in routes]
 
-    if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
-        raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
-                        "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
+        if sorted(routes) != sorted(tc_template):
+            raise Exception("V2 TEMPLATES do not match Mozharness's routes.json: "
+                            "(tc):%s vs (mh):%s" % (tc_template, routes))
+
 
 check_v2_routes()