Backed out changeset 7a27e1371c3e (bug 1301495) for breaking decision tasks a=backout
authorWes Kocher <wkocher@mozilla.com>
Tue, 17 Jan 2017 13:47:20 -0800
changeset 376973 80eac484366ad881c6a10bf81e8d9b8f7a676c75
parent 376972 7a27e1371c3e6bb8ca6d8000325ab6699d7a8dd1
child 377039 1214fccec674373d756441d7cc9375e636586306
child 377082 9baafb3d98464b04313a76059b38094707ed9464
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1301495
milestone53.0a1
backs out7a27e1371c3e6bb8ca6d8000325ab6699d7a8dd1
first release with
nightly linux32
80eac484366a / 53.0a1 / 20170118030214 / files
nightly linux64
80eac484366a / 53.0a1 / 20170118030214 / files
nightly mac
80eac484366a / 53.0a1 / 20170118030214 / files
nightly win32
80eac484366a / 53.0a1 / 20170118030214 / files
nightly win64
80eac484366a / 53.0a1 / 20170118030214 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 7a27e1371c3e (bug 1301495) for breaking decision tasks a=backout MozReview-Commit-ID: Gx21fdPIkyy
taskcluster/ci/build/android.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/build/macosx.yml
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -48,18 +48,17 @@ android-x86/opt:
         tooltool-downloads: internal
 
 android-x86-nightly/opt:
     description: "Android 4.2 x86 Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-x86-opt
-        type: nightly
+        job-name: android-x86-nightly-opt
     treeherder:
         platform: android-4-2-x86/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
@@ -103,18 +102,17 @@ android-api-15/opt:
         tooltool-downloads: internal
 
 android-api-15-nightly/opt:
     description: "Android 4.0 API15+ Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-api-15-opt
-        type: nightly
+        job-name: android-api-15-nightly-opt
     treeherder:
         platform: android-4-0-armv7-api15/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -150,18 +150,17 @@ linux/pgo:
         need-xvfb: true
 
 linux-nightly/opt:
     description: "Linux32 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux-opt
-        type: nightly
+        job-name: linux32-nightly-opt
     treeherder:
         platform: linux32/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
@@ -228,18 +227,17 @@ linux64-asan/debug:
         need-xvfb: true
 
 linux64-nightly/opt:
     description: "Linux64 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux64-opt
-        type: nightly
+        job-name: linux64-nightly-opt
     treeherder:
         platform: linux64/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -39,10 +39,8 @@ macosx64/opt:
         using: mozharness
         actions: [get-secrets build generate-build-stats update]
         config:
             - builds/releng_base_mac_64_cross_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: internal
-
-        type: nightly
\ No newline at end of file
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -32,27 +32,26 @@ job-template:
       by-build-platform:
          default: 36000
          android-api-15-l10n: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-l10n: internal
    index:
-      type: l10n
       product:
          by-build-platform:
             default: firefox
             android-api-15-l10n: mobile
       job-name:
          by-build-platform:
-            linux-l10n: linux-opt
-            linux64-l10n: linux64-opt
-            macosx64: macosx64-opt
-            android-api-15-l10n: android-api-15-opt
+            linux-l10n: linux32-l10n-opt
+            linux64-l10n: linux64-l10n-opt
+            macosx64: macosx64-l10n-opt
+            android-api-15-l10n: android-l10n-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc(L10n)
       tier:
          by-build-platform:
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -31,28 +31,27 @@ job-template:
    run-time:
       by-build-platform:
          default: 36000
          android-api-15-nightly: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-nightly: internal
-   index:
-      type: l10n
-      product:
-         by-build-platform:
-            default: firefox
-            android-api-15-nightly: mobile
-      job-name:
-         by-build-platform:
-            linux-nightly: linux-opt
-            linux64-nightly: linux64-opt
-            macosx64-nightly: macosx64-opt
-            android-api-15-nightly: android-api-15-opt
+#   index:
+#      product:
+#         by-build-platform:
+#            default: firefox
+#            android-api-15-nightly: mobile
+#      job-name:
+#         by-build-platform:
+#            linux-nightly: linux32-nightly-l10n-opt
+#            linux64-nightly: linux64-nightly-l10n-opt
+#            macosx64-nightly: macosx64-nightly-l10n-opt
+#            android-api-15-nightly: android-nightly-l10n-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android-api-15-nightly: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc-L10n(N)
       tier:
          by-build-platform:
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -63,19 +63,16 @@ l10n_description_schema = Schema({
     },
     # Items for the taskcluster index
     Optional('index'): {
         # Product to identify as in the taskcluster index
         Required('product'): _by_platform(basestring),
 
         # Job name to identify as in the taskcluster index
         Required('job-name'): _by_platform(basestring),
-
-        # Type of index
-        Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
     # task object of the dependent task
     Required('dependent-task'): object,
 
     # worker-type to utilize
@@ -353,17 +350,16 @@ def make_job_description(config, jobs):
             },
             'run-on-projects': [],
         }
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
-                'type': job['index'].get('type', 'generic'),
             }
 
         if job.get('dependencies'):
             job_description['dependencies'] = job['dependencies']
         if job.get('env'):
             job_description['worker']['env'] = job['env']
         if job.get('when', {}).get('files-changed'):
             job_description.setdefault('when', {})
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -87,19 +87,16 @@ task_description_schema = Schema({
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any('firefox', 'mobile', 'static-analysis'),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
-        # Type of gecko v2 index to use
-        'type': Any('generic', 'nightly', 'l10n'),
-
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
             # builds below tier-1 in the index.
@@ -372,29 +369,16 @@ GROUP_NAMES = {
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
-V2_NIGHTLY_TEMPLATES = [
-    "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
-    "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
-    "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
-    "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
-]
-
-V2_L10N_TEMPLATES = [
-    "index.gecko.v2.{project}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
-    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}",
-    "index.gecko.v2.{project}.latest.{product}-l10n.{job-name}.{locale}",
-]
-
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
 COALESCE_KEY = 'builds.{project}.{name}'
 
@@ -403,26 +387,16 @@ payload_builders = {}
 
 
 def payload_builder(name):
     def wrap(func):
         payload_builders[name] = func
         return func
     return wrap
 
-# define a collection of index builders, depending on the type implementation
-index_builders = {}
-
-
-def index_builder(name):
-    def wrap(func):
-        index_builders[name] = func
-        return func
-    return wrap
-
 
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
 
     image = worker['docker-image']
     if isinstance(image, dict):
         docker_image_task = 'build-docker-image-' + image['in-tree']
@@ -617,104 +591,38 @@ transforms = TransformSequence()
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             task_description_schema, task,
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
-@index_builder('generic')
-def add_generic_index_routes(config, task):
-    index = task.get('index')
-    routes = task.setdefault('routes', [])
-
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
-    subs = config.params.copy()
-    subs['job-name'] = job_name
-    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
-                                            time.gmtime(config.params['build_date']))
-    subs['product'] = index['product']
-
-    for tpl in V2_ROUTE_TEMPLATES:
-        routes.append(tpl.format(**subs))
-
-    return task
-
-
-@index_builder('nightly')
-def add_nightly_index_routes(config, task):
-    index = task.get('index')
-    routes = task.setdefault('routes', [])
-
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
-    subs = config.params.copy()
-    subs['job-name'] = job_name
-    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
-                                            time.gmtime(config.params['build_date']))
-    subs['build_date'] = time.strftime("%Y.%m.%d",
-                                       time.gmtime(config.params['build_date']))
-    subs['product'] = index['product']
-
-    for tpl in V2_NIGHTLY_TEMPLATES:
-        routes.append(tpl.format(**subs))
-
-    return task
-
-
-@index_builder('l10n')
-def add_l10n_index_routes(config, task):
-    index = task.get('index')
-    routes = task.setdefault('routes', [])
-
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
-    subs = config.params.copy()
-    subs['job-name'] = job_name
-    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
-                                            time.gmtime(config.params['build_date']))
-    subs['product'] = index['product']
-
-    locales = task['attributes'].get('chunk_locales',
-                                     task['attributes'].get('all_locales'))
-
-    if not locales:
-        raise Exception("Error: Unable to use l10n index for tasks without locales")
-
-    # If there are too many locales, we can't write a route for all of them
-    # See Bug 1323792
-    if len(locales) > 18:  # 18 * 3 = 54, max routes = 64
-        return task
-
-    for locale in locales:
-        for tpl in V2_L10N_TEMPLATES:
-            routes.append(tpl.format(locale=locale, **subs))
-
-    return task
-
-
 @transforms.add
 def add_index_routes(config, tasks):
     for task in tasks:
         index = task.get('index')
+        routes = task.setdefault('routes', [])
 
         if not index:
             yield task
             continue
 
-        index_type = index.get('type', 'generic')
-        task = index_builders[index_type](config, task)
+        job_name = index['job-name']
+        if job_name not in JOB_NAME_WHITELIST:
+            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+        subs = config.params.copy()
+        subs['job-name'] = job_name
+        subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                                time.gmtime(config.params['build_date']))
+        subs['product'] = index['product']
+
+        for tpl in V2_ROUTE_TEMPLATES:
+            routes.append(tpl.format(**subs))
 
         # The default behavior is to rank tasks according to their tier
         extra_index = task.setdefault('extra', {}).setdefault('index', {})
         rank = index.get('rank', 'by-tier')
 
         if rank == 'by-tier':
             # rank is zero for non-tier-1 tasks and based on pushid for others;
             # this sorts tier-{2,3} builds below tier-1 in the index
@@ -818,33 +726,24 @@ def build_task(config, tasks):
 
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open("testing/mozharness/configs/routes.json", "rb") as f:
         routes_json = json.load(f)
 
-    for key in ('routes', 'nightly', 'l10n'):
-        if key == 'routes':
-            tc_template = V2_ROUTE_TEMPLATES
-        elif key == 'nightly':
-            tc_template = V2_NIGHTLY_TEMPLATES
-        elif key == 'l10n':
-            tc_template = V2_L10N_TEMPLATES
-
-        routes = routes_json[key]
+    # we only deal with the 'routes' key here
+    routes = routes_json['routes']
 
-        # we use different variables than mozharness
-        for mh, tg in [
-                ('{index}', 'index'),
-                ('{build_product}', '{product}'),
-                ('{build_name}-{build_type}', '{job-name}'),
-                ('{year}.{month}.{day}.{pushdate}', '{build_date_long}'),
-                ('{year}.{month}.{day}', '{build_date}')]:
-            routes = [r.replace(mh, tg) for r in routes]
+    # we use different variables than mozharness
+    for mh, tg in [
+            ('{index}', 'index'),
+            ('{build_product}', '{product}'),
+            ('{build_name}-{build_type}', '{job-name}'),
+            ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
+        routes = [r.replace(mh, tg) for r in routes]
 
-        if sorted(routes) != sorted(tc_template):
-            raise Exception("V2 TEMPLATES do not match Mozharness's routes.json: "
-                            "(tc):%s vs (mh):%s" % (tc_template, routes))
-
+    if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
+        raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
+                        "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
 
 check_v2_routes()