bug 1415391 - move release indexes into index_builder('release'). r=dustin
authorAki Sasaki <asasaki@mozilla.com>
Tue, 07 Nov 2017 20:57:35 -0800
changeset 444599 a6108f6cc6a1859b2d9dfb705baefc044ea4871a
parent 444598 331c5f0d914ed183b52e5dcee72f8f31281cdde0
child 444600 136574f7a5e4a793a8306f020c9d4b15655e0d53
push id1618
push userCallek@gmail.com
push dateThu, 11 Jan 2018 17:45:48 +0000
treeherdermozilla-release@882ca853e05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1415391, 1412690
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1415391 - move release indexes into index_builder('release'). r=dustin In bug 1412690, Dustin noted that the scopes and routes don't belong at the worker level. To deal with the release indexes, we now have a new `index_builder`. We also add the default release bbb scope in `build_buildbot_bridge_payload`. We can and should still move the product info to attributes. I left that for another patch. MozReview-Commit-ID: 4ZqvnY577S7
taskcluster/ci/beetmover-cdns/kind.yml
taskcluster/ci/release-bouncer-aliases/kind.yml
taskcluster/ci/release-bouncer-sub/kind.yml
taskcluster/ci/release-mark-as-shipped/kind.yml
taskcluster/ci/release-uptake-monitoring/kind.yml
taskcluster/ci/release-version-bump/kind.yml
taskcluster/taskgraph/transforms/beetmover_cdns.py
taskcluster/taskgraph/transforms/job/buildbot.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/beetmover-cdns/kind.yml
+++ b/taskcluster/ci/beetmover-cdns/kind.yml
@@ -19,13 +19,16 @@ job-defaults:
          mozilla-beta: scriptworker-prov-v1/beetmoverworker-v1
          default: scriptworker-prov-v1/beetmoverworker-dev
    run-on-projects: []
 
 jobs:
    fennec-push-to-cdns:
       name: fennec_push_to_cdns
       product: fennec
-      run:
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
+      routes:
+        - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
+        - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
       treeherder-platform: Android/opt
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
--- a/taskcluster/ci/release-bouncer-aliases/kind.yml
+++ b/taskcluster/ci/release-bouncer-aliases/kind.yml
@@ -18,26 +18,30 @@ jobs:
       description: Update bouncer aliases job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_bouncer_aliases
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
          properties:
              tuxedo_server_url:
                 by-project:
                     mozilla-beta: https://bounceradmin.mozilla.com/api
                     mozilla-release: https://bounceradmin.mozilla.com/api
                     maple: https://admin-bouncer.stage.mozaws.net/api/
                     default: http://localhost/api
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -15,19 +15,23 @@ jobs:
       description: release bouncer submission job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_bncr_sub
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-mark-as-shipped/kind.yml
+++ b/taskcluster/ci/release-mark-as-shipped/kind.yml
@@ -21,19 +21,23 @@ jobs:
       description: mark release as shipped in Ship-It
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_mark_as_shipped
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-uptake-monitoring/kind.yml
+++ b/taskcluster/ci/release-uptake-monitoring/kind.yml
@@ -18,28 +18,32 @@ jobs:
       description: Uptake monitoring job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_uptake_monitoring
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.uptake_monitoring
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.uptake_monitoring
          properties:
              # TODO: Calculate "platforms" dynamically
              platforms: "android-api-16, android-x86"
              tuxedo_server_url:
                 by-project:
                     mozilla-beta: https://bounceradmin.mozilla.com/api
                     mozilla-release: https://bounceradmin.mozilla.com/api
                     maple: https://admin-bouncer.stage.mozaws.net/api/
                     default: http://localhost/api
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.uptake_monitoring
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.uptake_monitoring
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-version-bump/kind.yml
+++ b/taskcluster/ci/release-version-bump/kind.yml
@@ -18,19 +18,23 @@ jobs:
       description: Release Promotion version bump
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_version_bump
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.version_bump
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.version_bump
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/taskgraph/transforms/beetmover_cdns.py
+++ b/taskcluster/taskgraph/transforms/beetmover_cdns.py
@@ -35,16 +35,18 @@ beetmover_cdns_description_schema = Sche
     Optional('job-from'): task_description_schema['job-from'],
     Optional('run'): {basestring: object},
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Required('worker-type'): Any(
         job_description_schema['worker-type'],
         {'by-project': {basestring: job_description_schema['worker-type']}},
     ),
     Optional('dependencies'): {basestring: taskref_or_string},
+    Optional('index'): {basestring: basestring},
+    Optional('routes'): [basestring],
 })
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         label = job['name']
         yield validate_schema(
--- a/taskcluster/taskgraph/transforms/job/buildbot.py
+++ b/taskcluster/taskgraph/transforms/job/buildbot.py
@@ -25,52 +25,32 @@ buildbot_run_schema = Schema({
     # the buildername to use for buildbot-bridge, will expand {branch} in name from
     # the current project.
     Required('buildername'): basestring,
 
     # the product to use
     Required('product'): Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'),
 
     Optional('release-promotion'): bool,
-    Optional('routes'): [basestring],
     Optional('properties'): {basestring: optionally_keyed_by('project', basestring)},
 })
 
 
 def bb_release_worker(config, worker, run):
     # props
     release_props = get_release_config(config, force=True)
     repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
     revision = config.params['head_rev']
-    branch = config.params['project']
-    buildername = worker['buildername']
-    underscore_version = release_props['version'].replace('.', '_')
     release_props.update({
         'release_promotion': True,
         'repo_path': repo_path,
         'revision': revision,
         'script_repo_revision': revision,
     })
     worker['properties'].update(release_props)
-    # scopes
-    worker['scopes'] = [
-        "project:releng:buildbot-bridge:builder-name:{}".format(buildername)
-    ]
-    # routes
-    if run.get('routes'):
-        worker['routes'] = []
-        repl_dict = {
-            'branch': branch,
-            'build_number': str(release_props['build_number']),
-            'revision': revision,
-            'underscore_version': underscore_version,
-        }
-        for route in run['routes']:
-            route = route.format(**repl_dict)
-            worker['routes'].append(route)
 
 
 def bb_ci_worker(config, worker):
     worker['properties'].update({
         'who': config.params['owner'],
         'upload_to_task_id': slugid.nice(),
     })
 
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -127,27 +127,28 @@ task_description_schema = Schema({
     },
 
     # information for indexing this build so its artifacts can be discovered;
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any(
             'firefox',
+            'fennec',
             'mobile',
             'static-analysis',
             'devedition',
             'source',
         ),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
         # Type of gecko v2 index to use
-        'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n'),
+        'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n', 'release'),
 
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
@@ -406,18 +407,16 @@ task_description_schema = Schema({
         },
         Required('properties'): {
             'product': basestring,
             Optional('build_number'): int,
             Optional('release_promotion'): bool,
             Optional('tuxedo_server_url'): optionally_keyed_by('project', basestring),
             Extra: taskref_or_string,  # additional properties are allowed
         },
-        Optional('scopes'): [basestring],
-        Optional('routes'): [basestring],
     }, {
         Required('implementation'): 'native-engine',
         Required('os'): Any('macosx', 'linux'),
 
         # A link for an executable to download
         Optional('context'): basestring,
 
         # Tells the worker whether machine should reboot
@@ -660,16 +659,22 @@ def superseder_url(config, task):
     size = task['coalesce']['size']
     return SUPERSEDER_URL.format(
         age=age,
         size=size,
         key=key
     )
 
 
+def verify_index_job_name(index):
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
     level = int(config.params['level'])
 
     image = worker['docker-image']
     if isinstance(image, dict):
         if 'in-tree' in image:
@@ -1003,18 +1008,21 @@ def build_buildbot_bridge_payload(config
     task['extra'].pop('treeherder', None)
     task['extra'].pop('treeherderEnv', None)
     worker = task['worker']
     task_def['payload'] = {
         'buildername': worker['buildername'],
         'sourcestamp': worker['sourcestamp'],
         'properties': worker['properties'],
     }
-    task_def['scopes'].extend(worker.get('scopes', []))
-    task_def['routes'].extend(worker.get('routes', []))
+    task_def.setdefault('scopes', [])
+    if worker['properties'].get('release_promotion'):
+        task_def['scopes'].append(
+            "project:releng:buildbot-bridge:builder-name:{}".format(worker['buildername'])
+        )
 
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def task_name_from_label(config, tasks):
     for task in tasks:
@@ -1035,22 +1043,20 @@ def validate(config, tasks):
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
 @index_builder('generic')
 def add_generic_index_routes(config, task):
     index = task.get('index')
     routes = task.setdefault('routes', [])
 
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+    verify_index_job_name(index)
 
     subs = config.params.copy()
-    subs['job-name'] = job_name
+    subs['job-name'] = index['job-name']
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
     project = config.params.get('project')
 
     for tpl in V2_ROUTE_TEMPLATES:
         routes.append(tpl.format(**subs))
@@ -1064,55 +1070,74 @@ def add_generic_index_routes(config, tas
     return task
 
 
 @index_builder('nightly')
 def add_nightly_index_routes(config, task):
     index = task.get('index')
     routes = task.setdefault('routes', [])
 
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+    verify_index_job_name(index)
 
     subs = config.params.copy()
-    subs['job-name'] = job_name
+    subs['job-name'] = index['job-name']
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['build_date'] = time.strftime("%Y.%m.%d",
                                        time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
     for tpl in V2_NIGHTLY_TEMPLATES:
         routes.append(tpl.format(**subs))
 
     # Also add routes for en-US
     task = add_l10n_index_routes(config, task, force_locale="en-US")
 
     return task
 
 
+@index_builder('release')
+def add_release_index_routes(config, task):
+    index = task.get('index')
+    routes = []
+    release_config = get_release_config(config, force=True)
+
+    verify_index_job_name(index)
+
+    subs = config.params.copy()
+    subs['build_number'] = str(release_config['build_number'])
+    subs['revision'] = subs['head_rev']
+    subs['underscore_version'] = release_config['version'].replace('.', '_')
+    subs['product'] = index['product']
+    subs['branch'] = subs['project']
+
+    for rt in task.get('routes', []):
+        routes.append(rt.format(**subs))
+
+    task['routes'] = routes
+
+    return task
+
+
 @index_builder('nightly-with-multi-l10n')
 def add_nightly_multi_index_routes(config, task):
     task = add_nightly_index_routes(config, task)
     task = add_l10n_index_routes(config, task, force_locale="multi")
     return task
 
 
 @index_builder('l10n')
 def add_l10n_index_routes(config, task, force_locale=None):
     index = task.get('index')
     routes = task.setdefault('routes', [])
 
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+    verify_index_job_name(index)
 
     subs = config.params.copy()
-    subs['job-name'] = job_name
+    subs['job-name'] = index['job-name']
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
     locales = task['attributes'].get('chunk_locales',
                                      task['attributes'].get('all_locales'))
     # Some tasks has only one locale set
     if task['attributes'].get('locale'):