bug 1415391 - move release indexes into index_builder('release'). r=dustin
authorAki Sasaki <asasaki@mozilla.com>
Tue, 07 Nov 2017 20:57:35 -0800
changeset 696705 a6108f6cc6a1859b2d9dfb705baefc044ea4871a
parent 696704 331c5f0d914ed183b52e5dcee72f8f31281cdde0
child 696706 136574f7a5e4a793a8306f020c9d4b15655e0d53
push id88770
push userbmo:emilio@crisal.io
push dateSat, 11 Nov 2017 01:20:46 +0000
reviewersdustin
bugs1415391, 1412690
milestone58.0a1
bug 1415391 - move release indexes into index_builder('release'). r=dustin In bug 1412690, Dustin noted that the scopes and routes don't belong at the worker level. To deal with the release indexes, we now have a new `index_builder`. We also add the default release bbb scope in `build_buildbot_bridge_payload`. We can and should still move the product info to attributes. I left that for another patch. MozReview-Commit-ID: 4ZqvnY577S7
taskcluster/ci/beetmover-cdns/kind.yml
taskcluster/ci/release-bouncer-aliases/kind.yml
taskcluster/ci/release-bouncer-sub/kind.yml
taskcluster/ci/release-mark-as-shipped/kind.yml
taskcluster/ci/release-uptake-monitoring/kind.yml
taskcluster/ci/release-version-bump/kind.yml
taskcluster/taskgraph/transforms/beetmover_cdns.py
taskcluster/taskgraph/transforms/job/buildbot.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/beetmover-cdns/kind.yml
+++ b/taskcluster/ci/beetmover-cdns/kind.yml
@@ -19,13 +19,16 @@ job-defaults:
          mozilla-beta: scriptworker-prov-v1/beetmoverworker-v1
          default: scriptworker-prov-v1/beetmoverworker-dev
    run-on-projects: []
 
 jobs:
    fennec-push-to-cdns:
       name: fennec_push_to_cdns
       product: fennec
-      run:
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
+      routes:
+        - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
+        - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
       treeherder-platform: Android/opt
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
--- a/taskcluster/ci/release-bouncer-aliases/kind.yml
+++ b/taskcluster/ci/release-bouncer-aliases/kind.yml
@@ -18,26 +18,30 @@ jobs:
       description: Update bouncer aliases job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_bouncer_aliases
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
          properties:
              tuxedo_server_url:
                 by-project:
                     mozilla-beta: https://bounceradmin.mozilla.com/api
                     mozilla-release: https://bounceradmin.mozilla.com/api
                     maple: https://admin-bouncer.stage.mozaws.net/api/
                     default: http://localhost/api
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -15,19 +15,23 @@ jobs:
       description: release bouncer submission job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_bncr_sub
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-mark-as-shipped/kind.yml
+++ b/taskcluster/ci/release-mark-as-shipped/kind.yml
@@ -21,19 +21,23 @@ jobs:
       description: mark release as shipped in Ship-It
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_mark_as_shipped
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.mark_as_shipped
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.mark_as_shipped
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-uptake-monitoring/kind.yml
+++ b/taskcluster/ci/release-uptake-monitoring/kind.yml
@@ -18,28 +18,32 @@ jobs:
       description: Uptake monitoring job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_uptake_monitoring
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.uptake_monitoring
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.uptake_monitoring
          properties:
              # TODO: Calculate "platforms" dynamically
              platforms: "android-api-16, android-x86"
              tuxedo_server_url:
                 by-project:
                     mozilla-beta: https://bounceradmin.mozilla.com/api
                     mozilla-release: https://bounceradmin.mozilla.com/api
                     maple: https://admin-bouncer.stage.mozaws.net/api/
                     default: http://localhost/api
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.uptake_monitoring
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.uptake_monitoring
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/ci/release-version-bump/kind.yml
+++ b/taskcluster/ci/release-version-bump/kind.yml
@@ -18,19 +18,23 @@ jobs:
       description: Release Promotion version bump
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
       run:
          using: buildbot
          product: fennec
          buildername: release-{branch}-fennec_version_bump
          release-promotion: true
-         routes:
-            - index.releases.v1.{branch}.latest.fennec.latest.version_bump
-            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
+      routes:
+         - index.releases.v1.{branch}.latest.fennec.latest.version_bump
+         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.version_bump
+      index:
+         type: release
+         product: fennec
+         job-name: android-api-16-opt
       notifications:
          completed:
             by-project:
                maple:
                   - "release-drivers-staging"
                try:
                   #- "{task[tags][createdForUser]}"
                default:
--- a/taskcluster/taskgraph/transforms/beetmover_cdns.py
+++ b/taskcluster/taskgraph/transforms/beetmover_cdns.py
@@ -35,16 +35,18 @@ beetmover_cdns_description_schema = Sche
     Optional('job-from'): task_description_schema['job-from'],
     Optional('run'): {basestring: object},
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Required('worker-type'): Any(
         job_description_schema['worker-type'],
         {'by-project': {basestring: job_description_schema['worker-type']}},
     ),
     Optional('dependencies'): {basestring: taskref_or_string},
+    Optional('index'): {basestring: basestring},
+    Optional('routes'): [basestring],
 })
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         label = job['name']
         yield validate_schema(
--- a/taskcluster/taskgraph/transforms/job/buildbot.py
+++ b/taskcluster/taskgraph/transforms/job/buildbot.py
@@ -25,52 +25,32 @@ buildbot_run_schema = Schema({
     # the buildername to use for buildbot-bridge, will expand {branch} in name from
     # the current project.
     Required('buildername'): basestring,
 
     # the product to use
     Required('product'): Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'),
 
     Optional('release-promotion'): bool,
-    Optional('routes'): [basestring],
     Optional('properties'): {basestring: optionally_keyed_by('project', basestring)},
 })
 
 
 def bb_release_worker(config, worker, run):
     # props
     release_props = get_release_config(config, force=True)
     repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
     revision = config.params['head_rev']
-    branch = config.params['project']
-    buildername = worker['buildername']
-    underscore_version = release_props['version'].replace('.', '_')
     release_props.update({
         'release_promotion': True,
         'repo_path': repo_path,
         'revision': revision,
         'script_repo_revision': revision,
     })
     worker['properties'].update(release_props)
-    # scopes
-    worker['scopes'] = [
-        "project:releng:buildbot-bridge:builder-name:{}".format(buildername)
-    ]
-    # routes
-    if run.get('routes'):
-        worker['routes'] = []
-        repl_dict = {
-            'branch': branch,
-            'build_number': str(release_props['build_number']),
-            'revision': revision,
-            'underscore_version': underscore_version,
-        }
-        for route in run['routes']:
-            route = route.format(**repl_dict)
-            worker['routes'].append(route)
 
 
 def bb_ci_worker(config, worker):
     worker['properties'].update({
         'who': config.params['owner'],
         'upload_to_task_id': slugid.nice(),
     })
 
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -127,27 +127,28 @@ task_description_schema = Schema({
     },
 
     # information for indexing this build so its artifacts can be discovered;
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any(
             'firefox',
+            'fennec',
             'mobile',
             'static-analysis',
             'devedition',
             'source',
         ),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
         # Type of gecko v2 index to use
-        'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n'),
+        'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n', 'release'),
 
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
@@ -406,18 +407,16 @@ task_description_schema = Schema({
         },
         Required('properties'): {
             'product': basestring,
             Optional('build_number'): int,
             Optional('release_promotion'): bool,
             Optional('tuxedo_server_url'): optionally_keyed_by('project', basestring),
             Extra: taskref_or_string,  # additional properties are allowed
         },
-        Optional('scopes'): [basestring],
-        Optional('routes'): [basestring],
     }, {
         Required('implementation'): 'native-engine',
         Required('os'): Any('macosx', 'linux'),
 
         # A link for an executable to download
         Optional('context'): basestring,
 
         # Tells the worker whether machine should reboot
@@ -660,16 +659,22 @@ def superseder_url(config, task):
     size = task['coalesce']['size']
     return SUPERSEDER_URL.format(
         age=age,
         size=size,
         key=key
     )
 
 
+def verify_index_job_name(index):
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
     level = int(config.params['level'])
 
     image = worker['docker-image']
     if isinstance(image, dict):
         if 'in-tree' in image:
@@ -1003,18 +1008,21 @@ def build_buildbot_bridge_payload(config
     task['extra'].pop('treeherder', None)
     task['extra'].pop('treeherderEnv', None)
     worker = task['worker']
     task_def['payload'] = {
         'buildername': worker['buildername'],
         'sourcestamp': worker['sourcestamp'],
         'properties': worker['properties'],
     }
-    task_def['scopes'].extend(worker.get('scopes', []))
-    task_def['routes'].extend(worker.get('routes', []))
+    task_def.setdefault('scopes', [])
+    if worker['properties'].get('release_promotion'):
+        task_def['scopes'].append(
+            "project:releng:buildbot-bridge:builder-name:{}".format(worker['buildername'])
+        )
 
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def task_name_from_label(config, tasks):
     for task in tasks:
@@ -1035,22 +1043,20 @@ def validate(config, tasks):
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
 @index_builder('generic')
 def add_generic_index_routes(config, task):
     index = task.get('index')
     routes = task.setdefault('routes', [])
 
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+    verify_index_job_name(index)
 
     subs = config.params.copy()
-    subs['job-name'] = job_name
+    subs['job-name'] = index['job-name']
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
     project = config.params.get('project')
 
     for tpl in V2_ROUTE_TEMPLATES:
         routes.append(tpl.format(**subs))
@@ -1064,55 +1070,74 @@ def add_generic_index_routes(config, tas
     return task
 
 
 @index_builder('nightly')
 def add_nightly_index_routes(config, task):
     index = task.get('index')
     routes = task.setdefault('routes', [])
 
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+    verify_index_job_name(index)
 
     subs = config.params.copy()
-    subs['job-name'] = job_name
+    subs['job-name'] = index['job-name']
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['build_date'] = time.strftime("%Y.%m.%d",
                                        time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
     for tpl in V2_NIGHTLY_TEMPLATES:
         routes.append(tpl.format(**subs))
 
     # Also add routes for en-US
     task = add_l10n_index_routes(config, task, force_locale="en-US")
 
     return task
 
 
+@index_builder('release')
+def add_release_index_routes(config, task):
+    index = task.get('index')
+    routes = []
+    release_config = get_release_config(config, force=True)
+
+    verify_index_job_name(index)
+
+    subs = config.params.copy()
+    subs['build_number'] = str(release_config['build_number'])
+    subs['revision'] = subs['head_rev']
+    subs['underscore_version'] = release_config['version'].replace('.', '_')
+    subs['product'] = index['product']
+    subs['branch'] = subs['project']
+
+    for rt in task.get('routes', []):
+        routes.append(rt.format(**subs))
+
+    task['routes'] = routes
+
+    return task
+
+
 @index_builder('nightly-with-multi-l10n')
 def add_nightly_multi_index_routes(config, task):
     task = add_nightly_index_routes(config, task)
     task = add_l10n_index_routes(config, task, force_locale="multi")
     return task
 
 
 @index_builder('l10n')
 def add_l10n_index_routes(config, task, force_locale=None):
     index = task.get('index')
     routes = task.setdefault('routes', [])
 
-    job_name = index['job-name']
-    if job_name not in JOB_NAME_WHITELIST:
-        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+    verify_index_job_name(index)
 
     subs = config.params.copy()
-    subs['job-name'] = job_name
+    subs['job-name'] = index['job-name']
     subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                             time.gmtime(config.params['build_date']))
     subs['product'] = index['product']
 
     locales = task['attributes'].get('chunk_locales',
                                      task['attributes'].get('all_locales'))
     # Some tasks has only one locale set
     if task['attributes'].get('locale'):