Backed out 2 changesets (bug 1605275, bug 1605277) for build sign bustage on a CLOSED TREE.
authorGurzau Raul <rgurzau@mozilla.com>
Tue, 24 Dec 2019 03:37:17 +0200
changeset 508348 27a1becd947bec022c2fb02411b4d9b57a58c2a9
parent 508347 a336e2b237c7fb163642a2c7643ef9fa3170899c
child 508349 e9be8f3a3de25c851d77e22c908bb41e8829e6f7
push id103908
push userrgurzau@mozilla.com
push dateTue, 24 Dec 2019 01:37:53 +0000
treeherderautoland@27a1becd947b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1605275, 1605277
milestone73.0a1
backs outaf16e71c6922383e4ccd1bb214fc1e47e6a6e154
4534f3e46f6edabc135dbb84fbcfcd4dbc276e42
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 2 changesets (bug 1605275, bug 1605277) for build sign bustage on a CLOSED TREE. Backed out changeset af16e71c6922 (bug 1605277) Backed out changeset 4534f3e46f6e (bug 1605275)
taskcluster/ci/config.yml
taskcluster/taskgraph/config.py
taskcluster/taskgraph/transforms/geckodriver_signing.py
taskcluster/taskgraph/transforms/mar_signing.py
taskcluster/taskgraph/transforms/openh264_signing.py
taskcluster/taskgraph/transforms/release_generate_checksums_signing.py
taskcluster/taskgraph/transforms/repackage_signing.py
taskcluster/taskgraph/transforms/repackage_signing_partner.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/transforms/source_checksums_signing.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/scriptworker.py
taskcluster/taskgraph/util/workertypes.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -250,16 +250,21 @@ release-promotion:
             is-rc: true
             target-tasks-method: ship_desktop
             partial-updates: true
 
 scriptworker:
     # See additional configuration in taskcluster/taskgraph/util/scriptworker.py
     scope-prefix: 'project:releng'
     worker-types:
+        'linux-signing':
+            - 'project:releng:signing:cert:release-signing'
+            - 'project:releng:signing:cert:nightly-signing'
+        'linux-depsigning':
+            - 'project:releng:signing:cert:dep-signing'
         'scriptworker-k8s/gecko-3-beetmover':
             - 'project:releng:beetmover:bucket:release'
             - 'project:releng:beetmover:bucket:nightly'
             - 'project:releng:beetmover:bucket:partner'
             - 'project:releng:beetmover:bucket:maven-production'
         'scriptworker-k8s/gecko-1-beetmover':
             - 'project:releng:beetmover:bucket:dep'
             - 'project:releng:beetmover:bucket:dep-partner'
@@ -331,20 +336,17 @@ workers:
             provisioner: scriptworker-k8s
             implementation: scriptworker-signing
             os: linux
             worker-type: gecko-t-signing
         linux-signing:
             provisioner: scriptworker-k8s
             implementation: scriptworker-signing
             os: linux
-            worker-type:
-                by-release-level:
-                    production: gecko-3-signing
-                    staging: gecko-t-signing
+            worker-type: gecko-3-signing
         mac-depsigning:
             provisioner: scriptworker-prov-v1
             implementation: scriptworker-iscript
             os: macosx
             worker-type: depsigning-mac-v1
         mac-signing:
             provisioner: scriptworker-prov-v1
             implementation: scriptworker-iscript
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -77,17 +77,17 @@ graph_config_schema = Schema({
                                 Any(basestring, None)),
     },
     Required('workers'): {
         Required('aliases'): {
             text_type: {
                 Required('provisioner'): optionally_keyed_by('level', text_type),
                 Required('implementation'): text_type,
                 Required('os'): text_type,
-                Required('worker-type'): optionally_keyed_by('level', 'release-level', text_type),
+                Required('worker-type'): optionally_keyed_by('level', text_type),
             }
         },
     },
     Required('mac-notarization'): {
         Required('mac-behavior'):
             optionally_keyed_by('platform', 'release-type',
                                 Any('mac_notarize', 'mac_geckodriver', 'mac_sign',
                                     'mac_sign_and_pkg')),
--- a/taskcluster/taskgraph/transforms/geckodriver_signing.py
+++ b/taskcluster/taskgraph/transforms/geckodriver_signing.py
@@ -7,16 +7,17 @@ Transform the repackage signing task int
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
 )
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 repackage_signing_description_schema = schema.extend({
     Required('depname', default='geckodriver-repackage'): basestring,
     Optional('label'): basestring,
     Optional('treeherder'): task_description_schema['treeherder'],
@@ -63,17 +64,17 @@ def make_repackage_signing_description(c
 
         upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform)
 
         scopes = [signing_cert_scope]
 
         task = {
             'label': job['label'],
             'description': description,
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {
                 'implementation': 'scriptworker-signing',
                 'upstream-artifacts': upstream_artifacts,
             },
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
--- a/taskcluster/taskgraph/transforms/mar_signing.py
+++ b/taskcluster/taskgraph/transforms/mar_signing.py
@@ -7,16 +7,17 @@ Transform the {partials,mar}-signing tas
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job, sorted_unique_list
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
 )
 from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts_from_params
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.treeherder import join_symbol
 
 import logging
 logger = logging.getLogger(__name__)
 
@@ -142,17 +143,17 @@ def make_task_description(config, jobs):
         )
 
         scopes = [signing_cert_scope]
 
         task = {
             'label': label,
             'description': "{} {}".format(
                 dep_job.task["metadata"]["description"], job['description-suffix']),
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'dependencies': dependencies,
             'attributes': attributes,
             'scopes': scopes,
             'run-on-projects': job.get('run-on-projects',
                                        dep_job.attributes.get('run_on_projects')),
--- a/taskcluster/taskgraph/transforms/openh264_signing.py
+++ b/taskcluster/taskgraph/transforms/openh264_signing.py
@@ -7,16 +7,17 @@ Transform the repackage signing task int
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
 )
 from taskgraph.util.treeherder import inherit_treeherder_from_dep
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 transforms = TransformSequence()
 
 signing_description_schema = schema.extend({
@@ -76,17 +77,17 @@ def make_signing_description(config, job
         treeherder = inherit_treeherder_from_dep(job, dep_job)
         treeherder.setdefault('symbol', _generate_treeherder_symbol(
             dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol')
         ))
 
         task = {
             'label': job['label'],
             'description': description,
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': my_attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder
--- a/taskcluster/taskgraph/transforms/release_generate_checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/release_generate_checksums_signing.py
@@ -7,16 +7,17 @@ Transform the release-generate-checksums
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope,
+    get_worker_type_for_scope,
 )
 from taskgraph.util.taskcluster import get_artifact_path
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 release_generate_checksums_signing_schema = schema.extend({
     Required('depname', default='release-generate-checksums'): basestring,
     Optional('label'): basestring,
@@ -62,17 +63,17 @@ def make_release_generate_checksums_sign
             "formats": ["autograph_gpg"]
         }]
 
         signing_cert_scope = get_signing_cert_scope(config)
 
         task = {
             'label': label,
             'description': description,
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': [
                 signing_cert_scope,
             ],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 import os
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
 )
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 repackage_signing_description_schema = schema.extend({
     Required('depname', default='repackage'): basestring,
     Optional('label'): basestring,
     Optional('treeherder'): task_description_schema['treeherder'],
@@ -102,17 +103,17 @@ def make_repackage_signing_description(c
                     "taskType": "repackage",
                     "paths": [artifact],
                     "formats": SIGNING_FORMATS[os.path.basename(artifact)],
                 })
 
         task = {
             'label': label,
             'description': description,
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
--- a/taskcluster/taskgraph/transforms/repackage_signing_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing_partner.py
@@ -8,16 +8,17 @@ Transform the repackage signing task int
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.partners import check_if_partners_enabled, get_partner_config_by_kind
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
 )
 from taskgraph.util.taskcluster import get_artifact_path
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 transforms = TransformSequence()
 
 repackage_signing_description_schema = schema.extend({
@@ -114,17 +115,17 @@ def make_repackage_signing_description(c
                     get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)),
                 ],
                 "formats": ["autograph_gpg"]
             }]
 
         task = {
             'label': label,
             'description': description,
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'extra': {
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.keyed_by import evaluate_keyed_by
 from taskgraph.util.schema import taskref_or_string
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
 )
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 
 transforms = TransformSequence()
 
 signing_description_schema = schema.extend({
@@ -151,17 +152,17 @@ def make_task_description(config, jobs):
 
         if dep_job.attributes.get('chunk_locales'):
             # Used for l10n attribute passthrough
             attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
 
         signing_cert_scope = get_signing_cert_scope_per_platform(
             build_platform, is_nightly, config
         )
-        worker_type_alias = 'linux-signing' if is_nightly else 'linux-depsigning'
+        worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope)
         mac_behavior = None
         task = {
             'label': label,
             'description': description,
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': job.get('max-run-time', 3600)},
             'scopes': [signing_cert_scope] + signing_format_scopes,
--- a/taskcluster/taskgraph/transforms/source_checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/source_checksums_signing.py
@@ -1,21 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the checksums signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope,
+    get_worker_type_for_scope,
 )
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 checksums_signing_description_schema = schema.extend({
     Required('depname', default='beetmover'): basestring,
     Optional('label'): basestring,
     Optional('treeherder'): task_description_schema['treeherder'],
@@ -57,17 +59,17 @@ def make_checksums_signing_description(c
             "formats": ["autograph_gpg"]
         }]
 
         signing_cert_scope = get_signing_cert_scope(config)
 
         task = {
             'label': label,
             'description': description,
-            'worker-type': 'linux-signing',
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': [
                 signing_cert_scope,
             ],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1819,18 +1819,17 @@ def try_task_config_chemspill_prio(confi
 @transforms.add
 def build_task(config, tasks):
     for task in tasks:
         level = str(config.params['level'])
 
         provisioner_id, worker_type = get_worker_type(
             config.graph_config,
             task['worker-type'],
-            level=level,
-            release_level=config.params.release_level(),
+            level,
         )
         task['worker-type'] = '/'.join([provisioner_id, worker_type])
         project = config.params['project']
 
         routes = task.get('routes', [])
         scopes = [s.format(level=level, project=project) for s in task.get('scopes', [])]
 
         # set up extra
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -372,17 +372,17 @@ def get_release_config(config):
     release_config['next_version'] = str(config.params['next_version'])
     release_config['build_number'] = config.params['build_number']
     return release_config
 
 
 def get_signing_cert_scope_per_platform(build_platform, is_nightly, config):
     if 'devedition' in build_platform:
         return get_devedition_signing_cert_scope(config)
-    elif is_nightly:
+    elif is_nightly or build_platform in ('firefox-source', 'fennec-source', 'thunderbird-source'):
         return get_signing_cert_scope(config)
     else:
         return add_scope_prefix(config, 'signing:cert:dep-signing')
 
 
 def get_worker_type_for_scope(config, scope):
     """Get the scriptworker type that will accept the given scope.
 
--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -27,17 +27,17 @@ WORKER_TYPES = {
     "scriptworker-k8s/gecko-1-tree": ('treescript', None),
     'terraform-packet/gecko-t-linux': ('docker-worker', 'linux'),
     'releng-hardware/gecko-t-osx-1014': ('generic-worker', 'macosx'),
     'releng-hardware/gecko-t-osx-1014-power': ('generic-worker', 'macosx'),
 }
 
 
 @memoize
-def _get(graph_config, alias, level, release_level):
+def _get(graph_config, alias, level):
     """Get the configuration for this worker_type alias: {provisioner,
     worker-type, implementation, os}"""
     level = str(level)
 
     # handle the legacy (non-alias) format
     if '/' in alias:
         alias = alias.format(level=level)
         provisioner, worker_type = alias.split("/", 1)
@@ -64,28 +64,31 @@ def _get(graph_config, alias, level, rel
 
     worker_config['provisioner'] = evaluate_keyed_by(
         worker_config['provisioner'],
         "worker-type alias {} field provisioner".format(alias),
         {"level": level}).format(level=level, alias=alias)
     worker_config['worker-type'] = evaluate_keyed_by(
         worker_config['worker-type'],
         "worker-type alias {} field worker-type".format(alias),
-        {"level": level, 'release-level': release_level}).format(level=level, alias=alias)
+        {"level": level}).format(level=level, alias=alias)
 
     return worker_config
 
 
+@memoize
 def worker_type_implementation(graph_config, worker_type):
     """Get the worker implementation and OS for the given workerType, where the
     OS represents the host system, not the target OS, in the case of
     cross-compiles."""
-    worker_config = _get(graph_config, worker_type, '1', 'staging')
+    worker_config = _get(graph_config, worker_type, '1')
     return worker_config['implementation'], worker_config.get('os')
 
 
-def get_worker_type(graph_config, worker_type, level, release_level):
+@memoize
+def get_worker_type(graph_config, worker_type, level):
     """
     Get the worker type provisioner and worker-type, optionally evaluating
     aliases from the graph config.
     """
-    worker_config = _get(graph_config, worker_type, level, release_level)
+    worker_config = _get(graph_config, worker_type, level)
+
     return worker_config['provisioner'], worker_config['worker-type']