author | Gurzau Raul <rgurzau@mozilla.com> |
Tue, 24 Dec 2019 03:37:17 +0200 | |
changeset 508348 | 27a1becd947bec022c2fb02411b4d9b57a58c2a9 |
parent 508347 | a336e2b237c7fb163642a2c7643ef9fa3170899c |
child 508349 | e9be8f3a3de25c851d77e22c908bb41e8829e6f7 |
push id | 103908 |
push user | rgurzau@mozilla.com |
push date | Tue, 24 Dec 2019 01:37:53 +0000 |
treeherder | autoland@27a1becd947b [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
bugs | 1605275, 1605277 |
milestone | 73.0a1 |
backs out | af16e71c6922383e4ccd1bb214fc1e47e6a6e154 4534f3e46f6edabc135dbb84fbcfcd4dbc276e42 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -250,16 +250,21 @@ release-promotion: is-rc: true target-tasks-method: ship_desktop partial-updates: true scriptworker: # See additional configuration in taskcluster/taskgraph/util/scriptworker.py scope-prefix: 'project:releng' worker-types: + 'linux-signing': + - 'project:releng:signing:cert:release-signing' + - 'project:releng:signing:cert:nightly-signing' + 'linux-depsigning': + - 'project:releng:signing:cert:dep-signing' 'scriptworker-k8s/gecko-3-beetmover': - 'project:releng:beetmover:bucket:release' - 'project:releng:beetmover:bucket:nightly' - 'project:releng:beetmover:bucket:partner' - 'project:releng:beetmover:bucket:maven-production' 'scriptworker-k8s/gecko-1-beetmover': - 'project:releng:beetmover:bucket:dep' - 'project:releng:beetmover:bucket:dep-partner' @@ -331,20 +336,17 @@ workers: provisioner: scriptworker-k8s implementation: scriptworker-signing os: linux worker-type: gecko-t-signing linux-signing: provisioner: scriptworker-k8s implementation: scriptworker-signing os: linux - worker-type: - by-release-level: - production: gecko-3-signing - staging: gecko-t-signing + worker-type: gecko-3-signing mac-depsigning: provisioner: scriptworker-prov-v1 implementation: scriptworker-iscript os: macosx worker-type: depsigning-mac-v1 mac-signing: provisioner: scriptworker-prov-v1 implementation: scriptworker-iscript
--- a/taskcluster/taskgraph/config.py +++ b/taskcluster/taskgraph/config.py @@ -77,17 +77,17 @@ graph_config_schema = Schema({ Any(basestring, None)), }, Required('workers'): { Required('aliases'): { text_type: { Required('provisioner'): optionally_keyed_by('level', text_type), Required('implementation'): text_type, Required('os'): text_type, - Required('worker-type'): optionally_keyed_by('level', 'release-level', text_type), + Required('worker-type'): optionally_keyed_by('level', text_type), } }, }, Required('mac-notarization'): { Required('mac-behavior'): optionally_keyed_by('platform', 'release-type', Any('mac_notarize', 'mac_geckodriver', 'mac_sign', 'mac_sign_and_pkg')),
--- a/taskcluster/taskgraph/transforms/geckodriver_signing.py +++ b/taskcluster/taskgraph/transforms/geckodriver_signing.py @@ -7,16 +7,17 @@ Transform the repackage signing task int from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope_per_platform, + get_worker_type_for_scope, ) from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional repackage_signing_description_schema = schema.extend({ Required('depname', default='geckodriver-repackage'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], @@ -63,17 +64,17 @@ def make_repackage_signing_description(c upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform) scopes = [signing_cert_scope] task = { 'label': job['label'], 'description': description, - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'),
--- a/taskcluster/taskgraph/transforms/mar_signing.py +++ b/taskcluster/taskgraph/transforms/mar_signing.py @@ -7,16 +7,17 @@ Transform the {partials,mar}-signing tas from __future__ import absolute_import, print_function, unicode_literals import os from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job, sorted_unique_list from taskgraph.util.scriptworker import ( get_signing_cert_scope_per_platform, + get_worker_type_for_scope, ) from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts_from_params from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.treeherder import join_symbol import logging logger = logging.getLogger(__name__) @@ -142,17 +143,17 @@ def make_task_description(config, jobs): ) scopes = [signing_cert_scope] task = { 'label': label, 'description': "{} {}".format( dep_job.task["metadata"]["description"], job['description-suffix']), - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': job.get('run-on-projects', dep_job.attributes.get('run_on_projects')),
--- a/taskcluster/taskgraph/transforms/openh264_signing.py +++ b/taskcluster/taskgraph/transforms/openh264_signing.py @@ -7,16 +7,17 @@ Transform the repackage signing task int from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope_per_platform, + get_worker_type_for_scope, ) from taskgraph.util.treeherder import inherit_treeherder_from_dep from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional transforms = TransformSequence() signing_description_schema = schema.extend({ @@ -76,17 +77,17 @@ def make_signing_description(config, job treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault('symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol') )) task = { 'label': job['label'], 'description': description, - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': my_attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder
--- a/taskcluster/taskgraph/transforms/release_generate_checksums_signing.py +++ b/taskcluster/taskgraph/transforms/release_generate_checksums_signing.py @@ -7,16 +7,17 @@ Transform the release-generate-checksums from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope, + get_worker_type_for_scope, ) from taskgraph.util.taskcluster import get_artifact_path from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional release_generate_checksums_signing_schema = schema.extend({ Required('depname', default='release-generate-checksums'): basestring, Optional('label'): basestring, @@ -62,17 +63,17 @@ def make_release_generate_checksums_sign "formats": ["autograph_gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': [ signing_cert_scope, ], 'dependencies': dependencies, 'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/repackage_signing.py +++ b/taskcluster/taskgraph/transforms/repackage_signing.py @@ -9,16 +9,17 @@ from __future__ import absolute_import, import os from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope_per_platform, + get_worker_type_for_scope, ) from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional repackage_signing_description_schema = schema.extend({ Required('depname', default='repackage'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], @@ -102,17 +103,17 @@ def make_repackage_signing_description(c "taskType": "repackage", "paths": [artifact], "formats": SIGNING_FORMATS[os.path.basename(artifact)], }) task = { 'label': label, 'description': description, - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder,
--- a/taskcluster/taskgraph/transforms/repackage_signing_partner.py +++ b/taskcluster/taskgraph/transforms/repackage_signing_partner.py @@ -8,16 +8,17 @@ Transform the repackage signing task int from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.partners import check_if_partners_enabled, get_partner_config_by_kind from taskgraph.util.scriptworker import ( get_signing_cert_scope_per_platform, + get_worker_type_for_scope, ) from taskgraph.util.taskcluster import get_artifact_path from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional transforms = TransformSequence() repackage_signing_description_schema = schema.extend({ @@ -114,17 +115,17 @@ def make_repackage_signing_description(c get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"] }] task = { 'label': label, 'description': description, - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': {
--- a/taskcluster/taskgraph/transforms/signing.py +++ b/taskcluster/taskgraph/transforms/signing.py @@ -9,16 +9,17 @@ from __future__ import absolute_import, from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.keyed_by import evaluate_keyed_by from taskgraph.util.schema import taskref_or_string from taskgraph.util.scriptworker import ( get_signing_cert_scope_per_platform, + get_worker_type_for_scope, ) from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional transforms = TransformSequence() signing_description_schema = schema.extend({ @@ -151,17 +152,17 @@ def make_task_description(config, jobs): if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) - worker_type_alias = 'linux-signing' if is_nightly else 'linux-depsigning' + worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope) mac_behavior = None task = { 'label': label, 'description': description, 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600)}, 'scopes': [signing_cert_scope] + signing_format_scopes,
--- a/taskcluster/taskgraph/transforms/source_checksums_signing.py +++ b/taskcluster/taskgraph/transforms/source_checksums_signing.py @@ -1,21 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the checksums signing task into an actual task description. """ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope, + get_worker_type_for_scope, ) from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional checksums_signing_description_schema = schema.extend({ Required('depname', default='beetmover'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], @@ -57,17 +59,17 @@ def make_checksums_signing_description(c "formats": ["autograph_gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, - 'worker-type': 'linux-signing', + 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': [ signing_cert_scope, ], 'dependencies': dependencies, 'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -1819,18 +1819,17 @@ def try_task_config_chemspill_prio(confi @transforms.add def build_task(config, tasks): for task in tasks: level = str(config.params['level']) provisioner_id, worker_type = get_worker_type( config.graph_config, task['worker-type'], - level=level, - release_level=config.params.release_level(), + level, ) task['worker-type'] = '/'.join([provisioner_id, worker_type]) project = config.params['project'] routes = task.get('routes', []) scopes = [s.format(level=level, project=project) for s in task.get('scopes', [])] # set up extra
--- a/taskcluster/taskgraph/util/scriptworker.py +++ b/taskcluster/taskgraph/util/scriptworker.py @@ -372,17 +372,17 @@ def get_release_config(config): release_config['next_version'] = str(config.params['next_version']) release_config['build_number'] = config.params['build_number'] return release_config def get_signing_cert_scope_per_platform(build_platform, is_nightly, config): if 'devedition' in build_platform: return get_devedition_signing_cert_scope(config) - elif is_nightly: + elif is_nightly or build_platform in ('firefox-source', 'fennec-source', 'thunderbird-source'): return get_signing_cert_scope(config) else: return add_scope_prefix(config, 'signing:cert:dep-signing') def get_worker_type_for_scope(config, scope): """Get the scriptworker type that will accept the given scope.
--- a/taskcluster/taskgraph/util/workertypes.py +++ b/taskcluster/taskgraph/util/workertypes.py @@ -27,17 +27,17 @@ WORKER_TYPES = { "scriptworker-k8s/gecko-1-tree": ('treescript', None), 'terraform-packet/gecko-t-linux': ('docker-worker', 'linux'), 'releng-hardware/gecko-t-osx-1014': ('generic-worker', 'macosx'), 'releng-hardware/gecko-t-osx-1014-power': ('generic-worker', 'macosx'), } @memoize -def _get(graph_config, alias, level, release_level): +def _get(graph_config, alias, level): """Get the configuration for this worker_type alias: {provisioner, worker-type, implementation, os}""" level = str(level) # handle the legacy (non-alias) format if '/' in alias: alias = alias.format(level=level) provisioner, worker_type = alias.split("/", 1) @@ -64,28 +64,31 @@ def _get(graph_config, alias, level, rel worker_config['provisioner'] = evaluate_keyed_by( worker_config['provisioner'], "worker-type alias {} field provisioner".format(alias), {"level": level}).format(level=level, alias=alias) worker_config['worker-type'] = evaluate_keyed_by( worker_config['worker-type'], "worker-type alias {} field worker-type".format(alias), - {"level": level, 'release-level': release_level}).format(level=level, alias=alias) + {"level": level}).format(level=level, alias=alias) return worker_config +@memoize def worker_type_implementation(graph_config, worker_type): """Get the worker implementation and OS for the given workerType, where the OS represents the host system, not the target OS, in the case of cross-compiles.""" - worker_config = _get(graph_config, worker_type, '1', 'staging') + worker_config = _get(graph_config, worker_type, '1') return worker_config['implementation'], worker_config.get('os') -def get_worker_type(graph_config, worker_type, level, release_level): +@memoize +def get_worker_type(graph_config, worker_type, level): """ Get the worker type provisioner and worker-type, optionally evaluating aliases from the graph config. """ - worker_config = _get(graph_config, worker_type, level, release_level) + worker_config = _get(graph_config, worker_type, level) + return worker_config['provisioner'], worker_config['worker-type']