Bug 1567429 - remove checksums signing from release automation. r=tomprince
authorMihai Tabara <mtabara@mozilla.com>
Thu, 15 Aug 2019 17:31:40 +0000
changeset 488292 b57a45d3b47642f3f1294e3e0151f8b018b00453
parent 488291 43d96432677f15bb68ebc9edebdcf83ec59be7ec
child 488293 a8966ba20267de4cd27a8da1bcb61f39297d4047
push id113906
push userncsoregi@mozilla.com
push dateFri, 16 Aug 2019 04:07:24 +0000
treeherdermozilla-inbound@d887276421d3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1567429
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1567429 - remove checksums signing from release automation. r=tomprince Remove individual checksums signing in favor of big fat checksums Differential Revision: https://phabricator.services.mozilla.com/D40095
taskcluster/ci/beetmover-checksums/kind.yml
taskcluster/ci/checksums-signing/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/manifests/fennec_candidates_checksums.yml
taskcluster/taskgraph/manifests/fennec_nightly_checksums.yml
taskcluster/taskgraph/manifests/firefox_candidates_checksums.yml
taskcluster/taskgraph/manifests/firefox_nightly_checksums.yml
taskcluster/taskgraph/transforms/beetmover_checksums.py
taskcluster/taskgraph/transforms/checksums_signing.py
--- a/taskcluster/ci/beetmover-checksums/kind.yml
+++ b/taskcluster/ci/beetmover-checksums/kind.yml
@@ -5,17 +5,18 @@
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
     - taskgraph.transforms.name_sanity:transforms
     - taskgraph.transforms.beetmover_checksums:transforms
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-    - checksums-signing
+    - beetmover
+    - beetmover-repackage
 
 only-for-attributes:
     - nightly
     - shippable
 
 job-template:
     shipping-phase: promote
     attributes:
deleted file mode 100644
--- a/taskcluster/ci/checksums-signing/kind.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-loader: taskgraph.loader.single_dep:loader
-
-transforms:
-    - taskgraph.transforms.name_sanity:transforms
-    - taskgraph.transforms.checksums_signing:transforms
-    - taskgraph.transforms.task:transforms
-
-kind-dependencies:
-    - beetmover
-    - beetmover-l10n
-    - beetmover-repackage
-
-only-for-attributes:
-    - nightly
-    - shippable
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -219,22 +219,16 @@ Beetmover-source publishes Ubuntu's snap
 beetmover-source
 ----------------
 Beetmover-source publishes release source. This is part of release promotion.
 
 beetmover-geckoview
 -------------------
 Beetmover-geckoview publishes the Android library called "geckoview".
 
-checksums-signing
------------------
-Checksums-signing take as input the checksums file generated by beetmover tasks
-and sign it via the signing scriptworkers. Returns the same file signed and
-additional detached signature.
-
 release-source-checksums-signing
 --------------------------------
 release-source-checksums-signing take as input the checksums file generated by
 source-related beetmover task and sign it via the signing scriptworkers.
 Returns the same file signed and additional detached signature.
 
 beetmover-checksums
 -------------------
--- a/taskcluster/taskgraph/manifests/fennec_candidates_checksums.yml
+++ b/taskcluster/taskgraph/manifests/fennec_candidates_checksums.yml
@@ -2,17 +2,18 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 s3_bucket_paths:
     - pub/mobile/candidates
 default_locales:  # if given an empty locale, use these locales
     - en-US
 tasktype_map:  # Map task reference to task type.
-    checksums-signing: signing
+    beetmover: beetmover
+
 platform_names:
     path_platform:
         by-platform:
             android-x86-nightly: 'android-x86'
             android-x86_64-nightly: 'android-x86_64'
             android-api-16-nightly: 'android-api-16'
             android-aarch64-nightly: 'android-aarch64'
     filename_platform:
@@ -22,32 +23,26 @@ platform_names:
             android-api-16-nightly: 'android-arm'
             android-aarch64-nightly: 'android-aarch64'
 
 # A default entry, which the mappings below extend and override.
 # Final 'destinations' will be the product of:
 # s3_bucket_paths + destinations + locale_prefix + pretty_name
 default: &default
     from:
-        - checksums-signing
+        - beetmover
     all_locales: true
     description: "TO_BE_OVERRIDDEN"
     locale_prefix:
         by-locale:
             default: ''
             en-US: '${locale}/'
     source_path_modifier: ''
     destinations:
         - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}
-        - ${version}-candidates/build${build_number}/${path_platform}
 
 # Configuration for individual files. Extends 'default', above.
 mapping:
     target.checksums:
         <<: *default
         description: "Checksums file containing size, hash, sha algorithm and filename"
         pretty_name: fennec-${version}.${locale}.${filename_platform}.checksums.beet
         checksums_path: fennec-${version}.${locale}.${filename_platform}.checksums
-    target.checksums.asc:
-        <<: *default
-        description: "Detached signature for the checksums file"
-        pretty_name: fennec-${version}.${locale}.${filename_platform}.checksums.asc
-        checksums_path: fennec-${version}.${locale}.${filename_platform}.checksums.asc
--- a/taskcluster/taskgraph/manifests/fennec_nightly_checksums.yml
+++ b/taskcluster/taskgraph/manifests/fennec_nightly_checksums.yml
@@ -2,17 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 s3_bucket_paths:
     - pub/mobile/nightly
 default_locales:  # if given an empty locale, use these locales
     - en-US
 tasktype_map:  # Map task reference to task type.
-    checksums-signing: signing
+    beetmover: beetmover
 platform_names:
     path_platform:
         by-platform:
             android-x86-nightly: 'android-x86'
             android-x86_64-nightly: 'android-x86_64'
             android-api-16-nightly: 'android-api-16'
             android-aarch64-nightly: 'android-aarch64'
     filename_platform:
@@ -22,17 +22,17 @@ platform_names:
             android-api-16-nightly: 'android-arm'
             android-aarch64-nightly: 'android-aarch64'
 
 # A default entry, which the mappings below extend and override.
 # Final 'destinations' will be the product of:
 # s3_bucket_paths + destinations + locale_prefix + pretty_name
 default: &default
     from:
-        - checksums-signing
+        - beetmover
     all_locales: true
     description: "TO_BE_OVERRIDDEN"
     locale_prefix:
         by-locale:
             default: ''
             en-US: '${locale}/'
     source_path_modifier: ''
     destinations:  # locale_prefix is appended
@@ -49,13 +49,8 @@ default: &default
 
 # Configuration for individual files. Extends 'default', above.
 mapping:
     target.checksums:
         <<: *default
         description: "Checksums file containing size, hash, sha algorithm and filename"
         pretty_name: fennec-${version}.${locale}.${filename_platform}.checksums
         checksums_path: fennec-${version}.${locale}.${filename_platform}.checksums
-    target.checksums.asc:
-        <<: *default
-        description: "Detached signature for the checksums file"
-        pretty_name: fennec-${version}.${locale}.${filename_platform}.checksums.asc
-        checksums_path: fennec-${version}.${locale}.${filename_platform}.checksums.asc
--- a/taskcluster/taskgraph/manifests/firefox_candidates_checksums.yml
+++ b/taskcluster/taskgraph/manifests/firefox_candidates_checksums.yml
@@ -36,17 +36,17 @@ s3_bucket_paths:
     by-platform:
         .*devedition.*:
             - pub/devedition/candidates
         default:
             - pub/firefox/candidates
 default_locales:
     - en-US
 tasktype_map:
-    checksums-signing: signing
+    beetmover-repackage: beetmover
     release-beetmover-signed-langpacks: signing
 platform_names:
     path_platform:
         by-platform:
             linux-shippable: 'linux-i686'
             linux64-shippable: 'linux-x86_64'
             linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
             macosx64-shippable: 'mac'
@@ -57,35 +57,30 @@ platform_names:
             linux: 'linux-i686'
             linux64: 'linux-x86_64'
             macosx64: 'mac'
             win32: 'win32'
             win64: 'win64'
 
 default: &default
     from:
-        - checksums-signing
+        - beetmover-repackage
     all_locales: true
     description: "TO_BE_OVERRIDDEN"
     locale_prefix: '${locale}/'
     source_path_modifier: ''
     destinations:
         - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}
 
 mapping:
     target.checksums:
         <<: *default
         description: "Checksums file containing size, hash, sha algorithm and filename"
         pretty_name: firefox-${version}.checksums.beet
         checksums_path: beetmover-checksums/${path_platform}/${locale}/firefox-${version}.checksums.beet
-    target.checksums.asc:
-        <<: *default
-        description: "Detached signature for the checksums file"
-        pretty_name: firefox-${version}.checksums.asc
-        checksums_path: beetmover-checksums/${path_platform}/${locale}/firefox-${version}.checksums.asc
     target-langpack.checksums:
         <<: *default
         description: "Checksums file containing size, hash, sha algorithm and filename for the langpack"
         locale_prefix: ''
         from:
             - release-beetmover-signed-langpacks
         pretty_name: ${locale}.checksums.beet
         checksums_path: beetmover-checksums/${path_platform}/xpi/${locale}.checksums.beet
--- a/taskcluster/taskgraph/manifests/firefox_nightly_checksums.yml
+++ b/taskcluster/taskgraph/manifests/firefox_nightly_checksums.yml
@@ -6,17 +6,17 @@ s3_bucket_paths:
     by-platform:
         .*devedition.*:
             - pub/devedition/nightly
         default:
             - pub/firefox/nightly
 default_locales:  # if given an empty locale, use these locales
     - en-US
 tasktype_map:  # Map task reference to task type.
-    checksums-signing: signing
+    beetmover-repackage: beetmover
 platform_names:
     filename_platform:
         by-platform:
             linux-shippable: 'linux-i686'
             linux64-shippable: 'linux-x86_64'
             linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
             macosx64-shippable: 'mac'
             win32-shippable: 'win32'
@@ -24,17 +24,17 @@ platform_names:
             win64-aarch64-shippable: 'win64-aarch64'
             win64-asan-reporter-nightly: 'win64-asan-reporter'
 
 # A default entry, which the mappings below extend and override.
 # Final 'destinations' will be the product of:
 # s3_bucket_paths + destinations + locale_prefix + pretty_name
 default: &default
     from:
-        - checksums-signing
+        - beetmover-repackage
     all_locales: true
     description: "TO_BE_OVERRIDDEN"
     locale_prefix: ''
     source_path_modifier: ''
     destinations:  # locale_prefix is appended
         by-locale:
             en-US:
                 - ${year}/${month}/${upload_date}-${branch}
@@ -46,13 +46,8 @@ default: &default
 
 # Configuration for individual files. Extends 'default', above.
 mapping:
     target.checksums:
         <<: *default
         description: "Checksums file containing size, hash, sha algorithm and filename"
         pretty_name: firefox-${version}.${locale}.${filename_platform}.checksums
         checksums_path: firefox-${version}.${locale}.${filename_platform}.checksums
-    target.checksums.asc:
-        <<: *default
-        description: "Detached signature for the checksums file"
-        pretty_name: firefox-${version}.${locale}.${filename_platform}.checksums.asc
-        checksums_path: firefox-${version}.${locale}.${filename_platform}.checksums.asc
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -105,60 +105,51 @@ def make_beetmover_checksums_description
             task['shipping-phase'] = job['shipping-phase']
 
         if 'shipping-product' in job:
             task['shipping-product'] = job['shipping-product']
 
         yield task
 
 
-def generate_upstream_artifacts(refs, platform, locale=None):
-    # Until bug 1331141 is fixed, if you are adding any new artifacts here that
-    # need to be transfered to S3, please be aware you also need to follow-up
-    # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
-    # See example in bug 1348286
-    common_paths = [
-        "public/target.checksums",
-        "public/target.checksums.asc",
-    ]
-
+def generate_upstream_artifacts(locale=None):
+    # XXX: this function is called solely for Devedition, until we fix 1537713
+    # so that DevEdition uses in-tree manifests too. This will be fixed in Q3
     upstream_artifacts = [{
-        "taskId": {"task-reference": refs["signing"]},
-        "taskType": "signing",
-        "paths": common_paths,
+        "taskId": {"task-reference": "<beetmover-repackage>"},
+        "taskType": "beetmover",
+        "paths": [
+            "public/target.checksums",
+        ],
         "locale": locale or "en-US",
     }]
 
     return upstream_artifacts
 
 
 @transforms.add
 def make_beetmover_checksums_worker(config, jobs):
     for job in jobs:
         locale = job["attributes"].get("locale")
         platform = job["attributes"]["build_platform"]
 
-        refs = {
-            "signing": "<checksums-signing>",
-        }
-
         worker = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
         }
 
         if should_use_artifact_map(platform):
             upstream_artifacts = generate_beetmover_upstream_artifacts(
                 config, job, platform, locale
             )
             worker['artifact-map'] = generate_beetmover_artifact_map(
                 config, job, platform=platform, locale=locale)
         else:
             upstream_artifacts = generate_upstream_artifacts(
-                refs, platform, locale
+                locale
             )
             # Clean up un-used artifact map, to avoid confusion
             if job['attributes'].get('artifact_map'):
                 del job['attributes']['artifact_map']
 
         worker['upstream-artifacts'] = upstream_artifacts
 
         if locale:
deleted file mode 100644
--- a/taskcluster/taskgraph/transforms/checksums_signing.py
+++ /dev/null
@@ -1,97 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Transform the checksums signing task into an actual task description.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-from taskgraph.loader.single_dep import schema
-from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.scriptworker import (
-    get_signing_cert_scope,
-    get_worker_type_for_scope,
-)
-from taskgraph.util.treeherder import replace_group
-from taskgraph.transforms.task import task_description_schema
-from voluptuous import Required, Optional
-
-checksums_signing_description_schema = schema.extend({
-    Required('depname', default='beetmover'): basestring,
-    Optional('label'): basestring,
-    Optional('treeherder'): task_description_schema['treeherder'],
-    Optional('shipping-product'): task_description_schema['shipping-product'],
-    Optional('shipping-phase'): task_description_schema['shipping-phase'],
-})
-
-transforms = TransformSequence()
-transforms.add_validate(checksums_signing_description_schema)
-
-
-@transforms.add
-def make_checksums_signing_description(config, jobs):
-    for job in jobs:
-        dep_job = job['primary-dependency']
-        attributes = dep_job.attributes
-
-        treeherder = job.get('treeherder', {})
-        treeherder.setdefault(
-            'symbol',
-            replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs')
-        )
-        dep_th_platform = dep_job.task.get('extra', {}).get(
-            'treeherder', {}).get('machine', {}).get('platform', '')
-        treeherder.setdefault('platform',
-                              "{}/opt".format(dep_th_platform))
-        treeherder.setdefault(
-            'tier',
-            dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
-        )
-        treeherder.setdefault('kind', 'build')
-
-        label = job['label']
-        description = (
-            "Signing of Checksums file for locale '{locale}' for build '"
-            "{build_platform}/{build_type}'".format(
-                locale=attributes.get('locale', 'en-US'),
-                build_platform=attributes.get('build_platform'),
-                build_type=attributes.get('build_type')
-            )
-        )
-        dependencies = {"beetmover": dep_job.label}
-
-        attributes = copy_attributes_from_dependent_job(dep_job)
-
-        if dep_job.attributes.get('locale'):
-            treeherder['symbol'] = 'cs({})'.format(dep_job.attributes.get('locale'))
-            attributes['locale'] = dep_job.attributes.get('locale')
-
-        upstream_artifacts = [{
-            "taskId": {"task-reference": "<beetmover>"},
-            "taskType": "beetmover",
-            "paths": [
-                "public/target.checksums",
-            ],
-            "formats": ["autograph_gpg"]
-        }]
-
-        signing_cert_scope = get_signing_cert_scope(config)
-        task = {
-            'label': label,
-            'description': description,
-            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
-            'worker': {'implementation': 'scriptworker-signing',
-                       'upstream-artifacts': upstream_artifacts,
-                       'max-run-time': 3600},
-            'scopes': [
-                signing_cert_scope,
-            ],
-            'dependencies': dependencies,
-            'attributes': attributes,
-            'run-on-projects': dep_job.attributes.get('run_on_projects'),
-            'treeherder': treeherder,
-        }
-
-        yield task