Bug 1463749 - Langpack Checksums r=mtabara
authorJustin Wood <Callek@gmail.com>
Thu, 26 Jul 2018 15:16:45 -0400
changeset 485966 9a174db276a3c1e834bb8a5c22e44541deeaae46
parent 485965 185d367f1916d83ba4d0ff19f45437f5c47a212e
child 485967 dfdd922e68028b57b288fe2ecfac1a0b50ae6c2b
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1463749
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1463749 - Langpack Checksums r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D2446
taskcluster/ci/config.yml
taskcluster/ci/post-beetmover-checksums-dummy/kind.yml
taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -43,16 +43,17 @@ treeherder:
         'L10n-Rpk': 'Localized Repackaged Repacks'
         'BM-L10n': 'Beetmover for locales'
         'BMR-L10n': 'Beetmover repackages for locales'
         'c-Up': 'Balrog submission of complete updates'
         'cs': 'Checksum signing'
         'css': 'Checksum signing for source'
         'rs': 'Repackage signing'
         'BMcs': 'Beetmover checksums'
+        'BMcslang': 'Beetmover checksums for language packs'
         'BMcss': 'Beetmover checksums for source'
         'Aries': 'Aries Device Image'
         'Deb7': 'Packages for Debian 7'
         'Deb9': 'Packages for Debian 9'
         'Nexus 5-L': 'Nexus 5-L Device Image'
         'I': 'Docker Image Builds'
         'TL': 'Toolchain builds for Linux 64-bits'
         'TM': 'Toolchain builds for OSX'
--- a/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml
+++ b/taskcluster/ci/post-beetmover-checksums-dummy/kind.yml
@@ -7,16 +7,17 @@ loader: taskgraph.loader.transform:loade
 transforms:
    - taskgraph.transforms.reverse_chunk_deps:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - beetmover-checksums
    - release-beetmover-source-checksums
    - release-eme-free-repack-beetmover-checksums
+   - release-beetmover-signed-langpacks-checksums
 
 jobs:
    firefox-promote:
       name: post-beetmover-checksums-dummy
       description: Dummy task to deal with max_dependencies
       run-on-projects: []
       shipping-phase: promote
       shipping-product: firefox
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.beetmover_langpack_checksums:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-beetmover-signed-langpacks
+
+job-template:
+   shipping-phase: promote
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -333,16 +333,20 @@ Verifies the contents and package of rel
 release-sign-and-push-langpacks
 -------------------------------
 Sign a langpack XPI and publishes it onto addons.mozilla.org.
 
 release-beetmover-signed-langpacks
 ----------------------------------
 Publishes signed langpacks to archive.mozilla.org
 
+release-beetmover-signed-langpacks-checksums
+--------------------------------------------
+Publishes signed langpacks to archive.mozilla.org
+
 release-update-verify
 ---------------------
 Verifies the contents and package of release update MARs.
 
 release-secondary-update-verify
 -------------------------------
 Verifies the contents and package of release update MARs.
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
@@ -0,0 +1,165 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform release-beetmover-langpack-checksums into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+                                         get_beetmover_action_scope,
+                                         get_worker_type_for_scope)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+beetmover_checksums_description_schema = Schema({
+    Required('dependent-task'): object,
+    Required('depname', default='build'): basestring,
+    Optional('label'): basestring,
+    Optional('treeherder'): task_description_schema['treeherder'],
+    Optional('locale'): basestring,
+    Optional('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('shipping-product'): task_description_schema['shipping-product'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        validate_schema(
+            beetmover_checksums_description_schema, job,
+            "In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
+        yield job
+
+
+@transforms.add
+def make_beetmover_checksums_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        attributes = dep_job.attributes
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault(
+            'symbol',
+            'BMcslang(N{})'.format(attributes.get('l10n_chunk', ''))
+            )
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+
+        label = job['label']
+        build_platform = attributes.get('build_platform')
+
+        description = "Beetmover submission of checksums for langpack files"
+
+        extra = {}
+        if 'devedition' in build_platform:
+            extra['product'] = 'devedition'
+        else:
+            extra['product'] = 'firefox'
+
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        for k, v in dep_job.dependencies.items():
+            if k.startswith('beetmover'):
+                dependencies[k] = v
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+        if 'chunk_locales' in dep_job.attributes:
+            attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
+
+        bucket_scope = get_beetmover_bucket_scope(config)
+        action_scope = get_beetmover_action_scope(config)
+
+        task = {
+            'label': label,
+            'description': description,
+            'worker-type': get_worker_type_for_scope(config, bucket_scope),
+            'scopes': [bucket_scope, action_scope],
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+            'extra': extra,
+        }
+
+        if 'shipping-phase' in job:
+            task['shipping-phase'] = job['shipping-phase']
+
+        if 'shipping-product' in job:
+            task['shipping-product'] = job['shipping-product']
+
+        yield task
+
+
+def generate_upstream_artifacts(refs, platform, locales=None):
+    # Until bug 1331141 is fixed, if you are adding any new artifacts here that
+    # need to be transfered to S3, please be aware you also need to follow-up
+    # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
+    # See example in bug 1348286
+    common_paths = [
+        "public/target-langpack.checksums",
+    ]
+
+    upstream_artifacts = []
+    for locale in locales:
+        upstream_artifacts.append({
+            "taskId": {"task-reference": refs["beetmover"]},
+            "taskType": "signing",
+            "paths": common_paths,
+            "locale": locale or "en-US",
+        })
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_beetmover_checksums_worker(config, jobs):
+    for job in jobs:
+        valid_beetmover_job = (len(job["dependencies"]) == 1)
+        if not valid_beetmover_job:
+            raise NotImplementedError("Beetmover checksums must have one dependency.")
+
+        locales = job["attributes"].get("chunk_locales")
+        platform = job["attributes"]["build_platform"]
+
+        refs = {
+            "beetmover": None,
+        }
+        for dependency in job["dependencies"].keys():
+            if dependency.startswith("release-beetmover"):
+                refs['beetmover'] = "<{}>".format(dependency)
+        if None in refs.values():
+            raise NotImplementedError(
+                "Beetmover checksums must have a beetmover dependency!")
+
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                refs, platform, locales
+            ),
+        }
+
+        job["worker"] = worker
+
+        yield job