Bug 1441353 - part 2: Add beetmover job to publish signed langpacks r=Callek a=ritu
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 06 Apr 2018 16:04:27 +0200
changeset 463211 e34ed4b609e54848bb609d6be64d7300272c97f1
parent 463210 d0b45d0fc8ccdcedaeedb6d1b945bc8da15603e1
child 463212 5a573ad73b78509ff92daed4d161fb9244fb8dc1
push id1683
push usersfraser@mozilla.com
push dateThu, 26 Apr 2018 16:43:40 +0000
treeherdermozilla-release@5af6cb21869d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek, ritu
bugs1441353
milestone60.0
Bug 1441353 - part 2: Add beetmover job to publish signed langpacks r=Callek a=ritu MozReview-Commit-ID: HXjcclf3OeX
taskcluster/ci/release-beetmover-signed-langpacks/kind.yml
taskcluster/ci/release-generate-checksums/kind.yml
taskcluster/ci/release-snap-repackage/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-beetmover-signed-langpacks/kind.yml
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.release_beetmover_signed_addons:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-sign-and-push-langpacks
+
+only-for-attributes:
+   - nightly
+
+job-template:
+   description: Beetmover submission for platform-independent langpacks {locales} in {platform} directory
+   worker-type:
+      by-project:
+         mozilla-beta: scriptworker-prov-v1/beetmoverworker-v1
+         mozilla-release: scriptworker-prov-v1/beetmoverworker-v1
+         default: scriptworker-prov-v1/beetmoverworker-dev
+   run-on-projects: []
+   shipping-phase: promote
+   shipping-product: firefox
--- a/taskcluster/ci/release-generate-checksums/kind.yml
+++ b/taskcluster/ci/release-generate-checksums/kind.yml
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 kind-dependencies:
    - beetmover-source
    - post-beetmover-checksums-dummy
+   - release-beetmover-signed-langpacks
 
 transforms:
    - taskgraph.transforms.build:transforms
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.release_generate_checksums:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
--- a/taskcluster/ci/release-snap-repackage/kind.yml
+++ b/taskcluster/ci/release-snap-repackage/kind.yml
@@ -7,16 +7,17 @@ loader: taskgraph.loader.transform:loade
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.release_snap_repackage:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - post-beetmover-dummy
+   - release-beetmover-signed-langpacks
 
 job-defaults:
    description: Generates snap image
    run-on-projects: []  # to make sure this never runs as part of CI
    shipping-phase: promote
    scopes: []
    treeherder:
       platform: linux64/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -312,16 +312,20 @@ Verifies the contents and package of rel
 release-secondary-balrog-publishing
 ---------------------
 Schedule an RC release to go live in Balrog. Usually this will happen on the beta channel, to a smaller audience, before the RC goes live on the release channel.
 
 release-sign-and-push-langpacks
 -------------------------------
 Sign a langpack XPI and publishes it onto addons.mozilla.org.
 
+release-beetmover-signed-langpacks
+----------------------------------
+Publishes signed langpacks to archive.mozilla.org
+
 release-update-verify
 ---------------------
 Verifies the contents and package of release update MARs.
 
 release-secondary-update-verify
 ---------------------
 Verifies the contents and package of release update MARs.
 
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -49,44 +49,53 @@ logger = logging.getLogger(__name__)
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
     "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
     "mozharness.zip",
-    "target.langpack.xpi",
-]
-
-# Until bug 1331141 is fixed, if you are adding any new artifacts here that
-# need to be transfered to S3, please be aware you also need to follow-up
-# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
-# See example in bug 1348286
-_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
-    "target.langpack.xpi",
 ]
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
-    r'^(linux(|64)|macosx64)(|-devedition)-nightly$':
+    r'^(linux(|64)|macosx64)-nightly$':
         _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
             'host/bin/mar',
             'host/bin/mbsdiff',
         ],
-    r'^win(32|64)(|-devedition)-nightly$':
+    r'^(linux(|64)|macosx64)-devedition-nightly$':
+        _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+            'host/bin/mar',
+            'host/bin/mbsdiff',
+            # TODO Bug 1453033: Sign devedition langpacks
+            'target.langpack.xpi',
+        ],
+    r'^win(32|64)-nightly$':
         _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
-            "host/bin/mar.exe",
-            "host/bin/mbsdiff.exe",
+            'host/bin/mar.exe',
+            'host/bin/mbsdiff.exe',
         ],
-    r'^(linux(|64)|macosx64|win(32|64))(|-devedition)-nightly-l10n$':
-        _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+    r'^win(32|64)-devedition-nightly$':
+        _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+            'host/bin/mar.exe',
+            'host/bin/mbsdiff.exe',
+            # TODO Bug 1453033: Sign devedition langpacks
+            'target.langpack.xpi',
+        ],
+    r'^(linux(|64)|macosx64|win(32|64))-nightly-l10n$': [],
+    r'^(linux(|64)|macosx64|win(32|64))-devedition-nightly-l10n$':
+        [
+            # TODO Bug 1453033: Sign devedition langpacks
+            'target.langpack.xpi',
+        ],
 }
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     r'^linux(|64)(|-devedition)-nightly(|-l10n)$': ['target.tar.bz2', 'target.tar.bz2.asc'],
@@ -281,22 +290,23 @@ def generate_upstream_artifacts(build_ta
 
     for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
         plarform_was_previously_matched_by_regex = None
         for platform_regex, paths in mapping.iteritems():
             if platform_regex.match(platform) is not None:
                 _check_platform_matched_only_one_regex(
                     tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
                 )
-                upstream_artifacts.append({
-                    "taskId": {"task-reference": ref},
-                    "taskType": tasktype,
-                    "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
-                    "locale": locale or "en-US",
-                })
+                if paths:
+                    upstream_artifacts.append({
+                        "taskId": {"task-reference": ref},
+                        "taskType": tasktype,
+                        "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
+                        "locale": locale or "en-US",
+                    })
                 plarform_was_previously_matched_by_regex = platform_regex
 
     return upstream_artifacts
 
 
 def generate_partials_upstream_artifacts(artifacts, platform, locale=None):
     if not locale or locale == 'en-US':
         artifact_prefix = 'public/build'
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
@@ -0,0 +1,208 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+                                         get_beetmover_action_scope)
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.transforms.release_sign_and_push_langpacks import get_upstream_task_ref
+from voluptuous import Required, Optional
+
+import logging
+import copy
+
+logger = logging.getLogger(__name__)
+
+
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+
+transforms = TransformSequence()
+
+
+beetmover_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    Required('description'): basestring,
+    Required('worker-type'): optionally_keyed_by('project', basestring),
+    Required('run-on-projects'): [],
+
+    # locale is passed only for l10n beetmoving
+    Optional('locale'): basestring,
+    Optional('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('shipping-product'): task_description_schema['shipping-product'],
+})
+
+
+@transforms.add
+def set_label(config, jobs):
+    for job in jobs:
+        job['label'] = job['dependent-task'].label.replace(
+            'sign-and-push-langpacks', 'beetmover-signed-langpacks'
+        )
+
+        yield job
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover ({!r} kind) task for {!r}:".format(config.kind, job['label'])
+        )
+        yield job
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['label'], project=config.params['project']
+        )
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        attributes = dep_job.attributes
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk', '')))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+
+        job['attributes'] = copy_attributes_from_dependent_job(dep_job)
+        job['attributes']['chunk_locales'] = dep_job.attributes['chunk_locales']
+
+        job['description'] = job['description'].format(
+            locales='/'.join(job['attributes']['chunk_locales']),
+            platform=job['attributes']['build_platform']
+        )
+
+        job['scopes'] = [
+            get_beetmover_bucket_scope(config),
+            get_beetmover_action_scope(config),
+        ]
+
+        job['dependencies'] = {
+            str(dep_job.kind): dep_job.label
+        }
+
+        job['run-on-projects'] = dep_job.attributes['run_on_projects']
+        job['treeherder'] = treeherder
+        job['shipping-phase'] = dep_job.attributes['shipping_phase']
+        job['shipping-product'] = dep_job.attributes['shipping_product']
+
+        yield job
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        signing_task_ref = get_upstream_task_ref(
+            job, expected_kinds=('release-sign-and-push-langpacks',)
+        )
+
+        job['worker'] = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                signing_task_ref, job['attributes']['chunk_locales']
+            ),
+        }
+
+        yield job
+
+
+def generate_upstream_artifacts(upstream_task_ref, locales):
+    return [{
+        'taskId': {'task-reference': upstream_task_ref},
+        'taskType': 'scriptworker',
+        'locale': locale,
+        'paths': [
+            # addonscript uploads en-US XPI in the en-US folder
+            'public/build/{}/target.langpack.xpi'.format(locale)
+        ],
+    } for locale in locales]
+
+
+@transforms.add
+def strip_unused_data(config, jobs):
+    for job in jobs:
+        del job['dependent-task']
+
+        yield job
+
+
+@transforms.add
+def yield_all_platform_jobs(config, jobs):
+    # Even though langpacks are now platform independent, we keep beetmoving them at old
+    # platform-specific locations. That's why this transform exist
+    for job in jobs:
+        if 'ja-JP-mac' in job['label']:
+            # This locale must not be copied on any other platform than macos
+            yield job
+        else:
+            for platform in ('linux', 'linux64', 'macosx64', 'win32', 'win64'):
+                platform_job = copy.deepcopy(job)
+                if 'ja' in platform_job['attributes']['chunk_locales'] and platform == 'macosx64':
+                    platform_job = _strip_ja_data_from_linux_job(platform_job)
+
+                platform_job = _change_platform_data(platform_job, platform)
+
+                yield platform_job
+
+
+def _strip_ja_data_from_linux_job(platform_job):
+    # Let's take "ja" out the description. This locale is in a substring like "aa/bb/cc/dd", where
+    # "ja" could be any of "aa", "bb", "cc", "dd"
+    platform_job['description'] = platform_job['description'].replace('ja/', '')
+    platform_job['description'] = platform_job['description'].replace('/ja', '')
+
+    platform_job['worker']['upstream-artifacts'] = [
+        artifact
+        for artifact in platform_job['worker']['upstream-artifacts']
+        if artifact['locale'] != 'ja'
+    ]
+
+    return platform_job
+
+
+def _change_platform_data(platform_job, platform):
+    platform_job['attributes']['build_platform'] = platform
+    platform_job['label'] = platform_job['label'].replace('linux64', platform)
+    platform_job['description'] = platform_job['description'].replace('linux64', platform)
+    platform_job['treeherder']['platform'] = platform_job['treeherder']['platform'].replace(
+        'linux64', platform
+    )
+    platform_job['worker']['release-properties']['platform'] = platform
+
+    return platform_job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -453,17 +453,17 @@ task_description_schema = Schema({
             Required('formats'): [basestring],
         }],
     }, {
         Required('implementation'): 'binary-transparency',
     }, {
         Required('implementation'): 'beetmover',
 
         # the maximum time to run, in seconds
-        Required('max-run-time'): int,
+        Required('max-run-time', default=600): int,
 
         # locale key, if this is a locale beetmover job
         Optional('locale'): basestring,
 
         Required('release-properties'): {
             'app-name': basestring,
             'app-version': basestring,
             'branch': basestring,