Bug 1441353 - part 1: Add AMO tasks for signing and uploading r=Callek
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 30 Mar 2018 18:22:38 +0200
changeset 413045 fae9885d7b375793a987070307ef91ad7cbc54b9
parent 413012 7eb059869ed692e1dbe5ade88edb0aa7eff61f1d
child 413046 a7f18f20fab15f3c44ec317053444e9b7ab49a91
push id33832
push userrgurzau@mozilla.com
push dateThu, 12 Apr 2018 23:09:18 +0000
treeherdermozilla-central@da809ecceaf3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1441353
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1441353 - part 1: Add AMO tasks for signing and uploading r=Callek MozReview-Commit-ID: LplkEtjR2Mt
taskcluster/ci/config.yml
taskcluster/ci/release-sign-and-push-langpacks/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/release_sign_and_push_langpacks.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -53,16 +53,17 @@ treeherder:
         'WM64': 'MinGW builds for Windows 64-bits'
         'Searchfox': 'Searchfox builds'
         'SM': 'Spidermonkey builds'
         'pub': 'APK publishing'
         'p': 'Partial generation'
         'ps': 'Partials signing'
         'Rel': 'Release promotion'
         'Snap': 'Snap image generation'
+        'langpack': 'Langpack sigatures and uploads'
 
 index:
     products:
         - 'firefox'
         - 'fennec'
         - 'mobile'
         - 'static-analysis'
         - 'devedition'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-sign-and-push-langpacks/kind.yml
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.release_sign_and_push_langpacks:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - build
+   - nightly-l10n
+
+
+only-for-build-platforms:
+   - linux64-nightly/opt    # addons.mozilla.org only support 1 platform per locale. That's why we use linux64
+   - macosx64-nightly/opt   # Although, we need the special locale "ja-JP-Mac" from this platform
+   # TODO Activate devedition
+
+
+job-template:
+   description: Signs {locales} XPIs for platform via addons.mozilla.org and pushes them
+   worker-type:
+      by-project:
+         mozilla-beta: scriptworker-prov-v1/addon-v1
+         mozilla-release: scriptworker-prov-v1/addon-v1
+         default: scriptworker-prov-v1/addon-dev
+   worker:
+      implementation: sign-and-push-addons
+      channel:
+         by-project:
+            # Only release langpacks are listed publicly
+            mozilla-release: listed
+            default: unlisted
+      upstream-artifacts:   # See transforms
+   run-on-projects: []
+   scopes:
+      by-project:
+         mozilla-beta:
+            - project:releng:addons.mozilla.org:server:production
+         mozilla-release:
+            - project:releng:addons.mozilla.org:server:production
+         default:
+            - project:releng:addons.mozilla.org:server:staging
+   shipping-phase: promote
+   shipping-product: firefox
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -305,16 +305,20 @@ Submit to S3 the artifacts produced by t
 release-final-verify
 --------------------
 Verifies the contents and package of release update MARs.
 
 release-secondary-final-verify
 ------------------------------
 Verifies the contents and package of release update MARs for RC releases.
 
+release-sign-and-push-langpacks
+-------------------------------
+Sign a langpack XPI and publishes it onto addons.mozilla.org.
+
 release-update-verify
 ---------------------
 Verifies the contents and package of release update MARs.
 
 release-secondary-update-verify
 ---------------------
 Verifies the contents and package of release update MARs.
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/release_sign_and_push_langpacks.py
@@ -0,0 +1,175 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the release-sign-and-push task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import validate_schema, Schema, resolve_keyed_by, optionally_keyed_by
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required
+
+
+transforms = TransformSequence()
+
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+
+transforms = TransformSequence()
+
+
+langpack_sign_push_description_schema = Schema({
+    Required('dependent-task'): object,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('worker-type'): optionally_keyed_by('project', basestring),
+    Required('worker'): {
+        Required('implementation'): 'sign-and-push-addons',
+        Required('channel'): optionally_keyed_by('project', Any('listed', 'unlisted')),
+        Required('upstream-artifacts'): None,   # Processed here below
+    },
+
+    Required('run-on-projects'): [],
+    Required('scopes'): optionally_keyed_by('project', [basestring]),
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
+})
+
+
+@transforms.add
+def set_label(config, jobs):
+    for job in jobs:
+        label = 'sign-and-push-langpacks-{}'.format(job['dependent-task'].label)
+        job['label'] = label
+
+        yield job
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        validate_schema(
+            langpack_sign_push_description_schema, job,
+            'In sign-and-push-langpacks ({} kind) task for {}:'.format(config.kind, job['label'])
+        )
+        yield job
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['label'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'scopes', item_name=job['label'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'worker.channel', item_name=job['label'], project=config.params['project']
+        )
+
+        yield job
+
+
+@transforms.add
+def copy_attributes(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        job['attributes'] = copy_attributes_from_dependent_job(dep_job)
+        job['attributes']['chunk_locales'] = dep_job.attributes.get('chunk_locales', ['en-US'])
+
+        yield job
+
+
+@transforms.add
+def filter_out_macos_jobs_but_mac_only_locales(config, jobs):
+    for job in jobs:
+        build_platform = job['dependent-task'].attributes.get('build_platform')
+
+        if build_platform == 'linux64-nightly':
+            yield job
+        elif build_platform == 'macosx64-nightly' and \
+                'ja-JP-mac' in job['attributes']['chunk_locales']:
+            # Other locales of the same job shouldn't be processed
+            job['attributes']['chunk_locales'] = ['ja-JP-mac']
+            job['label'] = job['label'].replace(
+                job['attributes']['l10n_chunk'], 'ja-JP-mac'
+            )
+            yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'langpack(SnP{})'.format(
+            job['attributes'].get('l10n_chunk', '')
+        ))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+
+        job['description'] = job['description'].format(
+            locales='/'.join(job['attributes']['chunk_locales']),
+        )
+
+        job['dependencies'] = {
+            str(dep_job.kind): dep_job.label
+        }
+        job['treeherder'] = treeherder
+
+        yield job
+
+
+def generate_upstream_artifacts(upstream_task_ref, locales):
+    return [{
+        'taskId': {'task-reference': upstream_task_ref},
+        'taskType': 'build',
+        'paths': [
+            'public/build{locale}/target.langpack.xpi'.format(
+                locale='' if locale == 'en-US' else '/' + locale
+            )
+            for locale in locales
+        ],
+    }]
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        upstream_task_ref = get_upstream_task_ref(job, expected_kinds=('build', 'nightly-l10n'))
+
+        job['worker']['upstream-artifacts'] = generate_upstream_artifacts(
+            upstream_task_ref, job['attributes']['chunk_locales']
+        )
+
+        yield job
+
+
+def get_upstream_task_ref(job, expected_kinds):
+    upstream_tasks = [
+        job_kind
+        for job_kind in job['dependencies'].keys()
+        if job_kind in expected_kinds
+    ]
+
+    if len(upstream_tasks) > 1:
+        raise Exception('Only one dependency expected')
+
+    return '<{}>'.format(upstream_tasks[0])
+
+
+@transforms.add
+def strip_unused_data(config, jobs):
+    for job in jobs:
+        del job['dependent-task']
+
+        yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -546,16 +546,24 @@ task_description_schema = Schema({
     }, {
         Required('implementation'): 'push-snap',
         Required('upstream-artifacts'): [{
             Required('taskId'): taskref_or_string,
             Required('taskType'): basestring,
             Required('paths'): [basestring],
         }],
     }, {
+        Required('implementation'): 'sign-and-push-addons',
+        Required('channel'): Any('listed', 'unlisted'),
+        Required('upstream-artifacts'): [{
+            Required('taskId'): taskref_or_string,
+            Required('taskType'): basestring,
+            Required('paths'): [basestring],
+        }],
+    }, {
         Required('implementation'): 'shipit',
         Required('release-name'): basestring,
     }, {
         Required('implementation'): 'treescript',
         Required('tag'): bool,
         Required('bump'): bool,
         Optional('bump-files'): [basestring],
         Required('force-dry-run', default=True): bool,
@@ -1094,17 +1102,17 @@ def build_bouncer_submission_payload(con
 
 
 @payload_builder('push-apk')
 def build_push_apk_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'commit': worker['commit'],
-        'upstreamArtifacts':  worker['upstream-artifacts'],
+        'upstreamArtifacts': worker['upstream-artifacts'],
         'google_play_track': worker['google-play-track'],
     }
 
     if worker.get('rollout-percentage', None):
         task_def['payload']['rollout_percentage'] = worker['rollout-percentage']
 
 
 @payload_builder('push-snap')
@@ -1120,16 +1128,26 @@ def build_push_snap_payload(config, task
 def build_ship_it_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'release_name': worker['release-name']
     }
 
 
+@payload_builder('sign-and-push-addons')
+def build_sign_and_push_addons_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'channel': worker['channel'],
+        'upstreamArtifacts': worker['upstream-artifacts'],
+    }
+
+
 @payload_builder('treescript')
 def build_treescript_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
 
     task_def['payload'] = {}
     task_def.setdefault('scopes', [])
     if worker['tag']: