Bug 1470942 - part 2: Upload geckoview to maven repository via beetmover r=mtabara
authorJohan Lorenzo <jlorenzo@mozilla.com>
Thu, 16 Aug 2018 16:23:15 +0000
changeset 487007 237d45421fb53b69dca38c25baaff15f6a7ef5aa
parent 487006 056aaefaffac83e68417a62a048b046ab5e20f58
child 487008 0c42a54600831e65a4fb0bfdefdaa67324d78510
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1470942
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1470942 - part 2: Upload geckoview to maven repository via beetmover r=mtabara Depends on D2420 Differential Revision: https://phabricator.services.mozilla.com/D2421
taskcluster/ci/beetmover-geckoview/kind.yml
taskcluster/ci/config.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/beetmover_geckoview.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-geckoview/kind.yml
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.beetmover_geckoview:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - build  # geckoview builds aren't signed
+
+only-for-attributes:
+   - nightly
+
+not-for-build-platforms:
+   - linux-nightly/opt
+   - linux64-nightly/opt
+   - macosx64-nightly/opt
+   - win32-nightly/opt
+   - win64-nightly/opt
+   - linux-devedition-nightly/opt
+   - linux64-devedition-nightly/opt
+   - macosx64-devedition-nightly/opt
+   - win32-devedition-nightly/opt
+   - win64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
+   - win64-asan-reporter-nightly/opt
+
+job-template:
+   # Beetmoving geckoview makes it available to the official maven repo. So we want beetmover to
+   # act only when the release is greenlit.
+   shipping-phase: ship
+   bucket-scope:
+      by-project:
+         mozilla-central: 'project:releng:beetmover:bucket:maven-production'
+         mozilla-beta: 'project:releng:beetmover:bucket:maven-production'
+         mozilla-release: 'project:releng:beetmover:bucket:maven-production'
+         default: 'project:releng:beetmover:bucket:maven-staging'
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -183,19 +183,21 @@ scriptworker:
             - 'project:releng:signing:cert:release-signing'
             - 'project:releng:signing:cert:nightly-signing'
         'scriptworker-prov-v1/depsigning':
             - 'project:releng:signing:cert:dep-signing'
         'scriptworker-prov-v1/beetmoverworker-v1':
             - 'project:releng:beetmover:bucket:release'
             - 'project:releng:beetmover:bucket:nightly'
             - 'project:releng:beetmover:bucket:partner'
+            - 'project:releng:beetmover:bucket:maven-production'
         'scriptworker-prov-v1/beetmoverworker-dev':
             - 'project:releng:beetmover:bucket:dep'
             - 'project:releng:beetmover:bucket:dep-partner'
+            - 'project:releng:beetmover:bucket:maven-staging'
         'scriptworker-prov-v1/balrogworker-v1':
             - 'project:releng:balrog:server:nightly'
             - 'project:releng:balrog:server:aurora'
             - 'project:releng:balrog:server:beta'
             - 'project:releng:balrog:server:release'
             - 'project:releng:balrog:server:esr'
         'scriptworker-prov-v1/balrog-dev':
             - 'project:releng:balrog:server:dep'
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -198,16 +198,21 @@ release-beetmover-push-to-release publis
 candidates directory to the release directory. This is part of release
 promotion.
 
 beetmover-source
 ----------------
 
 Beetmover-source publishes release source. This is part of release promotion.
 
+beetmover-geckoview
+-------------------
+
+Beetmover-geckoview publishes the Android library called "geckoview".
+
 checksums-signing
 -----------------
 Checksums-signing take as input the checksums file generated by beetmover tasks
 and sign it via the signing scriptworkers. Returns the same file signed and
 additional detached signature.
 
 release-source-checksums-signing
 --------------------------------
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_geckoview.py
@@ -0,0 +1,156 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import \
+    craft_release_properties as beetmover_craft_release_properties
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import validate_schema, Schema, resolve_keyed_by, optionally_keyed_by
+from taskgraph.util.scriptworker import (get_phase,
+                                         get_worker_type_for_scope)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Required, Optional
+
+
+_ARTIFACT_ID_PER_PLATFORM = {
+    'android-aarch64': 'geckoview{update_channel}-arm64-v8a',
+    'android-api-16': 'geckoview{update_channel}-armeabi-v7a',
+    'android-x86': 'geckoview{update_channel}-x86',
+}
+
+_MOZ_UPDATE_CHANNEL_PER_BRANCH = {
+    'mozilla-release': '',
+    'mozilla-beta': '-beta',
+    'mozilla-central': '-nightly',
+    'maple': '-nightly-maple',
+}
+
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+beetmover_description_schema = Schema({
+    Required('dependent-task'): object,
+    Required('depname', default='build'): basestring,
+    Optional('label'): basestring,
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    Optional('bucket-scope'): optionally_keyed_by('project', basestring),
+    Optional('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('shipping-product'): task_description_schema['shipping-product'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover-geckoview ({!r} kind) task for {!r}:".format(config.kind, label))
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        attributes = dep_job.attributes
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'BM-gv')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              '{}/opt'.format(dep_th_platform))
+        treeherder.setdefault('tier', 3)
+        treeherder.setdefault('kind', 'build')
+        label = job['label']
+        description = (
+            "Beetmover submission for geckoview"
+            "{build_platform}/{build_type}'".format(
+                build_platform=attributes.get('build_platform'),
+                build_type=attributes.get('build_type')
+            )
+        )
+
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+
+        if job.get('locale'):
+            attributes['locale'] = job['locale']
+
+        resolve_keyed_by(
+            job, 'bucket-scope', item_name=job['label'],
+            project=config.params['project']
+        )
+
+        task = {
+            'label': label,
+            'description': description,
+            'worker-type': get_worker_type_for_scope(config, job['bucket-scope']),
+            'scopes': [job['bucket-scope'], 'project:releng:beetmover:action:push-to-maven'],
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': ['mozilla-central'],
+            'treeherder': treeherder,
+            'shipping-phase': job.get('shipping-phase', get_phase(config)),
+        }
+
+        yield task
+
+
+def generate_upstream_artifacts(build_task_ref):
+    return [{
+        'taskId': {'task-reference': build_task_ref},
+        'taskType': 'build',
+        'paths': ['public/build/target.maven.zip'],
+        'zipExtract': True,
+    }]
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        valid_beetmover_job = len(job['dependencies']) == 1 and 'build' in job['dependencies']
+        if not valid_beetmover_job:
+            raise NotImplementedError(
+                'Beetmover-geckoview must have a single dependency. Got: {}'.format(
+                    job['dependencies']
+                )
+            )
+
+        build_task = list(job["dependencies"].keys())[0]
+        build_task_ref = "<" + str(build_task) + ">"
+
+        worker = {
+            'implementation': 'beetmover-maven',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(build_task_ref)
+        }
+
+        job["worker"] = worker
+
+        yield job
+
+
+def craft_release_properties(config, job):
+    props = beetmover_craft_release_properties(config, job)
+
+    platform = props['platform']
+    update_channel = _MOZ_UPDATE_CHANNEL_PER_BRANCH.get(
+        props['branch'], '-UNKNOWN_MOZ_UPDATE_CHANNEL'
+    )
+    artifact_id = _ARTIFACT_ID_PER_PLATFORM[platform].format(update_channel=update_channel)
+    props['artifact-id'] = artifact_id
+    props['app-name'] = 'geckoview'     # this beetmover job is not about pushing Fennec
+
+    return props
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -506,16 +506,36 @@ task_description_schema = Schema({
         }],
     }, {
         Required('implementation'): 'beetmover-push-to-release',
 
         # the maximum time to run, in seconds
         Required('max-run-time'): int,
         Required('product'): basestring,
     }, {
+        Required('implementation'): 'beetmover-maven',
+
+        Required('max-run-time', default=600): int,
+        Required('release-properties'): {
+            'app-name': basestring,
+            'app-version': basestring,
+            'branch': basestring,
+            'build-id': basestring,
+            'artifact-id': basestring,
+            'hash-type': basestring,
+            'platform': basestring,
+        },
+
+        Required('upstream-artifacts'): [{
+            Required('taskId'): taskref_or_string,
+            Required('taskType'): basestring,
+            Required('paths'): [basestring],
+            Required('zipExtract', default=False): bool,
+        }],
+    }, {
         Required('implementation'): 'balrog',
         Required('balrog-action'): Any(*BALROG_ACTIONS),
         Optional('product'): basestring,
         Optional('platforms'): [basestring],
         Optional('release-eta'): basestring,
         Optional('channel-names'): optionally_keyed_by('project', [basestring]),
         Optional('require-mirrors'): bool,
         Optional('publish-rules'): optionally_keyed_by('project', [int]),
@@ -1072,16 +1092,26 @@ def build_beetmover_push_to_release_payl
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'product': worker['product'],
         'version': release_config['version'],
         'build_number': release_config['build_number'],
     }
 
 
+@payload_builder('beetmover-maven')
+def build_beetmover_maven_payload(config, task, task_def):
+    build_beetmover_payload(config, task, task_def)
+
+    task_def['payload']['artifact_id'] = task['worker']['release-properties']['artifact-id']
+
+    del task_def['payload']['releaseProperties']['hashType']
+    del task_def['payload']['releaseProperties']['platform']
+
+
 @payload_builder('balrog')
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
 
     if worker['balrog-action'] == 'submit-locale':
         task_def['payload'] = {
             'upstreamArtifacts':  worker['upstream-artifacts']
@@ -1308,21 +1338,19 @@ def set_defaults(config, tasks):
             worker.setdefault('env', {})
             if 'caches' in worker:
                 for c in worker['caches']:
                     c.setdefault('skip-untrusted', False)
         elif worker['implementation'] == 'generic-worker':
             worker.setdefault('env', {})
             worker.setdefault('os-groups', [])
             worker.setdefault('chain-of-trust', False)
-        elif worker['implementation'] == 'scriptworker-signing':
-            worker.setdefault('max-run-time', 600)
-        elif worker['implementation'] == 'beetmover':
-            worker.setdefault('max-run-time', 600)
-        elif worker['implementation'] == 'beetmover-push-to-release':
+        elif worker['implementation'] in (
+            'scriptworker-signing', 'beetmover', 'beetmover-push-to-release', 'beetmover-maven',
+        ):
             worker.setdefault('max-run-time', 600)
         elif worker['implementation'] == 'push-apk':
             worker.setdefault('commit', False)
 
         yield task
 
 
 @transforms.add