Bug 1317783 - Put PushApk tasks in-tree r=aki
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 04 Apr 2017 11:21:07 +0200
changeset 351236 8b96751fecb27248c7b6565c6320bdaac2e40ec0
parent 351235 d6496a5aac96e0ee9b1ad932d482ec47597f7e9b
child 351237 55921bdd83467b44631c48765d3c33d20292c91f
push id40108
push userjlorenzo@mozilla.com
push dateWed, 05 Apr 2017 08:36:16 +0000
treeherderautoland@8b96751fecb2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1317783
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1317783 - Put PushApk tasks in-tree r=aki MozReview-Commit-ID: 8uGIuj7OXwZ
taskcluster/ci/push-apk-breakpoint/kind.yml
taskcluster/ci/push-apk/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/push_apk.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/push_apk_breakpoint.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/push_apk.py
taskcluster/taskgraph/util/scriptworker.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk-breakpoint/kind.yml
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk_breakpoint:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+
+jobs:
+    android-push-apk-breakpoint/opt:
+        description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: # see transforms
+        worker:
+            implementation: push-apk-breakpoint
+        treeherder:
+            symbol: pub(Br)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+  - push-apk-breakpoint
+
+jobs:
+    push-apk/opt:
+        description: Publishes APK onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: scriptworker-prov-v1/pushapk-v1
+        worker:
+            upstream-artifacts: # see transforms
+            google-play-track: # see transforms
+            implementation: push-apk
+            # TODO unhardcode that line
+            dry-run: true
+        scopes: # see transforms
+        treeherder:
+            symbol: pub(gp)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -190,8 +190,20 @@ Checksums-signing take as input the chec
 and sign it via the signing scriptworkers. Returns the same file signed and
 additional detached signature.
 
 beetmover-checksums
 -------------------
 Beetmover, takes specific artifact checksums and pushes it to a location outside
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
+
+push-apk-breakpoint
+-------------------
+Decides whether or not APKs should be published onto Google Play Store. Jobs of this
+kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
+in order to make the decision.
+
+push-apk
+--------
+PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
+all the signed multi-locales (aka "multi") APKs for a given release and upload them
+all at once. They also depend on the breakpoint.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .transform import loader as base_loader
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
+    """
+    jobs = base_loader(kind, path, config, params, loaded_tasks)
+
+    for job in jobs:
+        job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+        yield job
+
+
+def get_dependent_loaded_tasks(config, loaded_tasks):
+    nightly_tasks = (
+        task for task in loaded_tasks if task.attributes.get('nightly')
+    )
+    tasks_with_matching_kind = (
+        task for task in nightly_tasks if task.kind in config.get('kind-dependencies')
+    )
+    android_tasks = [
+        task for task in tasks_with_matching_kind if 'android' in task.label
+    ]
+
+    return android_tasks
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -202,17 +202,17 @@ def target_tasks_code_coverage(full_task
 
 @_target_task('nightly_fennec')
 def target_tasks_nightly_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
-        if platform in ('android-api-15-nightly', 'android-x86-nightly'):
+        if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
             if not task.attributes.get('nightly', False):
                 return False
             return filter_for_project(task, parameters)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_linux')
 def target_tasks_nightly_linux(full_task_graph, parameters):
@@ -242,16 +242,17 @@ def target_tasks_mozilla_beta(full_task_
             return False
         if platform in ('linux64', 'linux'):
             if task.attributes['build_type'] == 'opt':
                 return False
         # skip l10n, beetmover, balrog
         if task.kind in [
             'balrog', 'beetmover', 'beetmover-checksums', 'beetmover-l10n',
             'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing',
+            'push-apk', 'push-apk-breakpoint',
         ]:
             return False
         return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('mozilla_release_tasks')
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('worker-type'): basestring,
+    Required('worker'): object,
+    Required('scopes'): None,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_description_schema,
+    'PushApk'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+        job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+        job['worker']['google-play-track'] = get_push_apk_track(config)
+        job['scopes'] = [get_push_apk_scope(config)]
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
+
+
+def generate_upstream_artifacts(dependencies):
+    return [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        'taskType': 'signing',
+        'paths': ['public/build/target.apk'],
+    } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk-breakpoint kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_breakpoint_worker_type
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_breakpoint_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('worker-type'): None,
+    Required('worker'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_breakpoint_description_schema,
+    'PushApkBreakpoint'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+
+        worker_type = get_push_apk_breakpoint_worker_type(config)
+        job['worker-type'] = worker_type
+
+        job['worker']['payload'] = {} if 'human' in worker_type else {
+                'image': 'ubuntu:16.10',
+                'command': [
+                    '/bin/bash',
+                    '-c',
+                    'echo "Dummy task while while bug 1351664 is implemented"'
+                ],
+                'maxRunTime': 600,
+            }
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -340,16 +340,38 @@ task_description_schema = Schema({
             Required('taskId'): taskref_or_string,
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
+    }, {
+        Required('implementation'): 'push-apk-breakpoint',
+        Required('payload'): object,
+
+    }, {
+        Required('implementation'): 'push-apk',
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+        }],
+
+        # "Invalid" is a noop for try and other non-supported branches
+        Required('google-play-track'): Any('production', 'beta', 'alpha', 'invalid'),
+        Required('dry-run', default=True): bool,
     }),
 })
 
 GROUP_NAMES = {
     'py': 'Python unit tests',
     'tc': 'Executed by TaskCluster',
     'tc-e10s': 'Executed by TaskCluster with e10s',
     'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
@@ -377,16 +399,17 @@ GROUP_NAMES = {
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',
     'TM': 'Toolchain builds for OSX',
     'TW32': 'Toolchain builds for Windows 32-bits',
     'TW64': 'Toolchain builds for Windows 64-bits',
     'SM-tc': 'Spidermonkey builds',
+    'pub': 'APK publishing',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
@@ -593,16 +616,32 @@ def build_beetmover_payload(config, task
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
+@payload_builder('push-apk')
+def build_push_apk_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'dry_run': worker['dry-run'],
+        'upstreamArtifacts':  worker['upstream-artifacts'],
+        'google_play_track': worker['google-play-track'],
+    }
+
+
+@payload_builder('push-apk-breakpoint')
+def build_push_apk_breakpoint_payload(config, task, task_def):
+    task_def['payload'] = task['worker']['payload']
+
+
 @payload_builder('native-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/push_apk.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common functions for both push-apk and push-apk-breakpoint.
+"""
+
+import re
+
+from taskgraph.util.schema import validate_schema
+
+REQUIRED_ARCHITECTURES = ('android-x86', 'android-api-15')
+PLATFORM_REGEX = re.compile(r'signing-android-(\S+)-nightly')
+
+
+def fill_labels_tranform(_, jobs):
+    for job in jobs:
+        job['label'] = job['name']
+
+        yield job
+
+
+def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        yield validate_schema(
+            description_schema, job,
+            "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
+        )
+
+
+def validate_dependent_tasks_transform(_, jobs):
+    for job in jobs:
+        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+        yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+    dependencies_labels = [task.label for task in dependent_tasks]
+
+    is_this_required_architecture_present = {
+        architecture: any(architecture in label for label in dependencies_labels)
+        for architecture in REQUIRED_ARCHITECTURES
+    }
+    are_all_required_achitectures_present = all(is_this_required_architecture_present.values())
+
+    if not are_all_required_achitectures_present:
+        raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+        )
+
+
+def delete_non_required_fields_transform(_, jobs):
+    for job in jobs:
+        del job['name']
+        del job['dependent-tasks']
+
+        yield job
+
+
+def generate_dependencies(dependent_tasks):
+    # Because we depend on several tasks that have the same kind, we introduce the platform
+    dependencies = {}
+    for task in dependent_tasks:
+        platform_match = PLATFORM_REGEX.match(task.label)
+        # platform_match is None when the breakpoint task is given
+        task_kind = task.kind if platform_match is None else \
+            '{}-{}'.format(task.kind, platform_match.group(1))
+        dependencies[task_kind] = task.label
+    return dependencies
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -183,16 +183,54 @@ BALROG_CHANNEL_SCOPES = {
         'project:releng:balrog:channel:release-cdntest'
         'project:releng:balrog:channel:esr',
         'project:releng:balrog:channel:esr-localtest',
         'project:releng:balrog:channel:esr-cdntest'
     ]
 }
 
 
+PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
+    'aurora', set([
+        'mozilla-aurora',
+    ])
+], [
+    'beta', set([
+        'mozilla-beta',
+    ])
+], [
+    'release', set([
+        'mozilla-release',
+    ])
+]]
+
+
+PUSH_APK_SCOPES = {
+    'aurora': 'project:releng:googleplay:aurora',
+    'beta': 'project:releng:googleplay:beta',
+    'release': 'project:releng:googleplay:release',
+    'default': 'project:releng:googleplay:invalid',
+}
+
+# See https://github.com/mozilla-releng/pushapkscript#aurora-beta-release-vs-alpha-beta-production
+PUSH_APK_GOOGLE_PLAY_TRACT = {
+    'aurora': 'beta',
+    'beta': 'production',
+    'release': 'production',
+    'default': 'invalid',
+}
+
+PUSH_APK_BREAKPOINT_WORKER_TYPE = {
+    'aurora': 'aws-provisioner-v1/taskcluster-generic',
+    'beta': 'null-provisioner/human-breakpoint',
+    'release': 'null-provisioner/human-breakpoint',
+    'default': 'invalid/invalid',
+}
+
+
 # scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
 
     Args:
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
@@ -281,16 +319,34 @@ get_balrog_server_scope = functools.part
 )
 
 get_balrog_channel_scopes = functools.partial(
     get_scope_from_project,
     BALROG_SCOPE_ALIAS_TO_PROJECT,
     BALROG_CHANNEL_SCOPES
 )
 
+get_push_apk_scope = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_SCOPES
+)
+
+get_push_apk_track = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_GOOGLE_PLAY_TRACT
+)
+
+get_push_apk_breakpoint_worker_type = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_BREAKPOINT_WORKER_TYPE
+)
+
 
 # release_config {{{1
 def get_release_config(config):
     """Get the build number and version for a release task.
 
     Currently only applies to beetmover tasks.
 
     Args: