Bug 1317783 - Put PushApk tasks in-tree r=aki a=release
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 31 Mar 2017 09:29:00 +0200
changeset 379358 4abe0595c4d45c758bdb4c9cf1ceffd3e11e1733
parent 379357 ac672b378a0446c28e1df642b89b74f1d4ad4e2f
child 379359 c3dba85c045e8f5cb15c3e9575e2db228c716c1e
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki, release
bugs1317783
milestone53.0
Bug 1317783 - Put PushApk tasks in-tree r=aki a=release
taskcluster/ci/push-apk-breakpoint/kind.yml
taskcluster/ci/push-apk/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/task/push_apk.py
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/push_apk_breakpoint.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/push_apk.py
taskcluster/taskgraph/util/scriptworker.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk-breakpoint/kind.yml
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.push_apk:PushApkTask
+
+transforms:
+   - taskgraph.transforms.push_apk_breakpoint:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+
+jobs:
+    android-push-apk-breakpoint/opt:
+        description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: # see transforms
+        worker:
+            implementation: push-apk-breakpoint
+        treeherder:
+            symbol: pub(Br)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.push_apk:PushApkTask
+
+transforms:
+   - taskgraph.transforms.push_apk:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+  - push-apk-breakpoint
+
+jobs:
+    push-apk/opt:
+        description: Publishes APK onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: scriptworker-prov-v1/pushapk-v1
+        worker:
+            upstream-artifacts: # see transforms
+            google-play-track: # see transforms
+            implementation: push-apk
+            # TODO unhardcode that line
+            dry-run: true
+        scopes: # see transforms
+        treeherder:
+            symbol: pub(gp)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -195,8 +195,20 @@ Checksums-signing take as input the chec
 and sign it via the signing scriptworkers. Returns the same file signed and
 additional detached signature.
 
 beetmover-checksums
 -------------------
 Beetmover, takes specific artifact checksums and pushes it to a location outside
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
+
+push-apk-breakpoint
+-------------------
+Decides whether or not APKs should be published onto Google Play Store. Jobs of this
+kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
+in order to make the decision.
+
+push-apk
+--------
+PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
+all the signed multi-locales (aka "multi") APKs for a given release and upload them
+all at once. They also depend on the breakpoint.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -118,23 +118,23 @@ def target_tasks_graphics(full_task_grap
     def filter(task):
         if task.attributes['kind'] == 'artifact-build':
             return False
         return True
     return [l for l in filtered_for_project if filter(full_task_graph[l])]
 
 
 @_target_task('nightly_fennec')
-def target_tasks_nightly(full_task_graph, parameters):
+def target_tasks_nightly_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
-        if platform in ('android-api-15-nightly', 'android-x86-nightly'):
+        if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
             return task.attributes.get('nightly', False)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_linux')
 def target_tasks_nightly_linux(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of linux. The
     nightly build process involves a pipeline of builds, signing,
@@ -162,16 +162,17 @@ def target_tasks_mozilla_beta(full_task_
             return False
         if platform in ('linux64', 'linux'):
             if task.attributes['build_type'] == 'opt':
                 return False
         # skip l10n, beetmover, balrog
         if task.kind in [
             'balrog', 'beetmover', 'beetmover-checksums', 'beetmover-l10n',
             'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing',
+            'push-apk', 'push-apk-breakpoint',
         ]:
             return False
         return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('mozilla_release_tasks')
@@ -182,15 +183,15 @@ def target_tasks_mozilla_release(full_ta
     return target_tasks_mozilla_beta(full_task_graph, parameters)
 
 
 @_target_task('candidates_fennec')
 def target_tasks_candidates_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a candidates build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
-    filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+    filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters)
 
     def filter(task):
         if task.kind not in ['balrog']:
             return task.attributes.get('nightly', False)
 
     return [l for l in filtered_for_project if filter(full_task_graph[l])]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/push_apk.py
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class PushApkTask(transform.TransformTask):
+    """
+    A task implementing a beetmover job.  These depend on nightly build and signing
+    jobs and transfer the artifacts to S3 after build and signing are completed.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        """
+        Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly
+        builds.
+        """
+        jobs = super(PushApkTask, cls).get_inputs(kind, path, config, params, loaded_tasks)
+
+        for job in jobs:
+            job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+            yield job
+
+
+def get_dependent_loaded_tasks(config, loaded_tasks):
+    nightly_tasks = (
+        task for task in loaded_tasks if task.attributes.get('nightly')
+    )
+    tasks_with_matching_kind = (
+        task for task in nightly_tasks if task.kind in config.get('kind-dependencies')
+    )
+    android_tasks = [
+        task for task in tasks_with_matching_kind if 'android' in task.label
+    ]
+
+    return android_tasks
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Schema, Required
+
+
+transforms = TransformSequence()
+
+push_apk_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('worker-type'): basestring,
+    Required('worker'): object,
+    Required('scopes'): None,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_description_schema,
+    'PushApk'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+        job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+        job['worker']['google-play-track'] = get_push_apk_track(config)
+        job['scopes'] = [get_push_apk_scope(config)]
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
+
+
+def generate_upstream_artifacts(dependencies):
+    return [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        'taskType': 'signing',
+        'paths': ['public/build/target.apk'],
+    } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk-breakpoint kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.scriptworker import get_push_apk_breakpoint_worker_type
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Schema, Required
+
+
+transforms = TransformSequence()
+
+push_apk_breakpoint_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('worker-type'): None,
+    Required('worker'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_breakpoint_description_schema,
+    'PushApkBreakpoint'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+
+        worker_type = get_push_apk_breakpoint_worker_type(config)
+        job['worker-type'] = worker_type
+
+        job['worker']['payload'] = {} if 'human' in worker_type else {
+                'image': 'ubuntu:16.10',
+                'command': [
+                    '/bin/bash',
+                    '-c',
+                    'echo "Dummy task while while bug 1351664 is implemented"'
+                ],
+                'maxRunTime': 600,
+            }
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -319,16 +319,38 @@ task_description_schema = Schema({
             Required('taskId'): taskref_or_string,
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
+    }, {
+        Required('implementation'): 'push-apk-breakpoint',
+        Required('payload'): object,
+
+    }, {
+        Required('implementation'): 'push-apk',
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+        }],
+
+        # "Invalid" is a noop for try and other non-supported branches
+        Required('google-play-track'): Any('production', 'beta', 'alpha', 'invalid'),
+        Required('dry-run', default=True): bool,
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -360,16 +382,17 @@ GROUP_NAMES = {
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'tc-cs': 'Checksum signing executed by Taskcluster',
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
+    'pub': 'APK publishing',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
@@ -570,16 +593,32 @@ def build_beetmover_payload(config, task
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
+@payload_builder('push-apk')
+def build_push_apk_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'dry_run': worker['dry-run'],
+        'upstreamArtifacts':  worker['upstream-artifacts'],
+        'google_play_track': worker['google-play-track'],
+    }
+
+
+@payload_builder('push-apk-breakpoint')
+def build_push_apk_breakpoint_payload(config, task, task_def):
+    task_def['payload'] = task['worker']['payload']
+
+
 @payload_builder('macosx-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/push_apk.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common functions for both push-apk and push-apk-breakpoint.
+"""
+
+import re
+
+from taskgraph.util.schema import validate_schema
+
+REQUIRED_ARCHITECTURES = ('android-x86', 'android-api-15')
+PLATFORM_REGEX = re.compile(r'signing-android-(\S+)-nightly')
+
+
+def fill_labels_tranform(_, jobs):
+    for job in jobs:
+        job['label'] = job['name']
+
+        yield job
+
+
+def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        yield validate_schema(
+            description_schema, job,
+            "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
+        )
+
+
+def validate_dependent_tasks_transform(_, jobs):
+    for job in jobs:
+        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+        yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+    dependencies_labels = [task.label for task in dependent_tasks]
+
+    is_this_required_architecture_present = {
+        architecture: any(architecture in label for label in dependencies_labels)
+        for architecture in REQUIRED_ARCHITECTURES
+    }
+    are_all_required_achitectures_present = all(is_this_required_architecture_present.values())
+
+    if not are_all_required_achitectures_present:
+        raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+        )
+
+
+def delete_non_required_fields_transform(_, jobs):
+    for job in jobs:
+        del job['name']
+        del job['dependent-tasks']
+
+        yield job
+
+
+def generate_dependencies(dependent_tasks):
+    # Because we depend on several tasks that have the same kind, we introduce the platform
+    dependencies = {}
+    for task in dependent_tasks:
+        platform_match = PLATFORM_REGEX.match(task.label)
+        # platform_match is None when the breakpoint task is given
+        task_kind = task.kind if platform_match is None else \
+            '{}-{}'.format(task.kind, platform_match.group(1))
+        dependencies[task_kind] = task.label
+    return dependencies
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -139,16 +139,54 @@ BALROG_SERVER_SCOPES = {
     'nightly': 'project:releng:balrog:nightly',
     'aurora': 'project:releng:balrog:nightly',
     'beta': 'project:releng:balrog:nightly',
     'release': 'project:releng:balrog:nightly',
     'default': 'project:releng:balrog:nightly',
 }
 
 
+PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
+    'aurora', set([
+        'mozilla-aurora',
+    ])
+], [
+    'beta', set([
+        'mozilla-beta',
+    ])
+], [
+    'release', set([
+        'mozilla-release',
+    ])
+]]
+
+
+PUSH_APK_SCOPES = {
+    'aurora': 'project:releng:googleplay:aurora',
+    'beta': 'project:releng:googleplay:beta',
+    'release': 'project:releng:googleplay:release',
+    'default': 'project:releng:googleplay:invalid',
+}
+
+# See https://github.com/mozilla-releng/pushapkscript#aurora-beta-release-vs-alpha-beta-production
+PUSH_APK_GOOGLE_PLAY_TRACT = {
+    'aurora': 'beta',
+    'beta': 'production',
+    'release': 'production',
+    'default': 'invalid',
+}
+
+PUSH_APK_BREAKPOINT_WORKER_TYPE = {
+    'aurora': 'aws-provisioner-v1/taskcluster-generic',
+    'beta': 'null-provisioner/human-breakpoint',
+    'release': 'null-provisioner/human-breakpoint',
+    'default': 'invalid/invalid',
+}
+
+
 # scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
 
     Args:
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
@@ -231,16 +269,34 @@ get_beetmover_action_scope = functools.p
 )
 
 get_balrog_server_scope = functools.partial(
     get_scope_from_project,
     BALROG_SCOPE_ALIAS_TO_PROJECT,
     BALROG_SERVER_SCOPES
 )
 
+get_push_apk_scope = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_SCOPES
+)
+
+get_push_apk_track = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_GOOGLE_PLAY_TRACT
+)
+
+get_push_apk_breakpoint_worker_type = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_BREAKPOINT_WORKER_TYPE
+)
+
 
 # release_config {{{1
 def get_release_config(config):
     """Get the build number and version for a release task.
 
     Currently only applies to beetmover tasks.
 
     Args: