Bug 1317783 - Put PushApk tasks in-tree r=aki
☠☠ backed out by e91496c7153e ☠ ☠
authorJohan Lorenzo <jlorenzo@mozilla.com>
Thu, 30 Mar 2017 12:13:01 +0200
changeset 350646 0edd9de2ca10487b98f3e7dc5667138d04b93121
parent 350645 cad1604f3dbf1a690e9299ca52775344f8f70695
child 350647 b4ee149b44d1cc11338d4c0c27b1f444cc5ec846
push id31579
push usercbook@mozilla.com
push dateFri, 31 Mar 2017 12:45:54 +0000
treeherdermozilla-central@13f5ae940c4e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1317783
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1317783 - Put PushApk tasks in-tree r=aki MozReview-Commit-ID: 8uGIuj7OXwZ
taskcluster/ci/push-apk-breakpoint/kind.yml
taskcluster/ci/push-apk/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/push_apk.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/push_apk_breakpoint.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/push_apk.py
taskcluster/taskgraph/util/scriptworker.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk-breakpoint/kind.yml
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk_breakpoint:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+
+jobs:
+    android-push-apk-breakpoint/opt:
+        description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: # see transforms
+        worker:
+            implementation: push-apk-breakpoint
+        treeherder:
+            symbol: pub(Br)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+  - push-apk-breakpoint
+
+jobs:
+    push-apk/opt:
+        description: Publishes APK onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: scriptworker-prov-v1/pushapk-v1
+        worker:
+            upstream-artifacts: # see transforms
+            google-play-track: # see transforms
+            implementation: push-apk
+            # TODO unhardcode that line
+            dry-run: true
+        scopes: # see transforms
+        treeherder:
+            symbol: pub(gp)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -190,8 +190,20 @@ Checksums-signing take as input the chec
 and sign it via the signing scriptworkers. Returns the same file signed and
 additional detached signature.
 
 beetmover-checksums
 -------------------
 Beetmover, takes specific artifact checksums and pushes it to a location outside
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
+
+push-apk-breakpoint
+-------------------
+Decides whether or not APKs should be published onto Google Play Store. Jobs of this
+kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
+in order to make the decision.
+
+push-apk
+--------
+PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
+all the signed multi-locales (aka "multi") APKs for a given release and upload them
+all at once. They also depend on the breakpoint.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .transform import loader as base_loader
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
+    """
+    jobs = base_loader(kind, path, config, params, loaded_tasks)
+
+    for job in jobs:
+        job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+        yield job
+
+
+def get_dependent_loaded_tasks(config, loaded_tasks):
+    nightly_tasks = (
+        task for task in loaded_tasks if task.attributes.get('nightly')
+    )
+    tasks_with_matching_kind = (
+        task for task in nightly_tasks if task.kind in config.get('kind-dependencies')
+    )
+    android_tasks = [
+        task for task in tasks_with_matching_kind if 'android' in task.label
+    ]
+
+    return android_tasks
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -179,23 +179,23 @@ def target_tasks_code_coverage(full_task
         platform = task.attributes.get('test_platform')
         if platform not in ('linux64-ccov', 'linux64-jsdcov'):
             return False
         return True
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_fennec')
-def target_tasks_nightly(full_task_graph, parameters):
+def target_tasks_nightly_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
-        if platform in ('android-api-15-nightly', 'android-x86-nightly'):
+        if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
             return task.attributes.get('nightly', False)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('nightly_linux')
 def target_tasks_nightly_linux(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of linux. The
     nightly build process involves a pipeline of builds, signing,
@@ -223,16 +223,17 @@ def target_tasks_mozilla_beta(full_task_
             return False
         if platform in ('linux64', 'linux'):
             if task.attributes['build_type'] == 'opt':
                 return False
         # skip l10n, beetmover, balrog
         if task.kind in [
             'balrog', 'beetmover', 'beetmover-checksums', 'beetmover-l10n',
             'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing',
+            'push-apk', 'push-apk-breakpoint',
         ]:
             return False
         return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('mozilla_release_tasks')
@@ -243,17 +244,17 @@ def target_tasks_mozilla_release(full_ta
     return target_tasks_mozilla_beta(full_task_graph, parameters)
 
 
 @_target_task('candidates_fennec')
 def target_tasks_candidates_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a candidates build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
-    filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+    filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters)
 
     def filter(task):
         if task.kind not in ['balrog']:
             return task.attributes.get('nightly', False)
 
     return [l for l in filtered_for_project if filter(full_task_graph[l])]
 
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('worker-type'): basestring,
+    Required('worker'): object,
+    Required('scopes'): None,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_description_schema,
+    'PushApk'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+        job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+        job['worker']['google-play-track'] = get_push_apk_track(config)
+        job['scopes'] = [get_push_apk_scope(config)]
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
+
+
+def generate_upstream_artifacts(dependencies):
+    return [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        'taskType': 'signing',
+        'paths': ['public/build/target.apk'],
+    } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk-breakpoint kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_breakpoint_worker_type
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_breakpoint_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('worker-type'): None,
+    Required('worker'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_breakpoint_description_schema,
+    'PushApkBreakpoint'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+
+        worker_type = get_push_apk_breakpoint_worker_type(config)
+        job['worker-type'] = worker_type
+
+        job['worker']['payload'] = {} if 'human' in worker_type else {
+                'image': 'ubuntu:16.10',
+                'command': [
+                    '/bin/bash',
+                    '-c',
+                    'echo "Dummy task while while bug 1351664 is implemented"'
+                ],
+                'maxRunTime': 600,
+            }
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -339,16 +339,38 @@ task_description_schema = Schema({
             Required('taskId'): taskref_or_string,
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
+    }, {
+        Required('implementation'): 'push-apk-breakpoint',
+        Required('payload'): object,
+
+    }, {
+        Required('implementation'): 'push-apk',
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+        }],
+
+        # "Invalid" is a noop for try and other non-supported branches
+        Required('google-play-track'): Any('production', 'beta', 'alpha', 'invalid'),
+        Required('dry-run', default=True): bool,
     }),
 })
 
 GROUP_NAMES = {
     'py': 'Python unit tests',
     'tc': 'Executed by TaskCluster',
     'tc-e10s': 'Executed by TaskCluster with e10s',
     'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
@@ -376,16 +398,17 @@ GROUP_NAMES = {
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',
     'TM': 'Toolchain builds for OSX',
     'TW32': 'Toolchain builds for Windows 32-bits',
     'TW64': 'Toolchain builds for Windows 64-bits',
     'SM-tc': 'Spidermonkey builds',
+    'pub': 'APK publishing',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
@@ -592,16 +615,32 @@ def build_beetmover_payload(config, task
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
+@payload_builder('push-apk')
+def build_push_apk_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'dry_run': worker['dry-run'],
+        'upstreamArtifacts':  worker['upstream-artifacts'],
+        'google_play_track': worker['google-play-track'],
+    }
+
+
+@payload_builder('push-apk-breakpoint')
+def build_push_apk_breakpoint_payload(config, task, task_def):
+    task_def['payload'] = task['worker']['payload']
+
+
 @payload_builder('native-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/push_apk.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common functions for both push-apk and push-apk-breakpoint.
+"""
+
+import re
+
+from taskgraph.util.schema import validate_schema
+
+REQUIRED_ARCHITECTURES = ('android-x86', 'android-api-15')
+PLATFORM_REGEX = re.compile(r'signing-android-(\S+)-nightly')
+
+
+def fill_labels_tranform(_, jobs):
+    for job in jobs:
+        job['label'] = job['name']
+
+        yield job
+
+
+def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        yield validate_schema(
+            description_schema, job,
+            "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
+        )
+
+
+def validate_dependent_tasks_transform(_, jobs):
+    for job in jobs:
+        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+        yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+    dependencies_labels = [task.label for task in dependent_tasks]
+
+    is_this_required_architecture_present = {
+        architecture: any(architecture in label for label in dependencies_labels)
+        for architecture in REQUIRED_ARCHITECTURES
+    }
+    are_all_required_achitectures_present = all(is_this_required_architecture_present.values())
+
+    if not are_all_required_achitectures_present:
+        raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+        )
+
+
+def delete_non_required_fields_transform(_, jobs):
+    for job in jobs:
+        del job['name']
+        del job['dependent-tasks']
+
+        yield job
+
+
+def generate_dependencies(dependent_tasks):
+    # Because we depend on several tasks that have the same kind, we introduce the platform
+    dependencies = {}
+    for task in dependent_tasks:
+        platform_match = PLATFORM_REGEX.match(task.label)
+        # platform_match is None when the breakpoint task is given
+        task_kind = task.kind if platform_match is None else \
+            '{}-{}'.format(task.kind, platform_match.group(1))
+        dependencies[task_kind] = task.label
+    return dependencies
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -139,16 +139,54 @@ BALROG_SERVER_SCOPES = {
     'nightly': 'project:releng:balrog:nightly',
     'aurora': 'project:releng:balrog:nightly',
     'beta': 'project:releng:balrog:nightly',
     'release': 'project:releng:balrog:nightly',
     'default': 'project:releng:balrog:nightly',
 }
 
 
+PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
+    'aurora', set([
+        'mozilla-aurora',
+    ])
+], [
+    'beta', set([
+        'mozilla-beta',
+    ])
+], [
+    'release', set([
+        'mozilla-release',
+    ])
+]]
+
+
+PUSH_APK_SCOPES = {
+    'aurora': 'project:releng:googleplay:aurora',
+    'beta': 'project:releng:googleplay:beta',
+    'release': 'project:releng:googleplay:release',
+    'default': 'project:releng:googleplay:invalid',
+}
+
+# See https://github.com/mozilla-releng/pushapkscript#aurora-beta-release-vs-alpha-beta-production
+PUSH_APK_GOOGLE_PLAY_TRACT = {
+    'aurora': 'beta',
+    'beta': 'production',
+    'release': 'production',
+    'default': 'invalid',
+}
+
+PUSH_APK_BREAKPOINT_WORKER_TYPE = {
+    'aurora': 'aws-provisioner-v1/taskcluster-generic',
+    'beta': 'null-provisioner/human-breakpoint',
+    'release': 'null-provisioner/human-breakpoint',
+    'default': 'invalid/invalid',
+}
+
+
 # scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
 
     Args:
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
@@ -231,16 +269,34 @@ get_beetmover_action_scope = functools.p
 )
 
 get_balrog_server_scope = functools.partial(
     get_scope_from_project,
     BALROG_SCOPE_ALIAS_TO_PROJECT,
     BALROG_SERVER_SCOPES
 )
 
+get_push_apk_scope = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_SCOPES
+)
+
+get_push_apk_track = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_GOOGLE_PLAY_TRACT
+)
+
+get_push_apk_breakpoint_worker_type = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_BREAKPOINT_WORKER_TYPE
+)
+
 
 # release_config {{{1
 def get_release_config(config):
     """Get the build number and version for a release task.
 
     Currently only applies to beetmover tasks.
 
     Args: