Bug 1432817 - [in-tree relpro] Delete push-apk-breakpoint task r=aki draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Thu, 22 Feb 2018 17:10:29 +0000
changeset 761747 4554c1b7ce44edd9f6cef3dd9e2990d7ff2e5b05
parent 761746 0f8f71b0b9d84e7732c07f841e395de516b31b66
push id100984
push userbmo:jlorenzo@mozilla.com
push dateThu, 01 Mar 2018 10:14:09 +0000
reviewersaki
bugs1432817
milestone60.0a1
Bug 1432817 - [in-tree relpro] Delete push-apk-breakpoint task r=aki MozReview-Commit-ID: G5nvGfNIVNN
taskcluster/ci/push-apk-breakpoint/kind.yml
taskcluster/ci/push-apk/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/push_apk.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/google_play_strings.py
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/push_apk_breakpoint.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/push_apk.py
taskcluster/taskgraph/util/workertypes.py
deleted file mode 100644
--- a/taskcluster/ci/push-apk-breakpoint/kind.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-loader: taskgraph.loader.push_apk:loader
-
-transforms:
-    - taskgraph.transforms.push_apk_breakpoint:transforms
-    - taskgraph.transforms.release_notifications:transforms
-    - taskgraph.transforms.task:transforms
-
-kind-dependencies:
-    - build-signing
-
-jobs:
-    android-push-apk-breakpoint/opt:
-        description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
-        attributes:
-            build_platform: android-nightly
-            nightly: true
-        shipping-phase: ship
-        shipping-product: fennec
-        worker-type:  # see transforms
-            by-project:
-                mozilla-central: aws-provisioner-v1/taskcluster-generic
-                mozilla-beta: null-provisioner/human-breakpoint
-                mozilla-release: null-provisioner/human-breakpoint
-                maple: aws-provisioner-v1/taskcluster-generic
-                default: invalid/invalid
-        worker:
-            implementation: push-apk-breakpoint
-        treeherder:
-            symbol: pub(Br)
-            platform: Android/opt
-            tier: 2
-            kind: other
-        run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
-        deadline-after: 5 days
--- a/taskcluster/ci/push-apk/kind.yml
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -7,17 +7,16 @@ loader: taskgraph.loader.push_apk:loader
 transforms:
    - taskgraph.transforms.push_apk:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - build-signing
    - google-play-strings
-   - push-apk-breakpoint
    - beetmover-checksums
 
 jobs:
    push-apk/opt:
       description: Publishes APK onto Google Play Store
       attributes:
          build_platform: android-nightly
          nightly: true
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -217,27 +217,21 @@ Beetmover, takes specific artifact check
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
 
 google-play-strings
 -------------------
 Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/.
 Artifact is then used by push-apk.
 
-push-apk-breakpoint
--------------------
-Decides whether or not APKs should be published onto Google Play Store. Jobs of this
-kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
-in order to make the decision.
-
 push-apk
 --------
 PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
-all at once. They also depend on the breakpoint.
+all at once.
 
 release-balrog-submit-toplevel
 ----------------------
 Push a top-level release blob to Balrog.
 
 release-secondary-balrog-submit-toplevel
 ----------------------
 Push a top-level RC release blob to Balrog.
--- a/taskcluster/taskgraph/loader/push_apk.py
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -16,16 +16,18 @@ def loader(kind, path, config, params, l
     for job in jobs:
         dependent_tasks = get_dependent_loaded_tasks(config, loaded_tasks)
         if not dependent_tasks:
             # PushApk must depend on signed APK. If no dependent task was found,
             # this means another plaform (like windows) is being processed
             continue
 
         job['dependent-tasks'] = dependent_tasks
+        job['label'] = job['name']
+
         yield job
 
 
 def get_dependent_loaded_tasks(config, loaded_tasks):
     nightly_tasks = (
         task for task in loaded_tasks if task.attributes.get('nightly')
     )
     tasks_with_matching_kind = (
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -47,18 +47,17 @@ def filter_beta_release_tasks(task, para
         return False
     if ignore_kinds is None:
         ignore_kinds = [
             'balrog',
             'beetmover', 'beetmover-checksums', 'beetmover-l10n',
             'beetmover-repackage', 'beetmover-repackage-signing',
             'checksums-signing',
             'nightly-l10n', 'nightly-l10n-signing',
-            'push-apk', 'push-apk-breakpoint',
-            'repackage-l10n',
+            'push-apk', 'repackage-l10n',
         ]
     platform = task.attributes.get('build_platform')
     if platform in (
             # On beta, Nightly builds are already PGOs
             'linux-pgo', 'linux64-pgo',
             'win32-pgo', 'win64-pgo',
             ):
         return False
@@ -481,17 +480,17 @@ def target_tasks_promote_fennec(full_tas
         if attr("locale") or attr("chunk_locales"):
             return False
         if task.label in filtered_for_project:
             # bug 1438023 - old-id should only run on central.
             # We can remove this hack when shippable builds land and we
             # are using run-on-projects properly here.
             if 'old-id' in task.label:
                 return False
-            if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'):
+            if task.kind not in ('balrog', 'push-apk'):
                 if task.attributes.get('nightly'):
                     return True
         if task.attributes.get('shipping_product') == 'fennec' and \
                 task.attributes.get('shipping_phase') == 'promote':
             return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
 
@@ -507,22 +506,19 @@ def target_tasks_ship_fennec(full_task_g
 
     def filter(task):
         # Include candidates build tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
         if task.attributes.get('shipping_product') != 'fennec' or \
                 task.attributes.get('shipping_phase') not in ('ship', 'push'):
             return False
-        # We always run push-apk* during ship
-        if task.kind in (
-            'push-apk',
-            'push-apk-breakpoint',
-        ):
-                return True
+        # We always run push-apk during ship
+        if task.kind == 'push-apk':
+            return True
         # secondary-notify-ship is only for RC
         if task.kind in (
             'release-secondary-notify-ship',
         ):
             return is_rc
 
         # Everything else is only for non-RC
         return not is_rc
--- a/taskcluster/taskgraph/transforms/google_play_strings.py
+++ b/taskcluster/taskgraph/transforms/google_play_strings.py
@@ -2,22 +2,19 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the push-apk kind into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import functools
-
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.task import task_description_schema
-from taskgraph.util.schema import resolve_keyed_by, Schema
-from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
+from taskgraph.util.schema import resolve_keyed_by, Schema, validate_schema
 
 from voluptuous import Required
 
 
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
@@ -32,24 +29,26 @@ google_play_description_schema = Schema(
     Required('treeherder'): task_description_schema['treeherder'],
     Required('run-on-projects'): task_description_schema['run-on-projects'],
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
     Required('worker-type'): task_description_schema['worker-type'],
     Required('worker'): object,
 })
 
-validate_jobs_schema_transform = functools.partial(
-    validate_jobs_schema_transform_partial,
-    google_play_description_schema,
-    'GooglePlayStrings'
-)
 
-transforms.add(fill_labels_tranform)
-transforms.add(validate_jobs_schema_transform)
+@transforms.add
+def validate_jobs_schema(config, jobs):
+    for job in jobs:
+        job['label'] = job['name']
+        validate_schema(
+            google_play_description_schema, job,
+            "In GooglePlayStrings ({!r} kind) task for {!r}:".format(config.kind, job['label'])
+        )
+        yield job
 
 
 @transforms.add
 def set_worker_data(config, jobs):
     for job in jobs:
         worker = job['worker']
 
         env = worker.setdefault('env', {})
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -2,24 +2,22 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the push-apk kind into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import functools
+import re
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.task import task_description_schema
-from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
 from taskgraph.util.scriptworker import get_push_apk_scope
-from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
-    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
 
 from voluptuous import Optional, Required
 
 
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
@@ -40,39 +38,64 @@ push_apk_description_schema = Schema({
     Required('scopes'): None,
     Required('requires'): task_description_schema['requires'],
     Required('deadline-after'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
     Optional('extra'): task_description_schema['extra'],
 })
 
-validate_jobs_schema_transform = functools.partial(
-    validate_jobs_schema_transform_partial,
-    push_apk_description_schema,
-    'PushApk'
-)
+
+REQUIRED_ARCHITECTURES = {
+    'android-x86-nightly',
+    'android-api-16-nightly',
+}
+PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
+
+
+@transforms.add
+def validate_jobs_schema_transform_partial(config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        validate_schema(
+            push_apk_description_schema, job,
+            "In PushApk ({!r} kind) task for {!r}:".format(config.kind, label)
+        )
+        yield job
 
-transforms.add(fill_labels_tranform)
-transforms.add(validate_jobs_schema_transform)
-transforms.add(validate_dependent_tasks_transform)
+
+@transforms.add
+def validate_dependent_tasks(_, jobs):
+    for job in jobs:
+        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+        yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+    dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks)
+    missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms
+    if missed_architectures:
+        raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+        )
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         job['dependencies'] = generate_dependencies(job['dependent-tasks'])
         job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
 
         # Use the rc-google-play-track and rc-rollout-percentage in RC relpro flavors
         if config.params['release_type'] == 'rc':
             job['worker']['google-play-track'] = job['worker']['rc-google-play-track']
             job['worker']['rollout-percentage'] = job['worker']['rc-rollout-percentage']
-        del(job['worker']['rc-google-play-track'])
-        del(job['worker']['rc-rollout-percentage'])
 
         resolve_keyed_by(
             job, 'worker.google-play-track', item_name=job['name'],
             project=config.params['project']
         )
         resolve_keyed_by(
             job, 'worker.commit', item_name=job['name'],
             project=config.params['project']
@@ -88,30 +111,51 @@ def make_task_description(config, jobs):
         resolve_keyed_by(
             job, 'worker-type', item_name=job['name'],
             project=config.params['project']
         )
 
         yield job
 
 
-transforms.add(delete_non_required_fields_transform)
+def generate_dependencies(dependent_tasks):
+    # Because we depend on several tasks that have the same kind, we introduce the platform
+    dependencies = {}
+    for task in dependent_tasks:
+        platform_match = PLATFORM_REGEX.match(task.label)
+        # platform_match is None when the google-play-string task is given, for instance
+        task_kind = task.kind if platform_match is None else \
+            '{}-{}'.format(task.kind, platform_match.group(1))
+        dependencies[task_kind] = task.label
+    return dependencies
 
 
 def generate_upstream_artifacts(dependencies):
     apks = [{
         'taskId': {'task-reference': '<{}>'.format(task_kind)},
         'taskType': 'signing',
         'paths': ['public/build/target.apk'],
     } for task_kind in dependencies.keys()
-      if task_kind not in ('push-apk-breakpoint', 'google-play-strings', 'beetmover-checksums')
+      if task_kind not in ('google-play-strings', 'beetmover-checksums')
     ]
 
     google_play_strings = [{
         'taskId': {'task-reference': '<{}>'.format(task_kind)},
         'taskType': 'build',
         'paths': ['public/google_play_strings.json'],
         'optional': True,
     } for task_kind in dependencies.keys()
       if 'google-play-strings' in task_kind
     ]
 
     return apks + google_play_strings
+
+
+@transforms.add
+def delete_non_required_fields(_, jobs):
+    for job in jobs:
+        del job['name']
+        del job['dependent-tasks']
+
+        del(job['worker']['rc-google-play-track'])
+        del(job['worker']['rc-rollout-percentage'])
+
+        yield job
deleted file mode 100644
--- a/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Transform the push-apk-breakpoint kind into an actual task description.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import functools
-
-from taskgraph.transforms.base import TransformSequence
-from taskgraph.transforms.task import task_description_schema
-from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
-from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
-    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
-from voluptuous import Required
-
-
-transforms = TransformSequence()
-
-# Voluptuous uses marker objects as dictionary *keys*, but they are not
-# comparable, so we cast all of the keys back to regular strings
-task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
-
-push_apk_breakpoint_description_schema = Schema({
-    # the dependent task (object) for this beetmover job, used to inform beetmover.
-    Required('dependent-tasks'): object,
-    Required('name'): basestring,
-    Required('label'): basestring,
-    Required('description'): basestring,
-    Required('job-from'): basestring,
-    Required('attributes'): object,
-    Required('worker-type'): optionally_keyed_by('project', basestring),
-    Required('worker'): object,
-    Required('treeherder'): object,
-    Required('run-on-projects'): list,
-    Required('deadline-after'): basestring,
-    Required('shipping-phase'): task_description_schema['shipping-phase'],
-    Required('shipping-product'): task_description_schema['shipping-product'],
-})
-
-validate_jobs_schema_transform = functools.partial(
-    validate_jobs_schema_transform_partial,
-    push_apk_breakpoint_description_schema,
-    'PushApkBreakpoint'
-)
-
-transforms.add(fill_labels_tranform)
-transforms.add(validate_jobs_schema_transform)
-transforms.add(validate_dependent_tasks_transform)
-
-
-@transforms.add
-def make_task_description(config, jobs):
-    for job in jobs:
-        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
-
-        resolve_keyed_by(
-            job, 'worker-type', item_name=job['name'],
-            project=config.params['project']
-        )
-
-        job['worker']['payload'] = {} if 'human' in job['worker-type'] else {
-                'image': 'ubuntu:16.10',
-                'command': [
-                    '/bin/bash',
-                    '-c',
-                    'echo "Dummy task while while bug 1351664 is implemented"'
-                ],
-                'maxRunTime': 600,
-            }
-
-        yield job
-
-
-transforms.add(delete_non_required_fields_transform)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -554,20 +554,16 @@ task_description_schema = Schema({
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
     }, {
-        Required('implementation'): 'push-apk-breakpoint',
-        Required('payload'): object,
-
-    }, {
         Required('implementation'): 'invalid',
         # an invalid task is one which should never actually be created; this is used in
         # release automation on branches where the task just doesn't make sense
         Extra: object,
 
     }, {
         Required('implementation'): 'always-optimized',
         Extra: object,
@@ -1114,21 +1110,16 @@ def build_push_apk_payload(config, task,
         'upstreamArtifacts':  worker['upstream-artifacts'],
         'google_play_track': worker['google-play-track'],
     }
 
     if worker.get('rollout-percentage', None):
         task_def['payload']['rollout_percentage'] = worker['rollout-percentage']
 
 
-@payload_builder('push-apk-breakpoint')
-def build_push_apk_breakpoint_payload(config, task, task_def):
-    task_def['payload'] = task['worker']['payload']
-
-
 @payload_builder('shipit')
 def build_ship_it_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'release_name': worker['release-name']
     }
 
deleted file mode 100644
--- a/taskcluster/taskgraph/util/push_apk.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Common functions for both push-apk and push-apk-breakpoint.
-"""
-from __future__ import absolute_import, print_function, unicode_literals
-
-import re
-
-from taskgraph.util.schema import validate_schema
-
-REQUIRED_ARCHITECTURES = {
-    'android-x86-nightly',
-    'android-api-16-nightly',
-}
-PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
-
-
-def fill_labels_tranform(_, jobs):
-    for job in jobs:
-        job['label'] = job['name']
-
-        yield job
-
-
-def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
-    for job in jobs:
-        label = job.get('label', '?no-label?')
-        validate_schema(
-            description_schema, job,
-            "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
-        )
-        yield job
-
-
-def validate_dependent_tasks_transform(_, jobs):
-    for job in jobs:
-        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
-        yield job
-
-
-def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
-    dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks)
-    missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms
-    if missed_architectures:
-        raise Exception('''One or many required architectures are missing.
-
-Required architectures: {}.
-Given dependencies: {}.
-'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
-        )
-
-
-def delete_non_required_fields_transform(_, jobs):
-    for job in jobs:
-        del job['name']
-        del job['dependent-tasks']
-
-        yield job
-
-
-def generate_dependencies(dependent_tasks):
-    # Because we depend on several tasks that have the same kind, we introduce the platform
-    dependencies = {}
-    for task in dependent_tasks:
-        platform_match = PLATFORM_REGEX.match(task.label)
-        # platform_match is None when the breakpoint task is given
-        task_kind = task.kind if platform_match is None else \
-            '{}-{}'.format(task.kind, platform_match.group(1))
-        dependencies[task_kind] = task.label
-    return dependencies
--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -35,17 +35,16 @@ WORKER_TYPES = {
     'releng-hardware/gecko-t-win10-64-hw': ('generic-worker', 'windows'),
     'aws-provisioner-v1/gecko-t-win7-32': ('generic-worker', 'windows'),
     'aws-provisioner-v1/gecko-t-win7-32-gpu': ('generic-worker', 'windows'),
     'releng-hardware/gecko-t-win7-32-hw': ('generic-worker', 'windows'),
     'aws-provisioner-v1/taskcluster-generic': ('docker-worker', 'linux'),
     'buildbot-bridge/buildbot-bridge': ('buildbot-bridge', None),
     'invalid/invalid': ('invalid', None),
     'invalid/always-optimized': ('always-optimized', None),
-    'null-provisioner/human-breakpoint': ('push-apk-breakpoint', None),
     'releng-hardware/gecko-t-linux-talos': ('native-engine', 'linux'),
     'scriptworker-prov-v1/balrog-dev': ('balrog', None),
     'scriptworker-prov-v1/balrogworker-v1': ('balrog', None),
     'scriptworker-prov-v1/beetmoverworker-v1': ('beetmover', None),
     'scriptworker-prov-v1/pushapk-v1': ('push-apk', None),
     "scriptworker-prov-v1/signing-linux-v1": ('scriptworker-signing', None),
     "scriptworker-prov-v1/shipit": ('shipit', None),
     "scriptworker-prov-v1/shipit-dev": ('shipit', None),