Bug 1385401 - Fetch Google Play listings outside of the push-apk task r=aki
☠☠ backed out by 814510bf9447 ☠ ☠
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 05 Jan 2018 13:49:59 +0100
changeset 449740 a32acea9d091d3bc3aa9476f2a1ee3e565c2a211
parent 449739 25d72de8349834612ba5ff06ce4dcf62801fa595
child 449741 c592bcac2149084ca3ad5a72f77b8cf3c60db970
push id8527
push userCallek@gmail.com
push dateThu, 11 Jan 2018 21:05:50 +0000
treeherdermozilla-beta@95342d212a7a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1385401
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1385401 - Fetch Google Play listings outside of the push-apk task r=aki MozReview-Commit-ID: 2PTvLx4Atj5
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/google-play-strings/kind.yml
taskcluster/ci/push-apk/kind.yml
taskcluster/docker/google-play-strings/Dockerfile
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/google_play_strings.py
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -30,14 +30,16 @@ jobs:
   lint:
     symbol: I(lnt)
   android-build:
     symbol: I(agb)
   index-task:
     symbol: I(idx)
   funsize-update-generator:
     symbol: I(pg)
+  google-play-strings:
+    symbol: I(gps)
   funsize-balrog-submitter:
     symbol: I(fbs)
   beet-mover:
     symbol: I(bm)
   update-verify:
     symbol: I(uv)
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/google-play-strings/kind.yml
@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.google_play_strings:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs:
+   google-play-strings:
+      description: Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/
+      attributes:
+         build_type: google_play_strings
+         build_platform: android-nightly
+         nightly: true
+      shipping-phase: promote
+      shipping-product: fennec
+      worker-type: aws-provisioner-v1/taskcluster-generic
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: {in-tree: google-play-strings}
+         chain-of-trust: true
+         max-run-time: 600
+         artifacts:
+            - name: 'public/google_play_strings.json'
+              # XXX The folder depends on the one defined in the Dockerfile
+              path: /builds/worker/google_play_strings.json
+              type: 'file'
+         env:
+            # TODO Use the branch name instead of the android package name
+            PACKAGE_NAME:
+               by-project:
+                  mozilla-central: org.mozilla.fennec_aurora
+                  mozilla-beta: org.mozilla.firefox_beta
+                  mozilla-release: org.mozilla.firefox_beta
+                  default: org.mozilla.fennec_aurora  # Fetches strings for mozilla-central
+            # XXX The folder depends on the one defined in the Dockerfile
+            GOOGLE_PLAY_STRING_FILE: /builds/worker/google_play_strings.json
+         command:
+            - bash
+            - -cx
+            - >
+              python3 ./mozapkpublisher/get_l10n_strings.py
+              --package-name "${PACKAGE_NAME}"
+              --output-file "${GOOGLE_PLAY_STRING_FILE}"
+      treeherder:
+         symbol: pub(gps)
+         platform: Android/opt
+         tier: 2
+         kind: other
+      run-on-projects: ['maple', 'mozilla-central', 'mozilla-beta', 'mozilla-release']
--- a/taskcluster/ci/push-apk/kind.yml
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -1,43 +1,43 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.push_apk:loader
 
 transforms:
-    - taskgraph.transforms.push_apk:transforms
-    - taskgraph.transforms.task:transforms
+   - taskgraph.transforms.push_apk:transforms
+   - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-    - build-signing
-    - push-apk-breakpoint
+   - build-signing
+   - google-play-strings
+   - push-apk-breakpoint
 
 jobs:
-    push-apk/opt:
-        description: Publishes APK onto Google Play Store
-        attributes:
-            build_platform: android-nightly
-            nightly: true
-        shipping-phase: ship
-        shipping-product: fennec
-        worker-type:
-            by-project:
-                mozilla-central: scriptworker-prov-v1/pushapk-v1
-                mozilla-beta: scriptworker-prov-v1/pushapk-v1
-                mozilla-release: scriptworker-prov-v1/pushapk-v1
-                default: scriptworker-prov-v1/dep-pushapk
-        worker:
-            upstream-artifacts:  # see transforms
-            google-play-track:  # see transforms
-            implementation: push-apk
-            commit:  # see transforms
-        scopes:  # see transforms
-        treeherder:
-            symbol: pub(gp)
-            platform: Android/opt
-            tier: 2
-            kind: other
-        run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
-        deadline-after: 5 days
-        extra:
-            product: fennec
+   push-apk/opt:
+      description: Publishes APK onto Google Play Store
+      attributes:
+         build_platform: android-nightly
+         nightly: true
+      shipping-phase: ship
+      shipping-product: fennec
+      worker-type:
+         by-project:
+            mozilla-central: scriptworker-prov-v1/pushapk-v1
+            mozilla-beta: scriptworker-prov-v1/pushapk-v1
+            mozilla-release: scriptworker-prov-v1/pushapk-v1
+            default: scriptworker-prov-v1/dep-pushapk
+      worker:
+         upstream-artifacts:  # see transforms
+         google-play-track:  # see transforms
+         implementation: push-apk
+         commit:  # see transforms
+      requires: all-resolved
+      scopes:  # see transforms
+      treeherder:
+         symbol: pub(gp)
+         platform: Android/opt
+         tier: 2
+         kind: other
+      run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release', 'maple']
+      deadline-after: 5 days
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/google-play-strings/Dockerfile
@@ -0,0 +1,19 @@
+FROM          ubuntu:16.04
+MAINTAINER    Johan Lorenzo <jlorenzo+tc@mozilla.com>
+
+RUN mkdir /builds
+RUN groupadd -g 500 worker
+RUN useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker
+
+RUN apt-get update
+RUN apt-get install --yes git python3-setuptools build-essential libssl-dev libffi-dev python3-dev
+
+WORKDIR /builds/worker/
+RUN git clone https://github.com/mozilla-releng/mozapkpublisher
+WORKDIR /builds/worker/mozapkpublisher
+RUN python3 setup.py develop
+
+RUN chown -R worker:worker /builds/worker
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -212,16 +212,21 @@ and sign it via the signing scriptworker
 additional detached signature.
 
 beetmover-checksums
 -------------------
 Beetmover, takes specific artifact checksums and pushes it to a location outside
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
 
+google-play-strings
+-------------------
+Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/.
+Artifact is then used by push-apk.
+
 push-apk-breakpoint
 -------------------
 Decides whether or not APKs should be published onto Google Play Store. Jobs of this
 kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
 in order to make the decision.
 
 push-apk
 --------
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/google_play_strings.py
@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.util.schema import resolve_keyed_by, Schema
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
+
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+google_play_description_schema = Schema({
+    Required('name'): basestring,
+    Required('label'): task_description_schema['label'],
+    Required('description'): task_description_schema['description'],
+    Required('job-from'): task_description_schema['job-from'],
+    Required('attributes'): task_description_schema['attributes'],
+    Required('treeherder'): task_description_schema['treeherder'],
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
+    Required('worker-type'): task_description_schema['worker-type'],
+    Required('worker'): object,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    google_play_description_schema,
+    'GooglePlayStrings'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+
+
+@transforms.add
+def set_worker_data(config, jobs):
+    for job in jobs:
+        worker = job['worker']
+
+        env = worker.setdefault('env', {})
+        resolve_keyed_by(
+            env, 'PACKAGE_NAME', item_name=job['name'],
+            project=config.params['project']
+        )
+
+        cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
+        cot.setdefault('inputs', {})['docker-image'] = {'task-reference': '<docker-image>'}
+
+        yield job
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -23,32 +23,32 @@ from voluptuous import Optional, Require
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 
 push_apk_description_schema = Schema({
-    # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-tasks'): object,
     Required('name'): basestring,
-    Required('label'): basestring,
-    Required('description'): basestring,
-    Required('job-from'): basestring,
-    Required('attributes'): object,
-    Required('treeherder'): object,
-    Required('run-on-projects'): list,
+    Required('label'): task_description_schema['label'],
+    Required('description'): task_description_schema['description'],
+    Required('job-from'): task_description_schema['job-from'],
+    Required('attributes'): task_description_schema['attributes'],
+    Required('treeherder'): task_description_schema['treeherder'],
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
     Required('worker-type'): optionally_keyed_by('project', basestring),
     Required('worker'): object,
     Required('scopes'): None,
+    Required('requires'): task_description_schema['requires'],
     Required('deadline-after'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
-    Optional('extra'): object,
+    Optional('extra'): task_description_schema['extra'],
 })
 
 validate_jobs_schema_transform = functools.partial(
     validate_jobs_schema_transform_partial,
     push_apk_description_schema,
     'PushApk'
 )
 
@@ -78,13 +78,26 @@ def make_task_description(config, jobs):
 
         yield job
 
 
 transforms.add(delete_non_required_fields_transform)
 
 
 def generate_upstream_artifacts(dependencies):
-    return [{
+    apks = [{
         'taskId': {'task-reference': '<{}>'.format(task_kind)},
         'taskType': 'signing',
         'paths': ['public/build/target.apk'],
-    } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
+    } for task_kind in dependencies.keys()
+      if task_kind not in ('push-apk-breakpoint', 'google-play-strings')
+    ]
+
+    google_play_strings = [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        'taskType': 'build',
+        'paths': ['public/google_play_strings.json'],
+        'optional': True,
+    } for task_kind in dependencies.keys()
+      if 'google-play-strings' in task_kind
+    ]
+
+    return apks + google_play_strings
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -88,16 +88,18 @@ task_description_schema = Schema({
     # relative path (from config.path) to the file task was defined in
     Optional('job-from'): basestring,
 
     # dependencies of this task, keyed by name; these are passed through
     # verbatim and subject to the interpretation of the Task's get_dependencies
     # method.
     Optional('dependencies'): {basestring: object},
 
+    Optional('requires'): Any('all-completed', 'all-resolved'),
+
     # expiration and deadline times, relative to task creation, with units
     # (e.g., "14 days").  Defaults are set based on the project.
     Optional('expires-after'): basestring,
     Optional('deadline-after'): basestring,
 
     # custom routes for this task; the default treeherder routes will be added
     # automatically
     Optional('routes'): [basestring],
@@ -562,16 +564,19 @@ task_description_schema = Schema({
             # taskId of the task with the artifact
             Required('taskId'): taskref_or_string,
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
+
+            # Artifact is optional to run the task
+            Optional('optional', default=False): bool,
         }],
 
         # "Invalid" is a noop for try and other non-supported branches
         Required('google-play-track'): Any('production', 'beta', 'alpha', 'rollout', 'invalid'),
         Required('commit', default=False): bool,
         Optional('rollout-percentage'): int,
     }),
 })
@@ -1417,16 +1422,19 @@ def build_task(config, tasks):
                     config.params['head_rev'],
                     config.path),
             },
             'extra': extra,
             'tags': tags,
             'priority': task['priority'],
         }
 
+        if task.get('requires', None):
+            task_def['requires'] = task['requires']
+
         if task_th:
             # link back to treeherder in description
             th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(
                 config.params['project'], treeherder_rev)
             task_def['metadata']['description'] += ' ([Treeherder push]({}))'.format(
                 th_push_link)
 
         # add the payload and adjust anything else as required (e.g., scopes)