Bug 1540152 - Run checks done in push-apk in promote-phase, instead of the very last task of the pipeline r=mtabara
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 09 Apr 2019 14:56:52 +0000
changeset 468575 b979e271d9550c9baa63c1e98f4db5bc7f5677c5
parent 468574 b53ad32487116413bb6fa068ce43af2eae217e69
child 468576 de1c51fce80e5bee60661bff92e2006970ecd760
push id35843
push usernbeleuzu@mozilla.com
push dateTue, 09 Apr 2019 22:08:13 +0000
treeherdermozilla-central@a31032a16330 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1540152
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540152 - Run checks done in push-apk in promote-phase, instead of the very last task of the pipeline r=mtabara Run checks done in push-apk in promote-phase, instead of the very last task of the pipeline Differential Revision: https://phabricator.services.mozilla.com/D26328
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/google-play-strings/kind.yml
taskcluster/ci/push-apk-checks/kind.yml
taskcluster/ci/push-apk/kind.yml
taskcluster/docker/google-play-strings/Dockerfile
taskcluster/docker/mozapkpublisher/Dockerfile
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/push_apk_checks.py
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -173,18 +173,18 @@ jobs:
     parent: android-build
   mingw32-build:
     symbol: I(mingw)
     parent: debian9-base
   index-task:
     symbol: I(idx)
   funsize-update-generator:
     symbol: I(pg)
-  google-play-strings:
-    symbol: I(gps)
+  mozapkpublisher:
+    symbol: I(apk)
   update-verify:
     symbol: I(uv)
   diffoscope:
     symbol: I(diff)
     parent: debian9-base
   partner-repack:
     symbol: I(PR)
     parent: debian9-base
--- a/taskcluster/ci/google-play-strings/kind.yml
+++ b/taskcluster/ci/google-play-strings/kind.yml
@@ -16,17 +16,17 @@ jobs:
          build_platform: android-nightly
          nightly: true
       shipping-phase: promote
       shipping-product: fennec
       worker-type: b-linux
       worker:
          implementation: docker-worker
          os: linux
-         docker-image: {in-tree: google-play-strings}
+         docker-image: {in-tree: mozapkpublisher}
          chain-of-trust: true
          max-run-time: 600
          artifacts:
             - name: 'public/google_play_strings.json'
               # XXX The folder depends on the one defined in the Dockerfile
               path: /builds/worker/google_play_strings.json
               type: 'file'
          env:
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/push-apk-checks/kind.yml
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk_checks:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - build-signing
+
+jobs:
+   push-apk-checks/opt:
+      description: Verify APKs are sane before uploading them onto Google Play Store
+      attributes:
+         build_platform: android-nightly
+         nightly: true
+      shipping-phase: promote
+      shipping-product: fennec
+      package-name:
+         by-project:
+            mozilla-central: org.mozilla.fennec_aurora
+            mozilla-beta: org.mozilla.firefox_beta
+            mozilla-release: org.mozilla.firefox
+            default: org.mozilla.fennec_aurora
+      worker-type: b-linux
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: {in-tree: mozapkpublisher}
+         chain-of-trust: true
+         max-run-time: 600
+      treeherder:
+         symbol: pub(gpc)
+         platform: Android/opt
+         tier: 1
+         kind: test
+      run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
--- a/taskcluster/ci/push-apk/kind.yml
+++ b/taskcluster/ci/push-apk/kind.yml
@@ -6,19 +6,21 @@ loader: taskgraph.loader.push_apk:loader
 
 transforms:
    - taskgraph.transforms.push_apk:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - build-signing
    - google-play-strings
+   - push-apk-checks
 
 jobs:
    push-apk/opt:
+      label: 'push-apk/opt'
       description: Publishes APK onto Google Play Store
       attributes:
          build_platform: android-nightly
          nightly: true
       shipping-phase: ship
       shipping-product: fennec
       worker-type:
          by-release-level:
@@ -48,10 +50,9 @@ jobs:
                default: null
       requires: all-resolved
       scopes:  # see transforms
       treeherder:
          symbol: pub(gp)
          platform: Android/opt
          tier: 2
          kind: other
-      run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release', 'maple']
-      deadline-after: 5 days
+      run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
rename from taskcluster/docker/google-play-strings/Dockerfile
rename to taskcluster/docker/mozapkpublisher/Dockerfile
--- a/taskcluster/docker/google-play-strings/Dockerfile
+++ b/taskcluster/docker/mozapkpublisher/Dockerfile
@@ -1,38 +1,42 @@
-FROM          ubuntu:16.04
+FROM          ubuntu:18.04
 MAINTAINER    Johan Lorenzo <jlorenzo+tc@mozilla.com>
 
 RUN mkdir /builds
 RUN groupadd -g 1000 worker
 RUN useradd -u 1000 -g 1000 -d /builds/worker -s /bin/bash -m worker
 
-RUN apt-get update && apt-get install -y \
+RUN apt-get update \
+  && apt-get install -y \
     build-essential \
     curl \
     libffi-dev \
     libfreetype6-dev \
-    libpng12-dev \
+    libpng-dev \
     libssl-dev \
     libxml2-dev \
     libxslt1-dev \
     pkg-config \
     python3-dev \
     python3-pip \
     python3-setuptools \
-    unzip
+    unzip \
+  && apt-get clean
 
 WORKDIR /builds/worker/
 # Change hash_for_given_tag to point to a newer tag. For more information, see
 # https://bugzilla.mozilla.org/show_bug.cgi?id=1459980#c3
-RUN curl --location https://github.com/mozilla-releng/mozapkpublisher/archive/8051c7f98959e2f041ce63a1911ade88827c3a66.zip > mozapkpublisher.zip
-RUN unzip mozapkpublisher.zip
-RUN mv mozapkpublisher-8051c7f98959e2f041ce63a1911ade88827c3a66 mozapkpublisher
+RUN export revision='0ac130cdece189f3913952a5c80913504413de7c' \
+  && curl --location "https://github.com/mozilla-releng/mozapkpublisher/archive/$revision.zip" > mozapkpublisher.zip \
+  && unzip mozapkpublisher.zip \
+  && mv "mozapkpublisher-$revision" mozapkpublisher \
+  && rm mozapkpublisher.zip
 
 WORKDIR /builds/worker/mozapkpublisher
-RUN pip3 install --upgrade pip
-RUN pip3 install --no-deps --require-hashes --requirement requirements.txt
-RUN pip3 install --no-deps --editable .
+RUN pip3 --no-cache-dir install --upgrade pip
+RUN pip3 --no-cache-dir install --no-deps --require-hashes --requirement requirements.txt
+RUN pip3 --no-cache-dir install --no-deps --editable .
 
 RUN chown -R worker:worker /builds/worker
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -238,16 +238,20 @@ Download strings to display on Google Pl
 Artifact is then used by push-apk.
 
 push-apk
 --------
 PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
 all at once.
 
+push-apk-checks
+---------------
+Runs the checks done in push-apk to ensure APKs are sane before submitting them
+
 release-balrog-submit-toplevel
 ------------------------------
 Toplevel tasks are responsible for submitting metadata to Balrog that is not specific to any
 particular platform+locale. For example: fileUrl templates, versions, and platform aliases.
 
 Toplevel tasks are also responsible for updating test channel rules to point at the Release
 being generated.
 
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -26,17 +26,16 @@ push_apk_description_schema = Schema({
     Required('job-from'): task_description_schema['job-from'],
     Required('attributes'): task_description_schema['attributes'],
     Required('treeherder'): task_description_schema['treeherder'],
     Required('run-on-projects'): task_description_schema['run-on-projects'],
     Required('worker-type'): optionally_keyed_by('release-level', basestring),
     Required('worker'): object,
     Required('scopes'): None,
     Required('requires'): task_description_schema['requires'],
-    Required('deadline-after'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
     Optional('extra'): task_description_schema['extra'],
 })
 
 
 PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/push_apk_checks.py
@@ -0,0 +1,85 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk-checks kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.transforms.push_apk import (
+    validate_dependent_tasks,
+    generate_dependencies,
+    delete_non_required_fields,
+)
+from taskgraph.transforms.google_play_strings import set_worker_data
+from taskgraph.transforms.job.mozharness_test import get_artifact_url
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
+
+from voluptuous import Required
+
+transforms = TransformSequence()
+transforms.add_validate(Schema({
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): task_description_schema['label'],
+    Required('description'): task_description_schema['description'],
+    Required('job-from'): task_description_schema['job-from'],
+    Required('attributes'): task_description_schema['attributes'],
+    Required('treeherder'): task_description_schema['treeherder'],
+    Required('package-name'): optionally_keyed_by('project', basestring),
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
+    Required('worker-type'): basestring,
+    Required('worker'): object,
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
+}))
+transforms.add(validate_dependent_tasks)
+transforms.add(set_worker_data)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+        dependencies_labels = job['dependencies'].keys()
+
+        resolve_keyed_by(
+            job, 'package-name', item_name=job['name'], project=config.params['project']
+        )
+
+        commands = [
+            'curl --location "{}" > "<{}>.apk"'.format(
+                get_artifact_url('<{}>'.format(task_label), 'public/build/target.apk'),
+                task_label,
+            )
+            for task_label in dependencies_labels
+        ]
+
+        check_apks_command = (
+            'python3 ./mozapkpublisher/check_apks.py --expected-package-name "{}" {}'.format(
+                job['package-name'],
+                ' '.join(['"<{}>.apk"'.format(task_label) for task_label in dependencies_labels])
+            )
+        )
+        commands.append(check_apks_command)
+
+        job['worker']['command'] = [
+            '/bin/bash',
+            '-cx',
+            {'task-reference': ' && '.join(commands)},
+        ]
+
+        yield job
+
+
+@transforms.add
+def delete_non_required_fields_(config, jobs):
+    for job in jobs:
+        del job['package-name']
+        yield job
+
+
+transforms.add(delete_non_required_fields)