Bug 1357867 - Add repackage tasks suited for generating a .dmg from a signed tarball of OSX binaries. r=dustin
authorJustin Wood <Callek@gmail.com>
Wed, 19 Apr 2017 15:18:44 -0400
changeset 566279 598dd73ca7f3c58fc957607c10089709af3bbdf4
parent 566278 83c08163f3111504a6d6c4cfeffdfe082dbbfbbf
child 566280 feb371151541d2a14a138926f2f277f80bff781f
push id55180
push userjjong@mozilla.com
push dateFri, 21 Apr 2017 09:36:13 +0000
reviewersdustin
bugs1357867
milestone55.0a1
Bug 1357867 - Add repackage tasks suited for generating a .dmg from a signed tarball of OSX binaries. r=dustin MozReview-Commit-ID: 7TdQxTlKm9D
taskcluster/ci/repackage-l10n/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_l10n.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.repackage_l10n:transforms
+   - taskgraph.transforms.repackage:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - nightly-l10n-signing
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/repackage/kind.yml
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.repackage:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -202,8 +202,19 @@ Decides whether or not APKs should be pu
 kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
 in order to make the decision.
 
 push-apk
 --------
 PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
 all at once. They also depend on the breakpoint.
+
+repackage
+---------
+Repackage tasks take a signed output and package them up into something suitable
+for shipping to our users. For example, on OSX we return a tarball as the signed output
+and this task would package that up as an Apple Disk Image (.dmg)
+
+
+repackage-l10n
+--------------
+Repackage-L10n is a ```Repackage``` task split up to be suitable for use after l10n repacks.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -0,0 +1,171 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the repackage task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+transforms = TransformSequence()
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+packaging_description_schema = Schema({
+    # the dependant task (object) for this  job, used to inform repackaging.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the signed things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this repackaging task
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for repackaging.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    # If a l10n task, the corresponding locale
+    Optional('locale'): basestring,
+
+    # Routes specific to this task, if defined
+    Optional('routes'): [basestring],
+
+    # passed through directly to the job description
+    Optional('extra'): task_description_schema['extra'],
+
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            packaging_description_schema, job,
+            "In packaging ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_repackage_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        label = job.get('label',
+                        dep_job.label.replace("signing-", "repackage-"))
+        job['label'] = label
+        cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
+        cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
+
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc(Nr)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+
+        attributes = {
+            'nightly': dep_job.attributes.get('nightly', False),
+            'build_platform': dep_job.attributes.get('build_platform'),
+            'build_type': dep_job.attributes.get('build_type'),
+        }
+
+        command = ['/home/worker/bin/run-task',
+                   # Various caches/volumes are default owned by root:root.
+                   '--chown-recursive', '/home/worker/workspace',
+                   '--chown-recursive', '/home/worker/tooltool-cache',
+                   '--vcs-checkout', '/home/worker/workspace/build/src',
+                   '--tools-checkout', '/home/worker/workspace/build/tools',
+                   '--',
+                   '/home/worker/workspace/build/src/taskcluster/scripts/builder/repackage.sh'
+                   ]
+
+        dependencies = {dep_job.attributes.get('kind'): dep_job.label}
+        if job.get('locale'):
+            input_string = 'https://queue.taskcluster.net/v1/task/<nightly-l10n-signing>/' + \
+                'artifacts/public/build/{}/target.tar.gz'
+            input_string = input_string.format(job['locale'])
+        else:
+            input_string = 'https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/' + \
+                'public/build/target.tar.gz'
+        signed_input = {'task-reference': input_string}
+        level = config.params['level']
+
+        task = {
+            'label': job['label'],
+            'description': "{} Repackage".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'aws-provisioner-v1/gecko-%s-b-macosx64' % level,
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+            'routes': job.get('routes', []),
+            'extra': job.get('extra', {}),
+            'scopes':
+            ['docker-worker:relengapi-proxy:tooltool.download.internal',
+             'secrets:get:project/taskcluster/gecko/hgfingerprint',
+             'docker-worker:relengapi-proxy:tooltool.download.public',
+             'project:releng:signing:format:dmg'],
+            'worker': {'implementation': 'docker-worker',
+                       'docker-image': {"in-tree": "desktop-build"},
+                       'caches': [{
+                                   'type': 'persistent',
+                                   'name': 'tooltool-cache',
+                                   'mount-point': '/home/worker/tooltool-cache',
+                                 }, {
+                                   'type': 'persistent',
+                                   'name': 'level-%s-checkouts-v1' % level,
+                                   'mount-point': '/home/worker/checkouts',
+                                 }],
+                       'artifacts': [{
+                                       'type': 'file',
+                                       'path': '/home/worker/workspace/build/upload/target.dmg',
+                                       'name': 'public/build/target.dmg',
+                                     }],
+                       'env': {
+                               'SIGNED_INPUT': signed_input,
+                               'MOZ_BUILD_DATE': config.params['moz_build_date'],
+                               'MH_BUILD_POOL': 'taskcluster',
+                               'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
+                               'GECKO_HEAD_REV': config.params['head_rev'],
+                               'MH_BRANCH': config.params['project'],
+                               'MOZ_SCM_LEVEL': level,
+                               'MOZHARNESS_ACTIONS': 'download_input setup repackage',
+                               'NEED_XVFB': 'true',
+                               'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+                               'TOOLTOOL_CACHE': '/home/worker/tooltool-cache',
+                               'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+                               'MOZHARNESS_CONFIG': 'repackage/osx_signed.py',
+                               'USE_SCCACHE': '1',
+                               'MOZHARNESS_SCRIPT': 'mozharness/scripts/repackage.py'
+                               },
+                       'command': command,
+                       'chain-of-trust': True,
+                       'relengapi-proxy': True,
+                       'max-run-time': 3600
+                       }
+        }
+        yield task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/repackage_l10n.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the repackage task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def split_locales(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        for locale in dep_job.attributes.get('chunk_locales', []):
+            job['locale'] = locale
+
+            label = dep_job.label.replace("signing-", "repackage-{}-".format(locale))
+            label = "repackage-{}-{}/{}".format(locale,
+                                                dep_job.attributes['build_platform'],
+                                                dep_job.attributes['build_type'],)
+            job['label'] = label
+
+            treeherder = job.get('treeherder', {})
+            treeherder['symbol'] = 'tc-L10n-Rpk({})'.format(locale)
+            job['treeherder'] = treeherder
+            yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -434,16 +434,17 @@ GROUP_NAMES = {
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
     'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
+    'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'tc-cs': 'Checksum signing executed by Taskcluster',
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',