Bug 1381577 - Part M; Refactor repackage taskgraph code to be more readable and make it easier to add windows. r=kmoir
authorJustin Wood <Callek@gmail.com>
Tue, 18 Jul 2017 11:46:55 -0400
changeset 418976 635129751d087f6ba2244f8a6dc4325b024003c2
parent 418975 0b62fc3730838ee2f37c024f3f1ec6d382de585e
child 418977 8fed3841b32a327f9fa0ed2e5261848de0e7b7df
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmoir
bugs1381577
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1381577 - Part M; Refactor repackage taskgraph code to be more readable and make it easier to add windows. r=kmoir Land date changes to support windows nightlies onto central MozReview-Commit-ID: Bo0wVjGPcCN
taskcluster/taskgraph/transforms/repackage.py
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -8,16 +8,19 @@ Transform the repackage task into an act
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
+_TC_ARTIFACT_LOCATION = \
+        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
+
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
@@ -108,70 +111,47 @@ def make_job_description(config, jobs):
                 dependencies[build_task][13:dependencies[build_task].rfind('-')])
             build_task = 'build'
         signing_task_ref = "<{}>".format(signing_task)
         build_task_ref = "<{}>".format(build_task)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage'
 
+        locale = None
         if job.get('locale'):
-            attributes['locale'] = job['locale']
+            locale = job['locale']
+            attributes['locale'] = locale
 
         level = config.params['level']
 
-        task_env = {}
-        locale_output_path = ""
-        mar_prefix = 'https://queue.taskcluster.net/v1/task/' + \
-            '{}/artifacts/public/build/host/bin/'.format(build_task_ref)
-        if attributes['build_platform'].startswith('macosx'):
-            if job.get('locale'):
-                input_string = 'https://queue.taskcluster.net/v1/task/' + \
-                    '{}/artifacts/public/build/{}/target.tar.gz'
-                input_string = input_string.format(signing_task_ref, job['locale'])
-                locale_output_path = "{}/".format(job['locale'])
-            else:
-                input_string = 'https://queue.taskcluster.net/v1/task/' + \
-                    '{}/artifacts/public/build/target.tar.gz'.format(signing_task_ref)
-            task_env.update(
-                SIGNED_INPUT={'task-reference': input_string},
-                UNSIGNED_MAR={'task-reference': "{}mar".format(mar_prefix)},
-            )
-            mozharness_config = ['repackage/osx_signed.py']
-            output_files = [{
-                'type': 'file',
-                'path': '/home/worker/workspace/build/artifacts/target.dmg',
-                'name': 'public/build/{}target.dmg'.format(locale_output_path),
-            }, {
-                'type': 'file',
-                'path': '/home/worker/workspace/build/artifacts/target.complete.mar',
-                'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
-            }]
-        else:
-            raise Exception("Unexpected build platform for repackage")
-
+        build_platform = attributes['build_platform']
         run = {
             'using': 'mozharness',
             'script': 'mozharness/scripts/repackage.py',
-            'config': mozharness_config,
+            'config': _generate_task_mozharness_config(build_platform),
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['download_input', 'setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
         }
 
-        if attributes["build_platform"].startswith('macosx'):
-            worker = {
-                'docker-image': {"in-tree": "desktop-build"},
-                'artifacts': output_files,
-                'env': task_env,
-                'chain-of-trust': True,
-                'max-run-time': 3600
-            }
-            run["tooltool-downloads"] = 'internal'
+        worker = {
+            'env': _generate_task_env(build_platform, build_task_ref,
+                                      signing_task_ref, locale=locale),
+            'artifacts': _generate_task_output_files(build_platform, locale=locale),
+            'chain-of-trust': True,
+            'max-run-time': 3600,
+        }
+
+        if build_platform.startswith('macosx'):
             worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
+
+            run['tooltool-downloads'] = 'internal'
+            worker['docker-image'] = {"in-tree": "desktop-build"},
+
             cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
             cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
         task = {
             'label': job['label'],
             'description': "{} Repackage".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': worker_type,
@@ -180,8 +160,51 @@ def make_job_description(config, jobs):
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
             'extra': job.get('extra', {}),
             'worker': worker,
             'run': run,
         }
         yield task
+
+
+def _generate_task_mozharness_config(build_platform):
+    if build_platform.startswith('macosx'):
+        return ['repackage/osx_signed.py']
+    else:
+        raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
+
+
+def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
+    mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
+    signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
+
+    if build_platform.startswith('macosx'):
+        return {
+            'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)},
+            'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
+        }
+    else:
+        raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
+
+
+def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
+    if locale:
+        postfix = '{}/{}'.format(locale, postfix)
+
+    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
+
+
+def _generate_task_output_files(build_platform, locale=None):
+    locale_output_path = '{}/'.format(locale) if locale else ''
+    if build_platform.startswith('macosx'):
+        return [{
+            'type': 'file',
+            'path': '/home/worker/workspace/build/artifacts/target.dmg',
+            'name': 'public/build/{}target.dmg'.format(locale_output_path),
+        }, {
+            'type': 'file',
+            'path': '/home/worker/workspace/build/artifacts/target.complete.mar',
+            'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
+        }]
+    else:
+        raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))