Bug 1381577 - Refactor repackage taskgraph code to be more readable and make it easier to add windows. r=kmoir draft
authorJustin Wood <Callek@gmail.com>
Tue, 18 Jul 2017 11:46:55 -0400
changeset 611727 2b8c585e8c0ab817f366f7a1704659656b68cd8f
parent 611726 c08ead65a8804d4f6a741daf923c279721c0f00e
child 611728 aee3d31c0bd2fb98be5a3f84535a9713d1da8e9c
push id69287
push userCallek@gmail.com
push dateThu, 20 Jul 2017 01:11:37 +0000
reviewerskmoir
bugs1381577
milestone56.0a1
Bug 1381577 - Refactor repackage taskgraph code to be more readable and make it easier to add windows. r=kmoir Land date changes to support windows nightlies onto central MozReview-Commit-ID: Bo0wVjGPcCN
taskcluster/taskgraph/transforms/repackage.py
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -8,16 +8,19 @@ Transform the repackage task into an act
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
+_TC_ARTIFACT_LOCATION = \
+        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
+
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
@@ -108,70 +111,47 @@ def make_job_description(config, jobs):
                 dependencies[build_task][13:dependencies[build_task].rfind('-')])
             build_task = 'build'
         signing_task_ref = "<{}>".format(signing_task)
         build_task_ref = "<{}>".format(build_task)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage'
 
+        locale = None
         if job.get('locale'):
-            attributes['locale'] = job['locale']
+            locale = job['locale']
+            attributes['locale'] = locale
 
         level = config.params['level']
 
-        task_env = {}
-        locale_output_path = ""
-        mar_prefix = 'https://queue.taskcluster.net/v1/task/' + \
-            '{}/artifacts/public/build/host/bin/'.format(build_task_ref)
-        if attributes['build_platform'].startswith('macosx'):
-            if job.get('locale'):
-                input_string = 'https://queue.taskcluster.net/v1/task/' + \
-                    '{}/artifacts/public/build/{}/target.tar.gz'
-                input_string = input_string.format(signing_task_ref, job['locale'])
-                locale_output_path = "{}/".format(job['locale'])
-            else:
-                input_string = 'https://queue.taskcluster.net/v1/task/' + \
-                    '{}/artifacts/public/build/target.tar.gz'.format(signing_task_ref)
-            task_env.update(
-                SIGNED_INPUT={'task-reference': input_string},
-                UNSIGNED_MAR={'task-reference': "{}mar".format(mar_prefix)},
-            )
-            mozharness_config = ['repackage/osx_signed.py']
-            output_files = [{
-                'type': 'file',
-                'path': '/home/worker/workspace/build/artifacts/target.dmg',
-                'name': 'public/build/{}target.dmg'.format(locale_output_path),
-            }, {
-                'type': 'file',
-                'path': '/home/worker/workspace/build/artifacts/target.complete.mar',
-                'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
-            }]
-        else:
-            raise Exception("Unexpected build platform for repackage")
-
+        build_platform = attributes['build_platform']
         run = {
             'using': 'mozharness',
             'script': 'mozharness/scripts/repackage.py',
-            'config': mozharness_config,
+            'config': _generate_task_mozharness_config(build_platform),
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['download_input', 'setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
         }
 
-        if attributes["build_platform"].startswith('macosx'):
-            worker = {
-                'docker-image': {"in-tree": "desktop-build"},
-                'artifacts': output_files,
-                'env': task_env,
-                'chain-of-trust': True,
-                'max-run-time': 3600
-            }
-            run["tooltool-downloads"] = 'internal'
+        worker = {
+            'env': _generate_task_env(build_platform, build_task_ref,
+                                      signing_task_ref, locale=locale),
+            'artifacts': _generate_task_output_files(build_platform, locale=locale),
+            'chain-of-trust': True,
+            'max-run-time': 3600,
+        }
+
+        if build_platform.startswith('macosx'):
             worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
+
+            run['tooltool-downloads'] = 'internal'
+            worker['docker-image'] = {"in-tree": "desktop-build"},
+
             cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
             cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
         task = {
             'label': job['label'],
             'description': "{} Repackage".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': worker_type,
@@ -180,8 +160,51 @@ def make_job_description(config, jobs):
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
             'extra': job.get('extra', {}),
             'worker': worker,
             'run': run,
         }
         yield task
+
+
+def _generate_task_mozharness_config(build_platform):
+    if build_platform.startswith('macosx'):
+        return ['repackage/osx_signed.py']
+    else:
+        raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
+
+
+def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
+    mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
+    signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
+
+    if build_platform.startswith('macosx'):
+        return {
+            'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)},
+            'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
+        }
+    else:
+        raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
+
+
+def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
+    if locale:
+        postfix = '{}/{}'.format(locale, postfix)
+
+    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
+
+
+def _generate_task_output_files(build_platform, locale=None):
+    locale_output_path = '{}/'.format(locale) if locale else ''
+    if build_platform.startswith('macosx'):
+        return [{
+            'type': 'file',
+            'path': '/home/worker/workspace/build/artifacts/target.dmg',
+            'name': 'public/build/{}target.dmg'.format(locale_output_path),
+        }, {
+            'type': 'file',
+            'path': '/home/worker/workspace/build/artifacts/target.complete.mar',
+            'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
+        }]
+    else:
+        raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))