Bug 1484264: [taskgraph] Populate repackage artifacts from repackaged formats; r=Callek
authorTom Prince <mozilla@hocat.ca>
Fri, 17 Aug 2018 10:27:50 -0600
changeset 433942 206ab6424edfa60b452fa5ca370132f463f07f0a
parent 433941 2c723487d70b82dbb84ee91e51c5f586e3cce6a1
child 433943 c832e6e8714145a51619fbcbdd562422c931e59f
push id34526
push useraiakab@mozilla.com
push dateWed, 29 Aug 2018 21:56:30 +0000
treeherdermozilla-central@2b50a2ad969a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1484264
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1484264: [taskgraph] Populate repackage artifacts from repackaged formats; r=Callek Differential Revision: https://phabricator.services.mozilla.com/D3623
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_partner.py
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -14,16 +14,17 @@ from taskgraph.util.attributes import co
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
     resolve_keyed_by,
     Schema,
 )
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.platforms import archive_format, executable_extension
+from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@@ -259,19 +260,16 @@ def make_job_description(config, jobs):
             'actions': ['setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
             'extra-config': {
                 'repackage_config': repackage_config,
             },
         })
 
         worker = {
-            'artifacts': _generate_task_output_files(dep_job, build_platform,
-                                                     locale=locale,
-                                                     project=config.params["project"]),
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
             # Don't add generic artifact directory.
             'skip-artifacts': True,
         }
 
         if locale:
             # Make sure we specify the locale-specific upload dir
@@ -286,16 +284,22 @@ def make_job_description(config, jobs):
             else:
                 raise NotImplementedError(
                     'Unsupported build_platform: "{}"'.format(build_platform)
                 )
 
             run['tooltool-downloads'] = 'internal'
             worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
 
+        worker['artifacts'] = _generate_task_output_files(
+            dep_job, worker_type_implementation(worker_type),
+            repackage_config=repackage_config,
+            locale=locale,
+        )
+
         description = (
             "Repackaging for locale '{locale}' for build '"
             "{build_platform}/{build_type}'".format(
                 locale=attributes.get('locale', 'en-US'),
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
@@ -360,55 +364,29 @@ def _generate_download_config(task, buil
         if use_stub:
             fetch_config[signing_task].append('{}setup-stub.exe'.format(locale_path))
 
         return fetch_config
 
     raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
 
 
-def _generate_task_output_files(task, build_platform, locale=None, project=None):
+def _generate_task_output_files(task, worker_implementation, repackage_config, locale=None):
     locale_output_path = '{}/'.format(locale) if locale else ''
     artifact_prefix = get_artifact_prefix(task)
 
-    if build_platform.startswith('linux') or build_platform.startswith('macosx'):
-        output_files = [{
-            'type': 'file',
-            'path': '/builds/worker/workspace/build/outputs/{}target.complete.mar'
-                    .format(locale_output_path),
-            'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path),
-        }]
+    if worker_implementation == ('docker-worker', 'linux'):
+        local_prefix = '/builds/worker/workspace/'
+    elif worker_implementation == ('generic-worker', 'windows'):
+        local_prefix = ''
+    else:
+        raise NotImplementedError(
+            'Unsupported worker implementation: "{}"'.format(worker_implementation))
 
-        if build_platform.startswith('macosx'):
-            output_files.append({
-                'type': 'file',
-                'path': '/builds/worker/workspace/build/outputs/{}target.dmg'
-                        .format(locale_output_path),
-                'name': '{}/{}target.dmg'.format(artifact_prefix, locale_output_path),
-            })
-
-    elif build_platform.startswith('win'):
-        output_files = [{
+    output_files = []
+    for config in repackage_config:
+        output_files.append({
             'type': 'file',
-            'path': 'build/outputs/{}target.installer.exe'.format(locale_output_path),
-            'name': '{}/{}target.installer.exe'.format(artifact_prefix, locale_output_path),
-        }, {
-            'type': 'file',
-            'path': 'build/outputs/{}target.complete.mar'.format(locale_output_path),
-            'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path),
-        }]
-
-        use_stub = task.attributes.get('stub-installer')
-        if use_stub:
-            output_files.append({
-                'type': 'file',
-                'path': 'build/outputs/{}target.stub-installer.exe'.format(
-                    locale_output_path
-                ),
-                'name': '{}/{}target.stub-installer.exe'.format(
-                    artifact_prefix, locale_output_path
-                ),
-            })
-
-    if output_files:
-        return output_files
-
-    raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
+            'path': '{}build/outputs/{}{}'
+                    .format(local_prefix, locale_output_path, config['output']),
+            'name': '{}/{}{}'.format(artifact_prefix, locale_output_path, config['output']),
+        })
+    return output_files
--- a/taskcluster/taskgraph/transforms/repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_partner.py
@@ -15,16 +15,17 @@ from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
     resolve_keyed_by,
     Schema,
 )
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.partners import check_if_partners_enabled
 from taskgraph.util.platforms import archive_format, executable_extension
+from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.task import task_description_schema
 from taskgraph.transforms.repackage import PACKAGE_FORMATS
 from voluptuous import Any, Required, Optional
 
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
@@ -174,17 +175,16 @@ def make_job_description(config, jobs):
             'actions': ['download_input', 'setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
             'extra-config': {
                 'repackage_config': repackage_config,
             },
         })
 
         worker = {
-            'artifacts': _generate_task_output_files(dep_job, build_platform, partner=repack_id),
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
             'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False,
             'env': {
                 'REPACK_ID': repack_id,
             },
             # Don't add generic artifact directory.
             'skip-artifacts': True,
@@ -199,16 +199,21 @@ def make_job_description(config, jobs):
             else:
                 raise NotImplementedError(
                     'Unsupported build_platform: "{}"'.format(build_platform)
                 )
 
             run['tooltool-downloads'] = 'internal'
             worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
 
+        worker['artifacts'] = _generate_task_output_files(
+            dep_job, worker_type_implementation(worker_type),
+            repackage_config, partner=repack_id,
+        )
+
         description = (
             "Repackaging for repack_id '{repack_id}' for build '"
             "{build_platform}/{build_type}'".format(
                 repack_id=job['extra']['repack_id'],
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
@@ -261,35 +266,33 @@ def _generate_download_config(task, buil
                 },
                 '{}setup.exe'.format(locale_path),
             ],
         }
 
     raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
 
 
-def _generate_task_output_files(task, build_platform, partner):
+def _generate_task_output_files(task, worker_implementation, repackage_config, partner):
     """We carefully generate an explicit list here, but there's an artifacts directory
     too, courtesy of generic_worker_add_artifacts() (windows) or docker_worker_add_artifacts().
     Any errors here are likely masked by that.
     """
     partner_output_path = '{}/'.format(partner)
     artifact_prefix = get_artifact_prefix(task)
 
-    if build_platform.startswith('macosx'):
-        output_files = [{
-            'type': 'file',
-            'path': '/builds/worker/workspace/build/outputs/{}target.dmg'
-                    .format(partner_output_path),
-            'name': '{}/{}target.dmg'.format(artifact_prefix, partner_output_path),
-        }]
+    if worker_implementation == ('docker-worker', 'linux'):
+        local_prefix = '/builds/worker/workspace/'
+    elif worker_implementation == ('generic-worker', 'windows'):
+        local_prefix = ''
+    else:
+        raise NotImplementedError(
+            'Unsupported worker implementation: "{}"'.format(worker_implementation))
 
-    elif build_platform.startswith('win'):
-        output_files = [{
+    output_files = []
+    for config in repackage_config:
+        output_files.append({
             'type': 'file',
-            'path': 'build/outputs/{}target.installer.exe'.format(partner_output_path),
-            'name': '{}/{}target.installer.exe'.format(artifact_prefix, partner_output_path),
-        }]
-
-    if output_files:
-        return output_files
-
-    raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
+            'path': '{}build/outputs/{}{}'
+                    .format(local_prefix, partner_output_path, config['output']),
+            'name': '{}/{}{}'.format(artifact_prefix, partner_output_path, config['output']),
+        })
+    return output_files