author | Tom Prince <mozilla@hocat.ca> |
Fri, 17 Aug 2018 10:27:50 -0600 | |
changeset 433942 | 206ab6424edfa60b452fa5ca370132f463f07f0a |
parent 433941 | 2c723487d70b82dbb84ee91e51c5f586e3cce6a1 |
child 433943 | c832e6e8714145a51619fbcbdd562422c931e59f |
push id | 34526 |
push user | aiakab@mozilla.com |
push date | Wed, 29 Aug 2018 21:56:30 +0000 |
treeherder | mozilla-central@2b50a2ad969a [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | Callek |
bugs | 1484264 |
milestone | 63.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
taskcluster/taskgraph/transforms/repackage.py | file | annotate | diff | comparison | revisions | |
taskcluster/taskgraph/transforms/repackage_partner.py | file | annotate | diff | comparison | revisions |
--- a/taskcluster/taskgraph/transforms/repackage.py +++ b/taskcluster/taskgraph/transforms/repackage.py @@ -14,16 +14,17 @@ from taskgraph.util.attributes import co from taskgraph.util.schema import ( validate_schema, optionally_keyed_by, resolve_keyed_by, Schema, ) from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.platforms import archive_format, executable_extension +from taskgraph.util.workertypes import worker_type_implementation from taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required, Optional transforms = TransformSequence() # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} @@ -259,19 +260,16 @@ def make_job_description(config, jobs): 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { - 'artifacts': _generate_task_output_files(dep_job, build_platform, - locale=locale, - project=config.params["project"]), 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, } if locale: # Make sure we specify the locale-specific upload dir @@ -286,16 +284,22 @@ def make_job_description(config, jobs): else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform) ) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} + worker['artifacts'] = _generate_task_output_files( + dep_job, worker_type_implementation(worker_type), + repackage_config=repackage_config, + locale=locale, + ) + description = ( "Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) @@ -360,55 +364,29 @@ def _generate_download_config(task, buil if use_stub: fetch_config[signing_task].append('{}setup-stub.exe'.format(locale_path)) return fetch_config raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) -def _generate_task_output_files(task, build_platform, locale=None, project=None): +def _generate_task_output_files(task, worker_implementation, repackage_config, locale=None): locale_output_path = '{}/'.format(locale) if locale else '' artifact_prefix = get_artifact_prefix(task) - if build_platform.startswith('linux') or build_platform.startswith('macosx'): - output_files = [{ - 'type': 'file', - 'path': '/builds/worker/workspace/build/outputs/{}target.complete.mar' - .format(locale_output_path), - 'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path), - }] + if worker_implementation == ('docker-worker', 'linux'): + local_prefix = '/builds/worker/workspace/' + elif worker_implementation == ('generic-worker', 'windows'): + local_prefix = '' + else: + raise NotImplementedError( + 'Unsupported worker implementation: "{}"'.format(worker_implementation)) - if build_platform.startswith('macosx'): - output_files.append({ - 'type': 'file', - 'path': '/builds/worker/workspace/build/outputs/{}target.dmg' - .format(locale_output_path), - 'name': '{}/{}target.dmg'.format(artifact_prefix, locale_output_path), - }) - - elif build_platform.startswith('win'): - output_files = [{ + output_files = [] + for config in repackage_config: + output_files.append({ 'type': 'file', - 'path': 'build/outputs/{}target.installer.exe'.format(locale_output_path), - 'name': '{}/{}target.installer.exe'.format(artifact_prefix, locale_output_path), - }, { - 'type': 'file', - 'path': 'build/outputs/{}target.complete.mar'.format(locale_output_path), - 'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path), - }] - - use_stub = task.attributes.get('stub-installer') - if use_stub: - output_files.append({ - 'type': 'file', - 'path': 'build/outputs/{}target.stub-installer.exe'.format( - locale_output_path - ), - 'name': '{}/{}target.stub-installer.exe'.format( - artifact_prefix, locale_output_path - ), - }) - - if output_files: - return output_files - - raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) + 'path': '{}build/outputs/{}{}' + .format(local_prefix, locale_output_path, config['output']), + 'name': '{}/{}{}'.format(artifact_prefix, locale_output_path, config['output']), + }) + return output_files
--- a/taskcluster/taskgraph/transforms/repackage_partner.py +++ b/taskcluster/taskgraph/transforms/repackage_partner.py @@ -15,16 +15,17 @@ from taskgraph.util.schema import ( validate_schema, optionally_keyed_by, resolve_keyed_by, Schema, ) from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.partners import check_if_partners_enabled from taskgraph.util.platforms import archive_format, executable_extension +from taskgraph.util.workertypes import worker_type_implementation from taskgraph.transforms.task import task_description_schema from taskgraph.transforms.repackage import PACKAGE_FORMATS from voluptuous import Any, Required, Optional transforms = TransformSequence() # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings @@ -174,17 +175,16 @@ def make_job_description(config, jobs): 'actions': ['download_input', 'setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { - 'artifacts': _generate_task_output_files(dep_job, build_platform, partner=repack_id), 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, 'env': { 'REPACK_ID': repack_id, }, # Don't add generic artifact directory. 'skip-artifacts': True, @@ -199,16 +199,21 @@ def make_job_description(config, jobs): else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform) ) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} + worker['artifacts'] = _generate_task_output_files( + dep_job, worker_type_implementation(worker_type), + repackage_config, partner=repack_id, + ) + description = ( "Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) @@ -261,35 +266,33 @@ def _generate_download_config(task, buil }, '{}setup.exe'.format(locale_path), ], } raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) -def _generate_task_output_files(task, build_platform, partner): +def _generate_task_output_files(task, worker_implementation, repackage_config, partner): """We carefully generate an explicit list here, but there's an artifacts directory too, courtesy of generic_worker_add_artifacts() (windows) or docker_worker_add_artifacts(). Any errors here are likely masked by that. """ partner_output_path = '{}/'.format(partner) artifact_prefix = get_artifact_prefix(task) - if build_platform.startswith('macosx'): - output_files = [{ - 'type': 'file', - 'path': '/builds/worker/workspace/build/outputs/{}target.dmg' - .format(partner_output_path), - 'name': '{}/{}target.dmg'.format(artifact_prefix, partner_output_path), - }] + if worker_implementation == ('docker-worker', 'linux'): + local_prefix = '/builds/worker/workspace/' + elif worker_implementation == ('generic-worker', 'windows'): + local_prefix = '' + else: + raise NotImplementedError( + 'Unsupported worker implementation: "{}"'.format(worker_implementation)) - elif build_platform.startswith('win'): - output_files = [{ + output_files = [] + for config in repackage_config: + output_files.append({ 'type': 'file', - 'path': 'build/outputs/{}target.installer.exe'.format(partner_output_path), - 'name': '{}/{}target.installer.exe'.format(artifact_prefix, partner_output_path), - }] - - if output_files: - return output_files - - raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) + 'path': '{}build/outputs/{}{}' + .format(local_prefix, partner_output_path, config['output']), + 'name': '{}/{}{}'.format(artifact_prefix, partner_output_path, config['output']), + }) + return output_files