author | Tom Prince <mozilla@hocat.ca> |
Wed, 15 Aug 2018 15:12:33 -0600 | |
changeset 491142 | 5c6511e28eff1859c269f7ab6032f0ef7950881b |
parent 491141 | a55c12f57d37a8264fded5783d4d8ebccbe9b037 |
child 491143 | 79b341f0c72e49fafe85251770b4073b54c04222 |
push id | 1815 |
push user | ffxbld-merge |
push date | Mon, 15 Oct 2018 10:40:45 +0000 |
treeherder | mozilla-release@18d4c09e9378 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | Callek |
bugs | 1484012 |
milestone | 63.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/taskcluster/taskgraph/transforms/repackage.py +++ b/taskcluster/taskgraph/transforms/repackage.py @@ -12,17 +12,17 @@ import copy from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.schema import ( validate_schema, optionally_keyed_by, resolve_keyed_by, Schema, ) -from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix, get_artifact_prefix +from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required, Optional transforms = TransformSequence() # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} @@ -160,18 +160,16 @@ def make_job_description(config, jobs): if job.get('locale'): # XXXCallek: todo: rewrite dependency finding # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt' # We need this additional dependency to support finding the mar binary # Which is needed in order to generate a new complete.mar dependencies['build'] = "build-{}/opt".format( dependencies[build_task][13:dependencies[build_task].rfind('-')]) build_task = 'build' - signing_task_ref = "<{}>".format(signing_task) - build_task_ref = "<{}>".format(build_task) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' locale = None if job.get('locale'): locale = job['locale'] attributes['locale'] = locale @@ -179,36 +177,37 @@ def make_job_description(config, jobs): level = config.params['level'] build_platform = attributes['build_platform'] run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', - 'actions': ['download_input', 'setup', 'repackage'], + 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', }) worker = { - 'env': _generate_task_env(dep_job, build_platform, build_task_ref, - signing_task_ref, locale=locale, - project=config.params["project"]), 'artifacts': _generate_task_output_files(dep_job, build_platform, locale=locale, project=config.params["project"]), 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, } if locale: # Make sure we specify the locale-specific upload dir - worker['env'].update(LOCALE=locale) + worker.setdefault('env', {}).update(LOCALE=locale) + + use_stub = attributes.get('stub-installer') + if not use_stub: + worker.setdefault('env', {})['NO_STUB_INSTALLER'] = '1' if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level run['use-magic-mh-args'] = False else: if build_platform.startswith(('linux', 'macosx')): worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level else: @@ -235,59 +234,66 @@ def make_job_description(config, jobs): 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, + 'fetches': _generate_download_config(dep_job, build_platform, build_task, + signing_task, locale=locale, + project=config.params["project"]), } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task -def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None, - project=None): - mar_prefix = get_taskcluster_artifact_prefix( - task, build_task_ref, postfix='host/bin/', locale=None - ) - signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale) +def _generate_download_config(task, build_platform, build_task, signing_task, locale=None, + project=None): + locale_path = '{}/'.format(locale) if locale else '' if build_platform.startswith('linux') or build_platform.startswith('macosx'): tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz' return { - 'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format( - signed_prefix, tarball_extension - )}, - 'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)}, + signing_task: [ + { + 'artifact': '{}target.tar.{}'.format(locale_path, tarball_extension), + 'extract': False, + }, + ], + build_task: [ + 'host/bin/mar', + ], } elif build_platform.startswith('win'): - task_env = { - 'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)}, - 'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)}, - 'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)}, + fetch_config = { + signing_task: [ + { + 'artifact': '{}target.zip'.format(locale_path), + 'extract': False, + }, + '{}setup.exe'.format(locale_path), + ], + build_task: [ + 'host/bin/mar.exe', + ], } use_stub = task.attributes.get('stub-installer') if use_stub: - task_env['SIGNED_SETUP_STUB'] = { - 'task-reference': '{}setup-stub.exe'.format(signed_prefix), - } - elif '32' in build_platform: - # Stub installer is only attempted on win32 - task_env['NO_STUB_INSTALLER'] = '1' + fetch_config[signing_task].append('{}setup-stub.exe'.format(locale_path)) - return task_env + return fetch_config raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) def _generate_task_output_files(task, build_platform, locale=None, project=None): locale_output_path = '{}/'.format(locale) if locale else '' artifact_prefix = get_artifact_prefix(task) @@ -317,18 +323,18 @@ def _generate_task_output_files(task, bu 'path': 'build/outputs/{}target.complete.mar'.format(locale_output_path), 'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path), }] use_stub = task.attributes.get('stub-installer') if use_stub: output_files.append({ 'type': 'file', - 'path': '{}/{}target.stub-installer.exe'.format( - artifact_prefix, locale_output_path + 'path': 'build/outputs/{}target.stub-installer.exe'.format( + locale_output_path ), 'name': '{}/{}target.stub-installer.exe'.format( artifact_prefix, locale_output_path ), }) if output_files: return output_files
--- a/taskcluster/taskgraph/transforms/repackage_partner.py +++ b/taskcluster/taskgraph/transforms/repackage_partner.py @@ -12,17 +12,17 @@ import copy from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.schema import ( validate_schema, optionally_keyed_by, resolve_keyed_by, Schema, ) -from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix, get_artifact_prefix +from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.partners import check_if_partners_enabled from taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required, Optional transforms = TransformSequence() # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings @@ -137,39 +137,37 @@ def make_job_description(config, jobs): dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith('signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith('repack'): signing_task = dependency - signing_task_ref = "<{}>".format(signing_task) attributes['repackage_type'] = 'repackage' level = config.params['level'] repack_id = job['extra']['repack_id'] run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['download_input', 'setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', }) worker = { - 'env': _generate_task_env(build_platform, signing_task, signing_task_ref, - partner=repack_id), 'artifacts': _generate_task_output_files(dep_job, build_platform, partner=repack_id), 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, + 'env': {}, # Don't add generic artifact directory. 'skip-artifacts': True, } worker['env'].update(REPACK_ID=repack_id) if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level @@ -201,49 +199,54 @@ def make_job_description(config, jobs): 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, + 'fetches': _generate_download_config(dep_job, build_platform, + signing_task, partner=repack_id, + project=config.params["project"]), } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task -def _generate_task_env(build_platform, signing_task, signing_task_ref, partner): - # Force private artifacts here, until we can populate our dependency map - # with actual task definitions rather than labels. - # (get_taskcluster_artifact_prefix requires the task definition to find - # the artifact_prefix attribute). - signed_prefix = get_taskcluster_artifact_prefix( - signing_task, signing_task_ref, locale=partner, force_private=True - ) - signed_prefix = signed_prefix.replace('public/build', 'releng/partner') +def _generate_download_config(task, build_platform, signing_task, partner=None, + project=None): + locale_path = '{}/'.format(partner) if partner else '' if build_platform.startswith('macosx'): return { - 'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)}, + signing_task: [ + { + 'artifact': '{}target.tar.gz'.format(locale_path), + 'extract': False, + }, + ], } elif build_platform.startswith('win'): - task_env = { - 'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)}, - 'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)}, + return { + signing_task: [ + { + 'artifact': '{}target.zip'.format(locale_path), + 'extract': False, + }, + '{}setup.exe'.format(locale_path), + ], } - return task_env - raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) def _generate_task_output_files(task, build_platform, partner): """We carefully generate an explicit list here, but there's an artifacts directory too, courtesy of generic_worker_add_artifacts() (windows) or docker_worker_add_artifacts(). Any errors here are likely masked by that. """
--- a/testing/mozharness/configs/repackage/linux32_signed.py +++ b/testing/mozharness/configs/repackage/linux32_signed.py @@ -1,23 +1,18 @@ import os platform = "linux32" config = { "locale": os.environ.get("LOCALE"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - "mar": os.environ.get("UNSIGNED_MAR"), - }, - "repackage_config": [[ "mar", - "-i", "{abs_input_dir}/target.tar.gz", + "-i", "{abs_input_dir}/target.tar.bz2", "--mar", "{abs_input_dir}/mar", "-o", "{abs_output_dir}/target.complete.mar" ]], # ToolTool "tooltool_url": 'http://relengapi/tooltool/', 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
--- a/testing/mozharness/configs/repackage/linux64_signed.py +++ b/testing/mozharness/configs/repackage/linux64_signed.py @@ -1,23 +1,18 @@ import os platform = "linux64" config = { "locale": os.environ.get("LOCALE"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - "mar": os.environ.get("UNSIGNED_MAR"), - }, - "repackage_config": [[ "mar", - "-i", "{abs_input_dir}/target.tar.gz", + "-i", "{abs_input_dir}/target.tar.bz2", "--mar", "{abs_input_dir}/mar", "-o", "{abs_output_dir}/target.complete.mar" ]], # ToolTool "tooltool_url": 'http://relengapi/tooltool/', 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
--- a/testing/mozharness/configs/repackage/osx_partner.py +++ b/testing/mozharness/configs/repackage/osx_partner.py @@ -1,19 +1,15 @@ import os config = { "src_mozconfig": "browser/config/mozconfigs/macosx64/repack", "repack_id": os.environ.get("REPACK_ID"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - }, - "repackage_config": [[ "dmg", "-i", "{abs_input_dir}/target.tar.gz", "-o", "{abs_output_dir}/target.dmg" ]], # ToolTool "tooltool_url": 'http://relengapi/tooltool/',
--- a/testing/mozharness/configs/repackage/osx_signed.py +++ b/testing/mozharness/configs/repackage/osx_signed.py @@ -1,20 +1,15 @@ import os config = { "src_mozconfig": "browser/config/mozconfigs/macosx64/repack", "locale": os.environ.get("LOCALE"), - "download_config": { - "target.tar.gz": os.environ.get("SIGNED_INPUT"), - "mar": os.environ.get("UNSIGNED_MAR"), - }, - "repackage_config": [[ "dmg", "-i", "{abs_input_dir}/target.tar.gz", "-o", "{abs_output_dir}/target.dmg" ], [ "mar", "-i", "{abs_input_dir}/target.tar.gz", "--mar", "{abs_input_dir}/mar",
--- a/testing/mozharness/configs/repackage/win32_partner.py +++ b/testing/mozharness/configs/repackage/win32_partner.py @@ -1,32 +1,25 @@ import os platform = "win32" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - } - repackage_config = [[ "installer", "--package-name", "firefox", "--package", "{abs_input_dir}\\target.zip", "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag", "--setupexe", "{abs_input_dir}\\setup.exe", "-o", "{abs_output_dir}\\target.installer.exe", "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx", ]] config = { "repack_id": os.environ.get("REPACK_ID"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), 'tooltool_url': 'https://tooltool.mozilla-releng.net/', 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), 'run_configure': False,
--- a/testing/mozharness/configs/repackage/win32_signed.py +++ b/testing/mozharness/configs/repackage/win32_signed.py @@ -1,25 +1,12 @@ import os platform = "win32" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - "mar.exe": os.environ.get("UNSIGNED_MAR"), -} - -if not os.environ.get("NO_STUB_INSTALLER"): - # Some channels, like esr don't build a stub installer - download_config.update({ - # stub installer is only built on win32 - "setup-stub.exe": os.environ.get("SIGNED_SETUP_STUB"), - }) - repackage_config = [[ "installer", "--package-name", "firefox", "--package", "{abs_input_dir}\\target.zip", "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag", "--setupexe", "{abs_input_dir}\\setup.exe", "-o", "{abs_output_dir}\\target.installer.exe", "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx", @@ -38,18 +25,16 @@ if not os.environ.get("NO_STUB_INSTALLER "--setupexe", "{abs_input_dir}\\setup-stub.exe", "-o", "{abs_output_dir}\\target.stub-installer.exe", "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx", ]) config = { "locale": os.environ.get("LOCALE"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), 'tooltool_url': 'https://tooltool.mozilla-releng.net/', 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), 'run_configure': False,
--- a/testing/mozharness/configs/repackage/win64_partner.py +++ b/testing/mozharness/configs/repackage/win64_partner.py @@ -1,32 +1,25 @@ import os platform = "win64" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - } - repackage_config = [[ "installer", "--package-name", "firefox", "--package", "{abs_input_dir}\\target.zip", "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag", "--setupexe", "{abs_input_dir}\\setup.exe", "-o", "{abs_output_dir}\\target.installer.exe", "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx", ]] config = { "repack_id": os.environ.get("REPACK_ID"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), 'tooltool_url': 'https://tooltool.mozilla-releng.net/', 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), 'run_configure': False,
--- a/testing/mozharness/configs/repackage/win64_signed.py +++ b/testing/mozharness/configs/repackage/win64_signed.py @@ -1,18 +1,12 @@ import os platform = "win64" -download_config = { - "target.zip": os.environ.get("SIGNED_ZIP"), - "setup.exe": os.environ.get("SIGNED_SETUP"), - "mar.exe": os.environ.get("UNSIGNED_MAR"), - } - repackage_config = [[ "installer", "--package-name", "firefox", "--package", "{abs_input_dir}\\target.zip", "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag", "--setupexe", "{abs_input_dir}\\setup.exe", "-o", "{abs_output_dir}\\target.installer.exe", "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx", @@ -21,18 +15,16 @@ repackage_config = [[ "-i", "{abs_input_dir}\\target.zip", "--mar", "{abs_input_dir}\\mar.exe", "-o", "{abs_output_dir}\\target.complete.mar", ]] config = { "locale": os.environ.get("LOCALE"), - "download_config": download_config, - "repackage_config": repackage_config, # ToolTool "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), 'tooltool_url': 'https://tooltool.mozilla-releng.net/', 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), 'run_configure': False,
--- a/testing/mozharness/scripts/repackage.py +++ b/testing/mozharness/scripts/repackage.py @@ -18,49 +18,44 @@ class Repackage(BaseScript): ], } BaseScript.__init__( self, require_config_file=require_config_file, **script_kwargs ) - def download_input(self): - config = self.config + def setup(self): dirs = self.query_abs_dirs() - input_dir = dirs['abs_input_dir'] + self._run_tooltool() - for path, url in config["download_config"].items(): - status = self.download_file(url=url, - file_name=path, - parent_dir=input_dir) - if not status: - self.fatal("Unable to fetch signed input from %s" % url) + mar_path = os.path.join(dirs['abs_input_dir'], 'mar') + if self._is_windows(): + mar_path += '.exe' + if mar_path: + self.chmod(mar_path, 0755) - if 'mar' in path: - # Ensure mar is executable - self.chmod(os.path.join(input_dir, path), 0755) - - def setup(self): - self._run_tooltool() if self.config.get("run_configure", True): self._get_mozconfig() self._run_configure() def query_abs_dirs(self): if self.abs_dirs: return self.abs_dirs abs_dirs = super(Repackage, self).query_abs_dirs() config = self.config dirs = {} dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools') dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src') - dirs['abs_input_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'inputs') + dirs['abs_input_dir'] = os.path.join( + abs_dirs['base_work_dir'], + os.environ.get('MOZ_FETCHES_DIR', 'fetches'), + ) output_dir_suffix = [] if config.get('locale'): output_dir_suffix.append(config['locale']) if config.get('repack_id'): output_dir_suffix.append(config['repack_id']) dirs['abs_output_dir'] = os.path.join( abs_dirs['abs_work_dir'], 'outputs', *output_dir_suffix) for key in dirs.keys():