--- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,9 +1,9 @@ --# This file is rendered via JSON-e by +# This file is rendered via JSON-e by # - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml # - cron tasks - taskcluster/taskgraph/cron/decision.py version: 1 tasks: $let: # sometimes the push user is just `ffxbld` or the like, but we want an email-like field.. ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'} # ensure there's no trailing `/` on the repo URL @@ -83,17 +83,17 @@ tasks: chainOfTrust: true # Note: This task is built server side without the context or tooling that # exist in tree so we must hard code the hash # XXX Changing this will break Chain of Trust without an associated puppet and # scriptworker patch! image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef' - maxRunTime: 2400 + maxRunTime: 1800 # TODO use mozilla-unified for the base repository once the tc-vcs # tar.gz archives are created or tc-vcs isn't being used. command: - /builds/worker/bin/run-task - '--vcs-checkout=/builds/worker/checkouts/gecko' - '--sparse-profile=build/sparse-profiles/taskgraph' - '--'
--- a/taskcluster/ci/balrog/kind.yml +++ b/taskcluster/ci/balrog/kind.yml @@ -8,13 +8,12 @@ transforms: - taskgraph.transforms.name_sanity:transforms - taskgraph.transforms.balrog:transforms - taskgraph.transforms.task:transforms kind-dependencies: - beetmover - beetmover-l10n - beetmover-repackage - - beetmover-partials only-for-attributes: - nightly - signed
deleted file mode 100644 --- a/taskcluster/ci/beetmover-partials/kind.yml +++ /dev/null @@ -1,15 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -loader: taskgraph.loader.single_dep:loader - -transforms: - - taskgraph.transforms.name_sanity:transforms - - taskgraph.transforms.beetmover_repackage_l10n:transforms - - taskgraph.transforms.beetmover_repackage:transforms - - taskgraph.transforms.beetmover_partials:transforms - - taskgraph.transforms.task:transforms - -kind-dependencies: - - partials-signing
--- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -21,10 +21,8 @@ jobs: valgrind-build: symbol: I(vb) lint: symbol: I(lnt) android-gradle-build: symbol: I(agb) index-task: symbol: I(idx) - partial-update-generator: - symbol: I(pg)
deleted file mode 100644 --- a/taskcluster/ci/partials-signing/kind.yml +++ /dev/null @@ -1,13 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -loader: taskgraph.loader.single_dep:loader - -transforms: - - taskgraph.transforms.name_sanity:transforms - - taskgraph.transforms.partials_signing:transforms - - taskgraph.transforms.task:transforms - -kind-dependencies: - - partials
deleted file mode 100644 --- a/taskcluster/ci/partials/kind.yml +++ /dev/null @@ -1,23 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -loader: taskgraph.loader.single_dep:loader - -transforms: - - taskgraph.transforms.name_sanity:transforms - - taskgraph.transforms.partials:transforms - - taskgraph.transforms.task:transforms - -kind-dependencies: - - repackage-signing - -only-for-attributes: - - nightly - -only-for-build-platforms: - - macosx64-nightly/opt - - win32-nightly/opt - - win64-nightly/opt - - linux-nightly/opt - - linux64-nightly/opt
rename from taskcluster/docker/partial-update-generator/Dockerfile rename to taskcluster/docker/funsize-update-generator/Dockerfile --- a/taskcluster/docker/partial-update-generator/Dockerfile +++ b/taskcluster/docker/funsize-update-generator/Dockerfile @@ -20,20 +20,17 @@ RUN for i in 1 2 3 4 5; do freshclam --v # python-pip installs a lot of dependencies increasing the size of an image # drastically. Using easy_install saves us almost 200M. RUN easy_install pip RUN pip install -r /tmp/requirements.txt # scripts RUN mkdir /home/worker/bin COPY scripts/* /home/worker/bin/ - COPY runme.sh /runme.sh RUN chmod 755 /home/worker/bin/* /runme.sh RUN mkdir /home/worker/keys COPY *.pubkey /home/worker/keys/ ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker - -CMD ["/runme.sh"]
rename from taskcluster/docker/partial-update-generator/Makefile rename to taskcluster/docker/funsize-update-generator/Makefile
rename from taskcluster/docker/partial-update-generator/README rename to taskcluster/docker/funsize-update-generator/README
rename from taskcluster/docker/partial-update-generator/dep.pubkey rename to taskcluster/docker/funsize-update-generator/dep.pubkey
rename from taskcluster/docker/partial-update-generator/nightly_sha1.pubkey rename to taskcluster/docker/funsize-update-generator/nightly_sha1.pubkey
rename from taskcluster/docker/partial-update-generator/nightly_sha384.pubkey rename to taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey
rename from taskcluster/docker/partial-update-generator/release_sha1.pubkey rename to taskcluster/docker/funsize-update-generator/release_sha1.pubkey
rename from taskcluster/docker/partial-update-generator/release_sha384.pubkey rename to taskcluster/docker/funsize-update-generator/release_sha384.pubkey
rename from taskcluster/docker/partial-update-generator/requirements.txt rename to taskcluster/docker/funsize-update-generator/requirements.txt
rename from taskcluster/docker/partial-update-generator/runme.sh rename to taskcluster/docker/funsize-update-generator/runme.sh
rename from taskcluster/docker/partial-update-generator/scripts/funsize.py rename to taskcluster/docker/funsize-update-generator/scripts/funsize.py --- a/taskcluster/docker/partial-update-generator/scripts/funsize.py +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -22,17 +22,16 @@ log = logging.getLogger(__name__) ALLOWED_URL_PREFIXES = [ "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", "http://download.cdn.mozilla.net/pub/firefox/nightly/", "https://mozilla-nightly-updates.s3.amazonaws.com", "https://queue.taskcluster.net/", "http://ftp.mozilla.org/", "http://download.mozilla.org/", "https://archive.mozilla.org/", - "https://queue.taskcluster.net/v1/task/", ] DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \ "{locale}-{from_buildid}-{to_buildid}.partial.mar" def verify_signature(mar, certs): log.info("Checking %s signature", mar) @@ -282,21 +281,17 @@ def main(): "previousBuildNumber", "toVersion", "toBuildNumber"): if field in e: mar_data[field] = e[field] mar_data.update(complete_mars) # if branch not set explicitly use repo-name mar_data["branch"] = e.get("branch", mar_data["repo"].rstrip("/").split("/")[-1]) - if 'dest_mar' in e: - mar_name = e['dest_mar'] - else: - # default to formatted name if not specified - mar_name = args.filename_template.format(**mar_data) + mar_name = args.filename_template.format(**mar_data) mar_data["mar"] = mar_name dest_mar = os.path.join(work_env.workdir, mar_name) # TODO: download these once work_env.download_buildsystem_bits(repo=mar_data["repo"], revision=mar_data["revision"]) generate_partial(work_env, from_path, path, dest_mar, mar_data["ACCEPTED_MAR_CHANNEL_IDS"], mar_data["version"],
rename from taskcluster/docker/partial-update-generator/scripts/mbsdiff_hook.sh rename to taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh
--- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -188,24 +188,16 @@ the language in the final artifact names beetmover-repackage ------------------- Beetmover-repackage is beetmover but for tasks that need an intermediate step between signing and packaging, such as OSX. For more details see the definitions of the Beetmover kind above and the repackage kind below. -beetmover-partials ------------------- - -Beetmover-partials is beetmover but for the partial updates that have been -generated. Not every build produces partial updates, and so these are kept -separate from the regular beetmover jobs to avoid situations where the completes -are not uploaded. - checksums-signing ----------------- Checksums-signing take as input the checksums file generated by beetmover tasks and sign it via the signing scriptworkers. Returns the same file signed and additional detached signature. beetmover-checksums ------------------- @@ -236,18 +228,8 @@ repackage-l10n -------------- Repackage-L10n is a ```Repackage``` task split up to be suitable for use after l10n repacks. repackage-signing ----------------- Repackage-signing take the repackaged installers (windows) and update packaging (with the signed internal bits) and signs them. - -partials --------- -Partials takes the complete.mar files produced in previous tasks and generates partial -updates between previous nightly releases and the new one. Requires a release_history -in the parameters. See ``mach release-history`` if doing this manually. - -partials-signing ----------------- -Partials-signing takes the partial updates produced in Partials and signs them.
--- a/taskcluster/docs/parameters.rst +++ b/taskcluster/docs/parameters.rst @@ -102,22 +102,16 @@ syntax or reading a project-specific con one of the functions in ``taskcluster/taskgraph/target_tasks.py``. ``optimize_target_tasks`` If true, then target tasks are eligible for optimization. ``include_nightly`` If true, then nightly tasks are eligible for optimization. -``release_history`` - History of recent releases by platform and locale, used when generating - partial updates for nightly releases. - Suitable contents can be generated with ``mach release-history``, - which will print to the console by default. - Morphed Set ----------- ``morph_templates`` Dict of JSON-e templates to apply to each task, keyed by template name. Values are extra context that will be available to the template under the ``input.<template>`` key. Available templates live in ``taskcluster/taskgraph/templates``. Enabled on try only.
--- a/taskcluster/mach_commands.py +++ b/taskcluster/mach_commands.py @@ -499,28 +499,8 @@ class TaskClusterImagesProvider(object): try: if context_only is None: build_image(image_name) else: build_context(image_name, context_only) except Exception: traceback.print_exc() sys.exit(1) - - -@CommandProvider -class TaskClusterPartialsData(object): - @Command('release-history', category="ci", - description="Query balrog for release history used by enable partials generation") - @CommandArgument('-b', '--branch', - help="The gecko project branch used in balrog, such as " - "mozilla-central, release, date") - @CommandArgument('--product', default='Firefox', - help="The product identifier, such as 'Firefox'") - def generate_partials_builds(self, product, branch): - from taskgraph.util.partials import populate_release_history - try: - import yaml - release_history = {'release_history': populate_release_history(product, branch)} - print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False)) - except Exception: - traceback.print_exc() - sys.exit(1)
--- a/taskcluster/taskgraph/decision.py +++ b/taskcluster/taskgraph/decision.py @@ -13,17 +13,16 @@ import re import time import yaml from .generator import TaskGraphGenerator from .create import create_tasks from .parameters import Parameters from .taskgraph import TaskGraph from .actions import render_actions_json -from taskgraph.util.partials import populate_release_history from . import GECKO from taskgraph.util.templates import Templates from taskgraph.util.time import ( json_time_from_now, current_json_time, ) @@ -103,17 +102,16 @@ def taskgraph_decision(options): * processing decision task command-line options into parameters * running task-graph generation exactly the same way the other `mach taskgraph` commands do * generating a set of artifacts to memorialize the graph * calling TaskCluster APIs to create the graph """ parameters = get_decision_parameters(options) - # create a TaskGraphGenerator instance tgg = TaskGraphGenerator( root_dir=options['root'], parameters=parameters) # write out the parameters used to generate this graph write_artifact('parameters.yml', dict(**parameters)) @@ -199,23 +197,16 @@ def get_decision_parameters(options): task_config = json.load(fh) parameters['morph_templates'] = task_config.get('templates', {}) parameters['target_task_labels'] = task_config.get('tasks') # `target_tasks_method` has higher precedence than `project` parameters if options.get('target_tasks_method'): parameters['target_tasks_method'] = options['target_tasks_method'] - # If the target method is nightly, we should build partials. This means - # knowing what has been released previously. - # An empty release_history is fine, it just means no partials will be built - parameters.setdefault('release_history', dict()) - if 'nightly' in parameters.get('target_tasks_method', ''): - parameters['release_history'] = populate_release_history('Firefox', project) - return Parameters(parameters) def write_artifact(filename, data): logger.info('writing artifact file `{}`'.format(filename)) if not os.path.isdir(ARTIFACTS_DIR): os.mkdir(ARTIFACTS_DIR) path = os.path.join(ARTIFACTS_DIR, filename)
--- a/taskcluster/taskgraph/parameters.py +++ b/taskcluster/taskgraph/parameters.py @@ -23,17 +23,16 @@ PARAMETER_NAMES = set([ 'message', 'morph_templates', 'moz_build_date', 'optimize_target_tasks', 'owner', 'project', 'pushdate', 'pushlog_id', - 'release_history', 'target_task_labels', 'target_tasks_method', ]) TRY_ONLY_PARAMETERS = set([ 'morph_templates', 'target_task_labels', ])
--- a/taskcluster/taskgraph/transforms/balrog.py +++ b/taskcluster/taskgraph/transforms/balrog.py @@ -51,38 +51,31 @@ def validate(config, jobs): @transforms.add def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'c-Up(N)') + treeherder.setdefault('symbol', 'tc-Up(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') attributes = copy_attributes_from_dependent_job(dep_job) - treeherder_job_symbol = dep_job.attributes.get('locale', 'N') - if dep_job.attributes.get('locale'): - treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol) + treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') label = job['label'] - if 'partials' in dep_job.kind: - label = "partials-{}".format(label) - treeherder['symbol'] = 'cp-Up({})'.format(treeherder_job_symbol) - treeherder['tier'] = 3 # remove once proven stable - description = ( "Balrog submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) @@ -96,16 +89,17 @@ def make_task_description(config, jobs): }] server_scope = get_balrog_server_scope(config) channel_scopes = get_balrog_channel_scopes(config) task = { 'label': label, 'description': description, + # do we have to define worker type somewhere? 'worker-type': 'scriptworker-prov-v1/balrogworker-v1', 'worker': { 'implementation': 'balrog', 'upstream-artifacts': upstream_artifacts, }, 'scopes': [server_scope] + channel_scopes, 'dependencies': {'beetmover': dep_job.label}, 'attributes': attributes,
deleted file mode 100644 --- a/taskcluster/taskgraph/transforms/beetmover_partials.py +++ /dev/null @@ -1,77 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Add partial update artifacts to a beetmover task. -""" - -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.partials import (get_balrog_platform_name, - get_partials_artifacts, - get_partials_artifact_map) - -import logging -logger = logging.getLogger(__name__) - -transforms = TransformSequence() - - -def generate_upstream_artifacts(release_history, platform, locale=None): - if not locale or locale == 'en-US': - artifact_prefix = 'public/build' - else: - artifact_prefix = 'public/build/{}'.format(locale) - - artifacts = get_partials_artifacts(release_history, platform, locale) - - upstream_artifacts = [{ - 'taskId': {'task-reference': '<partials-signing>'}, - 'taskType': 'signing', - 'paths': ["{}/{}".format(artifact_prefix, p) - for p in artifacts], - 'locale': locale or 'en-US', - }] - - return upstream_artifacts - - -@transforms.add -def make_partials_artifacts(config, jobs): - for job in jobs: - locale = job["attributes"].get("locale") - if locale: - job['treeherder']['symbol'] = 'pBM({})'.format(locale) - else: - locale = 'en-US' - job['treeherder']['symbol'] = 'pBM(N)' - - # Remove when proved reliable - job['treeherder']['tier'] = 3 - - platform = job["attributes"]["build_platform"] - - platform = get_balrog_platform_name(platform) - upstream_artifacts = generate_upstream_artifacts( - config.params.get('release_history'), platform, locale - ) - - job['worker']['upstream-artifacts'].extend(upstream_artifacts) - - extra = list() - - artifact_map = get_partials_artifact_map( - config.params.get('release_history'), platform, locale) - for artifact in artifact_map: - extra.append({ - 'locale': locale, - 'artifact_name': artifact, - 'buildid': artifact_map[artifact], - 'platform': platform, - }) - - job.setdefault('extra', {}) - job['extra']['partials'] = extra - - yield job
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py +++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py @@ -210,24 +210,16 @@ def make_task_description(config, jobs): repackage_name = "repackage" # repackage-l10n actually uses the repackage depname here repackage_dependencies = {"repackage": dep_job.dependencies[repackage_name] } dependencies.update(repackage_dependencies) - # If this isn't a direct dependency, it won't be in there. - if 'repackage-signing' not in dependencies: - repackage_signing_name = "repackage-signing" - repackage_signing_deps = {"repackage-signing": - dep_job.dependencies[repackage_signing_name] - } - dependencies.update(repackage_signing_deps) - attributes = copy_attributes_from_dependent_job(dep_job) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { @@ -276,16 +268,17 @@ def generate_upstream_artifacts(build_ta for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping): plarform_was_previously_matched_by_regex = None for platform_regex, paths in mapping.iteritems(): if platform_regex.match(platform) is not None: _check_platform_matched_only_one_regex( tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex ) + upstream_artifacts.append({ "taskId": {"task-reference": ref}, "taskType": tasktype, "paths": ["{}/{}".format(artifact_prefix, path) for path in paths], "locale": locale or "en-US", }) plarform_was_previously_matched_by_regex = platform_regex @@ -301,22 +294,18 @@ least 2 regular expressions. First match "{second_matched}"'.format( task_type=task_type, platform=platform, first_matched=plarform_was_previously_matched_by_regex.pattern, second_matched=platform_regex.pattern )) def is_valid_beetmover_job(job): - # beetmover after partials-signing should have six dependencies. - # windows builds w/o partials don't have docker-image, so fewer - # dependencies - if 'partials-signing' in job['dependencies'].keys(): - expected_dep_count = 6 - elif any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS): + # windows builds don't have docker-image, so fewer dependencies + if any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS): expected_dep_count = 4 else: expected_dep_count = 5 return (len(job["dependencies"]) == expected_dep_count and any(['repackage' in j for j in job['dependencies']])) @@ -327,17 +316,16 @@ def make_task_worker(config, jobs): raise NotImplementedError("Beetmover_repackage must have five dependencies.") locale = job["attributes"].get("locale") platform = job["attributes"]["build_platform"] build_task = None build_signing_task = None repackage_task = None repackage_signing_task = None - for dependency in job["dependencies"].keys(): if 'repackage-signing' in dependency: repackage_signing_task = dependency elif 'repackage' in dependency: repackage_task = dependency elif 'signing' in dependency: # catches build-signing and nightly-l10n-signing build_signing_task = dependency
deleted file mode 100644 --- a/taskcluster/taskgraph/transforms/partials.py +++ /dev/null @@ -1,134 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Transform the partials task into an actual task description. -""" -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.attributes import copy_attributes_from_dependent_job -from taskgraph.util.partials import get_balrog_platform_name, get_builds -from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix - -import logging -logger = logging.getLogger(__name__) - -transforms = TransformSequence() - - -def _generate_task_output_files(filenames, locale=None): - locale_output_path = '{}/'.format(locale) if locale else '' - - data = list() - for filename in filenames: - data.append({ - 'type': 'file', - 'path': '/home/worker/artifacts/{}'.format(filename), - 'name': 'public/build/{}{}'.format(locale_output_path, filename) - }) - data.append({ - 'type': 'file', - 'path': '/home/worker/artifacts/manifest.json', - 'name': 'public/build/{}manifest.json'.format(locale_output_path) - }) - return data - - -@transforms.add -def make_task_description(config, jobs): - # If no balrog release history, then don't generate partials - if not config.params.get('release_history'): - return - for job in jobs: - dep_job = job['dependent-task'] - - treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'p(N)') - - label = job.get('label', "partials-{}".format(dep_job.label)) - dep_th_platform = dep_job.task.get('extra', {}).get( - 'treeherder', {}).get('machine', {}).get('platform', '') - - treeherder.setdefault('platform', - "{}/opt".format(dep_th_platform)) - treeherder.setdefault('kind', 'build') - treeherder.setdefault('tier', 3) - - dependent_kind = str(dep_job.kind) - dependencies = {dependent_kind: dep_job.label} - signing_dependencies = dep_job.dependencies - # This is so we get the build task etc in our dependencies to - # have better beetmover support. - dependencies.update(signing_dependencies) - - attributes = copy_attributes_from_dependent_job(dep_job) - locale = dep_job.attributes.get('locale') - if locale: - attributes['locale'] = locale - treeherder['symbol'] = "p({})".format(locale) - - build_locale = locale or 'en-US' - - builds = get_builds(config.params['release_history'], dep_th_platform, - build_locale) - - # If the list is empty there's no available history for this platform - # and locale combination, so we can't build any partials. - if not builds: - continue - - signing_task = None - for dependency in dependencies.keys(): - if 'repackage-signing' in dependency: - signing_task = dependency - signing_task_ref = '<{}>'.format(signing_task) - - extra = {'funsize': {'partials': list()}} - update_number = 1 - artifact_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale) - artifact_path = "{}{}".format(artifact_prefix, 'target.complete.mar') - for build in builds: - extra['funsize']['partials'].append({ - 'locale': build_locale, - 'from_mar': builds[build]['mar_url'], - 'to_mar': {'task-reference': artifact_path}, - 'platform': get_balrog_platform_name(dep_th_platform), - 'branch': config.params['project'], - 'update_number': update_number, - 'dest_mar': build, - }) - update_number += 1 - - cot = extra.setdefault('chainOfTrust', {}) - cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"} - - worker = { - 'artifacts': _generate_task_output_files(builds.keys(), locale), - 'implementation': 'docker-worker', - 'docker-image': {'in-tree': 'partial-update-generator'}, - 'os': 'linux', - 'max-run-time': 3600, - 'chain-of-trust': True, - 'env': { - 'SHA1_SIGNING_CERT': 'nightly_sha1', - 'SHA384_SIGNING_CERT': 'nightly_sha384' - } - } - - level = config.params['level'] - - task = { - 'label': label, - 'description': "{} Partials".format( - dep_job.task["metadata"]["description"]), - 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, - 'dependencies': dependencies, - 'attributes': attributes, - 'run-on-projects': dep_job.attributes.get('run_on_projects'), - 'treeherder': treeherder, - 'extra': extra, - 'worker': worker, - } - - yield task
deleted file mode 100644 --- a/taskcluster/taskgraph/transforms/partials_signing.py +++ /dev/null @@ -1,96 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Transform the partials task into an actual task description. -""" -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.attributes import copy_attributes_from_dependent_job -from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform -from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts - -import logging -logger = logging.getLogger(__name__) - -transforms = TransformSequence() - - -def generate_upstream_artifacts(release_history, platform, locale=None): - artifact_prefix = 'public/build' - if locale: - artifact_prefix = 'public/build/{}'.format(locale) - else: - locale = 'en-US' - - artifacts = get_partials_artifacts(release_history, platform, locale) - - upstream_artifacts = [{ - "taskId": {"task-reference": '<partials>'}, - "taskType": 'partials', - "paths": ["{}/{}".format(artifact_prefix, p) - for p in artifacts], - "formats": ["mar_sha384"], - }] - - return upstream_artifacts - - -@transforms.add -def make_task_description(config, jobs): - for job in jobs: - dep_job = job['dependent-task'] - - treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'ps(N)') - - dep_th_platform = dep_job.task.get('extra', {}).get( - 'treeherder', {}).get('machine', {}).get('platform', '') - label = job.get('label', "partials-signing-{}".format(dep_job.label)) - dep_th_platform = dep_job.task.get('extra', {}).get( - 'treeherder', {}).get('machine', {}).get('platform', '') - treeherder.setdefault('platform', - "{}/opt".format(dep_th_platform)) - treeherder.setdefault('kind', 'build') - treeherder.setdefault('tier', 3) - - dependent_kind = str(dep_job.kind) - dependencies = {dependent_kind: dep_job.label} - signing_dependencies = dep_job.dependencies - # This is so we get the build task etc in our dependencies to - # have better beetmover support. - dependencies.update(signing_dependencies) - - attributes = copy_attributes_from_dependent_job(dep_job) - locale = dep_job.attributes.get('locale') - if locale: - attributes['locale'] = locale - treeherder['symbol'] = 'ps({})'.format(locale) - - balrog_platform = get_balrog_platform_name(dep_th_platform) - upstream_artifacts = generate_upstream_artifacts(config.params['release_history'], - balrog_platform, locale) - - build_platform = dep_job.attributes.get('build_platform') - is_nightly = dep_job.attributes.get('nightly') - signing_cert_scope = get_signing_cert_scope_per_platform( - build_platform, is_nightly, config - ) - scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384'] - task = { - 'label': label, - 'description': "{} Partials".format( - dep_job.task["metadata"]["description"]), - 'worker-type': 'scriptworker-prov-v1/signing-linux-v1', - 'worker': {'implementation': 'scriptworker-signing', - 'upstream-artifacts': upstream_artifacts, - 'max-run-time': 3600}, - 'dependencies': dependencies, - 'attributes': attributes, - 'scopes': scopes, - 'run-on-projects': dep_job.attributes.get('run_on_projects'), - 'treeherder': treeherder, - } - - yield task
--- a/taskcluster/taskgraph/transforms/repackage.py +++ b/taskcluster/taskgraph/transforms/repackage.py @@ -5,20 +5,22 @@ Transform the repackage task into an actual task description. """ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.schema import validate_schema, Schema -from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix from taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required, Optional +_TC_ARTIFACT_LOCATION = \ + 'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}' + transforms = TransformSequence() # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} # shortcut for a string where task references are allowed taskref_or_string = Any( @@ -196,18 +198,18 @@ def _generate_task_mozharness_config(bui return ['repackage/linux{}_signed.py'.format(bits)] elif build_platform.startswith('win'): return ['repackage/win{}_signed.py'.format(bits)] raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None): - mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None) - signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale) + mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None) + signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale) if build_platform.startswith('linux') or build_platform.startswith('macosx'): tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz' return { 'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format( signed_prefix, tarball_extension )}, 'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)}, @@ -224,16 +226,23 @@ def _generate_task_env(build_platform, b task_env['SIGNED_SETUP_STUB'] = { 'task-reference': '{}setup-stub.exe'.format(signed_prefix), } return task_env raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform)) +def _generate_taskcluster_prefix(task_id, postfix='', locale=None): + if locale: + postfix = '{}/{}'.format(locale, postfix) + + return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix) + + def _generate_task_output_files(build_platform, locale=None): locale_output_path = '{}/'.format(locale) if locale else '' if build_platform.startswith('linux') or build_platform.startswith('macosx'): output_files = [{ 'type': 'file', 'path': '/builds/worker/workspace/build/artifacts/{}target.complete.mar' .format(locale_output_path),
--- a/taskcluster/taskgraph/transforms/repackage_signing.py +++ b/taskcluster/taskgraph/transforms/repackage_signing.py @@ -124,14 +124,28 @@ def make_repackage_signing_description(c 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } + funsize_platforms = [ + 'linux-nightly', + 'linux64-nightly', + 'macosx64-nightly', + 'win32-nightly', + 'win64-nightly' + ] + if build_platform in funsize_platforms and is_nightly: + route_template = "project.releng.funsize.level-{level}.{project}" + task['routes'] = [ + route_template.format(project=config.params['project'], + level=config.params['level']) + ] + yield task def _generate_worker_type(signing_cert_scope): worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1' return 'scriptworker-prov-v1/{}'.format(worker_type)
--- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -494,33 +494,29 @@ GROUP_NAMES = { 'tc-W': 'Web platform tests executed by TaskCluster', 'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s', 'tc-X': 'Xpcshell tests executed by TaskCluster', 'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s', 'tc-L10n': 'Localised Repacks executed by Taskcluster', 'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster', 'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster', 'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster', - 'c-Up': 'Balrog submission of complete updates', + 'tc-Up': 'Balrog submission of updates, executed by Taskcluster', 'tc-cs': 'Checksum signing executed by Taskcluster', 'tc-rs': 'Repackage signing executed by Taskcluster', 'tc-BMcs': 'Beetmover checksums, executed by Taskcluster', 'Aries': 'Aries Device Image', 'Nexus 5-L': 'Nexus 5-L Device Image', 'I': 'Docker Image Builds', 'TL': 'Toolchain builds for Linux 64-bits', 'TM': 'Toolchain builds for OSX', 'TW32': 'Toolchain builds for Windows 32-bits', 'TW64': 'Toolchain builds for Windows 64-bits', 'SM-tc': 'Spidermonkey builds', 'pub': 'APK publishing', - 'p': 'Partial generation', - 'ps': 'Partials signing', - 'pBM': 'Beetmover for partials', - 'cp-Up': 'Balrog submission of updates, completes and partials', } UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__ V2_ROUTE_TEMPLATES = [ "index.gecko.v2.{project}.latest.{product}.{job-name}", "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}", "index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}", "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
deleted file mode 100644 --- a/taskcluster/taskgraph/util/partials.py +++ /dev/null @@ -1,193 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -import requests -import redo - -import logging -logger = logging.getLogger(__name__) - -BALROG_API_ROOT = 'https://aus5.mozilla.org/api/v1' - -PLATFORM_RENAMES = { - 'windows2012-32': 'win32', - 'windows2012-64': 'win64', - 'osx-cross': 'macosx64', -} - -BALROG_PLATFORM_MAP = { - "linux": [ - "Linux_x86-gcc3" - ], - "linux64": [ - "Linux_x86_64-gcc3" - ], - "macosx64": [ - "Darwin_x86_64-gcc3-u-i386-x86_64", - "Darwin_x86-gcc3-u-i386-x86_64", - "Darwin_x86-gcc3", - "Darwin_x86_64-gcc3" - ], - "win32": [ - "WINNT_x86-msvc", - "WINNT_x86-msvc-x86", - "WINNT_x86-msvc-x64" - ], - "win64": [ - "WINNT_x86_64-msvc", - "WINNT_x86_64-msvc-x64" - ] -} - - -def get_balrog_platform_name(platform): - """Convert build platform names into balrog platform names""" - if '-nightly' in platform: - platform = platform.replace('-nightly', '') - if '-devedition' in platform: - platform = platform.replace('-devedition', '') - return PLATFORM_RENAMES.get(platform, platform) - - -def _sanitize_platform(platform): - platform = get_balrog_platform_name(platform) - if platform not in BALROG_PLATFORM_MAP: - return platform - return BALROG_PLATFORM_MAP[platform][0] - - -def get_builds(release_history, platform, locale): - """Examine cached balrog release history and return the list of - builds we need to generate diffs from""" - platform = _sanitize_platform(platform) - return release_history.get(platform, {}).get(locale, {}) - - -def get_partials_artifacts(release_history, platform, locale): - platform = _sanitize_platform(platform) - return release_history.get(platform, {}).get(locale, {}).keys() - - -def get_partials_artifact_map(release_history, platform, locale): - platform = _sanitize_platform(platform) - return {k: release_history[platform][locale][k]['buildid'] - for k in release_history.get(platform, {}).get(locale, {})} - - -def _retry_on_http_errors(url, verify, params, errors): - if params: - params_str = "&".join("=".join([k, str(v)]) - for k, v in params.iteritems()) - else: - params_str = '' - logger.info("Connecting to %s?%s", url, params_str) - for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10): - try: - req = requests.get(url, verify=verify, params=params) - req.raise_for_status() - return req - except requests.HTTPError as e: - if e.response.status_code in errors: - logger.exception("Got HTTP %s trying to reach %s", - e.response.status_code, url) - else: - raise - else: - raise - - -def get_sorted_releases(product, branch): - """Returns a list of release names from Balrog. - :param product: product name, AKA appName - :param branch: branch name, e.g. mozilla-central - :return: a sorted list of release names, most recent first. - """ - url = "{}/releases".format(BALROG_API_ROOT) - params = { - "product": product, - # Adding -nightly-2 (2 stands for the beginning of build ID - # based on date) should filter out release and latest blobs. - # This should be changed to -nightly-3 in 3000 ;) - "name_prefix": "{}-{}-nightly-2".format(product, branch), - "names_only": True - } - req = _retry_on_http_errors( - url=url, verify=True, params=params, - errors=[500]) - releases = req.json()["names"] - releases = sorted(releases, reverse=True) - return releases - - -def get_release_builds(release): - url = "{}/releases/{}".format(BALROG_API_ROOT, release) - req = _retry_on_http_errors( - url=url, verify=True, params=None, - errors=[500]) - return req.json() - - -def populate_release_history(product, branch, maxbuilds=4, maxsearch=10): - """Find relevant releases in Balrog - Not all releases have all platforms and locales, due - to Taskcluster migration. - - Args: - product (str): capitalized product name, AKA appName, e.g. Firefox - branch (str): branch name (mozilla-central) - maxbuilds (int): Maximum number of historical releases to populate - maxsearch(int): Traverse at most this many releases, to avoid - working through the entire history. - Returns: - json object based on data from balrog api - - results = { - 'platform1': { - 'locale1': { - 'buildid1': mar_url, - 'buildid2': mar_url, - 'buildid3': mar_url, - }, - 'locale2': { - 'target.partial-1.mar': ('buildid1': 'mar_url'), - } - }, - 'platform2': { - } - } - """ - last_releases = get_sorted_releases(product, branch) - - partial_mar_tmpl = 'target.partial-{}.mar' - - builds = dict() - for release in last_releases[:maxsearch]: - # maxbuilds in all categories, don't make any more queries - full = len(builds) > 0 and all( - len(builds[platform][locale]) >= maxbuilds - for platform in builds for locale in builds[platform]) - if full: - break - history = get_release_builds(release) - - for platform in history['platforms']: - if 'alias' in history['platforms'][platform]: - continue - if platform not in builds: - builds[platform] = dict() - for locale in history['platforms'][platform]['locales']: - if locale not in builds[platform]: - builds[platform][locale] = dict() - if len(builds[platform][locale]) >= maxbuilds: - continue - buildid = history['platforms'][platform]['locales'][locale]['buildID'] - url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl'] - nextkey = len(builds[platform][locale]) + 1 - builds[platform][locale][partial_mar_tmpl.format(nextkey)] = { - 'buildid': buildid, - 'mar_url': url, - } - return builds
--- a/taskcluster/taskgraph/util/taskcluster.py +++ b/taskcluster/taskgraph/util/taskcluster.py @@ -8,19 +8,16 @@ from __future__ import absolute_import, import functools import yaml import requests from mozbuild.util import memoize from requests.packages.urllib3.util.retry import Retry from requests.adapters import HTTPAdapter -_TC_ARTIFACT_LOCATION = \ - 'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}' - @memoize def get_session(): session = requests.Session() retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) session.mount('http://', HTTPAdapter(max_retries=retry)) session.mount('https://', HTTPAdapter(max_retries=retry)) @@ -99,15 +96,8 @@ def get_task_url(task_id, use_proxy=Fals else: TASK_URL = 'https://queue.taskcluster.net/v1/task/{}' return TASK_URL.format(task_id) def get_task_definition(task_id, use_proxy=False): response = _do_request(get_task_url(task_id, use_proxy)) return response.json() - - -def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None): - if locale: - postfix = '{}/{}'.format(locale, postfix) - - return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)