☠☠ backed out by d0e13414d651 ☠ ☠ | |
author | Dustin J. Mitchell <dustin@mozilla.com> |
Tue, 25 Sep 2018 20:26:55 +0000 | |
changeset 451435 | 9c35dd209c6b407bc3a45ce7b4c27272ef1bb486 |
parent 451434 | a972d6b4434edcbce1f93fb1629a6b81ca7bb585 |
child 451436 | c77ae59aba41a65f0e9d4896f9dfe3bf8a0e8353 |
push id | 35238 |
push user | rmaries@mozilla.com |
push date | Thu, 20 Dec 2018 05:04:43 +0000 |
treeherder | mozilla-central@f42265a0fe6f [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | tomprince |
bugs | 1492664 |
milestone | 66.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/taskcluster/docs/optimization-process.rst +++ b/taskcluster/docs/optimization-process.rst @@ -65,11 +65,11 @@ simultaneously rewrites all dependencies To do so, it assigns a taskId to each retained task and uses the replacement taskId for all replaced tasks. The result is an optimized taskgraph with tasks named by taskId instead of label. At this phase, the edges in the task graph diverge from the ``task.dependencies`` attributes, as the latter may contain dependencies outside of the taskgraph (for replacement tasks). -As a side-effect, this phase also expands all ``{"task-reference": ".."}`` -objects within the task definitions. +As a side-effect, this phase also expands all ``{"task-reference": ".."}`` and +``{"artifact-reference": ".."}`` objects within the task definitions.
--- a/taskcluster/docs/taskgraph.rst +++ b/taskcluster/docs/taskgraph.rst @@ -171,16 +171,21 @@ using simple parameterized values, as fo ``{"task-reference": "string containing <dep-name>"}`` The task definition may contain "task references" of this form. These will be replaced during the optimization step, with the appropriate taskId for the named dependency substituted for ``<dep-name>`` in the string. Multiple labels may be substituted in a single string, and ``<<>`` can be used to escape a literal ``<``. +``{"artifact-reference": "..<dep-name/artifact/name>.."}`` + Similar to a ``task-reference``, but this substitutes a URL to the queue's + ``getLatestArtifact`` API method (for which a GET will redirect to the + artifact itself). + .. _taskgraph-graph-config: Graph Configuration ------------------- There are several configuration settings that are pertain to the entire taskgraph. These are specified in :file:`config.yml` at the root of the taskgraph configuration (typically :file:`taskcluster/ci/`). The available
--- a/taskcluster/docs/transforms.rst +++ b/taskcluster/docs/transforms.rst @@ -197,18 +197,19 @@ Signing Descriptions Signing kinds are passed a single dependent job (from its kind dependency) to act on. The transforms in ``taskcluster/taskgraph/transforms/signing.py`` implement this common functionality. They expect a "signing description", and produce a task definition. The schema for a signing description is defined at the top of ``signing.py``, with copious comments. -In particular you define a set of upstream artifact urls (that point at the dependent -task) and can optionally provide a dependent name (defaults to build) for use in -task-reference. You also need to provide the signing formats to use. +In particular you define a set of upstream artifact urls (that point at the +dependent task) and can optionally provide a dependent name (defaults to build) +for use in ``task-reference``/``artifact-reference``. You also need to provide +the signing formats to use. More Detail ----------- The source files provide lots of additional detail, both in the code itself and in the comments and docstrings. For the next level of detail beyond this file, consult the transform source under ``taskcluster/taskgraph/transforms``.
--- a/taskcluster/taskgraph/morph.py +++ b/taskcluster/taskgraph/morph.py @@ -6,20 +6,21 @@ Graph morphs are modifications to task-graphs that take place *after* the optimization phase. These graph morphs are largely invisible to developers running `./mach` locally, so they should be limited to changes that do not modify the meaning of the graph. """ -# Note that the translation of `{'task-reference': '..'}` is handled in the -# optimization phase (since optimization involves dealing with taskIds -# directly). Similarly, `{'relative-datestamp': '..'}` is handled at the last -# possible moment during task creation. +# Note that the translation of `{'task-reference': '..'}` and +# `artifact-reference` are handled in the optimization phase (since +# optimization involves dealing with taskIds directly). Similarly, +# `{'relative-datestamp': '..'}` is handled at the last possible moment during +# task creation. from __future__ import absolute_import, print_function, unicode_literals import logging import os import re import jsone
--- a/taskcluster/taskgraph/test/test_util_parameterization.py +++ b/taskcluster/taskgraph/test/test_util_parameterization.py @@ -1,16 +1,18 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals import unittest import datetime +import mock +import os from mozunit import main from taskgraph.util.parameterization import ( resolve_timestamps, resolve_task_references, ) @@ -83,10 +85,48 @@ class TestTaskRefs(unittest.TestCase): "resolve_task_references raises a KeyError on reference to an invalid task" self.assertRaisesRegexp( KeyError, "task 'subject' has no dependency named 'no-such'", lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {}) ) +class TestArtifactRefs(unittest.TestCase): + + def do(self, input, output): + taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)} + with mock.patch.dict(os.environ, {'TASKCLUSTER_ROOT_URL': 'https://tc-tests.localhost'}): + self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name), + output) + + def test_in_list(self): + "resolve_task_references resolves artifact references in a list" + self.do( + {'in-a-list': [ + 'stuff', {'artifact-reference': '<edge1/foo/bar>'}]}, + {'in-a-list': [ + 'stuff', 'https://tc-tests.localhost/api/queue/v1/task/tid1/artifacts/foo/bar']}) + + def test_in_dict(self): + "resolve_task_references resolves artifact references in a dict" + self.do( + {'in-a-dict': + {'stuff': {'artifact-reference': '<edge2/bar/foo>'}}}, + {'in-a-dict': + {'stuff': 'https://tc-tests.localhost/api/queue/v1/task/tid2/artifacts/bar/foo'}}) + + def test_in_string(self): + "resolve_task_references resolves artifact references embedded in a string" + self.do( + {'stuff': {'artifact-reference': '<edge1/filename> and <edge2/bar/foo>'}}, + {'stuff': 'https://tc-tests.localhost/api/queue/v1/task/tid1/artifacts/filename and ' + 'https://tc-tests.localhost/api/queue/v1/task/tid2/artifacts/bar/foo'}) + + def test_invalid(self): + "resolve_task_references ignores badly-formatted artifact references" + for inv in ['<edge1>', 'edge1/foo>', '<edge1>/foo', '<edge1>foo']: + resolved = resolve_task_references('subject', {'artifact-reference': inv}, {}) + self.assertEqual(resolved, inv) + + if __name__ == '__main__': main()
--- a/taskcluster/taskgraph/transforms/balrog_submit.py +++ b/taskcluster/taskgraph/transforms/balrog_submit.py @@ -12,28 +12,23 @@ from taskgraph.transforms.base import Tr from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.schema import ( optionally_keyed_by, resolve_keyed_by, ) from taskgraph.util.scriptworker import ( get_balrog_server_scope, get_worker_type_for_scope ) from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - balrog_description_schema = schema.extend({ # unique label to describe this balrog task, defaults to balrog-{dep.label} Optional('label'): basestring, Optional( 'update-no-wnp', description="Whether the parallel `-No-WNP` blob should be updated as well.",
--- a/taskcluster/taskgraph/transforms/beetmover.py +++ b/taskcluster/taskgraph/transforms/beetmover.py @@ -2,17 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the beetmover task into an actual task description. """ from __future__ import absolute_import, print_function, unicode_literals -from voluptuous import Any, Optional, Required +from voluptuous import Optional, Required from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.transforms.task import task_description_schema from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import (generate_beetmover_artifact_map, generate_beetmover_upstream_artifacts, get_beetmover_bucket_scope, @@ -114,21 +114,16 @@ UPSTREAM_SOURCE_ARTIFACTS = [ ] # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} transforms = TransformSequence() -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_description_schema = schema.extend({ # depname is used in taskref's to identify the taskID of the unsigned things Required('depname', default='build'): basestring, # unique label to describe this beetmover task, defaults to {dep.label}-beetmover Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for beetmover. See
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py +++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py @@ -13,27 +13,23 @@ from taskgraph.transforms.base import Tr from taskgraph.transforms.beetmover import craft_release_properties from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import (generate_beetmover_artifact_map, generate_beetmover_upstream_artifacts, get_beetmover_action_scope, get_beetmover_bucket_scope, get_worker_type_for_scope, should_use_artifact_map) -from voluptuous import Any, Optional, Required +from voluptuous import Optional, Required from taskgraph.transforms.task import task_description_schema # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_checksums_description_schema = schema.extend({ Required('depname', default='build'): basestring, Required('attributes'): {basestring: object}, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('locale'): basestring, Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'],
--- a/taskcluster/taskgraph/transforms/beetmover_emefree_checksums.py +++ b/taskcluster/taskgraph/transforms/beetmover_emefree_checksums.py @@ -7,26 +7,22 @@ Transform release-beetmover-source-check from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.transforms.beetmover import craft_release_properties from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_checksums_description_schema = schema.extend({ Required('depname', default='build'): basestring, Optional('label'): basestring, Optional('extra'): object, Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], })
--- a/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py +++ b/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py @@ -10,26 +10,22 @@ from __future__ import absolute_import, from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.transforms.beetmover import craft_release_properties from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import (get_beetmover_bucket_scope, get_beetmover_action_scope, get_worker_type_for_scope) from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_checksums_description_schema = schema.extend({ Required('depname', default='build'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('locale'): basestring, Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], })
--- a/taskcluster/taskgraph/transforms/beetmover_push_to_release.py +++ b/taskcluster/taskgraph/transforms/beetmover_push_to_release.py @@ -3,35 +3,34 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the beetmover-push-to-release task into a task description. """ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import Schema +from taskgraph.util.schema import ( + Schema, + taskref_or_string, +) from taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, add_scope_prefix, get_worker_type_for_scope, ) from taskgraph.transforms.job import job_description_schema from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_push_to_release_description_schema = Schema({ Required('name'): basestring, Required('product'): basestring, Required('treeherder-platform'): basestring, Optional('attributes'): {basestring: object}, Optional('job-from'): task_description_schema['job-from'], Optional('run'): {basestring: object}, Optional('run-on-projects'): task_description_schema['run-on-projects'],
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py +++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py @@ -14,17 +14,17 @@ from taskgraph.util.attributes import co from taskgraph.util.partials import (get_balrog_platform_name, get_partials_artifacts, get_partials_artifact_map) from taskgraph.util.scriptworker import (get_beetmover_bucket_scope, get_beetmover_action_scope, get_worker_type_for_scope) from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional import logging import re logger = logging.getLogger(__name__) def _compile_regex_mapping(mapping): @@ -141,21 +141,16 @@ UPSTREAM_ARTIFACT_SIGNED_MAR_PATHS = [ 'target.complete.mar', 'target.bz2.complete.mar', ] # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_description_schema = schema.extend({ # depname is used in taskref's to identify the taskID of the unsigned things Required('depname', default='build'): basestring, # unique label to describe this beetmover task, defaults to {dep.label}-beetmover Required('label'): basestring, # treeherder is allowed here to override any defaults we use for beetmover. See
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py +++ b/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py @@ -34,21 +34,16 @@ import logging logger = logging.getLogger(__name__) # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_description_schema = schema.extend({ # depname is used in taskref's to identify the taskID of the unsigned things Required('depname', default='build'): basestring, # unique label to describe this beetmover task, defaults to {dep.label}-beetmover Optional('label'): basestring, Required('partner-bucket-scope'): optionally_keyed_by('release-level', basestring),
--- a/taskcluster/taskgraph/transforms/beetmover_source_checksums.py +++ b/taskcluster/taskgraph/transforms/beetmover_source_checksums.py @@ -13,26 +13,22 @@ from taskgraph.transforms.beetmover impo from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import (generate_beetmover_artifact_map, generate_beetmover_upstream_artifacts, get_beetmover_bucket_scope, get_beetmover_action_scope, get_worker_type_for_scope, should_use_artifact_map) from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - beetmover_checksums_description_schema = schema.extend({ Required('depname', default='build'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('locale'): basestring, Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('attributes'): task_description_schema['attributes'],
--- a/taskcluster/taskgraph/transforms/checksums_signing.py +++ b/taskcluster/taskgraph/transforms/checksums_signing.py @@ -11,26 +11,22 @@ from taskgraph.loader.single_dep import from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope, get_worker_type_for_scope, add_scope_prefix, ) from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - checksums_signing_description_schema = schema.extend({ Required('depname', default='beetmover'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], })
--- a/taskcluster/taskgraph/transforms/l10n.py +++ b/taskcluster/taskgraph/transforms/l10n.py @@ -13,16 +13,17 @@ import json from mozbuild.chunkify import chunkify from taskgraph.loader.multi_dep import schema from taskgraph.transforms.base import ( TransformSequence, ) from taskgraph.util.schema import ( optionally_keyed_by, resolve_keyed_by, + taskref_or_string, ) from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.treeherder import add_suffix from taskgraph.transforms.job import job_description_schema from taskgraph.transforms.task import task_description_schema from voluptuous import ( Any, @@ -30,21 +31,16 @@ from voluptuous import ( Required, ) def _by_platform(arg): return optionally_keyed_by('build-platform', arg) -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()} task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} l10n_description_schema = schema.extend({ # Name for this job, inferred from the dependent job before validation Required('name'): basestring,
--- a/taskcluster/taskgraph/transforms/repackage.py +++ b/taskcluster/taskgraph/transforms/repackage.py @@ -15,28 +15,23 @@ from taskgraph.util.attributes import co from taskgraph.util.schema import ( optionally_keyed_by, resolve_keyed_by, ) from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.platforms import archive_format, executable_extension from taskgraph.util.workertypes import worker_type_implementation from taskgraph.transforms.job import job_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()} -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - packaging_description_schema = schema.extend({ # depname is used in taskref's to identify the taskID of the signed things Required('depname', default='build'): basestring, # unique label to describe this repackaging task Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for repackaging. See
--- a/taskcluster/taskgraph/transforms/repackage_partner.py +++ b/taskcluster/taskgraph/transforms/repackage_partner.py @@ -17,32 +17,27 @@ from taskgraph.util.schema import ( resolve_keyed_by, ) from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.partners import check_if_partners_enabled from taskgraph.util.platforms import archive_format, executable_extension from taskgraph.util.workertypes import worker_type_implementation from taskgraph.transforms.task import task_description_schema from taskgraph.transforms.repackage import PACKAGE_FORMATS -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} def _by_platform(arg): return optionally_keyed_by('build-platform', arg) -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - packaging_description_schema = schema.extend({ # depname is used in taskref's to identify the taskID of the signed things Required('depname', default='build'): basestring, # unique label to describe this repackaging task Optional('label'): basestring, # Routes specific to this task, if defined
--- a/taskcluster/taskgraph/transforms/signing.py +++ b/taskcluster/taskgraph/transforms/signing.py @@ -5,36 +5,32 @@ Transform the signing task into an actual task description. """ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.loader.single_dep import schema from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job +from taskgraph.util.schema import taskref_or_string from taskgraph.util.scriptworker import ( add_scope_prefix, get_signing_cert_scope_per_platform, get_worker_type_for_scope, ) from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} transforms = TransformSequence() -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - signing_description_schema = schema.extend({ # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py # because this is passed directly into the signingscript worker Required('upstream-artifacts'): [{ # taskId of the task with the artifact Required('taskId'): taskref_or_string, # type of signing task (for CoT)
--- a/taskcluster/taskgraph/transforms/source_checksums_signing.py +++ b/taskcluster/taskgraph/transforms/source_checksums_signing.py @@ -11,26 +11,22 @@ from taskgraph.loader.single_dep import from taskgraph.transforms.base import TransformSequence from taskgraph.util.attributes import copy_attributes_from_dependent_job from taskgraph.util.scriptworker import ( get_signing_cert_scope, get_worker_type_for_scope, add_scope_prefix, ) from taskgraph.transforms.task import task_description_schema -from voluptuous import Any, Required, Optional +from voluptuous import Required, Optional # Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}) - checksums_signing_description_schema = schema.extend({ Required('depname', default='beetmover'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], })
--- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -22,16 +22,17 @@ from taskgraph.util.hash import hash_pat from taskgraph.util.treeherder import split_symbol from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import ( validate_schema, Schema, optionally_keyed_by, resolve_keyed_by, OptimizationSchema, + taskref_or_string, ) from taskgraph.util.scriptworker import ( BALROG_ACTIONS, get_release_config, add_scope_prefix, ) from taskgraph.util.signed_artifacts import get_signed_artifacts from voluptuous import Any, Required, Optional, Extra @@ -42,22 +43,16 @@ RUN_TASK = os.path.join(GECKO, 'taskclus @memoize def _run_task_suffix(): """String to append to cache names under control of run-task.""" return hash_path(RUN_TASK)[0:20] -# shortcut for a string where task references are allowed -taskref_or_string = Any( - basestring, - {Required('task-reference'): basestring}, -) - # A task description is a general description of a TaskCluster task task_description_schema = Schema({ # the label for this task Required('label'): basestring, # description of the task (for metadata) Required('description'): basestring,
--- a/taskcluster/taskgraph/util/parameterization.py +++ b/taskcluster/taskgraph/util/parameterization.py @@ -1,48 +1,77 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals +import os import re +import taskcluster_urls from taskgraph.util.time import json_time_from_now TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>') +ARTIFACT_REFERENCE_PATTERN = re.compile('<([^/]+)/([^>]+)>') -def _recurse(val, param_name, param_fn): - param_keys = [param_name] - +def _recurse(val, param_fns): def recurse(val): if isinstance(val, list): return [recurse(v) for v in val] elif isinstance(val, dict): - if val.keys() == param_keys: - return param_fn(val[param_name]) - else: - return {k: recurse(v) for k, v in val.iteritems()} + if len(val) == 1: + for param_key, param_fn in param_fns.items(): + if val.keys() == [param_key]: + return param_fn(val[param_key]) + return {k: recurse(v) for k, v in val.iteritems()} else: return val return recurse(val) def resolve_timestamps(now, task_def): """Resolve all instances of `{'relative-datestamp': '..'}` in the given task definition""" - return _recurse(task_def, 'relative-datestamp', lambda v: json_time_from_now(v, now)) + return _recurse(task_def, { + 'relative-datestamp': lambda v: json_time_from_now(v, now), + }) def resolve_task_references(label, task_def, dependencies): - """Resolve all instances of `{'task-reference': '..<..>..'}` in the given task - definition, using the given dependencies""" - def repl(match): - key = match.group(1) - try: - return dependencies[key] - except KeyError: - # handle escaping '<' - if key == '<': - return key - raise KeyError("task '{}' has no dependency named '{}'".format(label, key)) + """Resolve all instances of + {'task-reference': '..<..>..'} + and + {'artifact-reference`: '..<dependency/artifact/path>..'} + in the given task definition, using the given dependencies""" + + def task_reference(val): + def repl(match): + key = match.group(1) + try: + return dependencies[key] + except KeyError: + # handle escaping '<' + if key == '<': + return key + raise KeyError("task '{}' has no dependency named '{}'".format(label, key)) + + return TASK_REFERENCE_PATTERN.sub(repl, val) - return _recurse(task_def, 'task-reference', lambda v: TASK_REFERENCE_PATTERN.sub(repl, v)) + def artifact_reference(val): + def repl(match): + dependency, artifact_name = match.group(1, 2) + + try: + dependency_task_id = dependencies[dependency] + except KeyError: + raise KeyError("task '{}' has no dependency named '{}'".format(label, dependency)) + + return taskcluster_urls.api( + os.environ['TASKCLUSTER_ROOT_URL'], 'queue', 'v1', + 'task/{}/artifacts/{}'.format(dependency_task_id, artifact_name)) + + return ARTIFACT_REFERENCE_PATTERN.sub(repl, val) + + return _recurse(task_def, { + 'task-reference': task_reference, + 'artifact-reference': artifact_reference, + })
--- a/taskcluster/taskgraph/util/schema.py +++ b/taskcluster/taskgraph/util/schema.py @@ -229,8 +229,15 @@ OptimizationSchema = voluptuous.Any( # skip this task if unless the change files' SCHEDULES contains any of these components {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)}, # skip if SETA or skip-unless-schedules says to {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)}, # only run this task if its dependencies will run (useful for follow-on tasks that # are unnecessary if the parent tasks are not run) {'only-if-dependencies-run': None} ) + +# shortcut for a string where task references are allowed +taskref_or_string = voluptuous.Any( + basestring, + {voluptuous.Required('task-reference'): basestring}, + {voluptuous.Required('artifact-reference'): basestring}, +)