--- a/taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml
+++ b/taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml
@@ -6,17 +6,17 @@ loader: taskgraph.loader.single_dep:load
transforms:
- taskgraph.transforms.chunk_partners:transforms
- taskgraph.transforms.name_sanity:transforms
# This transform sets build_platform to the same thing as the upstream task.
# We'd do it here, except single_dep doesn't pay attention to any
# per platform things that we set.
- taskgraph.transforms.copy_attributes_from_dependent_task:transforms
- # This transform is needed because task.py doesn't allow "dependent-task" to be
+ # This transform is needed because task.py doesn't allow "primary-dependency" to be
# set, but the single_dep loader sets it (and we need it for chunk_partners,
# name_sanity, and copy_build_platform_from_dependent_task to work).
- taskgraph.transforms.strip_dependent_task:transforms
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- release-partner-repack
--- a/taskcluster/taskgraph/loader/build_signing.py
+++ b/taskcluster/taskgraph/loader/build_signing.py
@@ -22,12 +22,12 @@ NON_NIGHTLY_LABELS_WHICH_SHOULD_SIGN_BUI
'release-partner-repack-macosx64-nightly',
)
def loader(kind, path, config, params, loaded_tasks):
jobs = base_loader(kind, path, config, params, loaded_tasks)
for job in jobs:
- dependent_task = job['dependent-task']
+ dependent_task = job['primary-dependency']
if dependent_task.attributes.get('nightly') or \
dependent_task.label in NON_NIGHTLY_LABELS_WHICH_SHOULD_SIGN_BUILDS:
yield job
--- a/taskcluster/taskgraph/loader/multi_dep.py
+++ b/taskcluster/taskgraph/loader/multi_dep.py
@@ -7,18 +7,18 @@ from __future__ import absolute_import,
import copy
from voluptuous import Required
from ..task import Task
from ..util.schema import Schema
schema = Schema({
- Required('dependent-tasks', 'primary dependency task'): {basestring: Task},
- Required('primary-dependency', 'dictionary of dependent tasks, keyed by kind'): Task,
+ Required('primary-dependency', 'primary dependency task'): Task,
+ Required('dependent-tasks', 'dictionary of dependent tasks, keyed by kind'): {basestring: Task},
})
# Define a collection of group_by functions
GROUP_BY_MAP = {}
def group_by(name):
--- a/taskcluster/taskgraph/loader/single_dep.py
+++ b/taskcluster/taskgraph/loader/single_dep.py
@@ -1,16 +1,25 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
+from voluptuous import Required
+
+from ..task import Task
+from ..util.schema import Schema
+
+schema = Schema({
+ Required('primary-dependency', 'primary dependency task'): Task,
+})
+
def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds.
The `only-for-build-platforms` kind configuration, if specified, will limit
the build platforms for which a job will be created. Alternatively there is
'not-for-build-platforms' kind configuration which will be consulted only after
@@ -45,17 +54,19 @@ def loader(kind, path, config, params, l
continue
if only_attributes:
config_attrs = set(only_attributes)
if config_attrs - set(task.attributes):
# make sure all attributes exist
continue
- job = {'dependent-task': task}
+ job = {
+ 'primary-dependency': task,
+ }
if job_template:
job.update(copy.deepcopy(job_template))
# copy shipping_product from upstream
product = task.attributes.get(
'shipping_product', task.task.get('shipping-product')
)
--- a/taskcluster/taskgraph/transforms/balrog_submit.py
+++ b/taskcluster/taskgraph/transforms/balrog_submit.py
@@ -2,19 +2,20 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the per-locale balrog task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_balrog_server_scope, get_worker_type_for_scope
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -23,20 +24,17 @@ task_description_schema = {str(k): v for
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-balrog_description_schema = Schema({
- # the dependent task (object) for this balrog job, used to inform balrogworker.
- Required('dependent-task'): object,
-
+balrog_description_schema = schema.extend({
# unique label to describe this balrog task, defaults to balrog-{dep.label}
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'): task_description_schema['treeherder'],
@@ -44,28 +42,28 @@ balrog_description_schema = Schema({
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
balrog_description_schema, job,
"In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'c-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault(
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -2,19 +2,20 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@@ -116,20 +117,17 @@ task_description_schema = {str(k): v for
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-beetmover_description_schema = Schema({
- # the dependent task (object) for this beetmover job, used to inform beetmover.
- Required('dependent-task'): object,
-
+beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
@@ -142,27 +140,27 @@ beetmover_description_schema = Schema({
Required('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-S')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -2,61 +2,61 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the checksums signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-beetmover_checksums_description_schema = Schema({
- Required('dependent-task'): object,
+beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/beetmover_emefree_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_emefree_checksums.py
@@ -2,57 +2,57 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform release-beetmover-source-checksums into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-beetmover_checksums_description_schema = Schema({
- Required('dependent-task'): object,
+beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('extra'): object,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
repack_id = dep_job.task.get('extra', {}).get('repack_id')
if not repack_id:
raise Exception("Cannot find repack id!")
--- a/taskcluster/taskgraph/transforms/beetmover_geckoview.py
+++ b/taskcluster/taskgraph/transforms/beetmover_geckoview.py
@@ -2,21 +2,22 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import \
craft_release_properties as beetmover_craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema, resolve_keyed_by, optionally_keyed_by
+from taskgraph.util.schema import validate_schema, resolve_keyed_by, optionally_keyed_by
from taskgraph.util.scriptworker import get_worker_type_for_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
_ARTIFACT_ID_PER_PLATFORM = {
'android-aarch64': 'geckoview{update_channel}-arm64-v8a',
'android-api-16': 'geckoview{update_channel}-armeabi-v7a',
@@ -31,42 +32,41 @@ from voluptuous import Required, Optiona
'try': '-nightly-try',
'maple': '-nightly-maple',
}
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-beetmover_description_schema = Schema({
- Required('dependent-task'): object,
+beetmover_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('bucket-scope'): optionally_keyed_by('release-level', basestring),
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover-geckoview ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-gv')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
'{}/opt'.format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/beetmover_l10n.py
+++ b/taskcluster/taskgraph/transforms/beetmover_l10n.py
@@ -11,27 +11,27 @@ from taskgraph.transforms.base import Tr
from taskgraph.util.treeherder import join_symbol
transforms = TransformSequence()
@transforms.add
def make_beetmover_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
for locale in dep_job.attributes.get('chunk_locales', []):
group = 'BM-L10n'
# add the locale code
symbol = locale
treeherder = {
'symbol': join_symbol(group, symbol),
}
beet_description = {
- 'dependent-task': dep_job,
+ 'primary-dependency': dep_job,
'treeherder': treeherder,
'locale': locale,
'shipping-phase': job['shipping-phase'],
}
yield beet_description
--- a/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
@@ -2,61 +2,61 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform release-beetmover-langpack-checksums into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-beetmover_checksums_description_schema = Schema({
- Required('dependent-task'): object,
+beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
'BMcslang(N{})'.format(attributes.get('l10n_chunk', ''))
)
dep_th_platform = dep_job.task.get('extra', {}).get(
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
@@ -2,26 +2,26 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partners import (
check_if_partners_enabled,
get_ftp_platform,
get_partner_config_by_kind,
)
from taskgraph.util.schema import (
- Schema,
optionally_keyed_by,
resolve_keyed_by,
validate_schema,
)
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_beetmover_bucket_scope,
get_worker_type_for_scope,
@@ -42,20 +42,17 @@ task_description_schema = {str(k): v for
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-beetmover_description_schema = Schema({
- # the dependent task (object) for this beetmover job, used to inform beetmover.
- Required('dependent-task'): object,
-
+beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
Required('partner-bucket-scope'): optionally_keyed_by('release-level', basestring),
Required('partner-public-path'): Any(None, basestring),
@@ -67,17 +64,17 @@ beetmover_description_schema = Schema({
})
transforms.add(check_if_partners_enabled)
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def resolve_keys(config, jobs):
@@ -87,17 +84,17 @@ def resolve_keys(config, jobs):
**{'release-level': config.params.release_level()}
)
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
repack_id = dep_job.task.get('extra', {}).get('repack_id')
if not repack_id:
raise Exception("Cannot find repack id!")
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
--- a/taskcluster/taskgraph/transforms/beetmover_source_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_source_checksums.py
@@ -2,61 +2,61 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform release-beetmover-source-checksums into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-beetmover_checksums_description_schema = Schema({
- Required('dependent-task'): object,
+beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcss(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -16,17 +16,17 @@ transforms = TransformSequence()
@transforms.add
def add_signed_routes(config, jobs):
"""Add routes corresponding to the routes of the build task
this corresponds to, with .signed inserted, for all gecko.v2 routes"""
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
job['routes'] = []
if dep_job.attributes.get('nightly'):
for dep_route in dep_job.task.get('routes', []):
if not dep_route.startswith('index.gecko.v2'):
continue
branch = dep_route.split(".")[3]
rest = ".".join(dep_route.split(".")[4:])
@@ -34,17 +34,17 @@ def add_signed_routes(config, jobs):
'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
yield job
@transforms.add
def define_upstream_artifacts(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
build_platform = dep_job.attributes.get('build_platform')
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
dep_job,
keep_locale_template=False,
kind=config.kind,
)
--- a/taskcluster/taskgraph/transforms/checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/checksums_signing.py
@@ -2,19 +2,20 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the checksums signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_signing_cert_scope,
get_worker_type_for_scope,
add_scope_prefix,
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@@ -23,40 +24,39 @@ from voluptuous import Any, Required, Op
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-checksums_signing_description_schema = Schema({
- Required('dependent-task'): object,
+checksums_signing_description_schema = schema.extend({
Required('depname', default='beetmover'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
checksums_signing_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_checksums_signing_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'cs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/chunk_partners.py
+++ b/taskcluster/taskgraph/transforms/chunk_partners.py
@@ -25,17 +25,17 @@ def _check_repack_ids_by_platform(platfo
repack_ids_by_platform.setdefault(platform, {})['repack_id'] = True
@transforms.add
def chunk_partners(config, jobs):
partner_configs = get_partner_config_by_kind(config, config.kind)
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
build_platform = dep_job.attributes["build_platform"]
# already chunked
if dep_job.task.get('extra', {}).get('repack_id'):
repack_id = dep_job.task['extra']['repack_id']
if _check_repack_ids_by_platform(build_platform, repack_id):
continue
partner_job = copy.deepcopy(job)
partner_job.setdefault('extra', {}).setdefault('repack_id', repack_id)
--- a/taskcluster/taskgraph/transforms/copy_attributes_from_dependent_task.py
+++ b/taskcluster/taskgraph/transforms/copy_attributes_from_dependent_task.py
@@ -12,11 +12,11 @@ from taskgraph.util.attributes import co
transforms = TransformSequence()
@transforms.add
def copy_attributes(config, jobs):
for job in jobs:
job.setdefault('attributes', {})
- job['attributes'].update(copy_attributes_from_dependent_job(job['dependent-task']))
+ job['attributes'].update(copy_attributes_from_dependent_job(job['primary-dependency']))
yield job
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -136,17 +136,17 @@ l10n_description_schema = schema.extend(
},
# Extra environment values to pass to the worker
Optional('env'): _by_platform({basestring: taskref_or_string}),
# Max number locales per chunk
Optional('locales-per-chunk'): _by_platform(int),
- # Task deps to chain this task with, added in transforms from dependent-task
+ # Task deps to chain this task with, added in transforms from primary-dependency
# if this is a nightly
Optional('dependencies'): {basestring: basestring},
# Run the task when the listed files change (if present).
Optional('when'): {
'files-changed': [basestring]
},
--- a/taskcluster/taskgraph/transforms/name_sanity.py
+++ b/taskcluster/taskgraph/transforms/name_sanity.py
@@ -1,14 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Generate labels for tasks without names, consistently.
-Uses attributes from `dependent-task` or `primary-dependency`.
+Uses attributes from `primary-dependency` or `primary-dependency`.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@@ -17,17 +17,17 @@ transforms = TransformSequence()
@transforms.add
def make_label(config, jobs):
""" Generate a sane label for a new task constructed from a dependency
Using attributes from the dependent job and the current task kind"""
for job in jobs:
if 'primary-dependency' in job:
dep_job = job['primary-dependency']
else:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attr = dep_job.attributes.get
if attr('locale', job.get('locale')):
template = "{kind}-{locale}-{build_platform}/{build_type}"
elif attr('l10n_chunk'):
template = "{kind}-{build_platform}-{l10n_chunk}/{build_type}"
elif config.kind.startswith("release-eme-free") or \
config.kind.startswith("release-partner-repack"):
--- a/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -14,33 +14,33 @@ from taskgraph.util.treeherder import jo
transforms = TransformSequence()
@transforms.add
def make_signing_description(config, jobs):
for job in jobs:
job['depname'] = 'unsigned-repack'
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
# add the chunk number to the TH symbol
symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk'))
group = 'L10n'
job['treeherder'] = {
'symbol': join_symbol(group, symbol),
}
yield job
@transforms.add
def define_upstream_artifacts(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
locale_specifications = generate_specifications_of_artifacts_to_sign(
dep_job,
keep_locale_template=True,
)
upstream_artifacts = []
for spec in locale_specifications:
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -37,17 +37,17 @@ def _generate_task_output_files(job, fil
@transforms.add
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
--- a/taskcluster/taskgraph/transforms/partials_signing.py
+++ b/taskcluster/taskgraph/transforms/partials_signing.py
@@ -60,17 +60,17 @@ def generate_upstream_artifacts(job, rel
upstream_artifacts.append(old_mar_upstream_artifacts)
return upstream_artifacts
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'ps(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "partials-signing-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
--- a/taskcluster/taskgraph/transforms/partner_signing.py
+++ b/taskcluster/taskgraph/transforms/partner_signing.py
@@ -18,17 +18,17 @@ transforms.add(check_if_partners_enabled
@transforms.add
def define_upstream_artifacts(config, jobs):
partner_configs = get_partner_config_by_kind(config, config.kind)
if not partner_configs:
return
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
repack_id = job['extra']['repack_id']
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
dep_job,
keep_locale_template=True,
kind=config.kind,
)
job['upstream-artifacts'] = [{
'taskId': {'task-reference': '<{}>'.format(job['depname'])},
--- a/taskcluster/taskgraph/transforms/per_platform_dummy.py
+++ b/taskcluster/taskgraph/transforms/per_platform_dummy.py
@@ -11,19 +11,19 @@ from taskgraph.transforms.base import Tr
transforms = TransformSequence()
@transforms.add
def one_task_per_product_and_platform(config, jobs):
unique_products_and_platforms = set()
for job in jobs:
- dep_task = job["dependent-task"]
- if 'dependent-task' in job:
- del job['dependent-task']
+ dep_task = job["primary-dependency"]
+ if 'primary-dependency' in job:
+ del job['primary-dependency']
product = dep_task.attributes.get("shipping_product")
platform = dep_task.attributes.get("build_platform")
if (product, platform) not in unique_products_and_platforms:
job.setdefault("attributes", {})
job["attributes"]["shipping_product"] = product
job["attributes"]["build_platform"] = platform
job["name"] = "{}-{}".format(product, platform)
yield job
--- a/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
+++ b/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
@@ -2,20 +2,21 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.schema import validate_schema, optionally_keyed_by, resolve_keyed_by
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.task import task_description_schema
from taskgraph.transforms.release_sign_and_push_langpacks import get_upstream_task_ref
from voluptuous import Required, Optional
import logging
import copy
@@ -24,20 +25,17 @@ logger = logging.getLogger(__name__)
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-beetmover_description_schema = Schema({
- # the dependent task (object) for this beetmover job, used to inform beetmover.
- Required('dependent-task'): object,
-
+beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
@@ -53,17 +51,17 @@ beetmover_description_schema = Schema({
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def set_label(config, jobs):
for job in jobs:
- job['label'] = job['dependent-task'].label.replace(
+ job['label'] = job['primary-dependency'].label.replace(
'sign-and-push-langpacks', 'beetmover-signed-langpacks'
)
yield job
@transforms.add
def validate(config, jobs):
@@ -83,17 +81,17 @@ def resolve_keys(config, jobs):
**{'release-level': config.params.release_level()}
)
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk', '')))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
@@ -153,17 +151,17 @@ def generate_upstream_artifacts(upstream
'public/build/{}/target.langpack.xpi'.format(locale)
],
} for locale in locales]
@transforms.add
def strip_unused_data(config, jobs):
for job in jobs:
- del job['dependent-task']
+ del job['primary-dependency']
yield job
@transforms.add
def yield_all_platform_jobs(config, jobs):
# Even though langpacks are now platform independent, we keep beetmoving them at old
# platform-specific locations. That's why this transform exist
--- a/taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
+++ b/taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
@@ -1,19 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the `release-generate-checksums-beetmover` task to also append `build` as dependency
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
@@ -35,20 +36,17 @@ CHECKSUMS_SIGNING_ARTIFACTS = [
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-release_generate_checksums_beetmover_schema = Schema({
- # the dependent task (object) for this beetmover job, used to inform beetmover.
- Required('dependent-task'): object,
-
+release_generate_checksums_beetmover_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
@@ -58,27 +56,27 @@ release_generate_checksums_beetmover_sch
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
release_generate_checksums_beetmover_schema, job,
"In ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/release_generate_checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/release_generate_checksums_signing.py
@@ -2,58 +2,58 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the release-generate-checksums-signing task into task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_signing_cert_scope,
get_worker_type_for_scope,
add_scope_prefix,
)
from taskgraph.util.taskcluster import get_artifact_path
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-release_generate_checksums_signing_schema = Schema({
- Required('dependent-task'): object,
+release_generate_checksums_signing_schema = schema.extend({
Required('depname', default='release-generate-checksums'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
release_generate_checksums_signing_schema, job,
"In ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/release_sign_and_push_langpacks.py
+++ b/taskcluster/taskgraph/transforms/release_sign_and_push_langpacks.py
@@ -2,33 +2,33 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the release-sign-and-push task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema, resolve_keyed_by, optionally_keyed_by
+from taskgraph.util.schema import validate_schema, resolve_keyed_by, optionally_keyed_by
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-langpack_sign_push_description_schema = Schema({
- Required('dependent-task'): object,
+langpack_sign_push_description_schema = schema.extend({
Required('label'): basestring,
Required('description'): basestring,
Required('worker-type'): optionally_keyed_by('release-level', basestring),
Required('worker'): {
Required('implementation'): 'sign-and-push-addons',
Required('channel'): optionally_keyed_by(
'project',
optionally_keyed_by('platform', Any('listed', 'unlisted'))),
@@ -40,17 +40,17 @@ langpack_sign_push_description_schema =
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def set_label(config, jobs):
for job in jobs:
- label = 'sign-and-push-langpacks-{}'.format(job['dependent-task'].label)
+ label = 'sign-and-push-langpacks-{}'.format(job['primary-dependency'].label)
job['label'] = label
yield job
@transforms.add
def validate(config, jobs):
for job in jobs:
@@ -70,53 +70,53 @@ def resolve_keys(config, jobs):
)
resolve_keyed_by(
job, 'scopes', item_name=job['label'],
**{'release-level': config.params.release_level()}
)
resolve_keyed_by(
job, 'worker.channel', item_name=job['label'],
project=config.params['project'],
- platform=job['dependent-task'].attributes['build_platform'],
+ platform=job['primary-dependency'].attributes['build_platform'],
)
yield job
@transforms.add
def copy_attributes(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
job['attributes'] = copy_attributes_from_dependent_job(dep_job)
job['attributes']['chunk_locales'] = dep_job.attributes.get('chunk_locales', ['en-US'])
yield job
@transforms.add
def filter_out_macos_jobs_but_mac_only_locales(config, jobs):
for job in jobs:
- build_platform = job['dependent-task'].attributes.get('build_platform')
+ build_platform = job['primary-dependency'].attributes.get('build_platform')
if build_platform in ('linux64-nightly', 'linux64-devedition-nightly'):
yield job
elif build_platform in ('macosx64-nightly', 'macosx64-devedition-nightly') and \
'ja-JP-mac' in job['attributes']['chunk_locales']:
# Other locales of the same job shouldn't be processed
job['attributes']['chunk_locales'] = ['ja-JP-mac']
job['label'] = job['label'].replace(
job['attributes']['l10n_chunk'], 'ja-JP-mac'
)
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'langpack(SnP{})'.format(
job['attributes'].get('l10n_chunk', '')
))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform))
@@ -171,11 +171,11 @@ def get_upstream_task_ref(job, expected_
raise Exception('Only one dependency expected')
return '<{}>'.format(upstream_tasks[0])
@transforms.add
def strip_unused_data(config, jobs):
for job in jobs:
- del job['dependent-task']
+ del job['primary-dependency']
yield job
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -4,23 +4,23 @@
"""
Transform the repackage task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
- Schema,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.platforms import archive_format, executable_extension
from taskgraph.util.workertypes import worker_type_implementation
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
transforms = TransformSequence()
@@ -30,20 +30,17 @@ transforms = TransformSequence()
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-packaging_description_schema = Schema({
- # the dependant task (object) for this job, used to inform repackaging.
- Required('dependent-task'): object,
-
+packaging_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the signed things
Required('depname', default='build'): basestring,
# unique label to describe this repackaging task
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for repackaging. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
@@ -139,28 +136,28 @@ PACKAGE_FORMATS = {
'output': 'target.stub-installer.exe',
},
}
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
packaging_description_schema, job,
"In packaging ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def copy_in_useful_magic(config, jobs):
"""Copy attributes from upstream task to be used for keyed configuration."""
for job in jobs:
- dep = job['dependent-task']
+ dep = job['primary-dependency']
job['build-platform'] = dep.attributes.get("build_platform")
yield job
@transforms.add
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
@@ -176,29 +173,29 @@ def handle_keyed_by(config, jobs):
item_name="?",
)
yield job
@transforms.add
def make_repackage_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
label = job.get('label',
dep_job.label.replace("signing-", "repackage-"))
job['label'] = label
yield job
@transforms.add
def make_job_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't repackage a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
# This is so we get the build task in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
--- a/taskcluster/taskgraph/transforms/repackage_l10n.py
+++ b/taskcluster/taskgraph/transforms/repackage_l10n.py
@@ -12,15 +12,15 @@ import copy
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@transforms.add
def split_locales(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
for locale in dep_job.attributes.get('chunk_locales', []):
locale_job = copy.deepcopy(job) # don't overwrite dict values here
treeherder = locale_job.setdefault('treeherder', {})
treeherder['symbol'] = 'L10n-Rpk({})'.format(locale)
locale_job['locale'] = locale
yield locale_job
--- a/taskcluster/taskgraph/transforms/repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_partner.py
@@ -4,23 +4,23 @@
"""
Transform the repackage task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
- Schema,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.partners import check_if_partners_enabled
from taskgraph.util.platforms import archive_format, executable_extension
from taskgraph.util.workertypes import worker_type_implementation
from taskgraph.transforms.task import task_description_schema
from taskgraph.transforms.repackage import PACKAGE_FORMATS
from voluptuous import Any, Required, Optional
@@ -36,20 +36,17 @@ def _by_platform(arg):
return optionally_keyed_by('build-platform', arg)
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-packaging_description_schema = Schema({
- # the dependant task (object) for this job, used to inform repackaging.
- Required('dependent-task'): object,
-
+packaging_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the signed things
Required('depname', default='build'): basestring,
# unique label to describe this repackaging task
Optional('label'): basestring,
# Routes specific to this task, if defined
Optional('routes'): [basestring],
@@ -79,28 +76,28 @@ packaging_description_schema = Schema({
})
transforms.add(check_if_partners_enabled)
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
packaging_description_schema, job,
"In packaging ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def copy_in_useful_magic(config, jobs):
"""Copy attributes from upstream task to be used for keyed configuration."""
for job in jobs:
- dep = job['dependent-task']
+ dep = job['primary-dependency']
job['build-platform'] = dep.attributes.get("build_platform")
yield job
@transforms.add
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
@@ -112,29 +109,29 @@ def handle_keyed_by(config, jobs):
for field in fields:
resolve_keyed_by(item=job, field=field, item_name="?")
yield job
@transforms.add
def make_repackage_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
label = job.get('label',
dep_job.label.replace("signing-", "repackage-"))
job['label'] = label
yield job
@transforms.add
def make_job_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
build_platform = attributes['build_platform']
if job['build-platform'].startswith('win'):
if dep_job.kind.endswith('signing'):
continue
if job['build-platform'].startswith('macosx'):
if dep_job.kind.endswith('repack'):
--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@@ -4,35 +4,35 @@
"""
Transform the repackage signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
import os
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
get_worker_type_for_scope,
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-repackage_signing_description_schema = Schema({
- Required('dependent-task'): object,
+repackage_signing_description_schema = schema.extend({
Required('depname', default='repackage'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
SIGNING_FORMATS = {
@@ -41,27 +41,27 @@ SIGNING_FORMATS = {
"target.installer.exe": ["sha2signcode"],
"target.stub-installer.exe": ["sha2signcodestub"],
}
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
repackage_signing_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_repackage_signing_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
--- a/taskcluster/taskgraph/transforms/repackage_signing_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing_partner.py
@@ -2,60 +2,60 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the repackage signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partners import check_if_partners_enabled
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
)
from taskgraph.util.taskcluster import get_artifact_path
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
-repackage_signing_description_schema = Schema({
- Required('dependent-task'): object,
+repackage_signing_description_schema = schema.extend({
Required('depname', default='repackage'): basestring,
Optional('label'): basestring,
Optional('extra'): object,
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
transforms.add(check_if_partners_enabled)
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
repackage_signing_description_schema, job,
"In repackage-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_repackage_signing_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -2,19 +2,20 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_signing_cert_scope_per_platform,
get_worker_type_for_scope,
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@@ -25,20 +26,17 @@ task_description_schema = {str(k): v for
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-signing_description_schema = Schema({
- # the dependant task (object) for this signing job, used to inform signing.
- Required('dependent-task'): object,
-
+signing_description_schema = schema.extend({
# Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py
# because this is passed directly into the signingscript worker
Required('upstream-artifacts'): [{
# taskId of the task with the artifact
Required('taskId'): taskref_or_string,
# type of signing task (for CoT)
Required('taskType'): basestring,
@@ -78,27 +76,27 @@ def set_defaults(config, jobs):
for job in jobs:
job.setdefault('depname', 'build')
yield job
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
signing_description_schema, job,
"In signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
--- a/taskcluster/taskgraph/transforms/source_checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/source_checksums_signing.py
@@ -2,19 +2,20 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the checksums signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import (
get_signing_cert_scope,
get_worker_type_for_scope,
add_scope_prefix,
)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
@@ -23,40 +24,39 @@ from voluptuous import Any, Required, Op
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
-checksums_signing_description_schema = Schema({
- Required('dependent-task'): object,
+checksums_signing_description_schema = schema.extend({
Required('depname', default='beetmover'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
- label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
validate_schema(
checksums_signing_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
@transforms.add
def make_checksums_signing_description(config, jobs):
for job in jobs:
- dep_job = job['dependent-task']
+ dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'css(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
--- a/taskcluster/taskgraph/transforms/strip_dependent_task.py
+++ b/taskcluster/taskgraph/transforms/strip_dependent_task.py
@@ -10,10 +10,10 @@ from __future__ import absolute_import,
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@transforms.add
def strip_dependent_task(config, jobs):
for job in jobs:
- del job['dependent-task']
+ del job['primary-dependency']
yield job
--- a/taskcluster/taskgraph/transforms/upload_generated_sources.py
+++ b/taskcluster/taskgraph/transforms/upload_generated_sources.py
@@ -13,18 +13,18 @@ from taskgraph.util.taskcluster import g
transforms = TransformSequence()
@transforms.add
def add_task_info(config, jobs):
for job in jobs:
- dep_task = job['dependent-task']
- del job['dependent-task']
+ dep_task = job['primary-dependency']
+ del job['primary-dependency']
# Add a dependency on the build task.
job['dependencies'] = {'build': dep_task.label}
# Label the job to match the build task it's uploading from.
job['label'] = dep_task.label.replace("build-", "upload-generated-sources-")
# Copy over some bits of metdata from the build task.
dep_th = dep_task.task['extra']['treeherder']
job.setdefault('attributes', {})
--- a/taskcluster/taskgraph/transforms/upload_symbols.py
+++ b/taskcluster/taskgraph/transforms/upload_symbols.py
@@ -21,29 +21,29 @@ transforms = TransformSequence()
@transforms.add
def check_nightlies(config, tasks):
"""Ensure that we upload symbols for all nightly builds, so that crash-stats can
resolve any reports sent to it. Try may enable full symbols but not upload them.
Putting this check here (instead of the transforms for the build kind) lets us
leverage the any not-for-build-platforms set in the update-symbols kind."""
for task in tasks:
- dep = task['dependent-task']
+ dep = task['primary-dependency']
if config.params['project'] in RELEASE_PROJECTS and \
dep.attributes.get('nightly') and \
not dep.attributes.get('enable-full-crashsymbols'):
raise Exception('Nightly job %s should have enable-full-crashsymbols attribute '
'set to true to enable symbol upload to crash-stats' % dep.label)
yield task
@transforms.add
def fill_template(config, tasks):
for task in tasks:
- dep = task['dependent-task']
+ dep = task['primary-dependency']
# Fill out the dynamic fields in the task description
task['label'] = dep.label + '-upload-symbols'
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get('enable-full-crashsymbols'):
logger.debug("Skipping upload symbols task for %s", task['label'])
continue
@@ -75,11 +75,11 @@ def fill_template(config, tasks):
# Disambiguate the treeherder symbol.
sym = 'Sym' + (th_symbol[1:] if th_symbol.startswith('B') else th_symbol)
treeherder.setdefault(
'symbol', join_symbol(th_groupsymbol, sym)
)
task['treeherder'] = treeherder
# clear out the stuff that's not part of a task description
- del task['dependent-task']
+ del task['primary-dependency']
yield task