Bug 1501878 - required_signoffs attribute. r=tomprince
authorAki Sasaki <aki@escapewindow.com>
Tue, 20 Nov 2018 21:26:18 +0000
changeset 504080 24c9ee68f82f94fb3a0a0b78cde4abddd85bdf3e
parent 504079 9be16b2dee8ef88be9d3658c0d83c9d8f183a967
child 504081 373839b9f787786aca3869614d71bcda66bfabf4
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1501878
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1501878 - required_signoffs attribute. r=tomprince If a task has `required_signoffs` that match a release's `required_signoffs`, we should defer running that task until we have a matching `signoff_url`. - add filter_out_missing_signoffs - add transforms changes to inherit upstream `required_signoffs` - add `mar-signing` `required_signoffs` to the `partials-signing`, `mar-signing`, and `mar-signing-l10n` kinds Differential Revision: https://phabricator.services.mozilla.com/D11734
taskcluster/ci/mar-signing-l10n/kind.yml
taskcluster/ci/mar-signing/kind.yml
taskcluster/ci/partials-signing/kind.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/loader/multi_dep.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/balrog_submit.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
taskcluster/taskgraph/transforms/mar_signing.py
taskcluster/taskgraph/transforms/per_platform_dummy.py
taskcluster/taskgraph/transforms/release_deps.py
taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
taskcluster/taskgraph/transforms/release_snap_push.py
taskcluster/taskgraph/util/attributes.py
--- a/taskcluster/ci/mar-signing-l10n/kind.yml
+++ b/taskcluster/ci/mar-signing-l10n/kind.yml
@@ -24,8 +24,10 @@ only-for-build-platforms:
     - win32-devedition-nightly/opt
     - win64-nightly/opt
     - win64-devedition-nightly/opt
 
 job-template:
     shipping-phase: promote
     treeherder-group: ms
     description-suffix: 'mar signing'
+    required_signoffs:
+        - mar-signing
--- a/taskcluster/ci/mar-signing/kind.yml
+++ b/taskcluster/ci/mar-signing/kind.yml
@@ -26,8 +26,10 @@ only-for-build-platforms:
     - win64-devedition-nightly/opt
     - linux64-asan-reporter-nightly/opt
     - win64-asan-reporter-nightly/opt
 
 job-template:
     shipping-phase: promote
     treeherder-group: ms
     description-suffix: 'mar signing'
+    required_signoffs:
+        - mar-signing
--- a/taskcluster/ci/partials-signing/kind.yml
+++ b/taskcluster/ci/partials-signing/kind.yml
@@ -11,8 +11,10 @@ transforms:
 
 kind-dependencies:
   - partials
 
 job-template:
   shipping-phase: promote
   treeherder-group: ps
   description-suffix: 'partial signing'
+  required_signoffs:
+    - mar-signing
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -261,8 +261,15 @@ cache_digest
 Some tasks generate artifacts that are cached between pushes. This is the unique string used
 to identify the current version of the artifacts. See :py:mod:`taskgraph.util.cached_task`.
 
 cache_type
 ==========
 Some tasks generate artifacts that are cached between pushes. This is the type of cache that is
 used for the this task. See :py:mod:`taskgraph.util.cached_task`.
 
+required_signoffs
+=================
+A list of release signoffs that this kind requires, should the release also
+require these signoffs. For example, ``mar-signing`` signoffs may be required
+by some releases in the future; for any releases that require ``mar-signing``
+signoffs, the kinds that also require that signoff are marked with this
+attribute.
--- a/taskcluster/taskgraph/loader/multi_dep.py
+++ b/taskcluster/taskgraph/loader/multi_dep.py
@@ -4,16 +4,17 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
 from voluptuous import Required
 
 from ..task import Task
+from ..util.attributes import sorted_unique_list
 from ..util.schema import Schema
 
 schema = Schema({
     Required('primary-dependency', 'primary dependency task'): Task,
     Required(
         'dependent-tasks',
         'dictionary of dependent tasks, keyed by kind',
     ): {basestring: Task},
@@ -61,16 +62,21 @@ def loader(kind, path, config, params, l
             job.update(copy.deepcopy(job_template))
         # copy shipping_product from upstream
         product = job['primary-dependency'].attributes.get(
             'shipping_product',
             job['primary-dependency'].task.get('shipping-product')
         )
         if product:
             job.setdefault('shipping-product', product)
+        job.setdefault('attributes', {})['required_signoffs'] = sorted_unique_list(
+            *[task.attributes.get('required_signoffs', [])
+              for task in dep_tasks.values()
+              ]
+        )
 
         yield job
 
 
 def group_tasks(config, tasks):
     group_by_fn = GROUP_BY_MAP[config['group-by']]
 
     groups = group_by_fn(config, tasks)
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -80,16 +80,24 @@ def filter_release_tasks(task, parameter
             return False
 
     if task.attributes.get('shipping_phase') not in (None, 'build'):
         return False
 
     return True
 
 
+def filter_out_missing_signoffs(task, parameters):
+    for signoff in parameters['required_signoffs']:
+        if signoff not in parameters['signoff_urls'] and \
+           signoff in task.attributes.get('required_signoffs', []):
+            return False
+    return True
+
+
 def standard_filter(task, parameters):
     return all(
         filter_func(task, parameters) for filter_func in
         (filter_out_cron, filter_for_project, filter_for_hg_branch)
     )
 
 
 def _try_task_config(full_task_graph, parameters, graph_config):
@@ -315,31 +323,36 @@ def target_tasks_promote_desktop(full_ta
         if task.attributes.get('shipping_product') != parameters['release_product']:
             return False
 
         # 'secondary' balrog/update verify/final verify tasks only run for RCs
         if parameters.get('release_type') != 'release-rc':
             if 'secondary' in task.kind:
                 return False
 
+        if not filter_out_missing_signoffs(task, parameters):
+            return False
+
         if task.attributes.get('shipping_phase') == 'promote':
             return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('push_desktop')
 def target_tasks_push_desktop(full_task_graph, parameters, graph_config):
     """Select the set of tasks required to push a build of desktop to cdns.
     Previous build deps will be optimized out via action task."""
     filtered_for_candidates = target_tasks_promote_desktop(
         full_task_graph, parameters, graph_config,
     )
 
     def filter(task):
+        if not filter_out_missing_signoffs(task, parameters):
+            return False
         # Include promotion tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
         if task.attributes.get('shipping_product') == parameters['release_product'] and \
                 task.attributes.get('shipping_phase') == 'push':
             return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
@@ -358,16 +371,18 @@ def target_tasks_ship_desktop(full_task_
         )
     else:
         # ship_firefox runs after `push`; include all push tasks.
         filtered_for_candidates = target_tasks_push_desktop(
             full_task_graph, parameters, graph_config,
         )
 
     def filter(task):
+        if not filter_out_missing_signoffs(task, parameters):
+            return False
         # Include promotion tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
         if task.attributes.get('shipping_product') != parameters['release_product'] or \
                 task.attributes.get('shipping_phase') != 'ship':
             return False
 
         if 'secondary' in task.kind:
--- a/taskcluster/taskgraph/transforms/balrog_submit.py
+++ b/taskcluster/taskgraph/transforms/balrog_submit.py
@@ -30,16 +30,18 @@ balrog_description_schema = schema.exten
     # unique label to describe this balrog task, defaults to balrog-{dep.label}
     Optional('label'): basestring,
 
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
+    Optional('attributes'): task_description_schema['attributes'],
+
     # Shipping product / phase
     Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
 })
 
 
 transforms = TransformSequence()
 transforms.add_validate(balrog_description_schema)
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -158,16 +158,18 @@ beetmover_description_schema = schema.ex
     # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
     Required('label'): basestring,
 
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
+    Optional('attributes'): task_description_schema['attributes'],
+
     # locale is passed only for l10n beetmoving
     Optional('locale'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     # Optional until we fix asan (run_on_projects?)
     Optional('shipping-product'): task_description_schema['shipping-product'],
 })
 
 transforms = TransformSequence()
@@ -221,16 +223,17 @@ def make_task_description(config, jobs):
         if 'partials-signing' in upstream_deps:
             dependencies['partials-signing'] = upstream_deps['partials-signing']
         if msi_signing_name in upstream_deps:
             dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
         if repackage_signing_name in upstream_deps:
             dependencies["repackage-signing"] = upstream_deps[repackage_signing_name]
 
         attributes = copy_attributes_from_dependent_job(dep_job)
+        attributes.update(job.get('attributes', {}))
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
         bucket_scope = get_beetmover_bucket_scope(config)
         action_scope = get_beetmover_action_scope(config)
 
         task = {
             'label': label,
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
@@ -31,14 +31,15 @@ def make_beetmover_description(config, j
         treeherder = {
             'symbol': join_symbol(group, symbol),
         }
 
         beet_description = {
             'label': job['label'],
             'primary-dependency': dep_job,
             'dependent-tasks': job['dependent-tasks'],
+            'attributes': job['attributes'],
             'treeherder': treeherder,
             'locale': locale,
             'shipping-phase': job['shipping-phase'],
             'shipping-product': job['shipping-product'],
         }
         yield beet_description
--- a/taskcluster/taskgraph/transforms/mar_signing.py
+++ b/taskcluster/taskgraph/transforms/mar_signing.py
@@ -4,17 +4,17 @@
 """
 Transform the {partials,mar}-signing task into an actual task description.
 """
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.attributes import copy_attributes_from_dependent_job, sorted_unique_list
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
     get_worker_type_for_scope,
 )
 from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.treeherder import join_symbol
 
@@ -105,16 +105,20 @@ def make_task_description(config, jobs):
         dependent_kind = str(dep_job.kind)
         dependencies = {dependent_kind: dep_job.label}
         signing_dependencies = dep_job.dependencies
         # This is so we get the build task etc in our dependencies to
         # have better beetmover support.
         dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
+        attributes['required_signoffs'] = sorted_unique_list(
+            attributes.get('required_signoffs', []),
+            job.pop('required_signoffs')
+        )
         attributes['shipping_phase'] = job['shipping-phase']
         if locale:
             attributes['locale'] = locale
 
         balrog_platform = get_balrog_platform_name(dep_th_platform)
         if config.kind == 'partials-signing':
             upstream_artifacts = generate_partials_artifacts(
                 dep_job, config.params['release_history'], balrog_platform, locale)
--- a/taskcluster/taskgraph/transforms/per_platform_dummy.py
+++ b/taskcluster/taskgraph/transforms/per_platform_dummy.py
@@ -3,28 +3,29 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the repackage task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def one_task_per_product_and_platform(config, jobs):
     unique_products_and_platforms = set()
     for job in jobs:
         dep_task = job["primary-dependency"]
         if 'primary-dependency' in job:
             del job['primary-dependency']
         product = dep_task.attributes.get("shipping_product")
         platform = dep_task.attributes.get("build_platform")
         if (product, platform) not in unique_products_and_platforms:
-            job.setdefault("attributes", {})
-            job["attributes"]["shipping_product"] = product
-            job["attributes"]["build_platform"] = platform
+            attributes = copy_attributes_from_dependent_job(dep_task)
+            attributes.update(job.get('attributes', {}))
+            job['attributes'] = attributes
             job["name"] = "{}-{}".format(product, platform)
             yield job
             unique_products_and_platforms.add((product, platform))
--- a/taskcluster/taskgraph/transforms/release_deps.py
+++ b/taskcluster/taskgraph/transforms/release_deps.py
@@ -19,16 +19,17 @@ def add_dependencies(config, jobs):
     for job in jobs:
         dependencies = {}
         # Add any kind_dependencies_tasks with matching product as dependencies
         product = job.get('shipping-product')
         phase = job.get('shipping-phase')
         if product is None:
             continue
 
+        required_signoffs = set(job.setdefault('attributes', {}).get('required_signoffs', []))
         for dep_task in config.kind_dependencies_tasks:
             # Weed out unwanted tasks.
             # XXX we have run-on-projects which specifies the on-push behavior;
             # we need another attribute that specifies release promotion,
             # possibly which action(s) each task belongs in.
             if product == 'fennec':
                 # Don't ship single locale fennec anymore - Bug 1408083
                 attr = dep_task.attributes.get
@@ -42,12 +43,15 @@ def add_dependencies(config, jobs):
             if dep_task.attributes.get("build_platform") and \
                job.get("attributes", {}).get("build_platform"):
                 if dep_task.attributes["build_platform"] != job["attributes"]["build_platform"]:
                     continue
             # Add matching product tasks to deps
             if dep_task.task.get('shipping-product') == product or \
                     dep_task.attributes.get('shipping_product') == product:
                 dependencies[dep_task.label] = dep_task.label
+                required_signoffs.update(dep_task.attributes.get('required_signoffs', []))
 
         job.setdefault('dependencies', {}).update(dependencies)
+        if required_signoffs:
+            job['attributes']['required_signoffs'] = sorted(required_signoffs)
 
         yield job
--- a/taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
+++ b/taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
@@ -47,16 +47,18 @@ release_generate_checksums_beetmover_sch
 
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
     Optional('shipping-product'): task_description_schema['shipping-product'],
+
+    Optional('attributes'): task_description_schema['attributes'],
 })
 
 transforms = TransformSequence()
 transforms.add_validate(release_generate_checksums_beetmover_schema)
 
 
 @transforms.add
 def make_task_description(config, jobs):
--- a/taskcluster/taskgraph/transforms/release_snap_push.py
+++ b/taskcluster/taskgraph/transforms/release_snap_push.py
@@ -25,16 +25,17 @@ push_snap_description_schema = Schema({
     Required('treeherder'): task_description_schema['treeherder'],
     Required('run-on-projects'): task_description_schema['run-on-projects'],
     Required('worker-type'): optionally_keyed_by('release-level', basestring),
     Required('worker'): object,
     Required('scopes'): optionally_keyed_by('project', [basestring]),
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
     Optional('extra'): task_description_schema['extra'],
+    Optional('attributes'): task_description_schema['attributes'],
 })
 
 transforms = TransformSequence()
 transforms.add_validate(push_snap_description_schema)
 
 
 @transforms.add
 def make_task_description(config, jobs):
--- a/taskcluster/taskgraph/util/attributes.py
+++ b/taskcluster/taskgraph/util/attributes.py
@@ -31,16 +31,17 @@ RELEASE_PROMOTION_PROJECTS = {
     'try-comm-central',
 } | RELEASE_PROJECTS
 
 _OPTIONAL_ATTRIBUTES = (
     'artifact_prefix',
     'l10n_chunk',
     'locale',
     'nightly',
+    'required_signoffs',
     'signed',
     'shipping_phase',
     'shipping_product',
     'stub-installer',
 )
 
 
 def attrmatch(attributes, **kwargs):
@@ -126,8 +127,14 @@ def copy_attributes_from_dependent_job(d
     }
 
     attributes.update({
         attr: dep_job.attributes[attr]
         for attr in _OPTIONAL_ATTRIBUTES if attr in dep_job.attributes
     })
 
     return attributes
+
+
+def sorted_unique_list(*args):
+    """Join one or more lists, and return a sorted list of unique members"""
+    combined = set().union(*args)
+    return sorted(combined)