Bug 1430006: Allow configuring scriptworkers per-graph config; r=aki
authorTom Prince <mozilla@hocat.ca>
Tue, 09 Jan 2018 10:44:04 -0700
changeset 399306 c5132727a5cfeafc6aeac6b10bb849c118838355
parent 399305 732a0fe9f2fb2be4fa05e7570273aeac8d2bbf44
child 399307 d9d1fb68d75d8e96dc3da6ca105c552a673db862
push id98936
push usermozilla@hocat.ca
push dateMon, 15 Jan 2018 20:53:46 +0000
treeherdermozilla-inbound@4da679fb17f7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1430006
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1430006: Allow configuring scriptworkers per-graph config; r=aki Differential Revision: https://phabricator.services.mozilla.com/D379
taskcluster/ci/config.yml
taskcluster/docs/taskgraph.rst
taskcluster/taskgraph/config.py
taskcluster/taskgraph/transforms/checksums_signing.py
taskcluster/taskgraph/transforms/repackage_signing.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -91,8 +91,16 @@ try:
             - 'sm-asan-linux64'
             - 'sm-mozjs-sys-linux64'
             - 'sm-msan-linux64'
             - 'sm-fuzzing-linux64'
             - 'sm-rust-bindings-linux64'
         'win32':
             - 'sm-plain-win32'
             - 'sm-compacting-win32'
+
+scriptworker:
+    worker-types:
+        'scriptworker-prov-v1/signing-linux-v1':
+            - 'project:releng:signing:cert:release-signing'
+            - 'project:releng:signing:cert:nightly-signing'
+        'scriptworker-prov-v1/depsigning':
+            - 'project:releng:signing:cert:dep-signing'
--- a/taskcluster/docs/taskgraph.rst
+++ b/taskcluster/docs/taskgraph.rst
@@ -171,31 +171,33 @@ using simple parameterized values, as fo
 
 ``{"task-reference": "string containing <dep-name>"}``
     The task definition may contain "task references" of this form.  These will
     be replaced during the optimization step, with the appropriate taskId for
     the named dependency substituted for ``<dep-name>`` in the string.
     Multiple labels may be substituted in a single string, and ``<<>`` can be
     used to escape a literal ``<``.
 
+.. _taskgraph-graph-config:
+
 Graph Configuration
 -------------------
 
 There are several configuration settings that are pertain to the entire
 taskgraph. These are specified in :file:`config.yml` at the root of the
 taskgraph configuration (typically :file:`taskcluster/ci`). The available
 settings are documented inline in `taskcluster/taskgraph/config.py
 <https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/config.py>`_.
 
 .. _taskgraph-trust-domain:
 
 Trust Domain
 ------------
 
 When publishing and signing releases, that tasks verify their definition and
 all upstream tasks come from a decision task based on a trusted tree. (see
-`chain-of-trust verification <http://scriptworker.readthedocs.io/en/latest/chain_of_trust.html>`).
+`chain-of-trust verification <http://scriptworker.readthedocs.io/en/latest/chain_of_trust.html>`_).
 Firefox and Thunderbird share the taskgraph code and in particular, they have
 separate taskgraph configurations and in particular distinct decision tasks.
 Although they use identical docker images and toolchains, in order to track the
 province of those artifacts when verifying the chain of trust, they use
 different index paths to cache those artifacts. The ``trust-domain`` graph
 configuration controls the base path for indexing these cached artifacts.
--- a/taskcluster/taskgraph/config.py
+++ b/taskcluster/taskgraph/config.py
@@ -21,13 +21,17 @@ graph_config_schema = Schema({
     },
     Required('try'): {
         # We have a few platforms for which we want to do some "extra" builds, or at
         # least build-ish things.  Sort of.  Anyway, these other things are implemented
         # as different "platforms".  These do *not* automatically ride along with "-p
         # all"
         Required('ridealong-builds', default={}): {basestring: [basestring]},
     },
+    Required('scriptworker'): {
+        # Mapping of scriptworker types to scopes they accept
+        Required('worker-types'): {basestring: [basestring]}
+    },
 })
 
 
 def validate_graph_config(config):
     return validate_schema(graph_config_schema, config, "Invalid graph configuration:")
--- a/taskcluster/taskgraph/transforms/checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/checksums_signing.py
@@ -5,17 +5,20 @@
 Transform the checksums signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope
+from taskgraph.util.scriptworker import (
+    get_signing_cert_scope,
+    get_worker_type_for_scope,
+)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 transforms = TransformSequence()
@@ -84,28 +87,23 @@ def make_checksums_signing_description(c
             ],
             "formats": ["gpg"]
         }]
 
         signing_cert_scope = get_signing_cert_scope(config)
         task = {
             'label': label,
             'description': description,
-            'worker-type': _generate_worker_type(signing_cert_scope),
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': [
                 signing_cert_scope,
                 "project:releng:signing:format:gpg"
             ],
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
-
-
-def _generate_worker_type(signing_cert_scope):
-    worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
-    return 'scriptworker-prov-v1/{}'.format(worker_type)
--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@@ -5,17 +5,20 @@
 Transform the repackage signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
+from taskgraph.util.scriptworker import (
+    get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
+)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 transforms = TransformSequence()
@@ -116,25 +119,20 @@ def make_repackage_signing_description(c
                     ],
                     "formats": ["sha2signcodestub"]
                 })
                 scopes.append("project:releng:signing:format:sha2signcodestub")
 
         task = {
             'label': label,
             'description': description,
-            'worker-type': _generate_worker_type(signing_cert_scope),
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
-
-
-def _generate_worker_type(signing_cert_scope):
-    worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
-    return 'scriptworker-prov-v1/{}'.format(worker_type)
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -5,17 +5,20 @@
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
+from taskgraph.util.scriptworker import (
+    get_signing_cert_scope_per_platform,
+    get_worker_type_for_scope,
+)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -130,17 +133,17 @@ def make_task_description(config, jobs):
 
         signing_cert_scope = get_signing_cert_scope_per_platform(
             dep_job.attributes.get('build_platform'), is_nightly, config
         )
 
         task = {
             'label': label,
             'description': description,
-            'worker-type': _generate_worker_type(signing_cert_scope),
+            'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': 3600},
             'scopes': [signing_cert_scope] + signing_format_scopes,
             'dependencies': {job['depname']: dep_job.label},
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
@@ -152,13 +155,8 @@ def make_task_description(config, jobs):
 
 def _generate_treeherder_platform(dep_th_platform, build_platform, build_type):
     actual_build_type = 'pgo' if '-pgo' in build_platform else build_type
     return '{}/{}'.format(dep_th_platform, actual_build_type)
 
 
 def _generate_treeherder_symbol(is_nightly):
     return 'tc(Ns)' if is_nightly else 'tc(Bs)'
-
-
-def _generate_worker_type(signing_cert_scope):
-    worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
-    return 'scriptworker-prov-v1/{}'.format(worker_type)
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -7,16 +7,18 @@ Scriptworker uses certain scopes to dete
 Certain scopes are restricted by branch in chain of trust verification, and are
 checked again at the script level.  This file provides functions to adjust
 these scopes automatically by project; this makes pushing to try, forking a
 project branch, and merge day uplifts more user friendly.
 
 In the future, we may adjust scopes by other settings as well, e.g. different
 scopes for `push-to-candidates` rather than `push-to-releases`, even if both
 happen on mozilla-beta and mozilla-release.
+
+Additional configuration is found in the :ref:`graph config <taskgraph-graph-config>`.
 """
 from __future__ import absolute_import, print_function, unicode_literals
 import functools
 import json
 import os
 
 
 # constants {{{1
@@ -291,17 +293,17 @@ PUSH_APK_ROLLOUT_PERCENTAGE = {
 # scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
 
     Args:
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
-        config (dict): the task config that defines the project.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         string: the scope to use.
     """
     for alias, projects in alias_to_project_map:
         if config.params['project'] in projects and alias in alias_to_scope_map:
             return alias_to_scope_map[alias]
     return alias_to_scope_map['default']
@@ -309,17 +311,17 @@ def get_scope_from_project(alias_to_proj
 
 def get_scope_from_target_method(alias_to_tasks_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['target_tasks_method']`.
 
     Args:
         alias_to_tasks_map (list of lists): each list pair contains the
             alias and the set of target methods that match. This is ordered.
         alias_to_scope_map (dict): the alias alias to scope
-        config (dict): the task config that defines the target task method.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         string: the scope to use.
     """
     for alias, tasks in alias_to_tasks_map:
         if config.params['target_tasks_method'] in tasks and alias in alias_to_scope_map:
             return alias_to_scope_map[alias]
     return alias_to_scope_map['default']
@@ -335,17 +337,17 @@ def get_scope_from_target_method_and_pro
     checks both.
 
     Args:
         alias_to_tasks_map (list of lists): each list pair contains the
             alias and the set of target methods that match. This is ordered.
         alias_to_project_map (list of lists): each list pair contains the
             alias and the set of projects that match.  This is ordered.
         aliases_to_scope_map (dict of dicts): the task alias to project alias to scope
-        config (dict): the task config that defines the target task method and project.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         string: the scope to use.
     """
     project = config.params['project']
     target = config.params['target_tasks_method']
     for alias1, tasks in alias_to_tasks_map:
         for alias2, projects in alias_to_project_map:
@@ -431,17 +433,17 @@ get_push_apk_rollout_percentage = functo
 
 # release_config {{{1
 def get_release_config(config):
     """Get the build number and version for a release task.
 
     Currently only applies to beetmover tasks.
 
     Args:
-        config (dict): the task config that defines the target task method.
+        config (TransformConfig): The configuration for the kind being transformed.
 
     Returns:
         dict: containing both `build_number` and `version`.  This can be used to
             update `task.payload`.
     """
     release_config = {}
 
     partial_updates = os.environ.get("PARTIAL_UPDATES", "")
@@ -479,8 +481,33 @@ def get_release_config(config):
 
 def get_signing_cert_scope_per_platform(build_platform, is_nightly, config):
     if 'devedition' in build_platform:
         return get_devedition_signing_cert_scope(config)
     elif is_nightly:
         return get_signing_cert_scope(config)
     else:
         return 'project:releng:signing:cert:dep-signing'
+
+
+def get_worker_type_for_scope(config, scope):
+    """Get the scriptworker type that will accept the given scope.
+
+    Args:
+        config (TransformConfig): The configuration for the kind being transformed.
+        scope (string): The scope being used.
+
+    Returns:
+        string: The worker-type to use.
+    """
+    for worker_type, scopes in config.graph_config['scriptworker']['worker-types'].items():
+        if scope in scopes:
+            return worker_type
+    raise RuntimeError(
+        "Unsupported scriptworker scope {scope}. (supported scopes: {available_scopes})".format(
+            scope=scope,
+            available_scopes=sorted(
+                scope
+                for scopes in config.graph_config['scriptworker']['worker-types'].values()
+                for scope in scopes
+            ),
+        )
+    )