bug 1340609 - toggle nightly scopes on-demand. r=dustin a=release
authorAki Sasaki <asasaki@mozilla.com>
Thu, 23 Feb 2017 16:38:06 -0800
changeset 378686 ddb1f78c74c155aabba863c176d9bd367aeba791
parent 378685 5fb1aafa8fd310af8d868076b42dbff28ef09c8c
child 378687 6d6a559d8cc128170baa2fd49f81e9656bfdf962
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, release
bugs1340609
milestone53.0a2
bug 1340609 - toggle nightly scopes on-demand. r=dustin a=release
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_checksums.py
taskcluster/taskgraph/transforms/checksums_signing.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -4,16 +4,17 @@
 """
 Transform the beetmover task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_balrog_server_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Schema, Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -85,27 +86,27 @@ def make_task_description(config, jobs):
         upstream_artifacts = [{
             "taskId": {"task-reference": "<beetmover>"},
             "taskType": "beetmover",
             "paths": [
                 "public/manifest.json"
             ],
         }]
 
+        server_scope = get_balrog_server_scope(config)
         task = {
             'label': label,
             'description': "{} Balrog".format(
                 dep_job.task["metadata"]["description"]),
             # do we have to define worker type somewhere?
             'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
             'worker': {
                 'implementation': 'balrog',
                 'upstream-artifacts': upstream_artifacts,
             },
-            # bump this to nightly / release when applicable+permitted
-            'scopes': ["project:releng:balrog:nightly"],
+            'scopes': [server_scope],
             'dependencies': {'beetmover': dep_job.label},
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -4,16 +4,17 @@
 """
 Transform the beetmover task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_beetmover_bucket_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Schema, Any, Required, Optional
 
 
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "balrog_props.json",
     "target.common.tests.zip",
     "target.cppunittest.tests.zip",
@@ -194,22 +195,24 @@ def make_task_description(config, jobs):
         attributes = {
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
+        bucket_scope = get_beetmover_bucket_scope(config)
+
         task = {
             'label': label,
             'description': "{} Beetmover".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
-            'scopes': ["project:releng:beetmover:nightly"],
+            'scopes': [bucket_scope],
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
 
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -4,16 +4,17 @@
 """
 Transform the checksums signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_beetmover_bucket_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Schema, Any, Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 transforms = TransformSequence()
@@ -65,22 +66,23 @@ def make_beetmover_checksums_description
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
         if dep_job.attributes.get('locale'):
             treeherder['symbol'] = 'tc-BMcs({})'.format(dep_job.attributes.get('locale'))
             attributes['locale'] = dep_job.attributes.get('locale')
 
+        bucket_scope = get_beetmover_bucket_scope(config)
         task = {
             'label': label,
             'description': "Beetmover {} ".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
-            'scopes': ["project:releng:beetmover:nightly"],
+            'scopes': [bucket_scope],
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
 
--- a/taskcluster/taskgraph/transforms/checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/checksums_signing.py
@@ -4,16 +4,17 @@
 """
 Transform the checksums signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_signing_cert_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Schema, Any, Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 transforms = TransformSequence()
@@ -69,26 +70,27 @@ def make_checksums_signing_description(c
             "taskId": {"task-reference": "<beetmover>"},
             "taskType": "beetmover",
             "paths": [
                 "public/target.checksums",
             ],
             "formats": ["gpg"]
         }]
 
+        signing_cert_scope = get_signing_cert_scope(config)
         task = {
             'label': label,
             'description': "Checksum signing {} ".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'scopes': [
-                "project:releng:signing:cert:nightly-signing",
+                signing_cert_scope,
                 "project:releng:signing:format:gpg"
             ],
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -4,16 +4,17 @@
 """
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
+from taskgraph.util.scriptworker import get_signing_cert_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Schema, Any, Required, Optional
 
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -103,25 +104,26 @@ def make_task_description(config, jobs):
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
         if dep_job.attributes.get('chunk_locales'):
             # Used for l10n attribute passthrough
             attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
 
+        signing_cert_scope = get_signing_cert_scope(config)
         task = {
             'label': label,
             'description': "{} Signing".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': 3600},
-            'scopes': ["project:releng:signing:cert:nightly-signing"] + signing_format_scopes,
+            'scopes': [signing_cert_scope] + signing_format_scopes,
             'dependencies': {job['depname']: dep_job.label},
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
         }
 
         if job.get('use-funsize-route', False):
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -0,0 +1,132 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""Make scriptworker.cot.verify more user friendly by making scopes dynamic.
+
+Scriptworker uses certain scopes to determine which sets of credentials to use.
+Certain scopes are restricted by branch in chain of trust verification, and are
+checked again at the script level.  This file provides functions to adjust
+these scopes automatically by project; this makes pushing to try, forking a
+project branch, and merge day uplifts more user friendly.
+
+In the future, we may adjust scopes by other settings as well, e.g. different
+scopes for `push-to-candidates` rather than `push-to-releases`, even if both
+happen on mozilla-beta and mozilla-release.
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+from copy import deepcopy
+import functools
+
+
+"""Map signing scope aliases to sets of projects.
+
+Currently m-c and m-a use nightly signing; m-b and m-r use release signing.
+We will need to add esr support at some point. Eventually we want to add
+nuance so certain m-b and m-r tasks use dep or nightly signing, and we only
+release sign when we have a signed-off set of candidate builds.  This current
+approach works for now, though.
+
+This is a list of list-pairs, for ordering.
+"""
+SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
+    'all-nightly-branches', set([
+        'mozilla-central',
+        'mozilla-aurora',
+    ])
+], [
+    'all-release-branches', set([
+        'mozilla-beta',
+        'mozilla-release',
+    ])
+]]
+
+"""Map the signing scope aliases to the actual scopes.
+"""
+SIGNING_CERT_SCOPES = {
+    'all-release-branches': 'project:releng:signing:cert:release-signing',
+    'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
+    'default': 'project:releng:signing:cert:dep-signing',
+}
+
+"""Map beetmover scope aliases to sets of projects.
+
+Currently this mirrors the signing scope alias behavior.
+"""
+BEETMOVER_SCOPE_ALIAS_TO_PROJECT = deepcopy(SIGNING_SCOPE_ALIAS_TO_PROJECT)
+
+"""Map the beetmover scope aliases to the actual scopes.
+"""
+BEETMOVER_BUCKET_SCOPES = {
+    'all-release-branches': 'project:releng:beetmover:bucket:release',
+    'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
+    'default': 'project:releng:beetmover:bucket:dep',
+}
+
+"""Map balrog scope aliases to sets of projects.
+
+This is a list of list-pairs, for ordering.
+"""
+BALROG_SCOPE_ALIAS_TO_PROJECT = [[
+    'nightly', set([
+        'mozilla-central',
+    ])
+], [
+    'aurora', set([
+        'mozilla-aurora',
+    ])
+], [
+    'beta', set([
+        'mozilla-beta',
+    ])
+], [
+    'release', set([
+        'mozilla-release',
+    ])
+]]
+
+"""Map the balrog scope aliases to the actual scopes.
+"""
+BALROG_SERVER_SCOPES = {
+    'nightly': 'project:releng:balrog:server:nightly',
+    'aurora': 'project:releng:balrog:server:aurora',
+    'beta': 'project:releng:balrog:server:beta',
+    'release': 'project:releng:balrog:server:release',
+    'default': 'project:releng:balrog:server:dep',
+}
+
+
+def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
+    """Determine the restricted scope from `config.params['project']`.
+
+    Args:
+        alias_to_project_map (list of lists): each list pair contains the
+            alias alias and the set of projects that match.  This is ordered.
+        alias_to_scope_map (dict): the alias alias to scope
+        config (dict): the task config that defines the project.
+
+    Returns:
+        string: the scope to use.
+    """
+    for alias, projects in alias_to_project_map:
+        if config.params['project'] in projects and alias in alias_to_scope_map:
+            return alias_to_scope_map[alias]
+    return alias_to_scope_map['default']
+
+
+get_signing_cert_scope = functools.partial(
+    get_scope_from_project,
+    SIGNING_SCOPE_ALIAS_TO_PROJECT,
+    SIGNING_CERT_SCOPES
+)
+
+get_beetmover_bucket_scope = functools.partial(
+    get_scope_from_project,
+    BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
+    BEETMOVER_BUCKET_SCOPES
+)
+
+get_balrog_server_scope = functools.partial(
+    get_scope_from_project,
+    BALROG_SCOPE_ALIAS_TO_PROJECT,
+    BALROG_SERVER_SCOPES
+)