Bug 1382204 - enable coalescing on macOS/win10-gpu tests and disable on linux builds,r=dustin
authorPeter Moore <pmoore@mozilla.com>
Wed, 13 Sep 2017 12:30:07 +0200
changeset 430204 641b46e5f932f7857b7129e0d1f76c696433cf15
parent 430203 fa8b67b47624e17900b25d99de5da83d733821ae
child 430205 afedf4a55687576bf44b22549e90fcaeb2524a66
push id7761
push userjlund@mozilla.com
push dateFri, 15 Sep 2017 00:19:52 +0000
treeherdermozilla-beta@c38455951db4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1382204
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1382204 - enable coalescing on macOS/win10-gpu tests and disable on linux builds,r=dustin
taskcluster/ci/build/linux.yml
taskcluster/ci/test/kind.yml
taskcluster/taskgraph/transforms/coalesce.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -64,17 +64,16 @@ linux64/pgo:
     treeherder:
         platform: linux64/pgo
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
-    coalesce-name: linux64-pgo
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         options: [enable-pgo]
         config:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
@@ -216,17 +215,16 @@ linux/opt:
     treeherder:
         platform: linux32/opt
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
-    coalesce-name: opt_linux32
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
@@ -245,17 +243,16 @@ linux/debug:
     treeherder:
         platform: linux32/debug
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
-    coalesce-name: dbg_linux32
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
@@ -275,17 +272,16 @@ linux/pgo:
     treeherder:
         platform: linux32/pgo
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
-    coalesce-name: linux32-pgo
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         options: [enable-pgo]
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -2,11 +2,12 @@ loader: taskgraph.loader.test:loader
 
 kind-dependencies:
     - build
     - build-signing
 
 transforms:
    - taskgraph.transforms.tests:transforms
    - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.coalesce:transforms
    - taskgraph.transforms.task:transforms
 
 parse-commit: taskgraph.try_option_syntax:parse_message
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/coalesce.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This transform enables coalescing for tasks with scm level > 1, by defining
+values for the coalesce settings of the job.
+"""
+
+from __future__ import absolute_import
+
+from taskgraph.transforms.base import TransformSequence
+from hashlib import sha256
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def enable_coalescing(config, jobs):
+    """
+    This transform enables coalescing for tasks with scm level > 1, setting the
+    name to be equal to the the first 20 chars of the sha256 of the task name
+    (label). The hashing is simply used to keep the name short, unique, and
+    alphanumeric, since it is used in AMQP routing keys.
+
+    Note, the coalescing keys themselves are not intended to be readable
+    strings or embed information. The tasks they represent contain all relevant
+    metadata.
+    """
+    for job in jobs:
+        if int(config.params['level']) > 1 and job['worker-type'] not in [
+            # These worker types don't currently support coalescing.
+            # This list can be removed when bug 1399401 is fixed:
+            #   https://bugzilla.mozilla.org/show_bug.cgi?id=1399401
+            'aws-provisioner-v1/gecko-t-win7-32',
+            'aws-provisioner-v1/gecko-t-win7-32-gpu',
+            'aws-provisioner-v1/gecko-t-win10-64',
+        ]:
+            job['coalesce'] = {
+                'job-identifier': sha256(job["label"]).hexdigest()[:20],
+                'age': 3600,
+                'size': 5,
+            }
+        yield job
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -53,17 +53,17 @@ job_description_schema = Schema({
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
-    Optional('coalesce-name'): task_description_schema['coalesce-name'],
+    Optional('coalesce'): task_description_schema['coalesce'],
     Optional('optimizations'): task_description_schema['optimizations'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
 
     # The "when" section contains descriptions of the circumstances
     # under which this task should be included in the task graph.  This
     # will be converted into an element in the `optimizations` list.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -22,18 +22,16 @@ from taskgraph.transforms.job.common imp
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_setup_secrets,
     docker_worker_add_public_artifacts,
     docker_worker_add_tooltool,
     generic_worker_add_public_artifacts,
     support_vcs_checkout,
 )
 
-COALESCE_KEY = 'builds.{project}.{name}'
-
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
     Required('script'): basestring,
 
     # the config files required for the task, relative to
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -141,20 +141,36 @@ task_description_schema = Schema({
         ),
     },
 
     # The `run_on_projects` attribute, defaulting to "all".  This dictates the
     # projects on which this task should be included in the target task set.
     # See the attributes documentation for details.
     Optional('run-on-projects'): [basestring],
 
-    # If the task can be coalesced, this is the name used in the coalesce key
-    # the project, etc. will be added automatically.  Note that try (level 1)
-    # tasks are never coalesced
-    Optional('coalesce-name'): basestring,
+    # Coalescing provides the facility for tasks to be superseded by the same
+    # task in a subsequent commit, if the current task backlog reaches an
+    # explicit threshold. Both age and size thresholds need to be met in order
+    # for coalescing to be triggered.
+    Optional('coalesce'): {
+        # A unique identifier per job (typically a hash of the job label) in
+        # order to partition tasks into appropriate sets for coalescing. This
+        # is combined with the project in order to generate a unique coalescing
+        # key for the coalescing service.
+        'job-identifier': basestring,
+
+        # The minimum amount of time in seconds between two pending tasks with
+        # the same coalescing key, before the coalescing service will return
+        # tasks.
+        'age': int,
+
+        # The minimum number of backlogged tasks with the same coalescing key,
+        # before the coalescing service will return tasks.
+        'size': int,
+    },
 
     # Optimizations to perform on this task during the optimization phase,
     # specified in order.  These optimizations are defined in
     # taskcluster/taskgraph/optimize.py.
     Optional('optimizations'): [Any(
         # search the index for the given index namespace, and replace this task if found
         ['index-search', basestring],
         # consult SETA and skip this task if it is low-value
@@ -542,17 +558,18 @@ V2_L10N_TEMPLATES = [
 ]
 
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
-COALESCE_KEY = 'builds.{project}.{name}'
+COALESCE_KEY = '{project}.{job-identifier}'
+SUPERSEDER_URL = 'https://coalesce.mozilla-releng.net/v1/list/{age}/{size}/{key}'
 
 DEFAULT_BRANCH_PRIORITY = 'low'
 BRANCH_PRIORITIES = {
     'mozilla-release': 'highest',
     'comm-esr45': 'highest',
     'comm-esr52': 'highest',
     'mozilla-esr45': 'very-high',
     'mozilla-esr52': 'very-high',
@@ -601,16 +618,34 @@ index_builders = {}
 
 def index_builder(name):
     def wrap(func):
         index_builders[name] = func
         return func
     return wrap
 
 
+def coalesce_key(config, task):
+    return COALESCE_KEY.format(**{
+               'project': config.params['project'],
+               'job-identifier': task['coalesce']['job-identifier'],
+           })
+
+
+def superseder_url(config, task):
+    key = coalesce_key(config, task)
+    age = task['coalesce']['age']
+    size = task['coalesce']['size']
+    return SUPERSEDER_URL.format(
+        age=age,
+        size=size,
+        key=key
+    )
+
+
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
     level = int(config.params['level'])
 
     image = worker['docker-image']
     if isinstance(image, dict):
         if 'in-tree' in image:
@@ -777,21 +812,18 @@ def build_docker_worker_payload(config, 
         payload['env']['TASKCLUSTER_UNTRUSTED_CACHES'] = '1'
 
     if features:
         payload['features'] = features
     if capabilities:
         payload['capabilities'] = capabilities
 
     # coalesce / superseding
-    if 'coalesce-name' in task and level > 1:
-        key = COALESCE_KEY.format(
-            project=config.params['project'],
-            name=task['coalesce-name'])
-        payload['supersederUrl'] = "https://coalesce.mozilla-releng.net/v1/list/" + key
+    if 'coalesce' in task:
+        payload['supersederUrl'] = superseder_url(config, task)
 
     check_caches_are_volumes(task)
 
 
 @payload_builder('generic-worker')
 def build_generic_worker_payload(config, task, task_def):
     worker = task['worker']
 
@@ -837,16 +869,20 @@ def build_generic_worker_payload(config,
     features = {}
 
     if worker.get('chain-of-trust'):
         features['chainOfTrust'] = True
 
     if features:
         task_def['payload']['features'] = features
 
+    # coalesce / superseding
+    if 'coalesce' in task:
+        task_def['payload']['supersederUrl'] = superseder_url(config, task)
+
 
 @payload_builder('scriptworker-signing')
 def build_scriptworker_signing_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upstreamArtifacts':  worker['upstream-artifacts']
@@ -1121,20 +1157,18 @@ def build_task(config, tasks):
             ])
 
         if 'expires-after' not in task:
             task['expires-after'] = '28 days' if config.params['project'] == 'try' else '1 year'
 
         if 'deadline-after' not in task:
             task['deadline-after'] = '1 day'
 
-        if 'coalesce-name' in task and int(config.params['level']) > 1:
-            key = COALESCE_KEY.format(
-                project=config.params['project'],
-                name=task['coalesce-name'])
+        if 'coalesce' in task:
+            key = coalesce_key(config, task)
             routes.append('coalesce.v1.' + key)
 
         if 'priority' not in task:
             task['priority'] = BRANCH_PRIORITIES.get(
                 config.params['project'],
                 DEFAULT_BRANCH_PRIORITY)
 
         tags = task.get('tags', {})