Bug 1620610 - Remove coalescing for tasks. r=catlee a=NPOTB
authorSebastian Hengst <archaeopteryx@coole-files.de>
Fri, 06 Mar 2020 15:52:51 +0000
changeset 579439 b5803347abb5f6920909b51dbbf18933908c22c1
parent 579438 0f23ca83fc9a788a307908fcc5577b3681008870
child 579440 7bbb74cde1d29549bdb3a4d9e37654bcb53eb0e4
push id2284
push userarchaeopteryx@coole-files.de
push dateMon, 09 Mar 2020 16:02:06 +0000
treeherdermozilla-release@bbfdcf44cbf6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscatlee, NPOTB
bugs1620610, 1602446
milestone74.0
Bug 1620610 - Remove coalescing for tasks. r=catlee a=NPOTB Bug 1602446 disabled coalescing for tasks which was designed to reduce load on the pools which ran those tasks by skipping some of them. After coalescing was affecting almost no tasks, it started to coalesce more tasks in November 2019 (bug 1602446) up to a level for which too few tasks ran and test coverage was impacted. Coalescing got disabled and the conclusion is that coalescing is currently not needed. Differential Revision: https://phabricator.services.mozilla.com/D65733
taskcluster/ci/test/kind.yml
taskcluster/taskgraph/transforms/coalesce.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/openh264.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -8,17 +8,16 @@ kind-dependencies:
     - build
     - build-signing
     - fetch
     - toolchain
 
 transforms:
     - taskgraph.transforms.tests:transforms
     - taskgraph.transforms.job:transforms
-    - taskgraph.transforms.coalesce:transforms
     - taskgraph.transforms.task:transforms
 
 # Each stanza in a file pointed to by 'jobs-from' describes a particular test
 # suite or sub-suite. These are processed through the transformations described
 # above to produce a bunch of tasks. See the schema in
 # `taskcluster/taskgraph/transforms/tests.py` for a description of the fields
 # used in these files.
 
deleted file mode 100644
--- a/taskcluster/taskgraph/transforms/coalesce.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-This transform enables coalescing for tasks with scm level > 1, by defining
-values for the coalesce settings of the job.
-"""
-
-from __future__ import absolute_import
-
-from taskgraph.transforms.base import TransformSequence
-# Caller disabled - see below.
-# from hashlib import sha256
-
-transforms = TransformSequence()
-
-
-@transforms.add
-def enable_coalescing(config, jobs):
-    """
-    This transform enables coalescing for tasks with scm level > 1, setting the
-    name to be equal to the the first 20 chars of the sha256 of the task name
-    (label). The hashing is simply used to keep the name short, unique, and
-    alphanumeric, since it is used in AMQP routing keys.
-
-    Note, the coalescing keys themselves are not intended to be readable
-    strings or embed information. The tasks they represent contain all relevant
-    metadata.
-    """
-    for job in jobs:
-        # TODO: Identify why coalescing became much more frequent (bug 1602446) or
-        #       disable it for backfills, retriggers and manually requested tasks
-        #       (bug 1397389).
-        # if int(config.params['level']) > 1:
-        #     job['coalesce'] = {
-        #         'job-identifier': sha256(job["label"]).hexdigest()[:20],
-        #         'age': 3600,
-        #         'size': 5,
-        #     }
-        yield job
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -60,17 +60,16 @@ job_description_schema = Schema({
     Optional('scopes'): task_description_schema['scopes'],
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
     Optional('shipping-product'): task_description_schema['shipping-product'],
-    Optional('coalesce'): task_description_schema['coalesce'],
     Optional('always-target'): task_description_schema['always-target'],
     Exclusive('optimization', 'optimization'): task_description_schema['optimization'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
     Optional('release-artifacts'): task_description_schema['release-artifacts'],
     Optional('priority'): task_description_schema['priority'],
 
     # The "when" section contains descriptions of the circumstances under which
     # this task should be included in the task graph.  This will be converted
--- a/taskcluster/taskgraph/transforms/openh264.py
+++ b/taskcluster/taskgraph/transforms/openh264.py
@@ -10,24 +10,17 @@ from __future__ import absolute_import, 
 from taskgraph.transforms.base import TransformSequence
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def set_mh_options(config, jobs):
     """
-    This transform enables coalescing for tasks with scm level > 1, setting the
-    name to be equal to the the first 20 chars of the sha256 of the task name
-    (label). The hashing is simply used to keep the name short, unique, and
-    alphanumeric, since it is used in AMQP routing keys.
-
-    Note, the coalescing keys themselves are not intended to be readable
-    strings or embed information. The tasks they represent contain all relevant
-    metadata.
+    This transform sets the 'openh264_rev' attribute.
     """
     for job in jobs:
         repo = job.pop('repo')
         rev = job.pop('revision')
         attributes = job.setdefault('attributes', {})
         attributes['openh264_rev'] = rev
         run = job.setdefault('run', {})
         options = run.setdefault('options', [])
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -187,37 +187,16 @@ task_description_schema = Schema({
 
     # The `shipping_product` attribute, defaulting to None. This specifies the
     # release promotion product that this task belongs to.
     Required('shipping-product'): Any(
         None,
         text_type
     ),
 
-    # Coalescing provides the facility for tasks to be superseded by the same
-    # task in a subsequent commit, if the current task backlog reaches an
-    # explicit threshold. Both age and size thresholds need to be met in order
-    # for coalescing to be triggered.
-    Optional('coalesce'): {
-        # A unique identifier per job (typically a hash of the job label) in
-        # order to partition tasks into appropriate sets for coalescing. This
-        # is combined with the project in order to generate a unique coalescing
-        # key for the coalescing service.
-        'job-identifier': text_type,
-
-        # The minimum amount of time in seconds between two pending tasks with
-        # the same coalescing key, before the coalescing service will return
-        # tasks.
-        'age': int,
-
-        # The minimum number of backlogged tasks with the same coalescing key,
-        # before the coalescing service will return tasks.
-        'size': int,
-    },
-
     # The `always-target` attribute will cause the task to be included in the
     # target_task_graph regardless of filtering. Tasks included in this manner
     # will be candidates for optimization even when `optimize_target_tasks` is
     # False, unless the task was also explicitly chosen by the target_tasks
     # method.
     Required('always-target'): bool,
 
     # Optimization to perform on this task during the optimization phase.
@@ -316,20 +295,16 @@ def get_branch_rev(config):
 
 
 def get_branch_repo(config):
     return config.params['{}head_repository'.format(
         config.graph_config['project-repo-param-prefix'],
     )]
 
 
-COALESCE_KEY = '{project}.{job-identifier}'
-SUPERSEDER_URL = 'https://coalesce.mozilla-releng.net/v1/list/{age}/{size}/{key}'
-
-
 @memoize
 def get_default_priority(graph_config, project):
     return evaluate_keyed_by(
         graph_config['task-priority'],
         "Graph Config",
         {'project': project}
     )
 
@@ -359,34 +334,16 @@ index_builders = {}
 
 def index_builder(name):
     def wrap(func):
         index_builders[name] = func
         return func
     return wrap
 
 
-def coalesce_key(config, task):
-    return COALESCE_KEY.format(**{
-               'project': config.params['project'],
-               'job-identifier': task['coalesce']['job-identifier'],
-           })
-
-
-def superseder_url(config, task):
-    key = coalesce_key(config, task)
-    age = task['coalesce']['age']
-    size = task['coalesce']['size']
-    return SUPERSEDER_URL.format(
-        age=age,
-        size=size,
-        key=key
-    )
-
-
 UNSUPPORTED_INDEX_PRODUCT_ERROR = """\
 The gecko-v2 product {product} is not in the list of configured products in
 `taskcluster/ci/config.yml'.
 """
 
 
 def verify_index(config, index):
     product = index['product']
@@ -680,20 +637,16 @@ def build_docker_worker_payload(config, 
     if payload.get('cache') and skip_untrusted:
         payload['env']['TASKCLUSTER_UNTRUSTED_CACHES'] = '1'
 
     if features:
         payload['features'] = features
     if capabilities:
         payload['capabilities'] = capabilities
 
-    # coalesce / superseding
-    if 'coalesce' in task:
-        payload['supersederUrl'] = superseder_url(config, task)
-
     check_caches_are_volumes(task)
 
 
 @payload_builder('generic-worker', schema={
     Required('os'): Any('windows', 'macosx', 'linux', 'linux-bitbar'),
     # see http://schemas.taskcluster.net/generic-worker/v1/payload.json
     # and https://docs.taskcluster.net/reference/workers/generic-worker/payload
 
@@ -881,20 +834,16 @@ def build_generic_worker_payload(config,
         features['runAsAdministrator'] = True
         task_def['scopes'].append(
             'generic-worker:run-as-administrator:{}'.format(task['worker-type']),
         )
 
     if features:
         task_def['payload']['features'] = features
 
-    # coalesce / superseding
-    if 'coalesce' in task:
-        task_def['payload']['supersederUrl'] = superseder_url(config, task)
-
 
 @payload_builder('scriptworker-signing', schema={
     # the maximum time to run, in seconds
     Required('max-run-time'): int,
 
     # list of artifact URLs for the artifacts that should be signed
     Required('upstream-artifacts'): [{
         # taskId of the task with the artifact
@@ -1895,20 +1844,16 @@ def build_task(config, tasks):
             )
 
         if 'expires-after' not in task:
             task['expires-after'] = '28 days' if config.params.is_try() else '1 year'
 
         if 'deadline-after' not in task:
             task['deadline-after'] = '1 day'
 
-        if 'coalesce' in task:
-            key = coalesce_key(config, task)
-            routes.append('coalesce.v1.' + key)
-
         if 'priority' not in task:
             task['priority'] = get_default_priority(config.graph_config, config.params['project'])
 
         tags = task.get('tags', {})
         attributes = task.get('attributes', {})
 
         tags.update({
             'createdForUser': config.params['owner'],