Backed out changeset 5428066436e7 (bug 1382204) for busting test on OS X. r=backout
authorSebastian Hengst <archaeopteryx@coole-files.de>
Wed, 06 Sep 2017 14:54:26 +0200
changeset 428802 6e81b409e0776a4674995a910ac3bf02084a9f02
parent 428801 b7726e9751ca5ade59f523aacc452b5c4f88193c
child 428803 7e1396b4b627f92f8c0e8e85319c6f38e304ea08
push id7761
push userjlund@mozilla.com
push dateFri, 15 Sep 2017 00:19:52 +0000
treeherdermozilla-beta@c38455951db4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1382204
milestone57.0a1
backs out5428066436e75195b347352486ea7ca9e826fb25
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 5428066436e7 (bug 1382204) for busting test on OS X. r=backout
taskcluster/ci/build/linux.yml
taskcluster/ci/test/kind.yml
taskcluster/taskgraph/transforms/coalesce.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -64,16 +64,17 @@ linux64/pgo:
     treeherder:
         platform: linux64/pgo
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+    coalesce-name: linux64-pgo
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         options: [enable-pgo]
         config:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
@@ -215,16 +216,17 @@ linux/opt:
     treeherder:
         platform: linux32/opt
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
+    coalesce-name: opt_linux32
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
@@ -243,16 +245,17 @@ linux/debug:
     treeherder:
         platform: linux32/debug
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
+    coalesce-name: dbg_linux32
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
@@ -272,16 +275,17 @@ linux/pgo:
     treeherder:
         platform: linux32/pgo
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
+    coalesce-name: linux32-pgo
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         options: [enable-pgo]
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -2,12 +2,11 @@ loader: taskgraph.loader.test:loader
 
 kind-dependencies:
     - build
     - build-signing
 
 transforms:
    - taskgraph.transforms.tests:transforms
    - taskgraph.transforms.job:transforms
-   - taskgraph.transforms.coalesce:transforms
    - taskgraph.transforms.task:transforms
 
 parse-commit: taskgraph.try_option_syntax:parse_message
deleted file mode 100644
--- a/taskcluster/taskgraph/transforms/coalesce.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-This transform enables coalescing for tasks with scm level > 1, by defining
-values for the coalesce settings of the job.
-"""
-
-from taskgraph.transforms.base import TransformSequence
-from hashlib import sha256
-
-transforms = TransformSequence()
-@transforms.add
-def enable_coalescing(config, jobs):
-    """
-    This transform enables coalescing for tasks with scm level > 1, setting the
-    name to be equal to the the first 20 chars of the sha256 of the task name
-    (label). The hashing is simply used to keep the name short, unique, and
-    alphanumeric, since it is used in AMQP routing keys.
-
-    Note, the coalescing keys themselves are not intended to be readable
-    strings or embed information. The tasks they represent contain all relevant
-    metadata.
-    """
-    if int(config.params['level']) <= 1:
-        return
-    for job in jobs:
-        job['coalesce'] = {
-            'job-identifier': sha256(job["label"]).hexdigest()[:20],
-            'age': 3600,
-            'size': 5,
-        }
-        yield job
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -52,17 +52,17 @@ job_description_schema = Schema({
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
-    Optional('coalesce'): task_description_schema['coalesce'],
+    Optional('coalesce-name'): task_description_schema['coalesce-name'],
     Optional('optimizations'): task_description_schema['optimizations'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
 
     # The "when" section contains descriptions of the circumstances
     # under which this task should be included in the task graph.  This
     # will be converted into an element in the `optimizations` list.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -22,16 +22,18 @@ from taskgraph.transforms.job.common imp
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_setup_secrets,
     docker_worker_add_public_artifacts,
     docker_worker_add_tooltool,
     generic_worker_add_public_artifacts,
     support_vcs_checkout,
 )
 
+COALESCE_KEY = 'builds.{project}.{name}'
+
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
     Required('script'): basestring,
 
     # the config files required for the task, relative to
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -137,36 +137,20 @@ task_description_schema = Schema({
         ),
     },
 
     # The `run_on_projects` attribute, defaulting to "all".  This dictates the
     # projects on which this task should be included in the target task set.
     # See the attributes documentation for details.
     Optional('run-on-projects'): [basestring],
 
-    # Coalescing provides the facility for tasks to be superseded by the same
-    # task in a subsequent commit, if the current task backlog reaches an
-    # explicit threshold. Both age and size thresholds need to be met in order
-    # for coalescing to be triggered.
-    Optional('coalesce'): {
-        # A unique identifier per job (typically a hash of the job label) in
-        # order to partition tasks into appropriate sets for coalescing. This
-        # is combined with the project in order to generate a unique coalescing
-        # key for the coalescing service.
-        'job-identifier': basestring,
-
-        # The minimum amount of time in seconds between two pending tasks with
-        # the same coalescing key, before the coalescing service will return
-        # tasks.
-        'age': int,
-
-        # The minimum number of backlogged tasks with the same coalescing key,
-        # before the coalescing service will return tasks.
-        'size': int,
-    },
+    # If the task can be coalesced, this is the name used in the coalesce key
+    # the project, etc. will be added automatically.  Note that try (level 1)
+    # tasks are never coalesced
+    Optional('coalesce-name'): basestring,
 
     # Optimizations to perform on this task during the optimization phase,
     # specified in order.  These optimizations are defined in
     # taskcluster/taskgraph/optimize.py.
     Optional('optimizations'): [Any(
         # search the index for the given index namespace, and replace this task if found
         ['index-search', basestring],
         # consult SETA and skip this task if it is low-value
@@ -554,18 +538,17 @@ V2_L10N_TEMPLATES = [
 ]
 
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
-COALESCE_KEY = '{project}.{job-identifier}'
-SUPERSEDER_URL = 'https://coalesce.mozilla-releng.net/v1/list/{age}/{size}/{key}'
+COALESCE_KEY = 'builds.{project}.{name}'
 
 DEFAULT_BRANCH_PRIORITY = 'low'
 BRANCH_PRIORITIES = {
     'mozilla-release': 'highest',
     'comm-esr45': 'highest',
     'comm-esr52': 'highest',
     'mozilla-esr45': 'very-high',
     'mozilla-esr52': 'very-high',
@@ -613,31 +596,16 @@ index_builders = {}
 
 
 def index_builder(name):
     def wrap(func):
         index_builders[name] = func
         return func
     return wrap
 
-def coalesce_key(config, task):
-    return COALESCE_KEY.format(**{
-               'project': config.params['project'],
-               'job-identifier': task['coalesce']['job-identifier'],
-           })
-
-def superseder_url(config, task):
-    key = coalesce_key(config, task)
-    age = task['coalesce']['age']
-    size = task['coalesce']['size']
-    return SUPERSEDER_URL.format(
-        age=age,
-        size=size,
-        key=key
-    )
 
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
     level = int(config.params['level'])
 
     image = worker['docker-image']
     if isinstance(image, dict):
@@ -778,18 +746,21 @@ def build_docker_worker_payload(config, 
         payload['env']['TASKCLUSTER_UNTRUSTED_CACHES'] = '1'
 
     if features:
         payload['features'] = features
     if capabilities:
         payload['capabilities'] = capabilities
 
     # coalesce / superseding
-    if 'coalesce' in task:
-        payload['supersederUrl'] = superseder_url(config, task)
+    if 'coalesce-name' in task and level > 1:
+        key = COALESCE_KEY.format(
+            project=config.params['project'],
+            name=task['coalesce-name'])
+        payload['supersederUrl'] = "https://coalesce.mozilla-releng.net/v1/list/" + key
 
     check_caches_are_volumes(task)
 
 
 @payload_builder('generic-worker')
 def build_generic_worker_payload(config, task, task_def):
     worker = task['worker']
 
@@ -835,20 +806,16 @@ def build_generic_worker_payload(config,
     features = {}
 
     if worker.get('chain-of-trust'):
         features['chainOfTrust'] = True
 
     if features:
         task_def['payload']['features'] = features
 
-    # coalesce / superseding
-    if 'coalesce' in task:
-        task_def['payload']['supersederUrl'] = superseder_url(config, task)
-
 
 @payload_builder('scriptworker-signing')
 def build_scriptworker_signing_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upstreamArtifacts':  worker['upstream-artifacts']
@@ -1123,18 +1090,20 @@ def build_task(config, tasks):
             ])
 
         if 'expires-after' not in task:
             task['expires-after'] = '28 days' if config.params['project'] == 'try' else '1 year'
 
         if 'deadline-after' not in task:
             task['deadline-after'] = '1 day'
 
-        if 'coalesce' in task:
-            key = coalesce_key(config, task)
+        if 'coalesce-name' in task and int(config.params['level']) > 1:
+            key = COALESCE_KEY.format(
+                project=config.params['project'],
+                name=task['coalesce-name'])
             routes.append('coalesce.v1.' + key)
 
         if 'priority' not in task:
             task['priority'] = BRANCH_PRIORITIES.get(
                 config.params['project'],
                 DEFAULT_BRANCH_PRIORITY)
 
         tags = task.get('tags', {})