Bug 1539631 - ignore 409 errors cancelling tasks; r=tomprince
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 30 Apr 2019 19:16:12 +0000
changeset 474065 8c1f903def0c4fae9cf3efb9735df8310404fea9
parent 474064 bcab46b227b6a31bc2df5cb83b482db2fc8df23a
child 474066 41f28d23024ebecf5445896cd3b6b39a6b650a9d
push id36022
push userncsoregi@mozilla.com
push dateThu, 16 May 2019 21:55:16 +0000
treeherdermozilla-central@96802be91766 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1539631
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1539631 - ignore 409 errors cancelling tasks; r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D28899
taskcluster/taskgraph/actions/cancel.py
taskcluster/taskgraph/actions/cancel_all.py
--- a/taskcluster/taskgraph/actions/cancel.py
+++ b/taskcluster/taskgraph/actions/cancel.py
@@ -1,27 +1,41 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import logging
+import requests
+
 from taskgraph.util.taskcluster import cancel_task
 from .registry import register_callback_action
 
+logger = logging.getLogger(__name__)
+
 
 @register_callback_action(
     title='Cancel Task',
     name='cancel',
     symbol='cx',
     generic=True,
     description=(
         'Cancel the given task'
     ),
     order=350,
     context=[{}]
 )
 def cancel_action(parameters, graph_config, input, task_group_id, task_id):
     # Note that this is limited by the scopes afforded to generic actions to
     # only cancel tasks with the level-specific schedulerId.
-    cancel_task(task_id, use_proxy=True)
+    try:
+        cancel_task(task_id, use_proxy=True)
+    except requests.HTTPError as e:
+        if e.response.status_code == 409:
+            # A 409 response indicates that this task is past its deadline.  It
+            # cannot be cancelled at this time, but it's also not running
+            # anymore, so we can ignore this error.
+            logger.info('Task is past its deadline and cannot be cancelled.'.format(task_id))
+            return
+        raise
--- a/taskcluster/taskgraph/actions/cancel_all.py
+++ b/taskcluster/taskgraph/actions/cancel_all.py
@@ -4,16 +4,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import concurrent.futures as futures
 import logging
 import os
+import requests
 
 from taskgraph.util.taskcluster import (
     list_task_group_incomplete_tasks,
     cancel_task,
     CONCURRENCY,
 )
 from .registry import register_callback_action
 
@@ -30,17 +31,27 @@ logger = logging.getLogger(__name__)
         'this action task is associated with.'
     ),
     order=400,
     context=[]
 )
 def cancel_all_action(parameters, graph_config, input, task_group_id, task_id):
     def do_cancel_task(task_id):
         logger.info('Cancelling task {}'.format(task_id))
-        cancel_task(task_id, use_proxy=True)
+        try:
+            cancel_task(task_id, use_proxy=True)
+        except requests.HTTPError as e:
+            if e.response.status_code == 409:
+                # A 409 response indicates that this task is past its deadline.  It
+                # cannot be cancelled at this time, but it's also not running
+                # anymore, so we can ignore this error.
+                logger.info(
+                    'Task {} is past its deadline and cannot be cancelled.'.format(task_id))
+                return
+            raise
 
     own_task_id = os.environ.get('TASK_ID', '')
     to_cancel = [t for t in list_task_group_incomplete_tasks(task_group_id) if t != own_task_id]
     logger.info("Cancelling {} tasks".format(len(to_cancel)))
     with futures.ThreadPoolExecutor(CONCURRENCY) as e:
         cancel_futs = [e.submit(do_cancel_task, t) for t in to_cancel]
         for f in futures.as_completed(cancel_futs):
             f.result()