Bug 1398277: special-case retriggering of tasks not in the taskgraph; r=bstack
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 04 Jul 2018 02:46:59 +0000
changeset 490807 ab58645e9230620ca45de3ec03ee9e61eb4a7cbf
parent 490806 4c20ba876af1c686aa454d8e8e216828df04216a
child 490808 aedd937e5d6a64b48b1458e408e2124376c15409
push id1815
push userffxbld-merge
push dateMon, 15 Oct 2018 10:40:45 +0000
treeherdermozilla-release@18d4c09e9378 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbstack
bugs1398277
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1398277: special-case retriggering of tasks not in the taskgraph; r=bstack This will apply to cron tasks, action tasks, and decision tasks. It is a distinct retrigger implementation because (a) we do not want to follow dependencies, and (b) it takes a lot of scopes to create a decision task, so we need to limit access to this action. MozReview-Commit-ID: 21DVSiagcrO
.taskcluster.yml
taskcluster/taskgraph/actions/retrigger.py
taskcluster/taskgraph/actions/util.py
taskcluster/taskgraph/taskgraph.py
taskcluster/taskgraph/test/python.ini
taskcluster/taskgraph/test/test_actions_util.py
taskcluster/taskgraph/test/test_taskgraph.py
taskcluster/taskgraph/test/test_util_taskcluster.py
taskcluster/taskgraph/util/taskcluster.py
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -43,22 +43,28 @@ tasks:
                   name: "Decision Task for cron job ${cron.job_name}"
                   description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
 
         provisionerId: "aws-provisioner-v1"
         workerType: "gecko-${repository.level}-decision"
 
         tags:
           $if: 'tasks_for == "hg-push"'
-          then: {createdForUser: "${ownerEmail}"}
+          then:
+            createdForUser: "${ownerEmail}"
+            kind: decision-task
           else:
             $if: 'tasks_for == "action"'
             then:
               createdForUser: '${ownerEmail}'
               kind: 'action-callback'
+            else:
+              $if: 'tasks_for == "cron"'
+              then:
+                kind: cron-task
 
         routes:
           $flatten:
             - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
             - $if: 'tasks_for == "hg-push"'
               then:
                 - "index.gecko.v2.${repository.project}.latest.taskgraph.decision"
                 - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
--- a/taskcluster/taskgraph/actions/retrigger.py
+++ b/taskcluster/taskgraph/actions/retrigger.py
@@ -3,23 +3,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import logging
+import textwrap
 
 from slugid import nice as slugid
 from .util import (
     combine_task_graph_files,
     create_tasks,
+    fetch_graph_and_labels,
+    relativize_datestamps,
     create_task_from_def,
-    fetch_graph_and_labels
 )
 from ..util.parameterization import resolve_task_references
 from .registry import register_callback_action
 
 logger = logging.getLogger(__name__)
 
 
 @register_callback_action(
@@ -143,21 +145,50 @@ def mochitest_retrigger_action(parameter
     create_task_from_def(new_task_id, new_task_definition, parameters['level'])
 
 
 @register_callback_action(
     title='Retrigger',
     name='retrigger',
     symbol='rt',
     kind='hook',
+    cb_name='retrigger-decision',
+    description=textwrap.dedent('''\
+        Create a clone of the task (retriggering decision, action, and cron tasks requires
+        special scopes).'''),
+    order=11,
+    context=[
+        {'kind': 'decision-task'},
+        {'kind': 'action-callback'},
+        {'kind': 'cron-task'},
+    ],
+)
+def retrigger_decision_action(parameters, graph_config, input, task_group_id, task_id, task):
+    decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
+        parameters, graph_config)
+    """For a single task, we try to just run exactly the same task once more.
+    It's quite possible that we don't have the scopes to do so (especially for
+    an action), but this is best-effort."""
+
+    # make all of the timestamps relative; they will then be turned back into
+    # absolute timestamps relative to the current time.
+    task = relativize_datestamps(task)
+    create_task_from_def(slugid(), task, parameters['level'])
+
+
+@register_callback_action(
+    title='Retrigger',
+    name='retrigger',
+    symbol='rt',
+    kind='hook',
     generic=True,
     description=(
-        'Create a clone of the task.\n\n'
+        'Create a clone of the task.'
     ),
-    order=11,  # must be greater than other orders in this file, as this is the fallback version
+    order=19,  # must be greater than other orders in this file, as this is the fallback version
     context=[{}],
     schema={
         'type': 'object',
         'properties': {
             'downstream': {
                 'type': 'boolean',
                 'description': (
                     'If true, downstream tasks from this one will be cloned as well. '
@@ -176,16 +207,17 @@ def mochitest_retrigger_action(parameter
         }
     }
 )
 def retrigger_action(parameters, graph_config, input, task_group_id, task_id, task):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
         parameters, graph_config)
 
     label = task['metadata']['name']
+
     with_downstream = ' '
     to_run = [label]
 
     if input.get('downstream'):
         to_run = full_task_graph.graph.transitive_closure(set(to_run), reverse=True).nodes
         to_run = to_run & set(label_to_taskid.keys())
         with_downstream = ' (with downstream) '
 
--- a/taskcluster/taskgraph/actions/util.py
+++ b/taskcluster/taskgraph/actions/util.py
@@ -5,24 +5,31 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import logging
 import requests
 import os
+import re
 
 from requests.exceptions import HTTPError
 
 from taskgraph import create
 from taskgraph.decision import read_artifact, write_artifact
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.optimize import optimize_task_graph
-from taskgraph.util.taskcluster import get_session, find_task_id, get_artifact, list_tasks
+from taskgraph.util.taskcluster import (
+    get_session,
+    find_task_id,
+    get_artifact,
+    list_tasks,
+    parse_time,
+)
 
 logger = logging.getLogger(__name__)
 
 PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1'
 
 
 def find_decision_task(parameters, graph_config):
     """Given the parameters for this action, find the taskId of the decision
@@ -159,8 +166,35 @@ def combine_task_graph_files(suffixes):
 
     Since Chain of Trust verification requires a task-graph.json file that
     contains all children tasks, we can combine the various task-graph-0.json
     type files into a master task-graph.json file at the end."""
     all = {}
     for suffix in suffixes:
         all.update(read_artifact('task-graph-{}.json'.format(suffix)))
     write_artifact('task-graph.json', all)
+
+
+def relativize_datestamps(task_def):
+    """
+    Given a task definition as received from the queue, convert all datestamps
+    to {relative_datestamp: ..} format, with the task creation time as "now".
+    The result is useful for handing to ``create_task``.
+    """
+    base = parse_time(task_def['created'])
+    # borrowed from https://github.com/epoberezkin/ajv/blob/master/lib/compile/formats.js
+    ts_pattern = re.compile(
+        r'^\d\d\d\d-[0-1]\d-[0-3]\d[t\s]'
+        r'(?:[0-2]\d:[0-5]\d:[0-5]\d|23:59:60)(?:\.\d+)?'
+        r'(?:z|[+-]\d\d:\d\d)$', re.I)
+
+    def recurse(value):
+        if isinstance(value, basestring):
+            if ts_pattern.match(value):
+                value = parse_time(value)
+                diff = value - base
+                return {'relative-datestamp': '{} seconds'.format(int(diff.total_seconds()))}
+        if isinstance(value, list):
+            return [recurse(e) for e in value]
+        if isinstance(value, dict):
+            return {k: recurse(v) for k, v in value.items()}
+        return value
+    return recurse(task_def)
--- a/taskcluster/taskgraph/taskgraph.py
+++ b/taskcluster/taskgraph/taskgraph.py
@@ -25,16 +25,19 @@ class TaskGraph(object):
         for task_label in self.graph.visit_postorder():
             task = self.tasks[task_label]
             f(task, self, *args, **kwargs)
 
     def __getitem__(self, label):
         "Get a task by label"
         return self.tasks[label]
 
+    def __contains__(self, label):
+        return label in self.tasks
+
     def __iter__(self):
         "Iterate over tasks in undefined order"
         return self.tasks.itervalues()
 
     def __repr__(self):
         return "<TaskGraph graph={!r} tasks={!r}>".format(self.graph, self.tasks)
 
     def __eq__(self, other):
--- a/taskcluster/taskgraph/test/python.ini
+++ b/taskcluster/taskgraph/test/python.ini
@@ -1,12 +1,13 @@
 [DEFAULT]
 subsuite = taskgraph
 skip-if = python == 3
 
+[test_actions_util.py]
 [test_create.py]
 [test_cron_util.py]
 [test_decision.py]
 [test_files_changed.py]
 [test_generator.py]
 [test_graph.py]
 [test_morph.py]
 [test_optimize.py]
@@ -16,12 +17,13 @@ skip-if = python == 3
 [test_transforms_base.py]
 [test_try_option_syntax.py]
 [test_util_attributes.py]
 [test_util_docker.py]
 [test_util_parameterization.py]
 [test_util_python_path.py]
 [test_util_runnable_jobs.py]
 [test_util_schema.py]
+[test_util_taskcluster.py]
 [test_util_templates.py]
 [test_util_time.py]
 [test_util_treeherder.py]
 [test_util_yaml.py]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_actions_util.py
@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+from mozunit import main
+from taskgraph.actions.util import (
+    relativize_datestamps
+)
+
+TASK_DEF = {
+    'created': '2017-10-10T18:33:03.460Z',
+    # note that this is not an even number of seconds off!
+    'deadline': '2017-10-11T18:33:03.461Z',
+    'dependencies': [],
+    'expires': '2018-10-10T18:33:04.461Z',
+    'payload': {
+        'artifacts': {
+            'public': {
+                'expires': '2018-10-10T18:33:03.463Z',
+                'path': '/builds/worker/artifacts',
+                'type': 'directory',
+            },
+        },
+        'maxRunTime': 1800,
+    },
+}
+
+
+class TestRelativize(unittest.TestCase):
+
+    def test_relativize(self):
+        rel = relativize_datestamps(TASK_DEF)
+        import pprint
+        pprint.pprint(rel)
+        assert rel['created'] == {'relative-datestamp': '0 seconds'}
+        assert rel['deadline'] == {'relative-datestamp': '86400 seconds'}
+        assert rel['expires'] == {'relative-datestamp': '31536001 seconds'}
+        assert rel['payload']['artifacts']['public']['expires'] == \
+            {'relative-datestamp': '31536000 seconds'}
+
+
+if __name__ == '__main__':
+    main()
--- a/taskcluster/taskgraph/test/test_taskgraph.py
+++ b/taskcluster/taskgraph/test/test_taskgraph.py
@@ -69,11 +69,32 @@ class TestTaskGraph(unittest.TestCase):
                 dependencies={},
                 optimization={'seta': None},
                 task={'task': 'def2'}),
         }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
 
         tasks, new_graph = TaskGraph.from_json(graph.to_json())
         self.assertEqual(graph, new_graph)
 
+    simple_graph = TaskGraph(tasks={
+        'a': Task(
+            kind='fancy',
+            label='a',
+            attributes={},
+            dependencies={'prereq': 'b'},  # must match edges, below
+            optimization={'seta': None},
+            task={'task': 'def'}),
+        'b': Task(
+            kind='pre',
+            label='b',
+            attributes={},
+            dependencies={},
+            optimization={'seta': None},
+            task={'task': 'def2'}),
+    }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
+
+    def test_contains(self):
+        assert 'a' in self.simple_graph
+        assert 'c' not in self.simple_graph
+
 
 if __name__ == '__main__':
     main()
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_taskcluster.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import datetime
+import unittest
+
+import mozunit
+from taskgraph.util.taskcluster import (
+    parse_time
+)
+
+
+class TestTCUtils(unittest.TestCase):
+
+    def test_parse_time(self):
+        exp = datetime.datetime(2018, 10, 10, 18, 33, 3, 463000)
+        assert parse_time('2018-10-10T18:33:03.463Z') == exp
+
+
+if __name__ == '__main__':
+    mozunit.main()
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -12,20 +12,20 @@ import yaml
 import requests
 import logging
 from mozbuild.util import memoize
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 from taskgraph.task import Task
 
 _PUBLIC_TC_ARTIFACT_LOCATION = \
-        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}'
+    'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}'
 
 _PRIVATE_TC_ARTIFACT_LOCATION = \
-        'http://taskcluster/queue/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}'
+    'http://taskcluster/queue/v1/task/{task_id}/artifacts/{artifact_prefix}/{postfix}'
 
 logger = logging.getLogger(__name__)
 
 # this is set to true for `mach taskgraph action-callback --test`
 testing = False
 
 
 @memoize
@@ -146,20 +146,25 @@ def list_tasks(index_path, use_proxy=Fal
             data = {'continuationToken': response.get('continuationToken')}
         else:
             break
 
     # We can sort on expires because in the general case
     # all of these tasks should be created with the same expires time so they end up in
     # order from earliest to latest action. If more correctness is needed, consider
     # fetching each task and sorting on the created date.
-    results.sort(key=lambda t: datetime.datetime.strptime(t['expires'], '%Y-%m-%dT%H:%M:%S.%fZ'))
+    results.sort(key=lambda t: parse_time(t['expires']))
     return [t['taskId'] for t in results]
 
 
+def parse_time(timestamp):
+    """Turn a "JSON timestamp" as used in TC APIs into a datetime"""
+    return datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
+
+
 def get_task_url(task_id, use_proxy=False):
     if use_proxy:
         TASK_URL = 'http://taskcluster/queue/v1/task/{}'
     else:
         TASK_URL = 'https://queue.taskcluster.net/v1/task/{}'
     return TASK_URL.format(task_id)