Bug 1414919 - [taskgraph] Create a 'rebuild' morph template, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 07 Nov 2017 10:22:35 -0500
changeset 695368 6d5a445dba8a7ee540eadc2019d29ce0353ac694
parent 695367 b38e8ccf12193bdd0372ec8bf1c73581c346557a
child 695369 7474dc45cbe36534ad063540b5fe4ae4a17eb59c
push id88406
push userbmo:mh+mozilla@glandium.org
push dateThu, 09 Nov 2017 05:17:33 +0000
reviewersdustin
bugs1414919
milestone58.0a1
Bug 1414919 - [taskgraph] Create a 'rebuild' morph template, r=dustin This template takes a single integer as input, and sets the 'task_duplicates' attribute on any tasks that were explicitly specified by try_task_config.json. This means dependent tasks or 'always_target' tasks will not be rebuilt. To support this template, the apply_jsone_templates morph now requires the entire try_task_config object instead of just the templates. MozReview-Commit-ID: DwxUtlC5VD5
taskcluster/docs/try.rst
taskcluster/taskgraph/morph.py
taskcluster/taskgraph/templates/rebuild.yml
taskcluster/taskgraph/test/test_morph.py
--- a/taskcluster/docs/try.rst
+++ b/taskcluster/docs/try.rst
@@ -109,16 +109,18 @@ using JSON-e `condition statements`_.
 The context available to the JSON-e render contains attributes from the
 :py:class:`taskgraph.task.Task` class. It looks like this:
 
 .. parsed-literal::
 
     {
       "attributes": task.attributes,
       "kind": task.kind,
+      "label": task.label,
+      "target_tasks": [<tasks from try_task_config.json>],
       "task": task.task,
       "taskId": task.task_id,
       "input": ...
     }
 
 The ``input`` context can be any arbitrary value or object. What it contains
 depends on each specific template. Templates must return objects that have have
 either ``attributes`` or ``task`` as a top level key. All other top level keys
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -248,33 +248,36 @@ def add_s3_uploader_task(taskgraph, labe
 class apply_jsone_templates(object):
     """Apply a set of JSON-e templates to each task's `task` attribute.
 
     :param templates: A dict with the template name as the key, and extra context
                       to use (in addition to task.to_json()) as the value.
     """
     template_dir = os.path.join(here, 'templates')
 
-    def __init__(self, templates):
-        self.templates = templates
+    def __init__(self, try_task_config):
+        self.templates = try_task_config.get('templates')
+        self.target_tasks = try_task_config.get('tasks')
 
     def __call__(self, taskgraph, label_to_taskid):
         if not self.templates:
             return taskgraph, label_to_taskid
 
         for task in taskgraph.tasks.itervalues():
             for template in sorted(self.templates):
                 context = {
                     'task': task.task,
                     'taskGroup': None,
                     'taskId': task.task_id,
                     'kind': task.kind,
                     'input': self.templates[template],
                     # The following context differs from action tasks
                     'attributes': task.attributes,
+                    'label': task.label,
+                    'target_tasks': self.target_tasks,
                 }
 
                 template_path = os.path.join(self.template_dir, template + '.yml')
                 with open(template_path) as f:
                     template = yaml.load(f)
                 result = jsone.render(template, context) or {}
                 for attr in ('task', 'attributes'):
                     if attr in result:
@@ -285,13 +288,13 @@ class apply_jsone_templates(object):
 
 def morph(taskgraph, label_to_taskid, parameters):
     """Apply all morphs"""
     morphs = [
         add_index_tasks,
         add_s3_uploader_task,
     ]
     if parameters['try_mode'] == 'try_task_config':
-        morphs.append(apply_jsone_templates(parameters['try_task_config'].get('templates')))
+        morphs.append(apply_jsone_templates(parameters['try_task_config']))
 
     for m in morphs:
         taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
     return taskgraph, label_to_taskid
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/templates/rebuild.yml
@@ -0,0 +1,8 @@
+---
+$if: label in target_tasks
+then:
+    attributes:
+        $merge:
+            - $eval: attributes
+            - task_duplicates:
+                $eval: input
--- a/taskcluster/taskgraph/test/test_morph.py
+++ b/taskcluster/taskgraph/test/test_morph.py
@@ -133,17 +133,22 @@ class TestApplyJSONeTemplates(MorphTestC
         },
     ]
 
     def test_template_artifact(self):
         tg, label_to_taskid = self.make_taskgraph({
             t['label']: Task(**t) for t in self.tasks[:]
         })
 
-        fn = morph.apply_jsone_templates({'artifact': {'enabled': 1}})
+        try_task_config = {
+            'templates': {
+                'artifact': {'enabled': 1}
+            },
+        }
+        fn = morph.apply_jsone_templates(try_task_config)
         morphed = fn(tg, label_to_taskid)[0]
 
         self.assertEqual(len(morphed.tasks), 2)
 
         for t in morphed.tasks.values():
             if t.kind == 'build':
                 self.assertEqual(t.task['extra']['treeherder']['group'], 'tc')
                 self.assertEqual(t.task['extra']['treeherder']['symbol'], 'Ba')
@@ -153,29 +158,62 @@ class TestApplyJSONeTemplates(MorphTestC
                 self.assertEqual(t.task['extra']['treeherder']['symbol'], 't')
                 self.assertNotIn('USE_ARTIFACT', t.task['payload']['env'])
 
     def test_template_env(self):
         tg, label_to_taskid = self.make_taskgraph({
             t['label']: Task(**t) for t in self.tasks[:]
         })
 
-        fn = morph.apply_jsone_templates({'env': {'ENABLED': 1, 'FOO': 'BAZ'}})
+        try_task_config = {
+            'templates': {
+                'env': {
+                    'ENABLED': 1,
+                    'FOO': 'BAZ',
+                }
+            },
+        }
+        fn = morph.apply_jsone_templates(try_task_config)
         morphed = fn(tg, label_to_taskid)[0]
 
         self.assertEqual(len(morphed.tasks), 2)
         for t in morphed.tasks.values():
             self.assertEqual(len(t.task['payload']['env']), 2)
             self.assertEqual(t.task['payload']['env']['ENABLED'], 1)
             self.assertEqual(t.task['payload']['env']['FOO'], 'BAZ')
 
-        fn = morph.apply_jsone_templates({'env': {'ENABLED': 0}})
+        try_task_config['templates']['env'] = {
+            'ENABLED': 0,
+        }
+        fn = morph.apply_jsone_templates(try_task_config)
         morphed = fn(tg, label_to_taskid)[0]
 
         self.assertEqual(len(morphed.tasks), 2)
         for t in morphed.tasks.values():
             self.assertEqual(len(t.task['payload']['env']), 2)
             self.assertEqual(t.task['payload']['env']['ENABLED'], 0)
             self.assertEqual(t.task['payload']['env']['FOO'], 'BAZ')
 
+    def test_template_rebuild(self):
+        tg, label_to_taskid = self.make_taskgraph({
+            t['label']: Task(**t) for t in self.tasks[:]
+        })
+
+        try_task_config = {
+            'tasks': ['b'],
+            'templates': {
+                'rebuild': 4,
+            },
+        }
+        fn = morph.apply_jsone_templates(try_task_config)
+        tasks = fn(tg, label_to_taskid)[0].tasks.values()
+        self.assertEqual(len(tasks), 2)
+
+        for t in tasks:
+            if t.label == 'a':
+                self.assertNotIn('task_duplicates', t.attributes)
+            elif t.label == 'b':
+                self.assertIn('task_duplicates', t.attributes)
+                self.assertEqual(t.attributes['task_duplicates'], 4)
+
 
 if __name__ == '__main__':
     main()