Bug 1414919 - [taskgraph] Allow morph templates to also modify task attributes, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 07 Nov 2017 10:18:56 -0500
changeset 444126 b38e8ccf12193bdd0372ec8bf1c73581c346557a
parent 444125 2e72bc4c19357e2aa1fc85d8132c17db0caae850
child 444127 6d5a445dba8a7ee540eadc2019d29ce0353ac694
push id1618
push userCallek@gmail.com
push dateThu, 11 Jan 2018 17:45:48 +0000
treeherdermozilla-release@882ca853e05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1414919
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1414919 - [taskgraph] Allow morph templates to also modify task attributes, r=dustin The motivation for this change is to create a 'rebuild' template that sets 'attributes.task_duplicates'. A nice side effect is that if any of 'task' or 'attributes' aren't explicitly set, we can ignore the result which means we don't need to have a whole bunch of 'else' clauses in the templates. MozReview-Commit-ID: DX3UyFGb4ff
taskcluster/docs/try.rst
taskcluster/moz.build
taskcluster/taskgraph/morph.py
taskcluster/taskgraph/templates/artifact.yml
taskcluster/taskgraph/templates/env.yml
--- a/taskcluster/docs/try.rst
+++ b/taskcluster/docs/try.rst
@@ -101,42 +101,33 @@ from the ``try_task_config.json`` like t
     }
 
 Each key in the templates object denotes a new template to apply, and the value
 denotes extra context to use while rendering. When specified, a template will
 be applied to every task no matter what. If the template should only be applied
 to certain kinds of tasks, this needs to be specified in the template itself
 using JSON-e `condition statements`_.
 
-The context available to the JSON-e render aims to match that of ``actions``.
-It looks like this:
+The context available to the JSON-e render contains attributes from the
+:py:class:`taskgraph.task.Task` class. It looks like this:
 
 .. parsed-literal::
 
     {
-      "task": {
-        "payload": {
-          "env": { ... },
-          ...
-        }
-        "extra": {
-          "treeherder": { ... },
-          ...
-        },
-        "tags": { "kind": "<kind>", ... },
-        ...
-      },
-      "input": {
-        "enabled": 1,
-        ...
-      },
-      "taskId": "<task id>"
+      "attributes": task.attributes,
+      "kind": task.kind,
+      "task": task.task,
+      "taskId": task.task_id,
+      "input": ...
     }
 
-See the `existing templates`_ for examples.
+The ``input`` context can be any arbitrary value or object. What it contains
+depends on each specific template. Templates must return objects that have have
+either ``attributes`` or ``task`` as a top level key. All other top level keys
+will be ignored. See the `existing templates`_ for examples.
 
 Empty Try
 :::::::::
 
 If there is no try syntax or ``try_task_config.json``, the ``try_mode``
 parameter is None and no tasks are selected to run.  The resulting push will
 only have a decision task, but one with an "add jobs" action that can be used
 to add the desired jobs to the try push.
--- a/taskcluster/moz.build
+++ b/taskcluster/moz.build
@@ -22,8 +22,9 @@ with Files('scripts/**'):
 
 with Files('taskgraph/**'):
     BUG_COMPONENT = ('Taskcluster', 'Task Configuration')
 
 PYTHON_UNITTEST_MANIFESTS += [
     'taskgraph/test/python.ini',
 ]
 SPHINX_TREES['taskcluster'] = 'docs'
+SPHINX_PYTHON_PACKAGE_DIRS += ['taskgraph']
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -263,22 +263,27 @@ class apply_jsone_templates(object):
         for task in taskgraph.tasks.itervalues():
             for template in sorted(self.templates):
                 context = {
                     'task': task.task,
                     'taskGroup': None,
                     'taskId': task.task_id,
                     'kind': task.kind,
                     'input': self.templates[template],
+                    # The following context differs from action tasks
+                    'attributes': task.attributes,
                 }
 
                 template_path = os.path.join(self.template_dir, template + '.yml')
                 with open(template_path) as f:
                     template = yaml.load(f)
-                task.task = jsone.render(template, context)
+                result = jsone.render(template, context) or {}
+                for attr in ('task', 'attributes'):
+                    if attr in result:
+                        setattr(task, attr, result[attr])
 
         return taskgraph, label_to_taskid
 
 
 def morph(taskgraph, label_to_taskid, parameters):
     """Apply all morphs"""
     morphs = [
         add_index_tasks,
--- a/taskcluster/taskgraph/templates/artifact.yml
+++ b/taskcluster/taskgraph/templates/artifact.yml
@@ -1,18 +1,15 @@
 ---
 $if: task["tags"]
 then:
     $if: task.tags["kind"] == "build"
     then:
-        $mergeDeep:
-            - $eval: task
-            - extra:
-                  treeherder:
-                      symbol: Ba
-            - payload:
-                  env:
-                      USE_ARTIFACT:
-                          $eval: input.enabled
-    else:
-        $eval: task
-else:
-    $eval: task
+        task:
+            $mergeDeep:
+                - $eval: task
+                - extra:
+                      treeherder:
+                          symbol: Ba
+                - payload:
+                      env:
+                          USE_ARTIFACT:
+                              $eval: input.enabled
--- a/taskcluster/taskgraph/templates/env.yml
+++ b/taskcluster/taskgraph/templates/env.yml
@@ -1,5 +1,6 @@
-$mergeDeep:
-    - $eval: task
-    - payload:
-          env:
-              $eval: input
+task:
+    $mergeDeep:
+        - $eval: task
+        - payload:
+              env:
+                  $eval: input