[mq]: action_tasks.patch
authorKalpesh Krishna <kalpeshk2011@gmail.com>
Thu, 23 Jun 2016 21:41:23 +0530
changeset 786878 063de0d244ec095b6b4c3f3b02e5bf3c88fcda39
parent 785302 0ffa18cfc8b4e9970275ab6dd94686792a5c8ae1
child 786879 a4f4f9cf1cb2f51e63d20e155d66d0a1dead38f8
push id130910
push userkalpeshk2011@gmail.com
push dateThu, 23 Jun 2016 16:12:46 +0000
treeherdertry@a4f4f9cf1cb2 [default view] [failures only]
milestone50.0a1
[mq]: action_tasks.patch
taskcluster/mach_commands.py
taskcluster/taskgraph/action.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/optimize.py
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -148,16 +148,37 @@ class MachCommands(MachCommandBase):
         import taskgraph.decision
         try:
             self.setup_logging()
             return taskgraph.decision.taskgraph_decision(options)
         except Exception as e:
             traceback.print_exc()
             sys.exit(1)
 
+    @SubCommand('taskgraph', 'action-task',
+                description="Run the action task")
+    @CommandArgument('--decision-id',
+        required=True,
+        help="Decision Task ID of the reference decision task")
+    @CommandArgument('--task-labels',
+        required=True,
+        help='Comma separated list of task labels to be scheduled')
+    def taskgraph_action(self, **options):
+        """Run the action task: Generates a task graph using the set of labels
+        provided in the task-labels parameter. It uses the full-task file of
+        the gecko decision task."""
+
+        import taskgraph.action
+        try:
+            self.setup_logging()
+            return taskgraph.action.taskgraph_action(options)
+        except Exception as e:
+            traceback.print_exc()
+            sys.exit(1)
+
 
     def setup_logging(self, quiet=False, verbose=True):
         """
         Set up Python logging for all loggers, sending results to stderr (so
         that command output can be redirected easily) and adding the typical
         mach timestamp.
         """
         # remove the old terminal handler
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/action.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import json
+import logging
+import requests
+import yaml
+
+from .decision import write_artifact
+from .generator import TaskGraphGenerator
+from .create import create_tasks
+from .parameters import Parameters
+from .target_tasks import get_method
+from .graph import Graph
+from .optimize import optimize_task_graph
+from .types import (
+    Task,
+    TaskGraph
+)
+
+logger = logging.getLogger(__name__)
+
+logger = logging.getLogger(__name__)
+
+
+def taskgraph_action(options):
+    """
+    Run the action task.  This function implements `mach taskgraph action`,
+    and is responsible for
+
+     * creating taskgraph of tasks asked for in parameters with respect to
+     a given gecko decision task and schedule these jobs.
+    """
+
+    parameters = get_action_parameters(options)
+
+    # write out the parameters used to generate this graph
+    write_artifact('parameters.yml', dict(**parameters))
+
+    # write out the full graph for reference
+    all_tasks, full_task_graph = get_full_task_graph(parameters['decision_id'])
+    target_tasks = set(parameters['task_labels'].split(','))
+
+    target_task_set = TaskGraph(
+        {l: all_tasks[l] for l in target_tasks},
+        Graph(target_tasks, set()))
+
+    target_graph = full_task_graph.graph.transitive_closure(target_tasks)
+    target_task_graph = TaskGraph(
+        {l: all_tasks[l] for l in target_graph.nodes},
+        target_graph)
+
+    optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph, set())
+    # write out the optimized task graph to describe what will actually happen,
+    # and the map of labels to taskids
+    write_artifact('task-graph.json', optimized_task_graph.to_json())
+    write_artifact('label-to-taskid.json', label_to_taskid)
+
+    # actually create the graph
+    create_tasks(optimized_task_graph, label_to_taskid)
+
+
+def get_action_parameters(options):
+    """
+    Load parameters from the command-line options for 'taskgraph action'.
+    """
+    parameters = {n: options[n] for n in [
+        'decision_id',
+        'task_labels',
+    ] if n in options}
+
+    return Parameters(parameters)
+
+
+def get_full_task_graph(decision_task_id):
+    """
+    This code is used to generate the full task graph using the decision
+    task id. The file is first downloaded and then converted to the
+    TaskGraph format.
+    """
+    url = "https://public-artifacts.taskcluster.net/" + decision_task_id + "/0/public/full-task-graph.json"
+    resp = requests.get(url=url)
+    full_tasks_dict = json.loads(resp.text)
+    all_tasks = {}
+    edges = set()
+    for key, value in full_tasks_dict.iteritems():
+        all_tasks[key] = Task(kind=None,
+                              label=value['label'],
+                              attributes=value['attributes'],
+                              task=value['task'])
+        for depname, dep in value['dependencies'].iteritems():
+            edges.add((key, dep, depname))
+    full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
+    full_task_graph = TaskGraph(all_tasks,
+                                Graph(full_task_set.graph.nodes, edges))
+    return all_tasks, full_task_graph
\ No newline at end of file
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -4,16 +4,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import json
 import logging
+import pystache
 import yaml
 
 from .generator import TaskGraphGenerator
 from .create import create_tasks
 from .parameters import Parameters
 from .target_tasks import get_method
 
 logger = logging.getLogger(__name__)
@@ -59,16 +60,20 @@ def taskgraph_decision(options):
     tgg = TaskGraphGenerator(
         root_dir=options['root'],
         parameters=parameters,
         target_tasks_method=target_tasks_method)
 
     # write out the parameters used to generate this graph
     write_artifact('parameters.yml', dict(**parameters))
 
+    with open('taskcluster/taskgraph/action.yml', 'r') as f:
+        c = pystache.render(f.read())
+        result = yaml.load(c)
+        write_artifact('action.yml', result)
     # write out the full graph for reference
     write_artifact('full-task-graph.json', tgg.full_task_graph.to_json())
 
     # write out the target task set to allow reproducing this as input
     write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
 
     # write out the optimized task graph to describe what will actually happen,
     # and the map of labels to taskids
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -81,17 +81,20 @@ def annotate_task_graph(target_task_grap
         replacement_task_id = None
         if label in do_not_optimize:
             optimized = False
         # if any dependencies can't be optimized, this task can't, either
         elif any(not t.optimized for t in dependencies):
             optimized = False
         # otherwise, examine the task itself (which may be an expensive operation)
         else:
-            optimized, replacement_task_id = task.kind.optimize_task(task, named_task_dependencies)
+            if task.kind is None:
+                optimized, replacement_task_id = False, None
+            else:
+                optimized, replacement_task_id = task.kind.optimize_task(task, named_task_dependencies)
 
         task.optimized = optimized
         task.task_id = replacement_task_id
         if replacement_task_id:
             label_to_taskid[label] = replacement_task_id
 
         if optimized:
             if replacement_task_id: