Bug 1333255: introduce graph morphs, use them to make index tasks; r=jonasfj
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 07 Mar 2017 20:39:27 +0000
changeset 398031 ddd3397bf197f4529f6a60c9138417211507e50e
parent 398030 cc28492c11440e1ba9626dec82967e00b77ca38d
child 398032 fe16f26427462c1bb243f401e822b01efa8fba52
push id1490
push usermtabara@mozilla.com
push dateMon, 31 Jul 2017 14:08:16 +0000
treeherdermozilla-release@70e32e6bf15e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjonasfj
bugs1333255
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1333255: introduce graph morphs, use them to make index tasks; r=jonasfj Graph morphs modify the graph after optimization, without changing its meaning. In this case, that means adding index tasks that will insert paths into the index beyond the relatively limited number afforded in task.routes. MozReview-Commit-ID: AJy4exX7q2v
taskcluster/docs/taskgraph.rst
taskcluster/mach_commands.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/morph.py
--- a/taskcluster/docs/taskgraph.rst
+++ b/taskcluster/docs/taskgraph.rst
@@ -94,17 +94,21 @@ Graph generation, as run via ``mach task
 #. Filter the target tasks (based on a series of filters, such as try syntax,
    tree-specific specifications, etc). The result is the "target task set".
 #. Based on the full task graph, calculate the transitive closure of the target
    task set.  That is, the target tasks and all requirements of those tasks.
    The result is the "target task graph".
 #. Optimize the target task graph using task-specific optimization methods.
    The result is the "optimized task graph" with fewer nodes than the target
    task graph.  See :ref:`optimization`.
-#. Create tasks for all tasks in the optimized task graph.
+#. Morph the graph. Morphs are like syntactic sugar: they keep the same meaning,
+   but express it in a lower-level way. These generally work around limitations
+   in the TaskCluster platform, such as number of dependencies or routes in
+   a task.
+#. Create tasks for all tasks in the morphed task graph.
 
 Transitive Closure
 ..................
 
 Transitive closure is a fancy name for this sort of operation:
 
  * start with a set of tasks
  * add all tasks on which any of those tasks depend
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -101,16 +101,21 @@ class MachCommands(MachCommandBase):
     def taskgraph_target_taskgraph(self, **options):
         return self.show_taskgraph('target_task_graph', options)
 
     @ShowTaskGraphSubCommand('taskgraph', 'optimized',
                              description="Show the optimized taskgraph")
     def taskgraph_optimized(self, **options):
         return self.show_taskgraph('optimized_task_graph', options)
 
+    @ShowTaskGraphSubCommand('taskgraph', 'morphed',
+                             description="Show the morphed taskgraph")
+    def taskgraph_morphed(self, **options):
+        return self.show_taskgraph('morphed_task_graph', options)
+
     @SubCommand('taskgraph', 'decision',
                 description="Run the decision task")
     @CommandArgument('--root', '-r',
                      default='taskcluster/ci',
                      help="root of the taskgraph definition relative to topsrcdir")
     @CommandArgument('--base-repository',
                      required=True,
                      help='URL for "base" repository to clone')
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -121,21 +121,21 @@ def taskgraph_decision(options):
     # this is just a test to check whether the from_json() function is working
     _, _ = TaskGraph.from_json(full_task_json)
 
     # write out the target task set to allow reproducing this as input
     write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
 
     # write out the optimized task graph to describe what will actually happen,
     # and the map of labels to taskids
-    write_artifact('task-graph.json', tgg.optimized_task_graph.to_json())
+    write_artifact('task-graph.json', tgg.morphed_task_graph.to_json())
     write_artifact('label-to-taskid.json', tgg.label_to_taskid)
 
     # actually create the graph
-    create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid, parameters)
+    create_tasks(tgg.morphed_task_graph, tgg.label_to_taskid, parameters)
 
 
 def get_decision_parameters(options):
     """
     Load parameters from the command-line options for 'taskgraph decision'.
     This also applies per-project parameters, based on the given project.
 
     """
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -8,16 +8,17 @@ import os
 import yaml
 import copy
 
 from . import filter_tasks
 from .graph import Graph
 from .taskgraph import TaskGraph
 from .task import Task
 from .optimize import optimize_task_graph
+from .morph import morph
 from .util.python_path import find_object
 from .transforms.base import TransformSequence, TransformConfig
 from .util.verify import (
     verify_docs,
     verify_task_graph_symbol,
     verify_gecko_v2_routes,
 )
 
@@ -162,16 +163,27 @@ class TaskGraphGenerator(object):
         """
         A dictionary mapping task label to assigned taskId.  This property helps
         in interpreting `optimized_task_graph`.
 
         @type: dictionary
         """
         return self._run_until('label_to_taskid')
 
+    @property
+    def morphed_task_graph(self):
+        """
+        The optimized task graph, with any subsequent morphs applied. This graph
+        will have the same meaning as the optimized task graph, but be in a form
+        more palatable to TaskCluster.
+
+        @type: TaskGraph
+        """
+        return self._run_until('morphed_task_graph')
+
     def _load_kinds(self):
         for path in os.listdir(self.root_dir):
             path = os.path.join(self.root_dir, path)
             if not os.path.isdir(path):
                 continue
             kind_name = os.path.basename(path)
 
             kind_yml = os.path.join(path, 'kind.yml')
@@ -255,18 +267,23 @@ class TaskGraphGenerator(object):
 
         logger.info("Generating optimized task graph")
         do_not_optimize = set()
         if not self.parameters.get('optimize_target_tasks', True):
             do_not_optimize = target_task_set.graph.nodes
         optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
                                                                     self.parameters,
                                                                     do_not_optimize)
+
+        yield 'optimized_task_graph', optimized_task_graph
+
+        morphed_task_graph, label_to_taskid = morph(optimized_task_graph, label_to_taskid)
+
         yield 'label_to_taskid', label_to_taskid
-        yield 'optimized_task_graph', optimized_task_graph
+        yield 'morphed_task_graph', morphed_task_graph
 
     def _run_until(self, name):
         while name not in self._run_results:
             try:
                 k, v = self._run.next()
             except StopIteration:
                 raise AttributeError("No such run result {}".format(name))
             self._run_results[k] = v
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/morph.py
@@ -0,0 +1,136 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Graph morphs are modifications to task-graphs that take place *after* the
+optimization phase.
+
+These graph morphs are largely invisible to developers running `./mach`
+locally, so they should be limited to changes that do not modify the meaning of
+the graph.
+"""
+
+# Note that the translation of `{'task-reference': '..'}` is handled in the
+# optimization phase (since optimization involves dealing with taskIds
+# directly).  Similarly, `{'relative-datestamp': '..'}` is handled at the last
+# possible moment during task creation.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from slugid import nice as slugid
+from .task import Task
+from .graph import Graph
+from .taskgraph import TaskGraph
+
+logger = logging.getLogger(__name__)
+MAX_ROUTES = 10
+
+
+def amend_taskgraph(taskgraph, label_to_taskid, to_add):
+    """Add the given tasks to the taskgraph, returning a new taskgraph"""
+    new_tasks = taskgraph.tasks.copy()
+    new_edges = taskgraph.graph.edges.copy()
+    for task in to_add:
+        new_tasks[task.task_id] = task
+        assert task.label not in label_to_taskid
+        label_to_taskid[task.label] = task.task_id
+        for depname, dep in task.dependencies.iteritems():
+            new_edges.add((task.task_id, dep, depname))
+
+    taskgraph = TaskGraph(new_tasks, Graph(set(new_tasks), new_edges))
+    return taskgraph, label_to_taskid
+
+
+def derive_misc_task(task, purpose, image, label_to_taskid):
+    """Create the shell of a task that depends on `task` and on the given docker
+    image."""
+    label = '{}-{}'.format(purpose, task.label)
+
+    # this is why all docker image tasks are included in the target task graph: we
+    # need to find them in label_to_taskid, if if nothing else required them
+    image_taskid = label_to_taskid['build-docker-image-' + image]
+
+    task_def = {
+        'provisionerId': 'aws-provisioner-v1',
+        'workerType': 'gecko-misc',
+        'dependencies': [task.task_id, image_taskid],
+        'created': {'relative-timestamp': '0 seconds'},
+        'deadline': task.task['deadline'],
+        # no point existing past the parent task's deadline
+        'expires': task.task['deadline'],
+        'metadata': {
+            'name': label,
+            'description': '{} for {}'.format(purpose, task.task['metadata']['description']),
+            'owner': task.task['metadata']['owner'],
+            'source': task.task['metadata']['source'],
+        },
+        'scopes': [],
+        'payload': {
+            'image': {
+                'path': 'public/image.tar.zst',
+                'taskId': image_taskid,
+                'type': 'task-image',
+            },
+            'features': {
+                'taskclusterProxy': True,
+            },
+            'maxRunTime': 600,
+        }
+    }
+    dependencies = {
+        'parent': task.task_id,
+        'docker-image': image_taskid,
+    }
+    task = Task(kind='misc', label=label, attributes={}, task=task_def,
+                dependencies=dependencies)
+    task.task_id = slugid()
+    return task
+
+
+def make_index_task(parent_task, label_to_taskid):
+    index_paths = [r.split('.', 1)[1] for r in parent_task.task['routes']
+                   if r.startswith('index.')]
+    parent_task.task['routes'] = [r for r in parent_task.task['routes']
+                                  if not r.startswith('index.')]
+
+    task = derive_misc_task(parent_task, 'index-task',
+                            'index-task', label_to_taskid)
+    task.task['scopes'] = [
+        'index:insert-task:{}'.format(path) for path in index_paths]
+    task.task['payload']['command'] = ['insert-indexes.js'] + index_paths
+    task.task['payload']['env'] = {
+        "TARGET_TASKID": parent_task.task_id,
+    }
+    return task
+
+
+def add_index_tasks(taskgraph, label_to_taskid):
+    """
+    The TaskCluster queue only allows 10 routes on a task, but we have tasks
+    with many more routes, for purposes of indexing. This graph morph adds
+    "index tasks" that depend on such tasks and do the index insertions
+    directly, avoiding the limits on task.routes.
+    """
+    logger.debug('Morphing: adding index tasks')
+
+    added = []
+    for label, task in taskgraph.tasks.iteritems():
+        if len(task.task.get('routes', [])) <= MAX_ROUTES:
+            continue
+        added.append(make_index_task(task, label_to_taskid))
+
+    if added:
+        taskgraph, label_to_taskid = amend_taskgraph(
+            taskgraph, label_to_taskid, added)
+        logger.info('Added {} index tasks'.format(len(added)))
+
+    return taskgraph, label_to_taskid
+
+
+def morph(taskgraph, label_to_taskid):
+    """Apply all morphs"""
+    taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid)
+    return taskgraph, label_to_taskid