Bug 1285755 - Adding a from_json function to TaskGraph and each Task subclass. r=dustin
authorKalpesh Krishna <kalpeshk2011@gmail.com>
Mon, 11 Jul 2016 22:39:04 +0530
changeset 304709 b1a86b2b81ff
parent 304708 de9a00cbb60c
child 304710 f9714d4cc47a
push id30439
push usercbook@mozilla.com
push date2016-07-13 15:24 +0000
treeherdermozilla-central@151aaa2db94d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1285755
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1285755 - Adding a from_json function to TaskGraph and each Task subclass. r=dustin MozReview-Commit-ID: 8fmALSP8nDs
taskcluster/docs/taskgraph.rst
taskcluster/taskgraph/kind/base.py
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/legacy.py
taskcluster/taskgraph/taskgraph.py
taskcluster/taskgraph/test/test_decision.py
taskcluster/taskgraph/test/test_taskgraph.py
--- a/taskcluster/docs/taskgraph.rst
+++ b/taskcluster/docs/taskgraph.rst
@@ -183,16 +183,20 @@ Each task has the following properties:
 
 ``dependencies``
    The task's in-graph dependencies, represented as an object mapping
    dependency name to label (or to taskId for optimized task graphs)
 
 ``task``
    The task's TaskCluster task definition.
 
+``kind_implementation``
+   The module and the class name which was used to implement this particular task.
+   It is always of the form ``<module-path>:<object-path>``
+
 The task definition may contain "relative datestamps" of the form
 ``{"relative-datestamp": "certain number of seconds/hours/days/years"}``.
 These will be replaced in the last step, while creating tasks.
 The UTC timestamp at that moment is noted, and all the relative datestamps
 are replaced with respect to this timestamp.
 
 The task definition may contain "task references" of the form
 ``{"task-reference": "string containing <task-label>"}``.  These will be
--- a/taskcluster/taskgraph/kind/base.py
+++ b/taskcluster/taskgraph/kind/base.py
@@ -38,16 +38,23 @@ class Task(object):
         self.optimized = False
 
         self.attributes['kind'] = kind
 
         if not (all(isinstance(x, basestring) for x in self.attributes.iterkeys()) and
                 all(isinstance(x, basestring) for x in self.attributes.itervalues())):
             raise TypeError("attribute names and values must be strings")
 
+    def __eq__(self, other):
+        return self.kind == other.kind and \
+               self.label == other.label and \
+               self.attributes == other.attributes and \
+               self.task == other.task and \
+               self.task_id == other.task_id
+
     @classmethod
     @abc.abstractmethod
     def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
         """
         Load the tasks for a given kind.
 
         The `kind` is the name of the kind; the configuration for that kind
         named this class.
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -2,39 +2,45 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 import json
 import os
+import re
 import urllib2
 import tarfile
 import time
 
 from . import base
 from taskgraph.util.docker import (
     docker_image,
     generate_context_hash
 )
 from taskgraph.util.templates import Templates
 
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+INDEX_REGEX = r'index\.(docker\.images\.v1\.(.+)\.(.+)\.hash\.(.+))'
 
 
 class DockerImageTask(base.Task):
 
     def __init__(self, *args, **kwargs):
         self.index_paths = kwargs.pop('index_paths')
         super(DockerImageTask, self).__init__(*args, **kwargs)
 
+    def __eq__(self, other):
+        return super(DockerImageTask, self).__eq__(other) and \
+               self.index_paths == other.index_paths
+
     @classmethod
     def load_tasks(cls, kind, path, config, params, loaded_tasks):
         # TODO: make this match the pushdate (get it from a parameter rather than vcs)
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
 
         parameters = {
             'pushlog_id': params.get('pushlog_id', 0),
             'pushdate': pushdate,
@@ -129,8 +135,27 @@ class DockerImageTask(base.Task):
     def create_context_tar(cls, context_dir, destination, image_name):
         'Creates a tar file of a particular context directory.'
         destination = os.path.abspath(destination)
         if not os.path.exists(os.path.dirname(destination)):
             os.makedirs(os.path.dirname(destination))
 
         with tarfile.open(destination, 'w:gz') as tar:
             tar.add(context_dir, arcname=image_name)
+
+    @classmethod
+    def from_json(cls, task_dict):
+        # Generating index_paths for optimization
+        routes = task_dict['task']['routes']
+        index_paths = []
+        for route in routes:
+            index_path_regex = re.compile(INDEX_REGEX)
+            result = index_path_regex.search(route)
+            if result is None:
+                continue
+            index_paths.append(result.group(1))
+            index_paths.append(result.group(1).replace(result.group(2), 'mozilla-central'))
+        docker_image_task = cls(kind='docker-image',
+                                label=task_dict['label'],
+                                attributes=task_dict['attributes'],
+                                task=task_dict['task'],
+                                index_paths=index_paths)
+        return docker_image_task
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -308,16 +308,20 @@ class LegacyTask(base.Task):
     generation, so this kind labels tasks using those slugids, with a prefix of
     "TaskLabel==".  These labels are unfortunately not stable from run to run.
     """
 
     def __init__(self, *args, **kwargs):
         self.task_dict = kwargs.pop('task_dict')
         super(LegacyTask, self).__init__(*args, **kwargs)
 
+    def __eq__(self, other):
+        return super(LegacyTask, self).__eq__(other) and \
+               self.task_dict == other.task_dict
+
     @classmethod
     def load_tasks(cls, kind, path, config, params, loaded_tasks):
         root = os.path.abspath(os.path.join(path, config['legacy_path']))
 
         project = params['project']
         # NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
         # resulting task graph later
         message = DEFAULT_TRY
@@ -647,8 +651,17 @@ class LegacyTask(base.Task):
             deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
                          'docker-image'))
 
         return deps
 
     def optimize(self):
         # no optimization for the moment
         return False, None
+
+    @classmethod
+    def from_json(cls, task_dict):
+        legacy_task = cls(kind='legacy',
+                          label=task_dict['label'],
+                          attributes=task_dict['attributes'],
+                          task=task_dict['task'],
+                          task_dict=task_dict)
+        return legacy_task
--- a/taskcluster/taskgraph/taskgraph.py
+++ b/taskcluster/taskgraph/taskgraph.py
@@ -1,14 +1,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import os
+
+from .graph import Graph
+from .util.python_path import find_object
+
+TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task/"
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+
 
 class TaskGraph(object):
     """
     Representation of a task graph.
 
     A task graph is a combination of a Graph and a dictionary of tasks indexed
     by label.  TaskGraph instances should be treated as immutable.
     """
@@ -20,21 +28,23 @@ class TaskGraph(object):
 
     def to_json(self):
         "Return a JSON-able object representing the task graph, as documented"
         named_links_dict = self.graph.named_links_dict()
         # this dictionary may be keyed by label or by taskid, so let's just call it 'key'
         tasks = {}
         for key in self.graph.visit_postorder():
             task = self.tasks[key]
+            implementation = task.__class__.__module__ + ":" + task.__class__.__name__
             task_json = {
                 'label': task.label,
                 'attributes': task.attributes,
                 'dependencies': named_links_dict.get(key, {}),
-                'task': task.task
+                'task': task.task,
+                'kind_implementation': implementation
             }
             if task.task_id:
                 task_json['task_id'] = task.task_id
             tasks[key] = task_json
         return tasks
 
     def __getitem__(self, label):
         "Get a task by label"
@@ -44,8 +54,29 @@ class TaskGraph(object):
         "Iterate over tasks in undefined order"
         return self.tasks.itervalues()
 
     def __repr__(self):
         return "<TaskGraph graph={!r} tasks={!r}>".format(self.graph, self.tasks)
 
     def __eq__(self, other):
         return self.tasks == other.tasks and self.graph == other.graph
+
+    @classmethod
+    def from_json(cls, tasks_dict, root):
+        """
+        This code is used to generate the a TaskGraph using a dictionary
+        which is representative of the TaskGraph.
+        """
+        tasks = {}
+        edges = set()
+        for key, value in tasks_dict.iteritems():
+            # We get the implementation from JSON
+            implementation = value['kind_implementation']
+            # Loading the module and creating a Task from a dictionary
+            task_kind = find_object(implementation)
+            tasks[key] = task_kind.from_json(value)
+            if 'task_id' in value:
+                tasks[key].task_id = value['task_id']
+            for depname, dep in value['dependencies'].iteritems():
+                edges.add((key, dep, depname))
+        task_graph = cls(tasks, Graph(set(tasks), edges))
+        return tasks, task_graph
--- a/taskcluster/taskgraph/test/test_decision.py
+++ b/taskcluster/taskgraph/test/test_decision.py
@@ -31,22 +31,24 @@ class TestDecision(unittest.TestCase):
         res = taskgraph.to_json()
 
         self.assertEqual(res, {
             'a': {
                 'label': 'a',
                 'attributes': {'attr': 'a-task', 'kind': 'test'},
                 'task': {},
                 'dependencies': {'edgelabel': 'b'},
+                'kind_implementation': 'taskgraph.test.util:TestTask',
             },
             'b': {
                 'label': 'b',
                 'attributes': {'kind': 'test'},
                 'task': {'task': 'def'},
                 'dependencies': {},
+                'kind_implementation': 'taskgraph.test.util:TestTask',
             }
         })
 
     def test_write_artifact_json(self):
         data = [{'some': 'data'}]
         tmpdir = tempfile.mkdtemp()
         try:
             decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_taskgraph.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..graph import Graph
+from ..kind.docker_image import DockerImageTask
+from ..kind.legacy import LegacyTask
+from ..taskgraph import TaskGraph
+from mozunit import main
+
+
+class TestTargetTasks(unittest.TestCase):
+
+    def test_from_json(self):
+        legacy_dict = {
+            'attributes': {'kind': 'legacy'},
+            'task': {},
+            'dependencies': {},
+            'label': 'a',
+            'kind_implementation': 'taskgraph.kind.legacy:LegacyTask'
+        }
+        graph = TaskGraph(tasks={
+            'a': LegacyTask(kind='legacy',
+                            label='a',
+                            attributes={},
+                            task={},
+                            task_dict=legacy_dict),
+            'b': DockerImageTask(kind='docker-image',
+                                 label='b',
+                                 attributes={},
+                                 task={"routes": []},
+                                 index_paths=[]),
+        }, graph=Graph(nodes={'a', 'b'}, edges=set()))
+
+        tasks, new_graph = TaskGraph.from_json(graph.to_json(), "taskcluster/ci")
+        self.assertEqual(graph, new_graph)
+
+if __name__ == '__main__':
+    main()