Bug 1280231: refactor task kinds to task classes; r=jonasfj
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 27 Jun 2016 22:57:44 +0000
changeset 302981 88119b527dd6a3130d2d53d1c4fde4b8a738ead4
parent 302980 6c3e768ce378d09d21721ed90f13b50363fa5b29
child 302982 fe1dcb3feb9aed3827de44011e6dadf21faf6d2a
push id30354
push usercbook@mozilla.com
push dateWed, 29 Jun 2016 14:23:15 +0000
treeherderautoland@0c56fe904602 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjonasfj
bugs1280231
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1280231: refactor task kinds to task classes; r=jonasfj MozReview-Commit-ID: 1cNukxBgfey
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/legacy/kind.yml
taskcluster/taskgraph/create.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/kind/base.py
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/legacy.py
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/test/test_create.py
taskcluster/taskgraph/test/test_decision.py
taskcluster/taskgraph/test/test_generator.py
taskcluster/taskgraph/test/test_kind_docker_image.py
taskcluster/taskgraph/test/test_kind_legacy.py
taskcluster/taskgraph/test/test_optimize.py
taskcluster/taskgraph/test/test_target_tasks.py
taskcluster/taskgraph/test/test_try_option_syntax.py
taskcluster/taskgraph/test/util.py
taskcluster/taskgraph/types.py
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -1,13 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-implementation: 'taskgraph.kind.docker_image:DockerImageKind'
+implementation: 'taskgraph.kind.docker_image:DockerImageTask'
 images_path: '../../../testing/docker'
 
 # make a task for each docker-image we might want.  For the moment, since we
 # write artifacts for each, these are whitelisted, but ideally that will change
 # (to use subdirectory clones of the proper directory), at which point we can
 # generate tasks for every docker image in the directory, secure in the
 # knowledge that unnecessary images will be omitted from the target task graph
 images:
--- a/taskcluster/ci/legacy/kind.yml
+++ b/taskcluster/ci/legacy/kind.yml
@@ -1,6 +1,6 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-implementation: 'taskgraph.kind.legacy:LegacyKind'
+implementation: 'taskgraph.kind.legacy:LegacyTask'
 legacy_path: '.'
--- a/taskcluster/taskgraph/create.py
+++ b/taskcluster/taskgraph/create.py
@@ -44,17 +44,18 @@ def create_tasks(taskgraph, label_to_tas
             # task so that it does not start immediately; and so that if this loop
             # fails halfway through, none of the already-created tasks run.
             if decision_task_id and not task_def.get('dependencies'):
                 task_def['dependencies'] = [decision_task_id]
 
             task_def['taskGroupId'] = task_group_id
 
             # Wait for dependencies before submitting this.
-            deps_fs = [fs[dep] for dep in task_def['dependencies'] if dep in fs]
+            deps_fs = [fs[dep] for dep in task_def.get('dependencies', [])
+                       if dep in fs]
             for f in futures.as_completed(deps_fs):
                 f.result()
 
             fs[task_id] = e.submit(_create_task, session, task_id,
                                    taskid_to_label[task_id], task_def)
 
         # Wait for all futures to complete.
         for f in futures.as_completed(fs.values()):
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -110,18 +110,18 @@ class TaskGraphGenerator(object):
         """
         return self._run_until('label_to_taskid')
 
     def _load_kinds(self):
         for path in os.listdir(self.root_dir):
             path = os.path.join(self.root_dir, path)
             if not os.path.isdir(path):
                 continue
-            name = os.path.basename(path)
-            logger.debug("loading kind `{}` from `{}`".format(name, path))
+            kind_name = os.path.basename(path)
+            logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
 
             kind_yml = os.path.join(path, 'kind.yml')
             with open(kind_yml) as f:
                 config = yaml.load(f)
 
             # load the class defined by implementation
             try:
                 impl = config['implementation']
@@ -133,34 +133,34 @@ class TaskGraphGenerator(object):
 
             impl_module, impl_object = impl.split(':')
             impl_class = __import__(impl_module)
             for a in impl_module.split('.')[1:]:
                 impl_class = getattr(impl_class, a)
             for a in impl_object.split('.'):
                 impl_class = getattr(impl_class, a)
 
-            yield impl_class(path, config)
+            for task in impl_class.load_tasks(kind_name, path, config, self.parameters):
+                yield task
 
     def _run(self):
         logger.info("Generating full task set")
         all_tasks = {}
-        for kind in self._load_kinds():
-            for task in kind.load_tasks(self.parameters):
-                if task.label in all_tasks:
-                    raise Exception("duplicate tasks with label " + task.label)
-                all_tasks[task.label] = task
+        for task in self._load_kinds():
+            if task.label in all_tasks:
+                raise Exception("duplicate tasks with label " + task.label)
+            all_tasks[task.label] = task
 
         full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
         yield 'full_task_set', full_task_set
 
         logger.info("Generating full task graph")
         edges = set()
         for t in full_task_set:
-            for dep, depname in t.kind.get_task_dependencies(t, full_task_set):
+            for dep, depname in t.get_dependencies(full_task_set):
                 edges.add((t.label, dep, depname))
 
         full_task_graph = TaskGraph(all_tasks,
                                     Graph(full_task_set.graph.nodes, edges))
         yield 'full_task_graph', full_task_graph
 
         logger.info("Generating target task set")
         target_tasks = set(self.target_tasks_method(full_task_graph, self.parameters))
--- a/taskcluster/taskgraph/kind/base.py
+++ b/taskcluster/taskgraph/kind/base.py
@@ -1,53 +1,82 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import os
 import abc
 
 
-class Kind(object):
+class Task(object):
     """
+    Representation of a task in a TaskGraph.  Each Task has, at creation:
+
+    - kind: the name of the task kind
+    - label; the label for this task
+    - attributes: a dictionary of attributes for this task (used for filtering)
+    - task: the task definition (JSON-able dictionary)
+
+    And later, as the task-graph processing proceeds:
+
+    - task_id -- TaskCluster taskId under which this task will be created
+    - optimized -- true if this task need not be performed
+
     A kind represents a collection of tasks that share common characteristics.
     For example, all build jobs.  Each instance of a kind is intialized with a
     path from which it draws its task configuration.  The instance is free to
     store as much local state as it needs.
     """
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, path, config):
-        self.name = os.path.basename(path)
-        self.path = path
-        self.config = config
+    def __init__(self, kind, label, attributes, task):
+        self.kind = kind
+        self.label = label
+        self.attributes = attributes
+        self.task = task
+
+        self.task_id = None
+        self.optimized = False
+
+        self.attributes['kind'] = kind
 
+        if not (all(isinstance(x, basestring) for x in self.attributes.iterkeys()) and
+                all(isinstance(x, basestring) for x in self.attributes.itervalues())):
+            raise TypeError("attribute names and values must be strings")
+
+    @classmethod
     @abc.abstractmethod
-    def load_tasks(self, parameters):
+    def load_tasks(cls, kind, path, config, parameters):
         """
-        Get the set of tasks of this kind.
+        Load the tasks for a given kind.
+
+        The `kind` is the name of the kind; the configuration for that kind
+        named this class.
+
+        The `path` is the path to the configuration directory for the kind.  This
+        can be used to load extra data, templates, etc.
 
         The `parameters` give details on which to base the task generation.
         See `taskcluster/docs/parameters.rst` for details.
 
         The return value is a list of Task instances.
         """
 
     @abc.abstractmethod
-    def get_task_dependencies(self, task, taskgraph):
+    def get_dependencies(self, taskgraph):
         """
-        Get the set of task labels this task depends on, by querying the task graph.
+        Get the set of task labels this task depends on, by querying the full
+        task set, given as `taskgraph`.
 
         Returns a list of (task_label, dependency_name) pairs describing the
         dependencies.
         """
 
-    def optimize_task(self, task):
+    def optimize(self):
         """
         Determine whether this task can be optimized, and if it can, what taskId
         it should be replaced with.
 
         The return value is a tuple `(optimized, taskId)`.  If `optimized` is
         true, then the task will be optimized (in other words, not included in
         the task graph).  If the second argument is a taskid, then any
         dependencies on this task will isntead depend on that taskId.  It is an
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -7,36 +7,40 @@ from __future__ import absolute_import, 
 import logging
 import json
 import os
 import urllib2
 import tarfile
 import time
 
 from . import base
-from ..types import Task
 from taskgraph.util.docker import (
     docker_image,
     generate_context_hash
 )
 from taskgraph.util.templates import Templates
 from taskgraph.util.time import (
     json_time_from_now,
     current_json_time,
 )
 
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
 
 
-class DockerImageKind(base.Kind):
+class DockerImageTask(base.Task):
 
-    def load_tasks(self, params):
+    def __init__(self, *args, **kwargs):
+        self.index_paths = kwargs.pop('index_paths')
+        super(DockerImageTask, self).__init__(*args, **kwargs)
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, params):
         # TODO: make this match the pushdate (get it from a parameter rather than vcs)
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
 
         parameters = {
             'pushlog_id': params.get('pushlog_id', 0),
             'pushdate': pushdate,
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
@@ -52,18 +56,18 @@ class DockerImageKind(base.Kind):
             'level': params['level'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
             'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
                       .format(repo=params['head_repository'], rev=params['head_rev']),
         }
 
         tasks = []
-        templates = Templates(self.path)
-        for image_name in self.config['images']:
+        templates = Templates(path)
+        for image_name in config['images']:
             context_path = os.path.join('testing', 'docker', image_name)
             context_hash = generate_context_hash(context_path)
 
             image_parameters = dict(parameters)
             image_parameters['context_hash'] = context_hash
             image_parameters['context_path'] = context_path
             image_parameters['artifact_path'] = 'public/image.tar'
             image_parameters['image_name'] = image_name
@@ -71,50 +75,47 @@ class DockerImageKind(base.Kind):
             image_artifact_path = \
                 "public/decision_task/image_contexts/{}/context.tar.gz".format(image_name)
             if os.environ.get('TASK_ID'):
                 destination = os.path.join(
                     os.environ['HOME'],
                     "artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
                 image_parameters['context_url'] = ARTIFACT_URL.format(
                     os.environ['TASK_ID'], image_artifact_path)
-                self.create_context_tar(context_path, destination, image_name)
+                cls.create_context_tar(context_path, destination, image_name)
             else:
                 # skip context generation since this isn't a decision task
                 # TODO: generate context tarballs using subdirectory clones in
                 # the image-building task so we don't have to worry about this.
                 image_parameters['context_url'] = 'file:///tmp/' + image_artifact_path
 
             image_task = templates.load('image.yml', image_parameters)
 
-            attributes = {
-                'kind': self.name,
-                'image_name': image_name,
-            }
+            attributes = {'image_name': image_name}
 
             # As an optimization, if the context hash exists for mozilla-central, that image
             # task ID will be used.  The reasoning behind this is that eventually everything ends
             # up on mozilla-central at some point if most tasks use this as a common image
             # for a given context hash, a worker within Taskcluster does not need to contain
             # the same image per branch.
             index_paths = ['docker.images.v1.{}.{}.hash.{}'.format(
                                 project, image_name, context_hash)
                            for project in ['mozilla-central', params['project']]]
 
-            tasks.append(Task(self, 'build-docker-image-' + image_name,
-                              task=image_task['task'], attributes=attributes,
-                              index_paths=index_paths))
+            tasks.append(cls(kind, 'build-docker-image-' + image_name,
+                             task=image_task['task'], attributes=attributes,
+                             index_paths=index_paths))
 
         return tasks
 
-    def get_task_dependencies(self, task, taskgraph):
+    def get_dependencies(self, taskgraph):
         return []
 
-    def optimize_task(self, task, taskgraph):
-        for index_path in task.extra['index_paths']:
+    def optimize(self):
+        for index_path in self.index_paths:
             try:
                 url = INDEX_URL.format(index_path)
                 existing_task = json.load(urllib2.urlopen(url))
                 # Only return the task ID if the artifact exists for the indexed
                 # task.  Otherwise, continue on looking at each of the branches.  Method
                 # continues trying other branches in case mozilla-central has an expired
                 # artifact, but 'project' might not. Only return no task ID if all
                 # branches have been tried
@@ -125,16 +126,17 @@ class DockerImageKind(base.Kind):
 
                 # HEAD success on the artifact is enough
                 return True, existing_task['taskId']
             except urllib2.HTTPError:
                 pass
 
         return False, None
 
-    def create_context_tar(self, context_dir, destination, image_name):
+    @classmethod
+    def create_context_tar(cls, context_dir, destination, image_name):
         'Creates a tar file of a particular context directory.'
         destination = os.path.abspath(destination)
         if not os.path.exists(os.path.dirname(destination)):
             os.makedirs(os.path.dirname(destination))
 
         with tarfile.open(destination, 'w:gz') as tar:
             tar.add(context_dir, arcname=image_name)
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -8,17 +8,16 @@ import copy
 import json
 import logging
 import os
 import re
 import time
 from collections import namedtuple
 
 from . import base
-from ..types import Task
 from mozpack.path import match as mozpackmatch
 from slugid import nice as slugid
 from taskgraph.util.legacy_commit_parser import parse_commit
 from taskgraph.util.time import (
     json_time_from_now,
     current_json_time,
 )
 from taskgraph.util.templates import Templates
@@ -66,20 +65,20 @@ def gaia_info():
 
     if gaia['git'] is None or \
        gaia['git']['remote'] == '' or \
        gaia['git']['git_revision'] == '' or \
        gaia['git']['branch'] == '':
 
         # Just use the hg params...
         return {
-          'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
-          'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
-          'gaia_ref': gaia['revision'],
-          'gaia_rev': gaia['revision']
+            'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+            'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+            'gaia_ref': gaia['revision'],
+            'gaia_rev': gaia['revision']
         }
 
     else:
         # Use git
         return {
             'gaia_base_repository': gaia['git']['remote'],
             'gaia_head_repository': gaia['git']['remote'],
             'gaia_rev': gaia['git']['git_revision'],
@@ -287,28 +286,33 @@ def validate_build_task(task):
         if 'build' not in locations:
             raise BuildTaskValidationException('task.extra.locations.build missing')
 
         if 'tests' not in locations and 'test_packages' not in locations:
             raise BuildTaskValidationException('task.extra.locations.tests or '
                                                'task.extra.locations.tests_packages missing')
 
 
-class LegacyKind(base.Kind):
+class LegacyTask(base.Task):
     """
     This kind generates a full task graph from the old YAML files in
     `testing/taskcluster/tasks`.  The tasks already have dependency links.
 
     The existing task-graph generation generates slugids for tasks during task
     generation, so this kind labels tasks using those slugids, with a prefix of
     "TaskLabel==".  These labels are unfortunately not stable from run to run.
     """
 
-    def load_tasks(self, params):
-        root = os.path.abspath(os.path.join(self.path, self.config['legacy_path']))
+    def __init__(self, *args, **kwargs):
+        self.task_dict = kwargs.pop('task_dict')
+        super(LegacyTask, self).__init__(*args, **kwargs)
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, params):
+        root = os.path.abspath(os.path.join(path, config['legacy_path']))
 
         project = params['project']
         # NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
         # resulting task graph later
         message = DEFAULT_TRY
 
         templates = Templates(root)
 
@@ -378,17 +382,17 @@ class LegacyKind(base.Kind):
             route = format_treeherder_route(TREEHERDER_ROUTES[env],
                                             parameters['project'],
                                             parameters['head_rev'],
                                             parameters['pushlog_id'])
             graph['scopes'].add("queue:route:{}".format(route))
 
         graph['metadata'] = {
             'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(
-                        repo=params['head_repository'], rev=params['head_rev']),
+                repo=params['head_repository'], rev=params['head_rev']),
             'owner': params['owner'],
             # TODO: Add full mach commands to this example?
             'description': 'Task graph generated via ./mach taskcluster-graph',
             'name': 'task graph local'
         }
 
         # Filter the job graph according to conditions met by this invocation run.
         def should_run(task):
@@ -492,17 +496,17 @@ class LegacyKind(base.Kind):
                             all_routes[route],
                         ))
                 all_routes[route] = build_task['task']['metadata']['name']
 
             graph['scopes'].add(define_task)
             graph['scopes'] |= set(build_task['task'].get('scopes', []))
             route_scopes = map(
                 lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])
-                )
+            )
             graph['scopes'] |= set(route_scopes)
 
             # Treeherder symbol configuration for the graph required for each
             # build so tests know which platform they belong to.
             build_treeherder_config = build_task['task']['extra']['treeherder']
 
             if 'machine' not in build_treeherder_config:
                 message = '({}), extra.treeherder.machine required for all builds'
@@ -608,33 +612,30 @@ class LegacyKind(base.Kind):
                         test_task['task']['workerType']
                     )
 
                     graph['scopes'].add(define_task)
                     graph['scopes'] |= set(test_task['task'].get('scopes', []))
 
         graph['scopes'] = sorted(graph['scopes'])
 
-        # save the graph for later, when taskgraph asks for additional information
-        # such as dependencies
-        self.graph = graph
-        self.tasks_by_label = {t['taskId']: t for t in self.graph['tasks']}
-
         # Convert to a dictionary of tasks.  The process above has invented a
         # taskId for each task, and we use those as the *labels* for the tasks;
         # taskgraph will later assign them new taskIds.
-        return [Task(self, t['taskId'], task=t['task'], attributes=t['attributes'])
-                for t in self.graph['tasks']]
+        return [
+            cls(kind, t['taskId'], task=t['task'], attributes=t['attributes'], task_dict=t)
+            for t in graph['tasks']
+        ]
 
-    def get_task_dependencies(self, task, taskgraph):
+    def get_dependencies(self, taskgraph):
         # fetch dependency information from the cached graph
-        taskdict = self.tasks_by_label[task.label]
-        deps = [(label, label) for label in taskdict.get('requires', [])]
+        deps = [(label, label) for label in self.task_dict.get('requires', [])]
 
         # add a dependency on an image task, if needed
-        if 'docker-image' in taskdict:
-            deps.append(('build-docker-image-{docker-image}'.format(**taskdict), 'docker-image'))
+        if 'docker-image' in self.task_dict:
+            deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
+                         'docker-image'))
 
         return deps
 
-    def optimize_task(self, task, taskgraph):
+    def optimize(self):
         # no optimization for the moment
         return False, None
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -83,17 +83,17 @@ def annotate_task_graph(target_task_grap
         replacement_task_id = None
         if label in do_not_optimize:
             optimized = False
         # if any dependencies can't be optimized, this task can't, either
         elif any(not t.optimized for t in dependencies):
             optimized = False
         # otherwise, examine the task itself (which may be an expensive operation)
         else:
-            optimized, replacement_task_id = task.kind.optimize_task(task, named_task_dependencies)
+            optimized, replacement_task_id = task.optimize()
 
         task.optimized = optimized
         task.task_id = replacement_task_id
         if replacement_task_id:
             label_to_taskid[label] = replacement_task_id
 
         if optimized:
             if replacement_task_id:
--- a/taskcluster/taskgraph/test/test_create.py
+++ b/taskcluster/taskgraph/test/test_create.py
@@ -4,30 +4,22 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 import os
 
 from .. import create
 from ..graph import Graph
-from ..types import Task, TaskGraph
+from ..types import TaskGraph
+from .util import TestTask
 
 from mozunit import main
 
 
-class FakeKind(object):
-
-    def get_task_definition(self, task, deps_by_name):
-        # sanity-check the deps_by_name
-        for k, v in deps_by_name.iteritems():
-            assert k == 'edge'
-        return {'payload': 'hello world'}
-
-
 class TestCreate(unittest.TestCase):
 
     def setUp(self):
         self.old_task_id = os.environ.get('TASK_ID')
         if 'TASK_ID' in os.environ:
             del os.environ['TASK_ID']
         self.created_tasks = {}
         self.old_create_task = create._create_task
@@ -39,21 +31,19 @@ class TestCreate(unittest.TestCase):
             os.environ['TASK_ID'] = self.old_task_id
         elif 'TASK_ID' in os.environ:
             del os.environ['TASK_ID']
 
     def fake_create_task(self, session, task_id, label, task_def):
         self.created_tasks[task_id] = task_def
 
     def test_create_tasks(self):
-        os.environ['TASK_ID'] = 'decisiontask'
-        kind = FakeKind()
         tasks = {
-            'tid-a': Task(kind=kind, label='a', task={'payload': 'hello world'}),
-            'tid-b': Task(kind=kind, label='b', task={'payload': 'hello world'}),
+            'tid-a': TestTask(label='a', task={'payload': 'hello world'}),
+            'tid-b': TestTask(label='b', task={'payload': 'hello world'}),
         }
         label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
         graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')})
         taskgraph = TaskGraph(tasks, graph)
 
         create.create_tasks(taskgraph, label_to_taskid)
 
         for tid, task in self.created_tasks.iteritems():
@@ -63,24 +53,23 @@ class TestCreate(unittest.TestCase):
                 if depid is 'decisiontask':
                     # Don't look for decisiontask here
                     continue
                 self.assertIn(depid, self.created_tasks)
 
     def test_create_task_without_dependencies(self):
         "a task with no dependencies depends on the decision task"
         os.environ['TASK_ID'] = 'decisiontask'
-        kind = FakeKind()
         tasks = {
-            'tid-a': Task(kind=kind, label='a', task={'payload': 'hello world'}),
+            'tid-a': TestTask(label='a', task={'payload': 'hello world'}),
         }
         label_to_taskid = {'a': 'tid-a'}
         graph = Graph(nodes={'tid-a'}, edges=set())
         taskgraph = TaskGraph(tasks, graph)
 
         create.create_tasks(taskgraph, label_to_taskid)
 
         for tid, task in self.created_tasks.iteritems():
-            self.assertEqual(task['dependencies'], [os.environ['TASK_ID']])
+            self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
 
 
 if __name__ == '__main__':
     main()
--- a/taskcluster/taskgraph/test/test_decision.py
+++ b/taskcluster/taskgraph/test/test_decision.py
@@ -8,42 +8,43 @@ import os
 import json
 import yaml
 import shutil
 import unittest
 import tempfile
 
 from .. import decision
 from ..graph import Graph
-from ..types import Task, TaskGraph
+from ..types import TaskGraph
+from .util import TestTask
 from mozunit import main
 
 
 class TestDecision(unittest.TestCase):
 
     def test_taskgraph_to_json(self):
         tasks = {
-            'a': Task(kind=None, label='a', attributes={'attr': 'a-task'}),
-            'b': Task(kind=None, label='b', task={'task': 'def'}),
+            'a': TestTask(label='a', attributes={'attr': 'a-task'}),
+            'b': TestTask(label='b', task={'task': 'def'}),
         }
         graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
         taskgraph = TaskGraph(tasks, graph)
 
         res = taskgraph.to_json()
 
         self.assertEqual(res, {
             'a': {
                 'label': 'a',
-                'attributes': {'attr': 'a-task'},
+                'attributes': {'attr': 'a-task', 'kind': 'test'},
                 'task': {},
                 'dependencies': {'edgelabel': 'b'},
             },
             'b': {
                 'label': 'b',
-                'attributes': {},
+                'attributes': {'kind': 'test'},
                 'task': {'task': 'def'},
                 'dependencies': {},
             }
         })
 
     def test_write_artifact_json(self):
         data = [{'some': 'data'}]
         tmpdir = tempfile.mkdtemp()
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -2,60 +2,61 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
 from ..generator import TaskGraphGenerator
-from .. import types
 from .. import graph
+from ..kind import base
 from mozunit import main
 
 
-class FakeKind(object):
+class FakeTask(base.Task):
+
+    def __init__(self, **kwargs):
+        self.i = kwargs.pop('i')
+        super(FakeTask, self).__init__(**kwargs)
 
-    def maketask(self, i):
-        return types.Task(
-            self,
-            label='t-{}'.format(i),
-            attributes={'tasknum': str(i)},
-            task={},
-            i=i)
+    @classmethod
+    def load_tasks(cls, kind, path, config, parameters):
+        return [cls(kind=kind,
+                    label='t-{}'.format(i),
+                    attributes={'tasknum': str(i)},
+                    task={},
+                    i=i)
+                for i in range(3)]
 
-    def load_tasks(self, parameters):
-        self.tasks = [self.maketask(i) for i in range(3)]
-        return self.tasks
-
-    def get_task_dependencies(self, task, full_task_set):
-        i = task.extra['i']
+    def get_dependencies(self, full_task_set):
+        i = self.i
         if i > 0:
             return [('t-{}'.format(i - 1), 'prev')]
         else:
             return []
 
-    def optimize_task(self, task, dependencies):
+    def optimize(self):
         return False, None
 
 
-class WithFakeKind(TaskGraphGenerator):
+class WithFakeTask(TaskGraphGenerator):
 
     def _load_kinds(self):
-        yield FakeKind()
+        return FakeTask.load_tasks('fake', '/fake', {}, {})
 
 
 class TestGenerator(unittest.TestCase):
 
     def setUp(self):
         self.target_tasks = []
 
         def target_tasks_method(full_task_graph, parameters):
             return self.target_tasks
-        self.tgg = WithFakeKind('/root', {}, target_tasks_method)
+        self.tgg = WithFakeTask('/root', {}, target_tasks_method)
 
     def test_full_task_set(self):
         "The full_task_set property has all tasks"
         self.assertEqual(self.tgg.full_task_set.graph,
                          graph.Graph({'t-0', 't-1', 't-2'}, set()))
         self.assertEqual(self.tgg.full_task_set.tasks.keys(),
                          ['t-0', 't-1', 't-2'])
 
--- a/taskcluster/taskgraph/test/test_kind_docker_image.py
+++ b/taskcluster/taskgraph/test/test_kind_docker_image.py
@@ -7,30 +7,36 @@ from __future__ import absolute_import, 
 import unittest
 import tempfile
 import os
 
 from ..kind import docker_image
 from mozunit import main
 
 
+KIND_PATH = os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image')
+
+
 class TestDockerImageKind(unittest.TestCase):
 
     def setUp(self):
-        self.kind = docker_image.DockerImageKind(
-                os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image'),
-                {})
+        self.task = docker_image.DockerImageTask(
+            'docker-image',
+            KIND_PATH,
+            {},
+            {},
+            index_paths=[])
 
     def test_get_task_dependencies(self):
         # this one's easy!
-        self.assertEqual(self.kind.get_task_dependencies(None, None), [])
+        self.assertEqual(self.task.get_dependencies(None), [])
 
     # TODO: optimize_task
 
     def test_create_context_tar(self):
         image_dir = os.path.join(docker_image.GECKO, 'testing', 'docker', 'image_builder')
         tarball = tempfile.mkstemp()[1]
-        self.kind.create_context_tar(image_dir, tarball, 'image_builder')
+        self.task.create_context_tar(image_dir, tarball, 'image_builder')
         self.failUnless(os.path.exists(tarball))
         os.unlink(tarball)
 
 if __name__ == '__main__':
     main()
--- a/taskcluster/taskgraph/test/test_kind_legacy.py
+++ b/taskcluster/taskgraph/test/test_kind_legacy.py
@@ -2,32 +2,22 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
 from ..kind.legacy import (
-    LegacyKind,
     validate_build_task,
     BuildTaskValidationException
 )
 from mozunit import main
 
 
-class TestLegacyKind(unittest.TestCase):
-    # NOTE: much of LegacyKind is copy-pasted from the old legacy code, which
-    # is emphatically *not* designed for testing, so this test class does not
-    # attempt to test the entire class.
-
-    def setUp(self):
-        self.kind = LegacyKind('/root', {})
-
-
 class TestValidateBuildTask(unittest.TestCase):
 
     def test_validate_missing_extra(self):
         with self.assertRaises(BuildTaskValidationException):
             validate_build_task({})
 
     def test_validate_valid(self):
         with self.assertRaises(BuildTaskValidationException):
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -5,16 +5,17 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
 from ..optimize import optimize_task_graph, resolve_task_references
 from ..optimize import annotate_task_graph, get_subgraph
 from .. import types
 from .. import graph
+from .util import TestTask
 
 
 class TestResolveTaskReferences(unittest.TestCase):
 
     def do(self, input, output):
         taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
         self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name), output)
 
@@ -44,130 +45,127 @@ class TestResolveTaskReferences(unittest
                 {'escape': '<tid3>'})
 
     def test_invalid(self):
         "resolve_task_references raises a KeyError on reference to an invalid task"
         self.assertRaisesRegexp(
             KeyError,
             "task 'subject' has no dependency with label 'no-such'",
             lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {})
-            )
+        )
 
 
-class FakeKind(object):
-
-    def __init__(self, optimize_task):
-        self.optimize_task = optimize_task
+class OptimizingTask(TestTask):
+    # the `optimize` method on this class is overridden direclty in the tests
+    # below.
+    pass
 
 
 class TestOptimize(unittest.TestCase):
 
     kind = None
 
-    def make_kind(self, optimize_task):
-        self.kind = FakeKind(optimize_task)
-
     def make_task(self, label, task_def=None, optimized=None, task_id=None):
         task_def = task_def or {'sample': 'task-def'}
-        task = types.Task(self.kind, label=label, task=task_def)
+        task = OptimizingTask(label=label, task=task_def)
         task.optimized = optimized
         task.task_id = task_id
         return task
 
     def make_graph(self, *tasks_and_edges):
-        tasks = {t.label: t for t in tasks_and_edges if isinstance(t, types.Task)}
-        edges = {e for e in tasks_and_edges if not isinstance(e, types.Task)}
+        tasks = {t.label: t for t in tasks_and_edges if isinstance(t, OptimizingTask)}
+        edges = {e for e in tasks_and_edges if not isinstance(e, OptimizingTask)}
         return types.TaskGraph(tasks, graph.Graph(set(tasks), edges))
 
     def assert_annotations(self, graph, **annotations):
         def repl(task_id):
             return 'SLUGID' if task_id and len(task_id) == 22 else task_id
         got_annotations = {
             t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
-            }
+        }
         self.assertEqual(got_annotations, annotations)
 
     def test_annotate_task_graph_no_optimize(self):
         "annotating marks everything as un-optimized if the kind returns that"
-        self.make_kind(lambda task, deps: (False, None))
+        OptimizingTask.optimize = lambda self: (False, None)
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
         annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None),
             task3=(False, None)
-            )
+        )
 
     def test_annotate_task_graph_taskid_without_optimize(self):
         "raises exception if kind returns a taskid without optimizing"
-        self.make_kind(lambda task, deps: (False, 'some-taskid'))
+        OptimizingTask.optimize = lambda self: (False, 'some-taskid')
         graph = self.make_graph(self.make_task('task1'))
         self.assertRaises(
             Exception,
             lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
-            )
+        )
 
     def test_annotate_task_graph_optimize_away_dependency(self):
         "raises exception if kind optimizes away a task on which another depends"
-        self.make_kind(lambda task, deps: (True, None) if task.label == 'task1' else (False, None))
+        OptimizingTask.optimize = \
+            lambda self: (True, None) if self.label == 'task1' else (False, None)
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             ('task2', 'task1', 'build'),
         )
         self.assertRaises(
             Exception,
             lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
-            )
+        )
 
     def test_annotate_task_graph_do_not_optimize(self):
         "annotating marks everything as un-optimized if in do_not_optimize"
-        self.make_kind(lambda task, deps: (True, 'taskid'))
+        OptimizingTask.optimize = lambda self: (True, 'taskid')
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             ('task2', 'task1', 'build'),
         )
         label_to_taskid = {}
         annotate_task_graph(graph, {'task1', 'task2'},
                             graph.graph.named_links_dict(), label_to_taskid)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None)
-            )
+        )
         self.assertEqual
 
     def test_annotate_task_graph_nos_propagate(self):
         "annotating marks a task with a non-optimized dependency as non-optimized"
-        self.make_kind(
-            lambda task, deps: (False, None) if task.label == 'task1' else (True, 'taskid')
-            )
+        OptimizingTask.optimize = \
+            lambda self: (False, None) if self.label == 'task1' else (True, 'taskid')
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
         annotate_task_graph(graph, set(),
                             graph.graph.named_links_dict(), {})
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None),  # kind would have returned (True, 'taskid') here
             task3=(True, 'taskid')
-            )
+        )
 
     def test_get_subgraph_single_dep(self):
         "when a single dependency is optimized, it is omitted from the graph"
         graph = self.make_graph(
             self.make_task('task1', optimized=True, task_id='dep1'),
             self.make_task('task2', optimized=False),
             self.make_task('task3', optimized=False),
             ('task2', 'task1', 'build'),
@@ -220,17 +218,17 @@ class TestOptimize(unittest.TestCase):
     def test_get_subgraph_refs_resolved(self):
         "get_subgraph resolves task references"
         graph = self.make_graph(
             self.make_task('task1', optimized=True, task_id='dep1'),
             self.make_task(
                 'task2',
                 optimized=False,
                 task_def={'payload': {'task-reference': 'http://<build>/<test>'}}
-                ),
+            ),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'test'),
             self.make_task('task3', optimized=False),
         )
         label_to_taskid = {'task1': 'dep1'}
         sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid)
         task2 = label_to_taskid['task2']
         task3 = label_to_taskid['task3']
@@ -238,19 +236,18 @@ class TestOptimize(unittest.TestCase):
         self.assertEqual(sub.graph.edges, {(task2, task3, 'test')})
         self.assertEqual(sub.tasks[task2].task_id, task2)
         self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1']))
         self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3)
         self.assertEqual(sub.tasks[task3].task_id, task3)
 
     def test_optimize(self):
         "optimize_task_graph annotates and extracts the subgraph from a simple graph"
-        self.make_kind(
-            lambda task, deps: (True, 'dep1') if task.label == 'task1' else (False, None)
-            )
+        OptimizingTask.optimize = \
+            lambda self: (True, 'dep1') if self.label == 'task1' else (False, None)
         input = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
         opt, label_to_taskid = optimize_task_graph(input, set())
--- a/taskcluster/taskgraph/test/test_target_tasks.py
+++ b/taskcluster/taskgraph/test/test_target_tasks.py
@@ -4,17 +4,18 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
 from .. import target_tasks
 from .. import try_option_syntax
 from ..graph import Graph
-from ..types import Task, TaskGraph
+from ..types import TaskGraph
+from .util import TestTask
 from mozunit import main
 
 
 class FakeTryOptionSyntax(object):
 
     def __init__(self, message, task_graph):
         pass
 
@@ -27,26 +28,26 @@ class TestTargetTasks(unittest.TestCase)
     def test_from_parameters(self):
         method = target_tasks.get_method('from_parameters')
         self.assertEqual(method(None, {'target_tasks': ['a', 'b']}),
                          ['a', 'b'])
 
     def test_all_builds_and_tests(self):
         method = target_tasks.get_method('all_builds_and_tests')
         graph = TaskGraph(tasks={
-            'a': Task(kind=None, label='a', attributes={'kind': 'legacy'}),
-            'b': Task(kind=None, label='b', attributes={'kind': 'legacy'}),
-            'boring': Task(kind=None, label='boring', attributes={'kind': 'docker-image'}),
+            'a': TestTask(kind='legacy', label='a'),
+            'b': TestTask(kind='legacy', label='b'),
+            'boring': TestTask(kind='docker', label='boring'),
         }, graph=Graph(nodes={'a', 'b', 'boring'}, edges=set()))
         self.assertEqual(sorted(method(graph, {})), sorted(['a', 'b']))
 
     def test_try_option_syntax(self):
         tasks = {
-            'a': Task(kind=None, label='a'),
-            'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}),
+            'a': TestTask(kind=None, label='a'),
+            'b': TestTask(kind=None, label='b', attributes={'at-at': 'yep'}),
         }
         graph = Graph(nodes=set('ab'), edges=set())
         tg = TaskGraph(tasks, graph)
         params = {'message': 'try me'}
 
         orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
         try:
             try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
--- a/taskcluster/taskgraph/test/test_try_option_syntax.py
+++ b/taskcluster/taskgraph/test/test_try_option_syntax.py
@@ -3,32 +3,33 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
 from ..try_option_syntax import TryOptionSyntax
 from ..graph import Graph
-from ..types import TaskGraph, Task
+from ..types import TaskGraph
+from .util import TestTask
 from mozunit import main
 
 # an empty graph, for things that don't look at it
 empty_graph = TaskGraph({}, Graph(set(), set()))
 
 
 def unittest_task(n, tp):
-    return (n, Task('test', n, {
+    return (n, TestTask('test', n, {
         'unittest_try_name': n,
         'test_platform': tp,
     }))
 
 
 def talos_task(n, tp):
-    return (n, Task('test', n, {
+    return (n, TestTask('test', n, {
         'talos_try_name': n,
         'test_platform': tp,
     }))
 
 tasks = {k: v for k, v in [
     unittest_task('mochitest-browser-chrome', 'linux'),
     unittest_task('mochitest-browser-chrome-e10s', 'linux64'),
     unittest_task('mochitest-chrome', 'linux'),
@@ -253,18 +254,18 @@ class TestTryOptionSyntax(unittest.TestC
     def test_t_single(self):
         "-t mochitest-webgl sets talos=[mochitest-webgl]"
         tos = TryOptionSyntax('try: -t mochitest-webgl', graph_with_jobs)
         self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}]))
 
     # -t shares an implementation with -u, so it's not tested heavily
 
     def test_trigger_tests(self):
-        "--trigger-tests 10 sets trigger_tests"
-        tos = TryOptionSyntax('try: --trigger-tests 10', empty_graph)
+        "--rebuild 10 sets trigger_tests"
+        tos = TryOptionSyntax('try: --rebuild 10', empty_graph)
         self.assertEqual(tos.trigger_tests, 10)
 
     def test_interactive(self):
         "--interactive sets interactive"
         tos = TryOptionSyntax('try: --interactive', empty_graph)
         self.assertEqual(tos.interactive, True)
 
 if __name__ == '__main__':
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/util.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from ..kind import base
+
+
+class TestTask(base.Task):
+
+    def __init__(self, kind=None, label=None, attributes=None, task=None):
+        super(TestTask, self).__init__(
+                kind or 'test',
+                label or 'test-label',
+                attributes or {},
+                task or {})
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, parameters):
+        return []
+
+    def get_dependencies(self, taskgraph):
+        return []
--- a/taskcluster/taskgraph/types.py
+++ b/taskcluster/taskgraph/types.py
@@ -1,51 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 
-class Task(object):
-    """
-    Representation of a task in a TaskGraph.
-
-    Each has, at creation:
-
-    - kind: Kind instance that created this task
-    - label; the label for this task
-    - attributes: a dictionary of attributes for this task (used for filtering)
-    - task: the task definition (JSON-able dictionary)
-    - extra: extra kind-specific metadata
-
-    And later, as the task-graph processing proceeds:
-
-    - optimization_key -- key for finding equivalent tasks in the TC index
-    - task_id -- TC taskId under which this task will be created
-    """
-
-    def __init__(self, kind, label, attributes=None, task=None, **extra):
-        self.kind = kind
-        self.label = label
-        self.attributes = attributes or {}
-        self.task = task or {}
-        self.extra = extra
-
-        self.task_id = None
-
-        if not (all(isinstance(x, basestring) for x in self.attributes.iterkeys()) and
-                all(isinstance(x, basestring) for x in self.attributes.itervalues())):
-            raise TypeError("attribute names and values must be strings")
-
-    def __str__(self):
-        return "{} ({})".format(self.task_id or self.label,
-                                self.task['metadata']['description'].strip())
-
-
 class TaskGraph(object):
     """
     Representation of a task graph.
 
     A task graph is a combination of a Graph and a dictionary of tasks indexed
     by label.  TaskGraph instances should be treated as immutable.
     """