Bug 1286075: rename taskgraph.transforms.make_task; r=wcosta
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 31 Aug 2016 15:24:54 +0000
changeset 354981 979cbd8c03c0f022bcd7b530c826dd0419cb0b34
parent 354980 e69476f9ce449ff019c66c9ab09b726f89847ab3
child 354982 f6ab21ff2b5b78eeea3d603e48e333d3e576ef5e
push id6570
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:26:13 +0000
treeherdermozilla-beta@f455459b2ae5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswcosta
bugs1286075
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1286075: rename taskgraph.transforms.make_task; r=wcosta Rename to taskgraph.transforms.task. This also adds some Required and Optional declarations to the schema to be explicit, and adjusts the transform to handle treeherder being optional. MozReview-Commit-ID: FuKYayvlwB9
taskcluster/ci/android-test/kind.yml
taskcluster/ci/desktop-test/kind.yml
taskcluster/docs/transforms.rst
taskcluster/taskgraph/transforms/make_task.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/transforms/tests/make_task_description.py
--- a/taskcluster/ci/android-test/kind.yml
+++ b/taskcluster/ci/android-test/kind.yml
@@ -4,9 +4,9 @@ kind-dependencies:
     - legacy
 
 transforms:
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.android_test:transforms
    - taskgraph.transforms.tests.all_kinds:transforms
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.make_task_description:transforms
-   - taskgraph.transforms.make_task:transforms
+   - taskgraph.transforms.task:transforms
--- a/taskcluster/ci/desktop-test/kind.yml
+++ b/taskcluster/ci/desktop-test/kind.yml
@@ -4,9 +4,9 @@ kind-dependencies:
     - legacy
 
 transforms:
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.desktop_test:transforms
    - taskgraph.transforms.tests.all_kinds:transforms
    - taskgraph.transforms.tests.test_description:validate
    - taskgraph.transforms.tests.make_task_description:transforms
-   - taskgraph.transforms.make_task:transforms
+   - taskgraph.transforms.task:transforms
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -76,20 +76,20 @@ using :func:`taskgraph.transform.base.ge
 Task-Generation Transforms
 --------------------------
 
 Every kind needs to create tasks, and all of those tasks have some things in
 common.  They all run on one of a small set of worker implementations, each
 with their own idiosyncracies.  And they all report to TreeHerder in a similar
 way.
 
-The transforms in ``taskcluster/taskgraph/transforms/make_task.py`` implement
+The transforms in ``taskcluster/taskgraph/transforms/task.py`` implement
 this common functionality.  They expect a "task description", and produce a
 task definition.  The schema for a task description is defined at the top of
-``make_task.py``, with copious comments.  The parts of the task description
+``task.py``, with copious comments.  The parts of the task description
 that are specific to a worker implementation are isolated in a ``worker``
 object which has an ``implementation`` property naming the worker
 implementation.  Thus the transforms that produce a task description must be
 aware of the worker implementation to be used, but need not be aware of the
 details of its payload format.
 
 The result is a dictionary with keys ``label``, ``attributes``, ``task``, and
 ``dependencies``, with the latter having the same format as the input
@@ -125,13 +125,13 @@ configuration in ``kind.yml``:
    the "shape" of the test description, and are still governed by the schema in
    ``test_description.py``.
 
  * The ``taskgraph.transforms.tests.make_task_description:transforms`` then
    take the test description and create a *task* description.  This transform
    embodies the specifics of how test runs work: invoking mozharness, various
    worker options, and so on.
 
- * Finally, the ``taskgraph.transforms.make_task:transforms``, described above
+ * Finally, the ``taskgraph.transforms.task:transforms``, described above
    under "Task-Generation Transforms", are applied.
 
 Test dependencies are produced in the form of a dictionary mapping dependency
 name to task label.
rename from taskcluster/taskgraph/transforms/make_task.py
rename to taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/transforms/make_task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -3,63 +3,67 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 These transformations take a task description and turn it into a TaskCluster
 task definition (along with attributes, label, etc.).  The input to these
 transformations is generic to any kind of task, but abstracts away some of the
 complexities of worker implementations, scopes, and treeherder annotations.
 """
 
+from __future__ import absolute_import, print_function, unicode_literals
+
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import (
     validate_schema,
     TransformSequence
 )
 from voluptuous import Schema, Any, Required, Optional, Extra
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring})
 
 # A task description is a general description of a TaskCluster task
 task_description_schema = Schema({
     # the label for this task
-    'label': basestring,
+    Required('label'): basestring,
 
     # description of the task (for metadata)
-    'description': basestring,
+    Required('description'): basestring,
 
     # attributes for this task
-    'attributes': {basestring: object},
+    Optional('attributes'): {basestring: object},
 
     # dependencies of this task, keyed by name; these are passed through
     # verbatim and subject to the interpretation of the Task's get_dependencies
     # method.
-    'dependencies': {basestring: object},
+    Optional('dependencies'): {basestring: object},
 
     # expiration and deadline times, relative to task creation, with units
-    # (e.g., "14 days")
-    'expires-after': basestring,
-    'deadline-after': basestring,
+    # (e.g., "14 days").  Defaults are set based on the project.
+    Optional('expires-after'): basestring,
+    Optional('deadline-after'): basestring,
 
     # custom routes for this task; the default treeherder routes will be added
     # automatically
-    'routes': [basestring],
+    Optional('routes'): [basestring],
 
     # custom scopes for this task; any scopes required for the worker will be
     # added automatically
-    'scopes': [basestring],
+    Optional('scopes'): [basestring],
 
     # custom "task.extra" content
-    'extra': {basestring: object},
+    Optional('extra'): {basestring: object},
 
     # treeherder-related information; see
     # https://schemas.taskcluster.net/taskcluster-treeherder/v1/task-treeherder-config.json
-    'treeherder': {
+    # If not specified, no treeherder extra information or routes will be
+    # added to the task
+    Optional('treeherder'): {
         # either a bare symbol, or "grp(sym)".
         'symbol': basestring,
 
         # the job kind
         'kind': Any('build', 'test', 'other'),
 
         # tier for this task
         'tier': int,
@@ -73,89 +77,92 @@ task_description_schema = Schema({
         Required('environments', default=['production', 'staging']): ['production', 'staging'],
     },
 
     # the provisioner-id/worker-type for the task
     'worker-type': basestring,
 
     # information specific to the worker implementation that will run this task
     'worker': Any({
-        'implementation': Any('docker-worker', 'docker-engine'),
+        Required('implementation'): Any('docker-worker', 'docker-engine'),
 
         # the docker image (in docker's `host/repo/image:tag` format) in which
         # to run the task; if omitted, this will be a reference to the image
         # generated by the 'docker-image' dependency, which must be defined in
         # 'dependencies'
         Optional('docker-image'): basestring,
 
         # worker features that should be enabled
         Required('relengapi-proxy', default=False): bool,
+        Required('taskcluster-proxy', default=False): bool,
         Required('allow-ptrace', default=False): bool,
         Required('loopback-video', default=False): bool,
         Required('loopback-audio', default=False): bool,
+        Optional('superseder-url'): basestring,
 
         # caches to set up for the task
-        'caches': [{
+        Optional('caches'): [{
             # only one type is supported by any of the workers right now
             'type': 'persistent',
 
             # name of the cache, allowing re-use by subsequent tasks naming the
             # same cache
             'name': basestring,
 
             # location in the task image where the cache will be mounted
             'mount-point': basestring,
         }],
 
         # artifacts to extract from the task image after completion
-        'artifacts': [{
+        Optional('artifacts'): [{
             # type of artifact -- simple file, or recursive directory
             'type': Any('file', 'directory'),
 
             # task image path from which to read artifact
             'path': basestring,
 
             # name of the produced artifact (root of the names for
             # type=directory)
             'name': basestring,
         }],
 
         # environment variables
-        'env': {basestring: taskref_or_string},
+        Required('env', default={}): {basestring: taskref_or_string},
 
         # the command to run
         'command': [taskref_or_string],
 
         # the maximum time to run, in seconds
         'max-run-time': int,
     }, {
-        'implementation': 'generic-worker',
+        Required('implementation'): 'generic-worker',
 
         # command is a list of commands to run, sequentially
         'command': [basestring],
 
         # artifacts to extract from the task image after completion; note that artifacts
         # for the generic worker cannot have names
-        'artifacts': [{
+        Optional('artifacts'): [{
             # type of artifact -- simple file, or recursive directory
             'type': Any('file', 'directory'),
 
             # task image path from which to read artifact
             'path': basestring,
         }],
 
         # environment variables
-        'env': {basestring: taskref_or_string},
+        Required('env', default={}): {basestring: taskref_or_string},
 
         # the maximum time to run, in seconds
         'max-run-time': int,
     }, {
-        'implementation': 'buildbot-bridge',
+        Required('implementation'): 'buildbot-bridge',
 
-        # see https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
+        # see
+        # https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
         'buildername': basestring,
         'sourcestamp': {
             'branch': basestring,
             Optional('revision'): basestring,
             Optional('repository'): basestring,
             Optional('project'): basestring,
         },
         'properties': {
@@ -293,47 +300,47 @@ def validate(config, tasks):
             task_description_schema, task,
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
 @transforms.add
 def build_task(config, tasks):
     for task in tasks:
         provisioner_id, worker_type = task['worker-type'].split('/', 1)
-        routes = task['routes']
-        scopes = task['scopes']
+        routes = task.get('routes', [])
+        scopes = task.get('scopes', [])
 
         # set up extra
-        extra = task['extra']
-        extra['treeherderEnv'] = task['treeherder']['environments']
-
-        task_th = task['treeherder']
-        treeherder = extra.setdefault('treeherder', {})
+        extra = task.get('extra', {})
+        task_th = task.get('treeherder')
+        if task_th:
+            extra['treeherderEnv'] = task_th['environments']
 
-        machine_platform, collection = task_th['platform'].split('/', 1)
-        treeherder['machine'] = {'platform': machine_platform}
-        treeherder['collection'] = {collection: True}
+            treeherder = extra.setdefault('treeherder', {})
 
-        groupSymbol, symbol = split_symbol(task_th['symbol'])
-        if groupSymbol != '?':
+            machine_platform, collection = task_th['platform'].split('/', 1)
+            treeherder['machine'] = {'platform': machine_platform}
+            treeherder['collection'] = {collection: True}
+
+            groupSymbol, symbol = split_symbol(task_th['symbol'])
             treeherder['groupSymbol'] = groupSymbol
             if groupSymbol not in GROUP_NAMES:
                 raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
             treeherder['groupName'] = GROUP_NAMES[groupSymbol]
-        treeherder['symbol'] = symbol
-        treeherder['jobKind'] = task_th['kind']
-        treeherder['tier'] = task_th['tier']
+            treeherder['symbol'] = symbol
+            treeherder['jobKind'] = task_th['kind']
+            treeherder['tier'] = task_th['tier']
 
-        routes.extend([
-            '{}.v2.{}.{}.{}'.format(root,
-                                    config.params['project'],
-                                    config.params['head_rev'],
-                                    config.params['pushlog_id'])
-            for root in 'tc-treeherder', 'tc-treeherder-stage'
-        ])
+            routes.extend([
+                '{}.v2.{}.{}.{}'.format(root,
+                                        config.params['project'],
+                                        config.params['head_rev'],
+                                        config.params['pushlog_id'])
+                for root in 'tc-treeherder', 'tc-treeherder-stage'
+            ])
 
         task_def = {
             'provisionerId': provisioner_id,
             'workerType': worker_type,
             'routes': routes,
             'created': {'relative-datestamp': '0 seconds'},
             'deadline': {'relative-datestamp': task['deadline-after']},
             'expires': {'relative-datestamp': task['expires-after']},
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -32,18 +32,18 @@ ARTIFACTS = [
 
 logger = logging.getLogger(__name__)
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def make_task_description(config, tests):
-    """Convert *test* descriptions to *task* descriptions, suitable for input
-    to make_task"""
+    """Convert *test* descriptions to *task* descriptions (input to
+    taskgraph.transforms.task)"""
 
     for test in tests:
         label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
         if test['chunks'] > 1:
             label += '-{}'.format(test['this-chunk'])
 
         build_label = test['build-label']