Bug 1318438 - [taskcluster] "job" tasks should have ability to run on multiple platforms, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 18 Nov 2016 15:07:56 -0500
changeset 324891 6461f9c75e80275cd4751c670f37efa4177315d5
parent 324890 748a0dd6b360607cb357fe2baea7f076f9374c22
child 324892 754947cc763ce9cee4574b04dd7a2c451163ebf8
push id24
push usermaklebus@msu.edu
push dateTue, 20 Dec 2016 03:11:33 +0000
reviewersdustin
bugs1318438
milestone53.0a1
Bug 1318438 - [taskcluster] "job" tasks should have ability to run on multiple platforms, r=dustin This adds an optional "platforms" key to the job description. It can be used in conjunction with "by-platform" like so: platforms: - linux - windows worker-type: by-platform: linux: ... windows: ... worker: by-platform: linux: ... windows: ... MozReview-Commit-ID: JwL1NAR4bnY
taskcluster/docs/transforms.rst
taskcluster/taskgraph/test/test_transforms_base.py
taskcluster/taskgraph/transforms/base.py
taskcluster/taskgraph/transforms/job/__init__.py
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -135,18 +135,20 @@ A job description says what to run in th
 ``run`` section and all of the fields from a task description.  The run section
 has a ``using`` property that defines how this task should be run; for example,
 ``mozharness`` to run a mozharness script, or ``mach`` to run a mach command.
 The remainder of the run section is specific to the run-using implementation.
 
 The effect of a job description is to say "run this thing on this worker".  The
 job description must contain enough information about the worker to identify
 the workerType and the implementation (docker-worker, generic-worker, etc.).
-Any other task-description information is passed along verbatim, although it is
-augmented by the run-using implementation.
+Alternatively, job descriptions can specify the ``platforms`` field in
+conjunction with the  ``by-platform`` key to specify multiple workerTypes and
+implementations. Any other task-description information is passed along
+verbatim, although it is augmented by the run-using implementation.
 
 The run-using implementations are all located in
 ``taskcluster/taskgraph/transforms/job``, along with the schemas for their
 implementations.  Those well-commented source files are the canonical
 documentation for what constitutes a job description, and should be considered
 part of the documentation.
 
 Task Descriptions
--- a/taskcluster/taskgraph/test/test_transforms_base.py
+++ b/taskcluster/taskgraph/test/test_transforms_base.py
@@ -103,41 +103,41 @@ class TestKeyedBy(unittest.TestCase):
                     'a': 10,
                     'default': 30,
                 },
             },
             'other-value': 'xxx',
         }
         self.assertEqual(get_keyed_by(test, 'option', 'x'), 30)
 
-    def test_by_value_invalid_dict(self):
+    def test_by_value_dict(self):
         test = {
             'test-name': 'tname',
             'option': {
                 'by-something-else': {},
                 'by-other-value': {},
             },
         }
-        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), test['option'])
 
     def test_by_value_invalid_no_default(self):
         test = {
             'test-name': 'tname',
             'option': {
                 'by-other-value': {
                     'a': 10,
                 },
             },
             'other-value': 'b',
         }
         self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
 
-    def test_by_value_invalid_no_by(self):
+    def test_by_value_no_by(self):
         test = {
             'test-name': 'tname',
             'option': {
                 'other-value': {},
             },
         }
-        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), test['option'])
 
 if __name__ == '__main__':
     main()
--- a/taskcluster/taskgraph/transforms/base.py
+++ b/taskcluster/taskgraph/transforms/base.py
@@ -98,31 +98,28 @@ def get_keyed_by(item, field, item_name,
     value = item[field]
     if not isinstance(value, dict):
         return value
     if subfield:
         value = item[field][subfield]
         if not isinstance(value, dict):
             return value
 
-    assert len(value) == 1, "Invalid attribute {} in {}".format(field, item_name)
     keyed_by = value.keys()[0]
+    if len(value) > 1 or not keyed_by.startswith('by-'):
+        return value
+
     values = value[keyed_by]
-    if keyed_by.startswith('by-'):
-        keyed_by = keyed_by[3:]  # extract just the keyed-by field name
-        if item[keyed_by] in values:
-            return values[item[keyed_by]]
-        for k in values.keys():
-            if re.match(k, item[keyed_by]):
-                return values[k]
-        if 'default' in values:
-            return values['default']
-        for k in item[keyed_by], 'default':
-            if k in values:
-                return values[k]
-        else:
-            raise Exception(
-                "Neither {} {} nor 'default' found while determining item {} in {}".format(
-                    keyed_by, item[keyed_by], field, item_name))
+    keyed_by = keyed_by[3:]  # strip 'by-' off the keyed-by field name
+    if item[keyed_by] in values:
+        return values[item[keyed_by]]
+    for k in values.keys():
+        if re.match(k, item[keyed_by]):
+            return values[k]
+    if 'default' in values:
+        return values['default']
+    for k in item[keyed_by], 'default':
+        if k in values:
+            return values[k]
     else:
         raise Exception(
-            "Invalid attribute {} keyed-by value {} in {}".format(
-                field, keyed_by, item_name))
+            "Neither {} {} nor 'default' found while determining item {} in {}".format(
+                keyed_by, item[keyed_by], field, item_name))
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -10,23 +10,24 @@ run-using handlers in `taskcluster/taskg
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import logging
 import os
 
-from taskgraph.transforms.base import validate_schema, TransformSequence
+from taskgraph.transforms.base import get_keyed_by, validate_schema, TransformSequence
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import (
+    Any,
+    Extra,
     Optional,
     Required,
     Schema,
-    Extra,
 )
 
 logger = logging.getLogger(__name__)
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -47,54 +48,95 @@ job_description_schema = Schema({
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('coalesce-name'): task_description_schema['coalesce-name'],
-    Optional('worker-type'): task_description_schema['worker-type'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
-    Required('worker'): task_description_schema['worker'],
     Optional('when'): task_description_schema['when'],
 
     # A description of how to run this job.
     'run': {
         # The key to a job implementation in a peer module to this one
         'using': basestring,
 
         # Any remaining content is verified against that job implementation's
         # own schema.
         Extra: object,
     },
+    Optional('platforms'): [basestring],
+    Required('worker-type'): Any(
+        task_description_schema['worker-type'],
+        {'by-platform': {basestring: task_description_schema['worker-type']}},
+    ),
+    Required('worker'): Any(
+        task_description_schema['worker'],
+        {'by-platform': {basestring: task_description_schema['worker']}},
+    ),
 })
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         yield validate_schema(job_description_schema, job,
                               "In job {!r}:".format(job['name']))
 
 
 @transforms.add
+def expand_platforms(config, jobs):
+    for job in jobs:
+        if 'platforms' not in job:
+            yield job
+            continue
+
+        for platform in job['platforms']:
+            pjob = copy.deepcopy(job)
+            pjob['platform'] = platform
+            del pjob['platforms']
+
+            platform, buildtype = platform.rsplit('/', 1)
+            pjob['name'] = '{}-{}-{}'.format(pjob['name'], platform, buildtype)
+            yield pjob
+
+
+@transforms.add
+def resolve_keyed_by(config, jobs):
+    fields = [
+        'worker-type',
+        'worker',
+    ]
+
+    for job in jobs:
+        for field in fields:
+            job[field] = get_keyed_by(item=job, field=field, item_name=job['name'])
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs
     import_all()
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job['name']:
             del job['name']
+        if 'platform' in job:
+            if 'treeherder' in job:
+                job['treeherder']['platform'] = job['platform']
+            del job['platform']
 
         taskdesc = copy.deepcopy(job)
 
         # fill in some empty defaults to make run implementations easier
         taskdesc.setdefault('attributes', {})
         taskdesc.setdefault('dependencies', {})
         taskdesc.setdefault('routes', [])
         taskdesc.setdefault('scopes', [])