Bug 1318438 - [taskcluster] "job" tasks should have ability to run on multiple platforms, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 18 Nov 2016 15:07:56 -0500
changeset 324926 6461f9c75e80275cd4751c670f37efa4177315d5
parent 324925 748a0dd6b360607cb357fe2baea7f076f9374c22
child 324927 754947cc763ce9cee4574b04dd7a2c451163ebf8
push id84553
push userryanvm@gmail.com
push dateThu, 01 Dec 2016 14:33:55 +0000
treeherdermozilla-inbound@07b9aab24f30 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1318438
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1318438 - [taskcluster] "job" tasks should have ability to run on multiple platforms, r=dustin This adds an optional "platforms" key to the job description. It can be used in conjunction with "by-platform" like so: platforms: - linux - windows worker-type: by-platform: linux: ... windows: ... worker: by-platform: linux: ... windows: ... MozReview-Commit-ID: JwL1NAR4bnY
taskcluster/docs/transforms.rst
taskcluster/taskgraph/test/test_transforms_base.py
taskcluster/taskgraph/transforms/base.py
taskcluster/taskgraph/transforms/job/__init__.py
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -135,18 +135,20 @@ A job description says what to run in th
 ``run`` section and all of the fields from a task description.  The run section
 has a ``using`` property that defines how this task should be run; for example,
 ``mozharness`` to run a mozharness script, or ``mach`` to run a mach command.
 The remainder of the run section is specific to the run-using implementation.
 
 The effect of a job description is to say "run this thing on this worker".  The
 job description must contain enough information about the worker to identify
 the workerType and the implementation (docker-worker, generic-worker, etc.).
-Any other task-description information is passed along verbatim, although it is
-augmented by the run-using implementation.
+Alternatively, job descriptions can specify the ``platforms`` field in
+conjunction with the  ``by-platform`` key to specify multiple workerTypes and
+implementations. Any other task-description information is passed along
+verbatim, although it is augmented by the run-using implementation.
 
 The run-using implementations are all located in
 ``taskcluster/taskgraph/transforms/job``, along with the schemas for their
 implementations.  Those well-commented source files are the canonical
 documentation for what constitutes a job description, and should be considered
 part of the documentation.
 
 Task Descriptions
--- a/taskcluster/taskgraph/test/test_transforms_base.py
+++ b/taskcluster/taskgraph/test/test_transforms_base.py
@@ -103,41 +103,41 @@ class TestKeyedBy(unittest.TestCase):
                     'a': 10,
                     'default': 30,
                 },
             },
             'other-value': 'xxx',
         }
         self.assertEqual(get_keyed_by(test, 'option', 'x'), 30)
 
-    def test_by_value_invalid_dict(self):
+    def test_by_value_dict(self):
         test = {
             'test-name': 'tname',
             'option': {
                 'by-something-else': {},
                 'by-other-value': {},
             },
         }
-        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), test['option'])
 
     def test_by_value_invalid_no_default(self):
         test = {
             'test-name': 'tname',
             'option': {
                 'by-other-value': {
                     'a': 10,
                 },
             },
             'other-value': 'b',
         }
         self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
 
-    def test_by_value_invalid_no_by(self):
+    def test_by_value_no_by(self):
         test = {
             'test-name': 'tname',
             'option': {
                 'other-value': {},
             },
         }
-        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), test['option'])
 
 if __name__ == '__main__':
     main()
--- a/taskcluster/taskgraph/transforms/base.py
+++ b/taskcluster/taskgraph/transforms/base.py
@@ -98,31 +98,28 @@ def get_keyed_by(item, field, item_name,
     value = item[field]
     if not isinstance(value, dict):
         return value
     if subfield:
         value = item[field][subfield]
         if not isinstance(value, dict):
             return value
 
-    assert len(value) == 1, "Invalid attribute {} in {}".format(field, item_name)
     keyed_by = value.keys()[0]
+    if len(value) > 1 or not keyed_by.startswith('by-'):
+        return value
+
     values = value[keyed_by]
-    if keyed_by.startswith('by-'):
-        keyed_by = keyed_by[3:]  # extract just the keyed-by field name
-        if item[keyed_by] in values:
-            return values[item[keyed_by]]
-        for k in values.keys():
-            if re.match(k, item[keyed_by]):
-                return values[k]
-        if 'default' in values:
-            return values['default']
-        for k in item[keyed_by], 'default':
-            if k in values:
-                return values[k]
-        else:
-            raise Exception(
-                "Neither {} {} nor 'default' found while determining item {} in {}".format(
-                    keyed_by, item[keyed_by], field, item_name))
+    keyed_by = keyed_by[3:]  # strip 'by-' off the keyed-by field name
+    if item[keyed_by] in values:
+        return values[item[keyed_by]]
+    for k in values.keys():
+        if re.match(k, item[keyed_by]):
+            return values[k]
+    if 'default' in values:
+        return values['default']
+    for k in item[keyed_by], 'default':
+        if k in values:
+            return values[k]
     else:
         raise Exception(
-            "Invalid attribute {} keyed-by value {} in {}".format(
-                field, keyed_by, item_name))
+            "Neither {} {} nor 'default' found while determining item {} in {}".format(
+                keyed_by, item[keyed_by], field, item_name))
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -10,23 +10,24 @@ run-using handlers in `taskcluster/taskg
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import logging
 import os
 
-from taskgraph.transforms.base import validate_schema, TransformSequence
+from taskgraph.transforms.base import get_keyed_by, validate_schema, TransformSequence
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import (
+    Any,
+    Extra,
     Optional,
     Required,
     Schema,
-    Extra,
 )
 
 logger = logging.getLogger(__name__)
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -47,54 +48,95 @@ job_description_schema = Schema({
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('coalesce-name'): task_description_schema['coalesce-name'],
-    Optional('worker-type'): task_description_schema['worker-type'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
-    Required('worker'): task_description_schema['worker'],
     Optional('when'): task_description_schema['when'],
 
     # A description of how to run this job.
     'run': {
         # The key to a job implementation in a peer module to this one
         'using': basestring,
 
         # Any remaining content is verified against that job implementation's
         # own schema.
         Extra: object,
     },
+    Optional('platforms'): [basestring],
+    Required('worker-type'): Any(
+        task_description_schema['worker-type'],
+        {'by-platform': {basestring: task_description_schema['worker-type']}},
+    ),
+    Required('worker'): Any(
+        task_description_schema['worker'],
+        {'by-platform': {basestring: task_description_schema['worker']}},
+    ),
 })
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         yield validate_schema(job_description_schema, job,
                               "In job {!r}:".format(job['name']))
 
 
 @transforms.add
+def expand_platforms(config, jobs):
+    for job in jobs:
+        if 'platforms' not in job:
+            yield job
+            continue
+
+        for platform in job['platforms']:
+            pjob = copy.deepcopy(job)
+            pjob['platform'] = platform
+            del pjob['platforms']
+
+            platform, buildtype = platform.rsplit('/', 1)
+            pjob['name'] = '{}-{}-{}'.format(pjob['name'], platform, buildtype)
+            yield pjob
+
+
+@transforms.add
+def resolve_keyed_by(config, jobs):
+    fields = [
+        'worker-type',
+        'worker',
+    ]
+
+    for job in jobs:
+        for field in fields:
+            job[field] = get_keyed_by(item=job, field=field, item_name=job['name'])
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs
     import_all()
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job['name']:
             del job['name']
+        if 'platform' in job:
+            if 'treeherder' in job:
+                job['treeherder']['platform'] = job['platform']
+            del job['platform']
 
         taskdesc = copy.deepcopy(job)
 
         # fill in some empty defaults to make run implementations easier
         taskdesc.setdefault('attributes', {})
         taskdesc.setdefault('dependencies', {})
         taskdesc.setdefault('routes', [])
         taskdesc.setdefault('scopes', [])