Bug 1286075: allow defaults in TransformTask; r=gps
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 01 Sep 2016 18:48:38 +0000
changeset 313706 40e59df093a7332f90a34de57a1d08766fc9a93c
parent 313705 315bae6d1488fa36f2f90501218159241866a686
child 313707 a1911eb6ed8b024fdad878de7bef5568c95e11c0
push id20528
push userryanvm@gmail.com
push dateWed, 14 Sep 2016 00:48:16 +0000
treeherderfx-team@8a996d7c71a2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
Bug 1286075: allow defaults in TransformTask; r=gps MozReview-Commit-ID: 5tdnxykLTZc
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -1,19 +1,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import, print_function, unicode_literals
 import logging
+import itertools
 from . import base
 from .. import files_changed
 from ..util.python_path import find_object
+from ..util.templates import merge
 from ..util.yaml import load_yaml
 from ..transforms.base import TransformSequence, TransformConfig
 logger = logging.getLogger(__name__)
 class TransformTask(base.Task):
@@ -26,27 +28,37 @@ class TransformTask(base.Task):
     def get_inputs(cls, kind, path, config, params, loaded_tasks):
         Get the input elements that will be transformed into tasks.  The
         elements themselves are free-form, and become the input to the first
         By default, this reads jobs from the `jobs` key, or from yaml files
-        named by `jobs-from`, but can be overridden in subclasses.  The
-        entities are read from mappings, and the keys to those mappings are
-        added in the `name` key of each entity.
+        named by `jobs-from`.  The entities are read from mappings, and the
+        keys to those mappings are added in the `name` key of each entity.
+        If there is a `job-defaults` config, then every job is merged with it.
+        This provides a simple way to set default values for all jobs of a
+        kind.  More complex defaults should be implemented with custom
+        transforms.
+        This method can be overridden in subclasses that need to perform more
+        complex calculations to generate the list of inputs.
         def jobs():
-            for name, job in config.get('jobs', {}).iteritems():
+            defaults = config.get('job-defaults')
+            jobs = config.get('jobs', {}).iteritems()
+            jobs_from = itertools.chain.from_iterable(
+                load_yaml(path, filename).iteritems()
+                for filename in config.get('jobs-from', {}))
+            for name, job in itertools.chain(jobs, jobs_from):
+                if defaults:
+                    job = merge(defaults, job)
                 yield name, job
-            for filename in config.get('jobs-from', {}):
-                jobs = load_yaml(path, filename)
-                for name, job in jobs.iteritems():
-                    yield name, job
         for name, job in jobs():
             job['name'] = name
             logger.debug("Generating tasks for {} {}".format(kind, name))
             yield job
     def load_tasks(cls, kind, path, config, params, loaded_tasks):