Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal
authorTom Prince <mozilla@hocat.ca>
Thu, 25 Oct 2018 18:39:22 +0000
changeset 491368 f08bde3d7956c7857dd2c0f50ebbbd895c0b386b
parent 491367 6c2ca1a524e6c126f25979c39f3ee9627efd3746
child 491369 4e1ac8b657be50febe961899ddeab9b001906fa0
push id247
push userfmarier@mozilla.com
push dateSat, 27 Oct 2018 01:06:44 +0000
Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal Differential Revision: https://phabricator.services.mozilla.com/D9680
--- a/taskcluster/docs/try.rst
+++ b/taskcluster/docs/try.rst
@@ -31,45 +31,46 @@ might look like:
 This gets parsed by ``taskgraph.try_option_syntax:TryOptionSyntax`` and returns
 a list of matching task labels. For more information see the
 `TryServer wiki page <https://wiki.mozilla.org/Try>`_.
 Try Task Config
 The second, more modern method specifies exactly the tasks to run.  That list
-of tasks is usually generated locally with some `local tool <tryselect>`_ and
-attached to the commit pushed to the try repository. This gives finer control
-over exactly what runs and enables growth of an ecosystem of tooling
-appropriate to varied circumstances.
+of tasks is usually generated locally with some :doc:`local tool </tools/try/selectors/fuzzy>`
+and attached to the commit pushed to the try repository. This gives
+finer control over exactly what runs and enables growth of an
+ecosystem of tooling appropriate to varied circumstances.
 This method uses a checked-in file called ``try_task_config.json`` which lives
 at the root of the source dir. The JSON object in this file contains a
 ``tasks`` key giving the labels of the tasks to run.  For example, the
 ``try_task_config.json`` file might look like:
 .. parsed-literal::
+      "version": 1,
       "tasks": [
 Very simply, this will run any task label that gets passed in as well as their
 dependencies. While it is possible to manually commit this file and push to
-try, it is mainly meant to be a generation target for various `tryselect`_
+try, it is mainly meant to be a generation target for various :doc:`tryselect </tools/try>`
 choosers.  For example:
 .. parsed-literal::
     $ ./mach try fuzzy
 A list of all possible task labels can be obtained by running:
@@ -89,16 +90,17 @@ Modifying Tasks in a Try Push
 It's possible to alter the definition of a task with templates. Templates are
 `JSON-e`_ files that live in the `taskgraph module`_. Templates can be specified
 from the ``try_task_config.json`` like this:
 .. parsed-literal::
+      "version": 1,
       "tasks": [...],
       "templates": {
         artifact: {"enabled": 1}
 Each key in the templates object denotes a new template to apply, and the value
 denotes extra context to use while rendering. When specified, a template will
@@ -129,14 +131,49 @@ will be ignored. See the `existing templ
 Empty Try
 If there is no try syntax or ``try_task_config.json``, the ``try_mode``
 parameter is None and no tasks are selected to run.  The resulting push will
 only have a decision task, but one with an "add jobs" action that can be used
 to add the desired jobs to the try push.
-.. _tryselect: https://dxr.mozilla.org/mozilla-central/source/tools/tryselect
+Complex Configuration
+If you need more control over the build configuration,
+(:doc:`staging releases </tools/try/selectors/release>`, for example),
+you can directly specify :doc:`parameters <parameters>`
+to override from the ``try_task_config.json`` like this:
+.. parsed-literal::
+   {
+       "version": 2,
+       "parameters": {
+           "optimize_target_tasks": true,
+           "release_type": "beta",
+           "target_tasks_method": "staging_release_builds"
+       }
+   }
+This format can express a superset of the version 1 format, as the
+version one configuration is equivalent to the following version 2
+.. parsed-literal::
+   {
+       "version": 2,
+       "parameters": {
+           "try_task_config": {...},
+           "try_mode": "try_task_config",
+       }
+   }
 .. _JSON-e: https://taskcluster.github.io/json-e/
 .. _taskgraph module: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
 .. _condition statements: https://taskcluster.github.io/json-e/#%60$if%60%20-%20%60then%60%20-%20%60else%60
 .. _existing templates: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
 .. _SCM Level: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -13,19 +13,21 @@ import time
 import yaml
 from .generator import TaskGraphGenerator
 from .create import create_tasks
 from .parameters import Parameters, get_version, get_app_version
 from .taskgraph import TaskGraph
 from .try_option_syntax import parse_message
 from .actions import render_actions_json
-from taskgraph.util.partials import populate_release_history
-from taskgraph.util.yaml import load_yaml
+from .util.partials import populate_release_history
+from .util.yaml import load_yaml
+from .util.schema import validate_schema, Schema
+from voluptuous import Required, Optional
 logger = logging.getLogger(__name__)
 ARTIFACTS_DIR = 'artifacts'
 # For each project, this gives a set of parameters specific to the project.
 # See `taskcluster/docs/parameters.rst` for information on parameters.
@@ -101,16 +103,26 @@ PER_PROJECT_PARAMETERS = {
     # the default parameters are used for projects that do not match above.
     'default': {
         'target_tasks_method': 'default',
         'optimize_target_tasks': True,
+try_task_config_schema = Schema({
+    Required('tasks'): [basestring],
+    Optional('templates'): {basestring: object},
+try_task_config_schema_v2 = Schema({
+    Optional('parameters'): {basestring: object},
 def full_task_graph_to_runnable_jobs(full_task_json):
     runnable_jobs = {}
     for label, node in full_task_json.iteritems():
         if not ('extra' in node['task'] and 'treeherder' in node['task']['extra']):
         th = node['task']['extra']['treeherder']
@@ -273,21 +285,29 @@ def get_decision_parameters(config, opti
     return result
 def set_try_config(parameters, task_config_file):
     if os.path.isfile(task_config_file):
         logger.info("using try tasks from {}".format(task_config_file))
         with open(task_config_file, 'r') as fh:
             task_config = json.load(fh)
-        task_config_version = task_config.get('version', 1)
+        task_config_version = task_config.pop('version', 1)
         if task_config_version == 1:
+            validate_schema(
+                try_task_config_schema, task_config,
+                "Invalid v1 `try_task_config.json`.",
+            )
             parameters['try_mode'] = 'try_task_config'
             parameters['try_task_config'] = task_config
         elif task_config_version == 2:
+            validate_schema(
+                try_task_config_schema_v2, task_config,
+                "Invalid v1 `try_task_config.json`.",
+            )
             raise Exception(
                 "Unknown `try_task_config.json` version: {}".format(task_config_version))
     if 'try:' in parameters['message']:
         parameters['try_mode'] = 'try_option_syntax'