Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal
authorTom Prince <mozilla@hocat.ca>
Thu, 25 Oct 2018 18:39:22 +0000
changeset 491368 f08bde3d7956c7857dd2c0f50ebbbd895c0b386b
parent 491367 6c2ca1a524e6c126f25979c39f3ee9627efd3746
child 491369 4e1ac8b657be50febe961899ddeab9b001906fa0
push id247
push userfmarier@mozilla.com
push dateSat, 27 Oct 2018 01:06:44 +0000
reviewersahal
bugs1497575
milestone65.0a1
Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal Differential Revision: https://phabricator.services.mozilla.com/D9680
taskcluster/docs/try.rst
taskcluster/taskgraph/decision.py
--- a/taskcluster/docs/try.rst
+++ b/taskcluster/docs/try.rst
@@ -31,45 +31,46 @@ might look like:
 This gets parsed by ``taskgraph.try_option_syntax:TryOptionSyntax`` and returns
 a list of matching task labels. For more information see the
 `TryServer wiki page <https://wiki.mozilla.org/Try>`_.
 
 Try Task Config
 :::::::::::::::
 
 The second, more modern method specifies exactly the tasks to run.  That list
-of tasks is usually generated locally with some `local tool <tryselect>`_ and
-attached to the commit pushed to the try repository. This gives finer control
-over exactly what runs and enables growth of an ecosystem of tooling
-appropriate to varied circumstances.
+of tasks is usually generated locally with some :doc:`local tool </tools/try/selectors/fuzzy>`
+and attached to the commit pushed to the try repository. This gives
+finer control over exactly what runs and enables growth of an
+ecosystem of tooling appropriate to varied circumstances.
 
 Implementation
 ,,,,,,,,,,,,,,
 
 This method uses a checked-in file called ``try_task_config.json`` which lives
 at the root of the source dir. The JSON object in this file contains a
 ``tasks`` key giving the labels of the tasks to run.  For example, the
 ``try_task_config.json`` file might look like:
 
 .. parsed-literal::
 
     {
+      "version": 1,
       "tasks": [
         "test-windows10-64/opt-web-platform-tests-12",
         "test-windows7-32/opt-reftest-1",
         "test-windows7-32/opt-reftest-2",
         "test-windows7-32/opt-reftest-3",
         "build-linux64/debug",
         "source-test-mozlint-eslint"
       ]
     }
 
 Very simply, this will run any task label that gets passed in as well as their
 dependencies. While it is possible to manually commit this file and push to
-try, it is mainly meant to be a generation target for various `tryselect`_
+try, it is mainly meant to be a generation target for various :doc:`tryselect </tools/try>`
 choosers.  For example:
 
 .. parsed-literal::
 
     $ ./mach try fuzzy
 
 A list of all possible task labels can be obtained by running:
 
@@ -89,16 +90,17 @@ Modifying Tasks in a Try Push
 
 It's possible to alter the definition of a task with templates. Templates are
 `JSON-e`_ files that live in the `taskgraph module`_. Templates can be specified
 from the ``try_task_config.json`` like this:
 
 .. parsed-literal::
 
     {
+      "version": 1,
       "tasks": [...],
       "templates": {
         artifact: {"enabled": 1}
       }
     }
 
 Each key in the templates object denotes a new template to apply, and the value
 denotes extra context to use while rendering. When specified, a template will
@@ -129,14 +131,49 @@ will be ignored. See the `existing templ
 Empty Try
 :::::::::
 
 If there is no try syntax or ``try_task_config.json``, the ``try_mode``
 parameter is None and no tasks are selected to run.  The resulting push will
 only have a decision task, but one with an "add jobs" action that can be used
 to add the desired jobs to the try push.
 
-.. _tryselect: https://dxr.mozilla.org/mozilla-central/source/tools/tryselect
+
+Complex Configuration
+:::::::::::::::::::::
+
+If you need more control over the build configuration,
+(:doc:`staging releases </tools/try/selectors/release>`, for example),
+you can directly specify :doc:`parameters <parameters>`
+to override from the ``try_task_config.json`` like this:
+
+.. parsed-literal::
+
+   {
+       "version": 2,
+       "parameters": {
+           "optimize_target_tasks": true,
+           "release_type": "beta",
+           "target_tasks_method": "staging_release_builds"
+       }
+   }
+
+This format can express a superset of the version 1 format, as the
+version one configuration is equivalent to the following version 2
+config.
+
+.. parsed-literal::
+
+   {
+       "version": 2,
+       "parameters": {
+           "try_task_config": {...},
+           "try_mode": "try_task_config",
+       }
+   }
+
 .. _JSON-e: https://taskcluster.github.io/json-e/
 .. _taskgraph module: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
 .. _condition statements: https://taskcluster.github.io/json-e/#%60$if%60%20-%20%60then%60%20-%20%60else%60
 .. _existing templates: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
 .. _SCM Level: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/
+
+
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -13,19 +13,21 @@ import time
 import yaml
 
 from .generator import TaskGraphGenerator
 from .create import create_tasks
 from .parameters import Parameters, get_version, get_app_version
 from .taskgraph import TaskGraph
 from .try_option_syntax import parse_message
 from .actions import render_actions_json
-from taskgraph.util.partials import populate_release_history
-from taskgraph.util.yaml import load_yaml
+from .util.partials import populate_release_history
+from .util.yaml import load_yaml
 
+from .util.schema import validate_schema, Schema
+from voluptuous import Required, Optional
 
 logger = logging.getLogger(__name__)
 
 ARTIFACTS_DIR = 'artifacts'
 
 # For each project, this gives a set of parameters specific to the project.
 # See `taskcluster/docs/parameters.rst` for information on parameters.
 PER_PROJECT_PARAMETERS = {
@@ -101,16 +103,26 @@ PER_PROJECT_PARAMETERS = {
 
     # the default parameters are used for projects that do not match above.
     'default': {
         'target_tasks_method': 'default',
         'optimize_target_tasks': True,
     }
 }
 
+try_task_config_schema = Schema({
+    Required('tasks'): [basestring],
+    Optional('templates'): {basestring: object},
+})
+
+
+try_task_config_schema_v2 = Schema({
+    Optional('parameters'): {basestring: object},
+})
+
 
 def full_task_graph_to_runnable_jobs(full_task_json):
     runnable_jobs = {}
     for label, node in full_task_json.iteritems():
         if not ('extra' in node['task'] and 'treeherder' in node['task']['extra']):
             continue
 
         th = node['task']['extra']['treeherder']
@@ -273,21 +285,29 @@ def get_decision_parameters(config, opti
     return result
 
 
 def set_try_config(parameters, task_config_file):
     if os.path.isfile(task_config_file):
         logger.info("using try tasks from {}".format(task_config_file))
         with open(task_config_file, 'r') as fh:
             task_config = json.load(fh)
-        task_config_version = task_config.get('version', 1)
+        task_config_version = task_config.pop('version', 1)
         if task_config_version == 1:
+            validate_schema(
+                try_task_config_schema, task_config,
+                "Invalid v1 `try_task_config.json`.",
+            )
             parameters['try_mode'] = 'try_task_config'
             parameters['try_task_config'] = task_config
         elif task_config_version == 2:
+            validate_schema(
+                try_task_config_schema_v2, task_config,
+                "Invalid v1 `try_task_config.json`.",
+            )
             parameters.update(task_config['parameters'])
             return
         else:
             raise Exception(
                 "Unknown `try_task_config.json` version: {}".format(task_config_version))
 
     if 'try:' in parameters['message']:
         parameters['try_mode'] = 'try_option_syntax'