Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal
authorTom Prince <mozilla@hocat.ca>
Thu, 25 Oct 2018 18:39:22 +0000
changeset 443009 f08bde3d7956c7857dd2c0f50ebbbd895c0b386b
parent 443008 6c2ca1a524e6c126f25979c39f3ee9627efd3746
child 443010 4e1ac8b657be50febe961899ddeab9b001906fa0
push id34933
push userccoroiu@mozilla.com
push dateThu, 25 Oct 2018 21:55:02 +0000
treeherdermozilla-central@4e1ac8b657be [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1497575
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal Differential Revision: https://phabricator.services.mozilla.com/D9680
taskcluster/docs/try.rst
taskcluster/taskgraph/decision.py
--- a/taskcluster/docs/try.rst
+++ b/taskcluster/docs/try.rst
@@ -31,45 +31,46 @@ might look like:
 This gets parsed by ``taskgraph.try_option_syntax:TryOptionSyntax`` and returns
 a list of matching task labels. For more information see the
 `TryServer wiki page <https://wiki.mozilla.org/Try>`_.
 
 Try Task Config
 :::::::::::::::
 
 The second, more modern method specifies exactly the tasks to run.  That list
-of tasks is usually generated locally with some `local tool <tryselect>`_ and
-attached to the commit pushed to the try repository. This gives finer control
-over exactly what runs and enables growth of an ecosystem of tooling
-appropriate to varied circumstances.
+of tasks is usually generated locally with some :doc:`local tool </tools/try/selectors/fuzzy>`
+and attached to the commit pushed to the try repository. This gives
+finer control over exactly what runs and enables growth of an
+ecosystem of tooling appropriate to varied circumstances.
 
 Implementation
 ,,,,,,,,,,,,,,
 
 This method uses a checked-in file called ``try_task_config.json`` which lives
 at the root of the source dir. The JSON object in this file contains a
 ``tasks`` key giving the labels of the tasks to run.  For example, the
 ``try_task_config.json`` file might look like:
 
 .. parsed-literal::
 
     {
+      "version": 1,
       "tasks": [
         "test-windows10-64/opt-web-platform-tests-12",
         "test-windows7-32/opt-reftest-1",
         "test-windows7-32/opt-reftest-2",
         "test-windows7-32/opt-reftest-3",
         "build-linux64/debug",
         "source-test-mozlint-eslint"
       ]
     }
 
 Very simply, this will run any task label that gets passed in as well as their
 dependencies. While it is possible to manually commit this file and push to
-try, it is mainly meant to be a generation target for various `tryselect`_
+try, it is mainly meant to be a generation target for various :doc:`tryselect </tools/try>`
 choosers.  For example:
 
 .. parsed-literal::
 
     $ ./mach try fuzzy
 
 A list of all possible task labels can be obtained by running:
 
@@ -89,16 +90,17 @@ Modifying Tasks in a Try Push
 
 It's possible to alter the definition of a task with templates. Templates are
 `JSON-e`_ files that live in the `taskgraph module`_. Templates can be specified
 from the ``try_task_config.json`` like this:
 
 .. parsed-literal::
 
     {
+      "version": 1,
       "tasks": [...],
       "templates": {
         artifact: {"enabled": 1}
       }
     }
 
 Each key in the templates object denotes a new template to apply, and the value
 denotes extra context to use while rendering. When specified, a template will
@@ -129,14 +131,49 @@ will be ignored. See the `existing templ
 Empty Try
 :::::::::
 
 If there is no try syntax or ``try_task_config.json``, the ``try_mode``
 parameter is None and no tasks are selected to run.  The resulting push will
 only have a decision task, but one with an "add jobs" action that can be used
 to add the desired jobs to the try push.
 
-.. _tryselect: https://dxr.mozilla.org/mozilla-central/source/tools/tryselect
+
+Complex Configuration
+:::::::::::::::::::::
+
+If you need more control over the build configuration,
+(:doc:`staging releases </tools/try/selectors/release>`, for example),
+you can directly specify :doc:`parameters <parameters>`
+to override from the ``try_task_config.json`` like this:
+
+.. parsed-literal::
+
+   {
+       "version": 2,
+       "parameters": {
+           "optimize_target_tasks": true,
+           "release_type": "beta",
+           "target_tasks_method": "staging_release_builds"
+       }
+   }
+
+This format can express a superset of the version 1 format, as the
+version one configuration is equivalent to the following version 2
+config.
+
+.. parsed-literal::
+
+   {
+       "version": 2,
+       "parameters": {
+           "try_task_config": {...},
+           "try_mode": "try_task_config",
+       }
+   }
+
 .. _JSON-e: https://taskcluster.github.io/json-e/
 .. _taskgraph module: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
 .. _condition statements: https://taskcluster.github.io/json-e/#%60$if%60%20-%20%60then%60%20-%20%60else%60
 .. _existing templates: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
 .. _SCM Level: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/
+
+
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -13,19 +13,21 @@ import time
 import yaml
 
 from .generator import TaskGraphGenerator
 from .create import create_tasks
 from .parameters import Parameters, get_version, get_app_version
 from .taskgraph import TaskGraph
 from .try_option_syntax import parse_message
 from .actions import render_actions_json
-from taskgraph.util.partials import populate_release_history
-from taskgraph.util.yaml import load_yaml
+from .util.partials import populate_release_history
+from .util.yaml import load_yaml
 
+from .util.schema import validate_schema, Schema
+from voluptuous import Required, Optional
 
 logger = logging.getLogger(__name__)
 
 ARTIFACTS_DIR = 'artifacts'
 
 # For each project, this gives a set of parameters specific to the project.
 # See `taskcluster/docs/parameters.rst` for information on parameters.
 PER_PROJECT_PARAMETERS = {
@@ -101,16 +103,26 @@ PER_PROJECT_PARAMETERS = {
 
     # the default parameters are used for projects that do not match above.
     'default': {
         'target_tasks_method': 'default',
         'optimize_target_tasks': True,
     }
 }
 
+try_task_config_schema = Schema({
+    Required('tasks'): [basestring],
+    Optional('templates'): {basestring: object},
+})
+
+
+try_task_config_schema_v2 = Schema({
+    Optional('parameters'): {basestring: object},
+})
+
 
 def full_task_graph_to_runnable_jobs(full_task_json):
     runnable_jobs = {}
     for label, node in full_task_json.iteritems():
         if not ('extra' in node['task'] and 'treeherder' in node['task']['extra']):
             continue
 
         th = node['task']['extra']['treeherder']
@@ -273,21 +285,29 @@ def get_decision_parameters(config, opti
     return result
 
 
 def set_try_config(parameters, task_config_file):
     if os.path.isfile(task_config_file):
         logger.info("using try tasks from {}".format(task_config_file))
         with open(task_config_file, 'r') as fh:
             task_config = json.load(fh)
-        task_config_version = task_config.get('version', 1)
+        task_config_version = task_config.pop('version', 1)
         if task_config_version == 1:
+            validate_schema(
+                try_task_config_schema, task_config,
+                "Invalid v1 `try_task_config.json`.",
+            )
             parameters['try_mode'] = 'try_task_config'
             parameters['try_task_config'] = task_config
         elif task_config_version == 2:
+            validate_schema(
+                try_task_config_schema_v2, task_config,
+                "Invalid v1 `try_task_config.json`.",
+            )
             parameters.update(task_config['parameters'])
             return
         else:
             raise Exception(
                 "Unknown `try_task_config.json` version: {}".format(task_config_version))
 
     if 'try:' in parameters['message']:
         parameters['try_mode'] = 'try_option_syntax'