Bug 1418058: [taskcluster] Pass a graph_config through action task generation; r=dustin
authorTom Prince <mozilla@hocat.ca>
Wed, 11 Apr 2018 10:24:14 -0600
changeset 413270 9fbb232ef1f16e423f55a8b0082f1dca58df1081
parent 413269 2efe54944e8cb91f1198d0c55ccb440de5b37997
child 413271 f23ff7f39f9a39acd8e8df718be9475f064a8652
push id33840
push userapavel@mozilla.com
push dateFri, 13 Apr 2018 21:56:54 +0000
treeherdermozilla-central@6547c27303bc [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1418058
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1418058: [taskcluster] Pass a graph_config through action task generation; r=dustin Differential Revision: https://phabricator.services.mozilla.com/D912
taskcluster/docs/actions.rst
taskcluster/taskgraph/actions/registry.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/generator.py
--- a/taskcluster/docs/actions.rst
+++ b/taskcluster/docs/actions.rst
@@ -240,17 +240,17 @@ The example below illustrates how to cre
       input={
           'title': 'priority'
           'description': 'Priority that should be given to the tasks',
           'type': 'string',
           'enum': ['low', 'normal', 'high'],
           'default': 'low',
       },
   )
-  def task_template_builder(parameters):
+  def task_template_builder(parameters, graph_config):
       # The task template builder may return None to signal that the action
       # isn't available.
       if parameters.get('project', None) != 'try':
         return None
       return {
           'created': {'$fromNow': ''},
           'deadline': {'$fromNow': '1 hour'},
           'expires': {'$fromNow': '14 days'},
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -6,20 +6,20 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
 import re
 import yaml
 from slugid import nice as slugid
-from mozbuild.util import memoize
 from types import FunctionType
 from collections import namedtuple
 from taskgraph import create, GECKO
+from taskgraph.generator import load_graph_config
 from taskgraph.util import taskcluster
 from taskgraph.parameters import Parameters
 
 
 actions = []
 callbacks = {}
 
 Action = namedtuple('Action', [
@@ -168,17 +168,17 @@ def register_callback_action(name, title
         assert isinstance(symbol, basestring), 'symbol must be a string'
         # Allow for json-e > 25 chars in the symbol.
         if '$' not in symbol:
             assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters'
         assert not mem['registered'], 'register_callback_action must be used as decorator'
         assert cb.__name__ not in callbacks, 'callback name {} is not unique'.format(cb.__name__)
 
         @register_task_action(name, title, description, order, context, schema)
-        def build_callback_action_task(parameters):
+        def build_callback_action_task(parameters, graph_config):
             if not available(parameters):
                 return None
 
             match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository'])
             if not match:
                 raise Exception('Unrecognized head_repository')
             repo_scope = 'assume:repo:{}/{}:branch:default'.format(
                 match.group(1), match.group(2))
@@ -220,34 +220,34 @@ def register_callback_action(name, title
                     'in': taskcluster_yml['tasks'][0]
                 }
 
         mem['registered'] = True
         callbacks[cb.__name__] = cb
     return register_callback
 
 
-def render_actions_json(parameters):
+def render_actions_json(parameters, graph_config):
     """
     Render JSON object for the ``public/actions.json`` artifact.
 
     Parameters
     ----------
     parameters : taskgraph.parameters.Parameters
         Decision task parameters.
 
     Returns
     -------
     dict
         JSON object representation of the ``public/actions.json`` artifact.
     """
     assert isinstance(parameters, Parameters), 'requires instance of Parameters'
     result = []
-    for action in sorted(get_actions(), key=lambda action: action.order):
-        task = action.task_template_builder(parameters)
+    for action in sorted(_get_actions(graph_config), key=lambda action: action.order):
+        task = action.task_template_builder(parameters, graph_config)
         if task:
             assert is_json(task), 'task must be a JSON compatible object'
             res = {
                 'kind': 'task',
                 'name': action.name,
                 'title': action.title,
                 'description': action.description,
                 'context': action.context,
@@ -267,41 +267,42 @@ def render_actions_json(parameters):
 
 
 def trigger_action_callback(task_group_id, task_id, task, input, callback, parameters,
                             test=False):
     """
     Trigger action callback with the given inputs. If `test` is true, then run
     the action callback in testing mode, without actually creating tasks.
     """
-    cb = get_callbacks().get(callback, None)
+    graph_config = load_graph_config("taskcluster/ci")
+    callbacks = _get_callbacks(graph_config)
+    cb = callbacks.get(callback, None)
     if not cb:
         raise Exception('Unknown callback: {}. Known callbacks: {}'.format(
-            callback, get_callbacks().keys()))
+            callback, callbacks))
 
     if test:
         create.testing = True
         taskcluster.testing = True
 
     cb(Parameters(**parameters), input, task_group_id, task_id, task)
 
 
-@memoize
-def _load():
+def _load(graph_config):
     # Load all modules from this folder, relying on the side-effects of register_
     # functions to populate the action registry.
     actions_dir = os.path.dirname(__file__)
     for f in os.listdir(actions_dir):
         if f.endswith('.py') and f not in ('__init__.py', 'registry.py', 'util.py'):
             __import__('taskgraph.actions.' + f[:-3])
         if f.endswith('.yml'):
             with open(os.path.join(actions_dir, f), 'r') as d:
                 frontmatter, template = yaml.safe_load_all(d)
-                register_task_action(**frontmatter)(lambda _: template)
+                register_task_action(**frontmatter)(lambda _p, _g: template)
     return callbacks, actions
 
 
-def get_callbacks():
-    return _load()[0]
+def _get_callbacks(graph_config):
+    return _load(graph_config)[0]
 
 
-def get_actions():
-    return _load()[1]
+def _get_actions(graph_config):
+    return _load(graph_config)[1]
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -126,17 +126,17 @@ def taskgraph_decision(options, paramete
     tgg = TaskGraphGenerator(
         root_dir=options.get('root'),
         parameters=parameters)
 
     # write out the parameters used to generate this graph
     write_artifact('parameters.yml', dict(**parameters))
 
     # write out the public/actions.json file
-    write_artifact('actions.json', render_actions_json(parameters))
+    write_artifact('actions.json', render_actions_json(parameters, tgg.graph_config))
 
     # write out the full graph for reference
     full_task_json = tgg.full_task_graph.to_json()
     write_artifact('full-task-graph.json', full_task_json)
 
     # write out the public/runnable-jobs.json.gz file
     write_artifact('runnable-jobs.json.gz', full_task_graph_to_runnable_jobs(full_task_json))
 
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -208,27 +208,38 @@ class TaskGraphGenerator(object):
         The optimized task graph, with any subsequent morphs applied. This graph
         will have the same meaning as the optimized task graph, but be in a form
         more palatable to TaskCluster.
 
         @type: TaskGraph
         """
         return self._run_until('morphed_task_graph')
 
+    @property
+    def graph_config(self):
+        """
+        The configuration for this graph.
+
+        @type: TaskGraph
+        """
+        return self._run_until('graph_config')
+
     def _load_kinds(self, graph_config):
         for kind_name in os.listdir(self.root_dir):
             try:
                 yield Kind.load(self.root_dir, graph_config, kind_name)
             except KindNotFound:
                 continue
 
     def _run(self):
         logger.info("Loading graph configuration.")
         graph_config = load_graph_config(self.root_dir)
 
+        yield verifications('graph_config', graph_config)
+
         logger.info("Loading kinds")
         # put the kinds into a graph and sort topologically so that kinds are loaded
         # in post-order
         kinds = {kind.name: kind for kind in self._load_kinds(graph_config)}
         self.verify_kinds(kinds)
 
         edges = set()
         for kind in kinds.itervalues():