author | Tom Prince <mozilla@hocat.ca> |
Wed, 11 Apr 2018 10:24:14 -0600 | |
changeset 413270 | 9fbb232ef1f16e423f55a8b0082f1dca58df1081 |
parent 413269 | 2efe54944e8cb91f1198d0c55ccb440de5b37997 |
child 413271 | f23ff7f39f9a39acd8e8df718be9475f064a8652 |
push id | 33840 |
push user | apavel@mozilla.com |
push date | Fri, 13 Apr 2018 21:56:54 +0000 |
treeherder | mozilla-central@6547c27303bc [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | dustin |
bugs | 1418058 |
milestone | 61.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/taskcluster/docs/actions.rst +++ b/taskcluster/docs/actions.rst @@ -240,17 +240,17 @@ The example below illustrates how to cre input={ 'title': 'priority' 'description': 'Priority that should be given to the tasks', 'type': 'string', 'enum': ['low', 'normal', 'high'], 'default': 'low', }, ) - def task_template_builder(parameters): + def task_template_builder(parameters, graph_config): # The task template builder may return None to signal that the action # isn't available. if parameters.get('project', None) != 'try': return None return { 'created': {'$fromNow': ''}, 'deadline': {'$fromNow': '1 hour'}, 'expires': {'$fromNow': '14 days'},
--- a/taskcluster/taskgraph/actions/registry.py +++ b/taskcluster/taskgraph/actions/registry.py @@ -6,20 +6,20 @@ from __future__ import absolute_import, print_function, unicode_literals import json import os import re import yaml from slugid import nice as slugid -from mozbuild.util import memoize from types import FunctionType from collections import namedtuple from taskgraph import create, GECKO +from taskgraph.generator import load_graph_config from taskgraph.util import taskcluster from taskgraph.parameters import Parameters actions = [] callbacks = {} Action = namedtuple('Action', [ @@ -168,17 +168,17 @@ def register_callback_action(name, title assert isinstance(symbol, basestring), 'symbol must be a string' # Allow for json-e > 25 chars in the symbol. if '$' not in symbol: assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters' assert not mem['registered'], 'register_callback_action must be used as decorator' assert cb.__name__ not in callbacks, 'callback name {} is not unique'.format(cb.__name__) @register_task_action(name, title, description, order, context, schema) - def build_callback_action_task(parameters): + def build_callback_action_task(parameters, graph_config): if not available(parameters): return None match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository']) if not match: raise Exception('Unrecognized head_repository') repo_scope = 'assume:repo:{}/{}:branch:default'.format( match.group(1), match.group(2)) @@ -220,34 +220,34 @@ def register_callback_action(name, title 'in': taskcluster_yml['tasks'][0] } mem['registered'] = True callbacks[cb.__name__] = cb return register_callback -def render_actions_json(parameters): +def render_actions_json(parameters, graph_config): """ Render JSON object for the ``public/actions.json`` artifact. Parameters ---------- parameters : taskgraph.parameters.Parameters Decision task parameters. Returns ------- dict JSON object representation of the ``public/actions.json`` artifact. """ assert isinstance(parameters, Parameters), 'requires instance of Parameters' result = [] - for action in sorted(get_actions(), key=lambda action: action.order): - task = action.task_template_builder(parameters) + for action in sorted(_get_actions(graph_config), key=lambda action: action.order): + task = action.task_template_builder(parameters, graph_config) if task: assert is_json(task), 'task must be a JSON compatible object' res = { 'kind': 'task', 'name': action.name, 'title': action.title, 'description': action.description, 'context': action.context, @@ -267,41 +267,42 @@ def render_actions_json(parameters): def trigger_action_callback(task_group_id, task_id, task, input, callback, parameters, test=False): """ Trigger action callback with the given inputs. If `test` is true, then run the action callback in testing mode, without actually creating tasks. """ - cb = get_callbacks().get(callback, None) + graph_config = load_graph_config("taskcluster/ci") + callbacks = _get_callbacks(graph_config) + cb = callbacks.get(callback, None) if not cb: raise Exception('Unknown callback: {}. Known callbacks: {}'.format( - callback, get_callbacks().keys())) + callback, callbacks)) if test: create.testing = True taskcluster.testing = True cb(Parameters(**parameters), input, task_group_id, task_id, task) -@memoize -def _load(): +def _load(graph_config): # Load all modules from this folder, relying on the side-effects of register_ # functions to populate the action registry. actions_dir = os.path.dirname(__file__) for f in os.listdir(actions_dir): if f.endswith('.py') and f not in ('__init__.py', 'registry.py', 'util.py'): __import__('taskgraph.actions.' + f[:-3]) if f.endswith('.yml'): with open(os.path.join(actions_dir, f), 'r') as d: frontmatter, template = yaml.safe_load_all(d) - register_task_action(**frontmatter)(lambda _: template) + register_task_action(**frontmatter)(lambda _p, _g: template) return callbacks, actions -def get_callbacks(): - return _load()[0] +def _get_callbacks(graph_config): + return _load(graph_config)[0] -def get_actions(): - return _load()[1] +def _get_actions(graph_config): + return _load(graph_config)[1]
--- a/taskcluster/taskgraph/decision.py +++ b/taskcluster/taskgraph/decision.py @@ -126,17 +126,17 @@ def taskgraph_decision(options, paramete tgg = TaskGraphGenerator( root_dir=options.get('root'), parameters=parameters) # write out the parameters used to generate this graph write_artifact('parameters.yml', dict(**parameters)) # write out the public/actions.json file - write_artifact('actions.json', render_actions_json(parameters)) + write_artifact('actions.json', render_actions_json(parameters, tgg.graph_config)) # write out the full graph for reference full_task_json = tgg.full_task_graph.to_json() write_artifact('full-task-graph.json', full_task_json) # write out the public/runnable-jobs.json.gz file write_artifact('runnable-jobs.json.gz', full_task_graph_to_runnable_jobs(full_task_json))
--- a/taskcluster/taskgraph/generator.py +++ b/taskcluster/taskgraph/generator.py @@ -208,27 +208,38 @@ class TaskGraphGenerator(object): The optimized task graph, with any subsequent morphs applied. This graph will have the same meaning as the optimized task graph, but be in a form more palatable to TaskCluster. @type: TaskGraph """ return self._run_until('morphed_task_graph') + @property + def graph_config(self): + """ + The configuration for this graph. + + @type: TaskGraph + """ + return self._run_until('graph_config') + def _load_kinds(self, graph_config): for kind_name in os.listdir(self.root_dir): try: yield Kind.load(self.root_dir, graph_config, kind_name) except KindNotFound: continue def _run(self): logger.info("Loading graph configuration.") graph_config = load_graph_config(self.root_dir) + yield verifications('graph_config', graph_config) + logger.info("Loading kinds") # put the kinds into a graph and sort topologically so that kinds are loaded # in post-order kinds = {kind.name: kind for kind in self._load_kinds(graph_config)} self.verify_kinds(kinds) edges = set() for kind in kinds.itervalues():