author Dustin J. Mitchell <dustin@mozilla.com>
Wed, 01 Jun 2016 20:29:08 +0000
changeset 375507 e0f07eab0aa8fd4a95d44564283e09dcb30edd46
parent 375504 01002e8a7eb2e422a60832a1709f3e8c8c1830c0
child 375934 e48ea22314b8d67d3c58aa09551a9da6150501a6
child 376335 767a3268c1fec871b0174e1e50e7e4365dc26c35
permissions -rw-r--r--
Bug 1274611: spell target-tasks.json with `-`; r?garndt MozReview-Commit-ID: Aij7wL3onop

# -*- coding: utf-8 -*-

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function, unicode_literals

import os
import json
import logging
import yaml

from .generator import TaskGraphGenerator
from .create import create_tasks
from .parameters import Parameters
from .target_tasks import get_method

logger = logging.getLogger(__name__)
ARTIFACTS_DIR = 'artifacts'

logger = logging.getLogger(__name__)

# For each project, this gives a set of parameters specific to the project.
# See `taskcluster/docs/parameters.rst` for information on parameters.
    'try': {
        'target_tasks_method': 'try_option_syntax',
        # for try, if a task was specified as a target, it should
        # not be optimized away
        'optimize_target_tasks': False,

    # the default parameters are used for projects that do not match above.
    'default': {
        'target_tasks_method': 'all_builds_and_tests',
        'optimize_target_tasks': True,

def taskgraph_decision(options):
    Run the decision task.  This function implements `mach taskgraph decision`,
    and is responsible for

     * processing decision task command-line options into parameters
     * running task-graph generation exactly the same way the other `mach
       taskgraph` commands do
     * generating a set of artifacts to memorialize the graph
     * calling TaskCluster APIs to create the graph

    parameters = get_decision_parameters(options)

    # create a TaskGraphGenerator instance
    target_tasks_method = parameters.get('target_tasks_method', 'all_tasks')
    target_tasks_method = get_method(target_tasks_method)
    tgg = TaskGraphGenerator(

    # write out the parameters used to generate this graph
    write_artifact('parameters.yml', dict(**parameters))

    # write out the full graph for reference

    # write out the target task set to allow reproducing this as input

    # write out the optimized task graph to describe what will actually happen,
    # and the map of labels to taskids
    write_artifact('label-to-taskid.json', tgg.label_to_taskid)

    # actually create the graph
    create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid)

def get_decision_parameters(options):
    Load parameters from the command-line options for 'taskgraph decision'.
    This also applies per-project parameters, based on the given project.

    parameters = {n: options[n] for n in [
    ] if n in options}

    project = parameters['project']
    except KeyError:
        logger.warning("using default project parameters; add {} to "
              "PER_PROJECT_PARAMETERS in {} to customize behavior "
              "for this project".format(project, __file__))

    return Parameters(parameters)

def taskgraph_to_json(taskgraph):
    tasks = taskgraph.tasks

    def tojson(task):
        return {
            'label': task.label,
            'task': task.task,
            'attributes': task.attributes,
            'dependencies': []
    rv = {label: tojson(tasks[label]) for label in taskgraph.graph.nodes}

    # add dependencies with one trip through the graph edges
    for (left, right, name) in taskgraph.graph.edges:
        rv[left]['dependencies'].append((name, right))

    return rv

def write_artifact(filename, data):
    logger.info('writing artifact file `{}`'.format(filename))
    if not os.path.isdir(ARTIFACTS_DIR):
    path = os.path.join(ARTIFACTS_DIR, filename)
    if filename.endswith('.yml'):
        with open(path, 'w') as f:
            yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
    elif filename.endswith('.json'):
        with open(path, 'w') as f:
            json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': '))
        raise TypeError("Don't know how to write to {}".format(filename))