author Dustin J. Mitchell <>
Tue, 25 Sep 2018 20:18:19 +0000
changeset 451433 5ea6f03f845e49d503f5d0283557f54561c41654
parent 451261 28f92797e661694da8f10e3a9a8fbcfff49c919c
child 451467 d0e13414d6512c9fe84911a0dd730e4fb4a28c27
permissions -rw-r--r--
Bug 1492664 - set TASKCLUSTER_ROOT_URL and TASKCLUSTER_PROXY_URL; r=tomprince,glandium Eventually, workers will provide these variables directly ( But for now, this ensures that TASKCLUSTER_ROOT_URL is set everywhere, and TASKCLUSTER_PROXY_URL is set wherever the proxy is active. The setup for the mach commands defaults to for user convenience. When the production instance's URL changes, we can simply change that default. This changes the docker build process propagate TASKCLUSTER_ROOT_URL into the docker images where necessary (using %ARG), specifically to create URLs for debian repo paths.

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

from __future__ import absolute_import, print_function, unicode_literals

import glob
import hashlib
import json
import os
import re
import shutil
import sys
from collections import defaultdict

from mozboot.util import get_state_dir
from mozbuild.base import MozbuildObject
from mozpack.files import FileFinder
from moztest.resolve import TestResolver, get_suite_definition

import taskgraph
from taskgraph.generator import TaskGraphGenerator
from taskgraph.parameters import (
from taskgraph.taskgraph import TaskGraph

here = os.path.abspath(os.path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here)

ERROR - The parameters being used to generate tasks differ from those defined
in your working copy:


To fix this, either rebase onto the latest mozilla-central or pass in
-p/--parameters. For more information on how to define parameters, see:

def invalidate(cache, root):
    if not os.path.isfile(cache):

    tc_dir = os.path.join(root, 'taskcluster')
    tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
    cmod = os.path.getmtime(cache)

    if tmod > cmod:

def generate_tasks(params, full, root):
    # Ensure that TASKCLUSTER_ROOT_URL is set

    params = params or "project=mozilla-central"

    # Try to delete the old taskgraph cache directory.
    old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
    if os.path.isdir(old_cache_dir):

    root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
    cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash, 'taskgraph')

    # Cleanup old cache files
    for path in glob.glob(os.path.join(cache_dir, '*_set')):

    attr = 'full_task_graph' if full else 'target_task_graph'
    cache = os.path.join(cache_dir, attr)

    invalidate(cache, root)
    if os.path.isfile(cache):
        with open(cache, 'r') as fh:
            return TaskGraph.from_json(json.load(fh))[1]

    if not os.path.isdir(cache_dir):

    print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
        params = load_parameters_file(params, strict=False, overrides={'try_mode': 'try_select'})
    except ParameterMismatch as e:
        sys.exit(1) = True
    cwd = os.getcwd()

    root = os.path.join(root, 'taskcluster', 'ci')
    tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)


    with open(cache, 'w') as fh:
        json.dump(tg.to_json(), fh)
    return tg

def filter_tasks_by_paths(tasks, paths):
    resolver = TestResolver.from_environment(cwd=here)
    run_suites, run_tests = resolver.resolve_metadata(paths)
    flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])

    task_regexes = set()
    for flavor, subsuite in flavors:
        suite = get_suite_definition(flavor, subsuite, strict=True)
        if 'task_regex' not in suite:
            print("warning: no tasks could be resolved from flavor '{}'{}".format(
                    flavor, " and subsuite '{}'".format(subsuite) if subsuite else ""))


    def match_task(task):
        return any(, task) for pattern in task_regexes)

    return filter(match_task, tasks)

def resolve_tests_by_suite(paths):
    resolver = TestResolver.from_environment(cwd=here)
    _, run_tests = resolver.resolve_metadata(paths)

    suite_to_tests = defaultdict(list)
    for test in run_tests:
        key = test['flavor']
        subsuite = test.get('subsuite')
        if subsuite:
            key += '-' + subsuite

    return suite_to_tests