taskcluster/taskgraph/transforms/source_test.py
author Sebastian Hengst <archaeopteryx@coole-files.de>
Fri, 16 Aug 2019 11:19:54 +0200
changeset 488476 8da8443e0bcb7a6d9766d179332443660c926d8b
parent 488356 056d9515483cd593dc1a931b4ade7ba7b143b540
child 488548 eaf06f57c25010d146477f6535116a91eebfbec6
permissions -rw-r--r--
Backed out 10 changesets (bug 1568277) for breaking ./mach bootstrap. a=backout Backed out changeset 056d9515483c (bug 1568277) Backed out changeset 632d943c947b (bug 1568277) Backed out changeset d7e8f80e2c85 (bug 1568277) Backed out changeset 7b59ed5d703d (bug 1568277) Backed out changeset f8b41cbaaf8e (bug 1568277) Backed out changeset 391a90f3f02b (bug 1568277) Backed out changeset d2b1d6c0a732 (bug 1568277) Backed out changeset cb35fd836621 (bug 1568277) Backed out changeset c8f797a19731 (bug 1568277) Backed out changeset 9b89f970d46b (bug 1568277)

# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Source-test jobs can run on multiple platforms.  These transforms allow jobs
with either `platform` or a list of `platforms`, and set the appropriate
treeherder configuration and attributes for that platform.
"""

from __future__ import absolute_import, print_function, unicode_literals

import copy
import os

from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.job import job_description_schema
from taskgraph.util.attributes import keymatch
from taskgraph.util.schema import (
    resolve_keyed_by,
)
from taskgraph.util.treeherder import join_symbol, split_symbol

from voluptuous import (
    Any,
    Extra,
    Optional,
    Required,
    Schema,
)

source_test_description_schema = Schema({
    # most fields are passed directly through as job fields, and are not
    # repeated here
    Extra: object,

    # The platform on which this task runs.  This will be used to set up attributes
    # (for try selection) and treeherder metadata (for display).  If given as a list,
    # the job will be "split" into multiple tasks, one with each platform.
    Required('platform'): Any(basestring, [basestring]),

    # Whether the job requires a build artifact or not. If True, the task will
    # depend on a build task and the installer url will be saved to the
    # GECKO_INSTALLER_URL environment variable. Build labels are determined by the
    # `dependent-build-platforms` config in kind.yml.
    Required('require-build'): bool,

    # These fields can be keyed by "platform", and are otherwise identical to
    # job descriptions.
    Required('worker-type'): Any(
        job_description_schema['worker-type'],
        {'by-platform': {basestring: job_description_schema['worker-type']}},
    ),
    Required('worker'): Any(
        job_description_schema['worker'],
        {'by-platform': {basestring: job_description_schema['worker']}},
    ),
    Optional('python-version'): [int],
})

transforms = TransformSequence()


@transforms.add
def set_defaults(config, jobs):
    for job in jobs:
        job.setdefault('require-build', False)
        yield job


transforms.add_validate(source_test_description_schema)


@transforms.add
def set_job_name(config, jobs):
    for job in jobs:
        if 'job-from' in job and job['job-from'] != 'kind.yml':
            from_name = os.path.splitext(job['job-from'])[0]
            job['name'] = '{}-{}'.format(from_name, job['name'])
        yield job


@transforms.add
def expand_platforms(config, jobs):
    for job in jobs:
        if isinstance(job['platform'], basestring):
            yield job
            continue

        for platform in job['platform']:
            pjob = copy.deepcopy(job)
            pjob['platform'] = platform

            if 'name' in pjob:
                pjob['name'] = '{}-{}'.format(pjob['name'], platform)
            else:
                pjob['label'] = '{}-{}'.format(pjob['label'], platform)
            yield pjob


@transforms.add
def split_python(config, jobs):
    for job in jobs:
        key = 'python-version'
        versions = job.pop(key, [])
        if not versions:
            yield job
            continue
        for version in versions:
            group = 'py{0}'.format(version)
            pyjob = copy.deepcopy(job)
            if 'name' in pyjob:
                pyjob['name'] += '-{0}'.format(group)
            else:
                pyjob['label'] += '-{0}'.format(group)
            symbol = split_symbol(pyjob['treeherder']['symbol'])[1]
            pyjob['treeherder']['symbol'] = join_symbol(group, symbol)
            pyjob['run'][key] = version
            yield pyjob


@transforms.add
def split_jsshell(config, jobs):
    all_shells = {
        'sm': "Spidermonkey",
        'v8': "Google V8"
    }

    for job in jobs:
        if not job['name'].startswith('jsshell'):
            yield job
            continue

        test = job.pop('test')
        for shell in job.get('shell', all_shells.keys()):
            assert shell in all_shells

            new_job = copy.deepcopy(job)
            new_job['name'] = '{}-{}'.format(new_job['name'], shell)
            new_job['description'] = '{} on {}'.format(new_job['description'], all_shells[shell])
            new_job['shell'] = shell

            group = 'js-bench-{}'.format(shell)
            symbol = split_symbol(new_job['treeherder']['symbol'])[1]
            new_job['treeherder']['symbol'] = join_symbol(group, symbol)

            run = new_job['run']
            run['mach'] = run['mach'].format(shell=shell, SHELL=shell.upper(), test=test)
            yield new_job


def add_build_dependency(config, job):
    """
    Add build dependency to the job and installer_url to env.
    """
    key = job['platform']
    build_labels = config.config.get('dependent-build-platforms', {})
    matches = keymatch(build_labels, key)
    if not matches:
        raise Exception("No build platform found for '{}'. "
                        "Define 'dependent-build-platforms' in the kind config.".format(key))

    if len(matches) > 1:
        raise Exception("More than one build platform found for '{}'.".format(key))

    label = matches[0]
    deps = job.setdefault('dependencies', {})
    deps.update({'build': label})


@transforms.add
def handle_platform(config, jobs):
    """
    Handle the 'platform' property, setting up treeherder context as well as
    try-related attributes.
    """
    fields = [
        'worker-type',
        'worker',
    ]

    for job in jobs:
        platform = job['platform']

        for field in fields:
            resolve_keyed_by(job, field, item_name=job['name'])

        if 'treeherder' in job:
            job['treeherder']['platform'] = platform

        if job.pop('require-build'):
            add_build_dependency(config, job)

        del job['platform']
        yield job


@transforms.add
def handle_shell(config, jobs):
    """
    Handle the 'shell' property.
    """
    fields = [
        'run-on-projects',
        'worker.env',
    ]

    for job in jobs:
        if not job.get('shell'):
            yield job
            continue

        for field in fields:
            resolve_keyed_by(job, field, item_name=job['name'])

        del job['shell']
        yield job