taskcluster/taskgraph/transforms/job/toolchain.py
author Brindusan Cristian <cbrindusan@mozilla.com>
Mon, 24 Jun 2019 06:09:11 +0300
changeset 479903 48f71175c99922f25dc5311f14db5a72c1b3f726
parent 459713 a4d10484972af18e49105238b8001ae6402f3d69
permissions -rw-r--r--
Backed out changeset c872cc627115 (bug 1558937) for reftest failures at 1558937-1.html.

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for running toolchain-building jobs via dedicated scripts
"""

from __future__ import absolute_import, print_function, unicode_literals

from mozbuild.shellutil import quote as shell_quote

from taskgraph.util.schema import Schema
from voluptuous import Optional, Required, Any

from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
    docker_worker_add_artifacts,
    docker_worker_add_tooltool,
    generic_worker_hg_commands,
    support_vcs_checkout,
)
from taskgraph.util.hash import hash_paths
from taskgraph import GECKO
import taskgraph


CACHE_TYPE = 'toolchains.v3'

toolchain_run_schema = Schema({
    Required('using'): 'toolchain-script',

    # The script (in taskcluster/scripts/misc) to run.
    # Python scripts are invoked with `mach python` so vendored libraries
    # are available.
    Required('script'): basestring,

    # Arguments to pass to the script.
    Optional('arguments'): [basestring],

    # If not false, tooltool downloads will be enabled via relengAPIProxy
    # for either just public files, or all files.  Not supported on Windows
    Required('tooltool-downloads'): Any(
        False,
        'public',
        'internal',
    ),

    # Sparse profile to give to checkout using `run-task`.  If given,
    # a filename in `build/sparse-profiles`.  Defaults to
    # "toolchain-build", i.e., to
    # `build/sparse-profiles/toolchain-build`.  If `None`, instructs
    # `run-task` to not use a sparse profile at all.
    Required('sparse-profile'): Any(basestring, None),

    # Paths/patterns pointing to files that influence the outcome of a
    # toolchain build.
    Optional('resources'): [basestring],

    # Path to the artifact produced by the toolchain job
    Required('toolchain-artifact'): basestring,

    # An alias that can be used instead of the real toolchain job name in
    # the toolchains list for build jobs.
    Optional('toolchain-alias'): basestring,

    # Base work directory used to set up the task.
    Required('workdir'): basestring,
})


def get_digest_data(config, run, taskdesc):
    files = list(run.get('resources', []))
    # This file
    files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
    # The script
    files.append('taskcluster/scripts/misc/{}'.format(run['script']))
    # Tooltool manifest if any is defined:
    tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST')
    if tooltool_manifest:
        files.append(tooltool_manifest)

    # Accumulate dependency hashes for index generation.
    data = [hash_paths(GECKO, files)]

    # If the task uses an in-tree docker image, we want it to influence
    # the index path as well. Ideally, the content of the docker image itself
    # should have an influence, but at the moment, we can't get that
    # information here. So use the docker image name as a proxy. Not a lot of
    # changes to docker images actually have an impact on the resulting
    # toolchain artifact, so we'll just rely on such important changes to be
    # accompanied with a docker image name change.
    image = taskdesc['worker'].get('docker-image', {}).get('in-tree')
    if image:
        data.append(image)

    # Likewise script arguments should influence the index.
    args = run.get('arguments')
    if args:
        data.extend(args)
    return data


toolchain_defaults = {
    'tooltool-downloads': False,
    'sparse-profile': 'toolchain-build',
}


@run_job_using("docker-worker", "toolchain-script",
               schema=toolchain_run_schema, defaults=toolchain_defaults)
def docker_worker_toolchain(config, job, taskdesc):
    run = job['run']

    worker = taskdesc['worker']
    worker['chain-of-trust'] = True

    # If the task doesn't have a docker-image, set a default
    worker.setdefault('docker-image', {'in-tree': 'toolchain-build'})

    # Allow the job to specify where artifacts come from, but add
    # public/build if it's not there already.
    artifacts = worker.setdefault('artifacts', [])
    if not any(artifact.get('name') == 'public/build' for artifact in artifacts):
        docker_worker_add_artifacts(config, job, taskdesc)

    support_vcs_checkout(config, job, taskdesc, sparse=('sparse-profile' in run))

    # Toolchain checkouts don't live under {workdir}/checkouts
    workspace = '{workdir}/workspace/build'.format(**run)
    gecko_path = '{}/src'.format(workspace)

    env = worker['env']
    env.update({
        'MOZ_BUILD_DATE': config.params['moz_build_date'],
        'MOZ_SCM_LEVEL': config.params['level'],
        'TOOLS_DISABLE': 'true',
        'MOZ_AUTOMATION': '1',
        'MOZ_FETCHES_DIR': workspace,
        'GECKO_PATH': gecko_path,
    })

    if run['tooltool-downloads']:
        internal = run['tooltool-downloads'] == 'internal'
        docker_worker_add_tooltool(config, job, taskdesc, internal=internal)

    # Use `mach` to invoke python scripts so in-tree libraries are available.
    if run['script'].endswith('.py'):
        wrapper = '{}/mach python '.format(gecko_path)
    else:
        wrapper = ''

    args = run.get('arguments', '')
    if args:
        args = ' ' + shell_quote(*args)

    sparse_profile = []
    if run.get('sparse-profile'):
        sparse_profile = ['--gecko-sparse-profile=build/sparse-profiles/{}'
                          .format(run['sparse-profile'])]

    worker['command'] = [
        '{workdir}/bin/run-task'.format(**run),
        '--gecko-checkout={}'.format(gecko_path),
    ] + sparse_profile + [
        '--',
        'bash',
        '-c',
        'cd {} && '
        '{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
            run['workdir'], wrapper, run['script'], args)
    ]

    attributes = taskdesc.setdefault('attributes', {})
    attributes['toolchain-artifact'] = run['toolchain-artifact']
    if 'toolchain-alias' in run:
        attributes['toolchain-alias'] = run['toolchain-alias']

    if not taskgraph.fast:
        name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
        taskdesc['cache'] = {
            'type': CACHE_TYPE,
            'name': name,
            'digest-data': get_digest_data(config, run, taskdesc),
        }


@run_job_using("generic-worker", "toolchain-script",
               schema=toolchain_run_schema, defaults=toolchain_defaults)
def windows_toolchain(config, job, taskdesc):
    run = job['run']

    worker = taskdesc['worker']

    worker['artifacts'] = [{
        'path': r'public\build',
        'type': 'directory',
    }]
    worker['chain-of-trust'] = True

    # There were no caches on generic-worker before bug 1519472, and they cause
    # all sorts of problems with toolchain tasks, disable them until
    # tasks are ready.
    run['use-caches'] = False
    support_vcs_checkout(config, job, taskdesc, sparse=('sparse-profile' in run))

    env = worker['env']
    env.update({
        'MOZ_BUILD_DATE': config.params['moz_build_date'],
        'MOZ_SCM_LEVEL': config.params['level'],
        'MOZ_AUTOMATION': '1',
    })

    sparse_profile = run.get('sparse-profile')
    if sparse_profile:
        sparse_profile = 'build/sparse-profiles/{}'.format(run['sparse-profile'])

    hg_command = generic_worker_hg_commands(
        'https://hg.mozilla.org/mozilla-unified',
        env['GECKO_HEAD_REPOSITORY'],
        env['GECKO_HEAD_REV'],
        r'.\build\src', sparse_profile=sparse_profile)[0]

    # Use `mach` to invoke python scripts so in-tree libraries are available.
    if run['script'].endswith('.py'):
        raise NotImplementedError("Python scripts don't work on Windows")

    args = run.get('arguments', '')
    if args:
        args = ' ' + shell_quote(*args)

    bash = r'c:\mozilla-build\msys\bin\bash'
    worker['command'] = [
        hg_command,
        # do something intelligent.
        r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
            bash, run['script'], args)
    ]

    attributes = taskdesc.setdefault('attributes', {})
    attributes['toolchain-artifact'] = run['toolchain-artifact']
    if 'toolchain-alias' in run:
        attributes['toolchain-alias'] = run['toolchain-alias']

    if not taskgraph.fast:
        name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
        taskdesc['cache'] = {
            'type': CACHE_TYPE,
            'name': name,
            'digest-data': get_digest_data(config, run, taskdesc),
        }