taskcluster/taskgraph/transforms/l10n.py
author Justin Wood <Callek@gmail.com>
Fri, 27 Jan 2017 15:47:30 -0500
changeset 378211 e9ebff9c36960e9ffb4407ba073ef28a8a8f665b
parent 377163 b3774461acc6bee2216c5f57e167f9e5795fb09d
child 378601 0ef6ce3951925a1972396f61db87aedfb3c9004a
child 381179 3cb163fd021d30f2e6e4a9eab8f3d1c70dfd5c2c
permissions -rw-r--r--
Bug 1333531 - chunkify.chunkify breaks if less locales are in all-locales than chunks in decision task. r=aki a=gchang MozReview-Commit-ID: Hr7KgyHIMlc

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Do transforms specific to l10n kind
"""

from __future__ import absolute_import, print_function, unicode_literals

import copy

from mozbuild.chunkify import chunkify
from taskgraph.transforms.base import (
    TransformSequence,
    resolve_keyed_by,
    optionally_keyed_by,
    validate_schema
)
from taskgraph.util.treeherder import split_symbol, join_symbol
from voluptuous import (
    Any,
    Extra,
    Optional,
    Required,
    Schema,
)


def _by_platform(arg):
    return optionally_keyed_by('build-platform', arg)

# shortcut for a string where task references are allowed
taskref_or_string = Any(
    basestring,
    {Required('task-reference'): basestring})

l10n_description_schema = Schema({
    # Name for this job, inferred from the dependent job before validation
    Required('name'): basestring,

    # build-platform, inferred from dependent job before validation
    Required('build-platform'): basestring,

    # max run time of the task
    Required('run-time'): _by_platform(int),

    # Data used by chain of trust (see `chain_of_trust` in this file)
    Optional('chainOfTrust'): {Extra: object},

    # All l10n jobs use mozharness
    Required('mozharness'): {
        # Script to invoke for mozharness
        Required('script'): _by_platform(basestring),

        # Config files passed to the mozharness script
        Required('config'): _by_platform([basestring]),

        # Options to pass to the mozharness script
        Required('options'): _by_platform([basestring]),

        # Action commands to provide to mozharness script
        Required('actions'): _by_platform([basestring]),
    },
    # Items for the taskcluster index
    Optional('index'): {
        # Product to identify as in the taskcluster index
        Required('product'): _by_platform(basestring),

        # Job name to identify as in the taskcluster index
        Required('job-name'): _by_platform(basestring),

        # Type of index
        Optional('type'): basestring,
    },
    # Description of the localized task
    Required('description'): _by_platform(basestring),

    # task object of the dependent task
    Required('dependent-task'): object,

    # worker-type to utilize
    Required('worker-type'): _by_platform(basestring),

    # File which contains the used locales
    Required('locales-file'): _by_platform(basestring),

    # Tooltool visibility required for task.
    Required('tooltool'): _by_platform(Any('internal', 'public')),

    # Information for treeherder
    Required('treeherder'): {
        # Platform to display the task on in treeherder
        Required('platform'): _by_platform(basestring),

        # Symbol to use
        Required('symbol'): basestring,

        # Tier this task is
        Required('tier'): _by_platform(int),
    },
    Required('attributes'): {
        # Is this a nightly task, inferred from dependent job before validation
        Optional('nightly'): bool,

        # build_platform of this task, inferred from dependent job before validation
        Required('build_platform'): basestring,

        # build_type for this task, inferred from dependent job before validation
        Required('build_type'): basestring,
        Extra: object,
    },

    # Extra environment values to pass to the worker
    Optional('env'): _by_platform({basestring: taskref_or_string}),

    # Number of chunks to split the locale repacks up into
    Optional('chunks'): _by_platform(int),

    # Task deps to chain this task with, added in transforms from dependent-task
    # if this is a nightly
    Optional('dependencies'): {basestring: basestring},

    # Run the task when the listed files change (if present).
    Optional('when'): {
        'files-changed': [basestring]
    }
})

transforms = TransformSequence()


def _parse_locales_file(locales_file, platform=None):
    """ Parse the passed locales file for a list of locales.
        If platform is unset matches all platforms.
    """
    locales = []
    if locales_file.endswith('json'):
        # Release process uses .json for locale files sometimes.
        raise NotImplementedError("Don't know how to parse a .json locales file")
    else:
        with open(locales_file, mode='r') as lf:
            locales = lf.read().split()
    return locales


@transforms.add
def setup_name(config, jobs):
    for job in jobs:
        dep = job['dependent-task']
        if dep.attributes.get('nightly'):
            # Set the name to the same as the dep task, without kind name.
            # Label will get set automatically with this kinds name.
            job['name'] = job.get('name',
                                  dep.task['metadata']['name'][
                                    len(dep.kind) + 1:])
        else:
            # Set to match legacy use at the moment (to support documented try
            # syntax). Set the name to same as dep task + '-l10n' but without the
            # kind name attached, since that gets added when label is generated
            name, jobtype = dep.task['metadata']['name'][len(dep.kind) + 1:].split('/')
            job['name'] = "{}-l10n/{}".format(name, jobtype)
        yield job


@transforms.add
def copy_in_useful_magic(config, jobs):
    for job in jobs:
        dep = job['dependent-task']
        attributes = job.setdefault('attributes', {})
        # build-platform is needed on `job` for by-build-platform
        job['build-platform'] = dep.attributes.get("build_platform")
        attributes['build_type'] = dep.attributes.get("build_type")
        if dep.attributes.get("nightly"):
            attributes['nightly'] = dep.attributes.get("nightly")
        else:
            # set build_platform to have l10n as well, to match older l10n setup
            # for now
            job['build-platform'] = "{}-l10n".format(job['build-platform'])

        attributes['build_platform'] = job['build-platform']
        yield job


@transforms.add
def validate_early(config, jobs):
    for job in jobs:
        yield validate_schema(l10n_description_schema, job,
                              "In job {!r}:".format(job.get('name', 'unknown')))


@transforms.add
def setup_nightly_dependency(config, jobs):
    """ Sets up a task dependency to the signing job this relates to """
    for job in jobs:
        if not job['attributes'].get('nightly'):
            yield job
            continue  # do not add a dep unless we're a nightly
        job['dependencies'] = {'unsigned-build': job['dependent-task'].label}
        yield job


@transforms.add
def handle_keyed_by(config, jobs):
    """Resolve fields that can be keyed by platform, etc."""
    fields = [
        "locales-file",
        "chunks",
        "worker-type",
        "description",
        "run-time",
        "tooltool",
        "env",
        "mozharness.config",
        "mozharness.options",
        "mozharness.actions",
        "mozharness.script",
        "treeherder.tier",
        "treeherder.platform",
        "index.product",
        "index.job-name",
        "when.files-changed",
    ]
    for job in jobs:
        job = copy.deepcopy(job)  # don't overwrite dict values here
        for field in fields:
            resolve_keyed_by(item=job, field=field, item_name=job['name'])
        yield job


@transforms.add
def all_locales_attribute(config, jobs):
    for job in jobs:
        locales = set(_parse_locales_file(job["locales-file"]))
        # ja-JP-mac is a mac-only locale, but there are no
        # mac builds being repacked, so just omit it unconditionally
        locales = locales - set(("ja-JP-mac", ))
        # Convert to mutable list.
        locales = list(sorted(locales))
        attributes = job.setdefault('attributes', {})
        attributes["all_locales"] = locales
        yield job


@transforms.add
def chunk_locales(config, jobs):
    """ Utilizes chunking for l10n stuff """
    for job in jobs:
        chunks = job.get('chunks')
        all_locales = job['attributes']['all_locales']
        if chunks:
            if chunks > len(all_locales):
                # Reduce chunks down to the number of locales
                chunks = len(all_locales)
            for this_chunk in range(1, chunks + 1):
                chunked = copy.deepcopy(job)
                chunked['name'] = chunked['name'].replace(
                    '/', '-{}/'.format(this_chunk), 1
                )
                chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
                my_locales = []
                my_locales = chunkify(all_locales, this_chunk, chunks)
                chunked['mozharness']['options'].extend([
                    "locale={}".format(locale) for locale in my_locales
                    ])
                chunked['attributes']['l10n_chunk'] = str(this_chunk)
                chunked['attributes']['chunk_locales'] = my_locales

                # add the chunk number to the TH symbol
                group, symbol = split_symbol(
                    chunked.get('treeherder', {}).get('symbol', ''))
                symbol += str(this_chunk)
                chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                yield chunked
        else:
            job['mozharness']['options'] = job['mozharness'].get('options', [])
            job['mozharness']['options'].extend([
                "locale={}".format(locale) for locale in all_locales
                ])
            yield job


@transforms.add
def mh_config_replace_project(config, jobs):
    """ Replaces {project} in mh config entries with the current project """
    # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
    for job in jobs:
        job['mozharness']['config'] = map(
            lambda x: x.format(project=config.params['project']),
            job['mozharness']['config']
            )
        yield job


@transforms.add
def mh_options_replace_project(config, jobs):
    """ Replaces {project} in mh option entries with the current project """
    # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
    for job in jobs:
        job['mozharness']['options'] = map(
            lambda x: x.format(project=config.params['project']),
            job['mozharness']['options']
            )
        yield job


@transforms.add
def chain_of_trust(config, jobs):
    for job in jobs:
        job.setdefault('chainOfTrust', {})
        job['chainOfTrust'].setdefault('inputs', {})
        job['chainOfTrust']['inputs']['docker-image'] = {
            "task-reference": "<docker-image>"
        }
        yield job


@transforms.add
def validate_again(config, jobs):
    for job in jobs:
        yield validate_schema(l10n_description_schema, job,
                              "In job {!r}:".format(job.get('name', 'unknown')))


@transforms.add
def make_job_description(config, jobs):
    for job in jobs:
        job_description = {
            'name': job['name'],
            'worker': {
                'implementation': 'docker-worker',
                'docker-image': {'in-tree': 'desktop-build'},
                'max-run-time': job['run-time'],
                'chain-of-trust': True,
            },
            'extra': {
                'chainOfTrust': job['chainOfTrust'],
            },
            'worker-type': job['worker-type'],
            'description': job['description'],
            'run': {
                'using': 'mozharness',
                'job-script': 'taskcluster/scripts/builder/build-l10n.sh',
                'config': job['mozharness']['config'],
                'script': job['mozharness']['script'],
                'actions': job['mozharness']['actions'],
                'options': job['mozharness']['options'],
                'tooltool-downloads': job['tooltool'],
                'need-xvfb': True,
            },
            'attributes': job['attributes'],
            'treeherder': {
                'kind': 'build',
                'tier': job['treeherder']['tier'],
                'symbol': job['treeherder']['symbol'],
                'platform': job['treeherder']['platform'],
            },
            'run-on-projects': [],
        }

        if job.get('index'):
            job_description['index'] = {
                'product': job['index']['product'],
                'job-name': job['index']['job-name'],
                'type': job['index'].get('type', 'generic'),
            }

        if job.get('dependencies'):
            job_description['dependencies'] = job['dependencies']
        if job.get('env'):
            job_description['worker']['env'] = job['env']
        if job.get('when', {}).get('files-changed'):
            job_description.setdefault('when', {})
            job_description['when']['files-changed'] = \
                [job['locales-file']] + job['when']['files-changed']
        yield job_description