author | Johan Lorenzo <jlorenzo@mozilla.com> |
Fri, 17 Feb 2017 15:57:11 +0100 | |
changeset 343604 | 17d5fa614caa8d11f8718b9cc7e602d1d07d4c28 |
parent 343603 | d226bf0c5b39384fec0a48da7a7d5a8da92b7ef5 |
child 343605 | e952a5cc4a140c1c4015d6a3311732f944a2b90f |
push id | 31382 |
push user | kwierso@gmail.com |
push date | Fri, 17 Feb 2017 21:41:52 +0000 |
treeherder | mozilla-central@0930fdc4cf8e [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | aki |
bugs | 1337825 |
milestone | 54.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
taskcluster/docs/attributes.rst | file | annotate | diff | comparison | revisions | |
taskcluster/taskgraph/transforms/l10n.py | file | annotate | diff | comparison | revisions |
--- a/taskcluster/docs/attributes.rst +++ b/taskcluster/docs/attributes.rst @@ -124,16 +124,21 @@ Signals whether the task is part of a ni out nightly tasks from full task set at target stage. all_locales =========== For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list of relevant locales for the platform. +all_locales_with_changesets +=========================== + +Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``). + l10n_chunk ========== For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk number of the job. Note that this is a string! chunk_locales ============= For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py +++ b/taskcluster/taskgraph/transforms/l10n.py @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Do transforms specific to l10n kind """ from __future__ import absolute_import, print_function, unicode_literals import copy +import json from mozbuild.chunkify import chunkify from taskgraph.transforms.base import ( TransformSequence, ) from taskgraph.util.schema import ( validate_schema, optionally_keyed_by, @@ -131,25 +132,41 @@ l10n_description_schema = Schema({ transforms = TransformSequence() def _parse_locales_file(locales_file, platform=None): """ Parse the passed locales file for a list of locales. If platform is unset matches all platforms. """ locales = [] - if locales_file.endswith('json'): - # Release process uses .json for locale files sometimes. - raise NotImplementedError("Don't know how to parse a .json locales file") - else: - with open(locales_file, mode='r') as lf: - locales = lf.read().split() + + with open(locales_file, mode='r') as f: + if locales_file.endswith('json'): + all_locales = json.load(f) + # XXX Only single locales are fetched + locales = { + locale: data['revision'] + for locale, data in all_locales.items() + if 'android' in data['platforms'] + } + else: + all_locales = f.read().split() + # 'default' is the hg revision at the top of hg repo, in this context + locales = {locale: 'default' for locale in all_locales} return locales +def _remove_ja_jp_mac_locale(locales): + # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, + # so just omit it unconditionally + return { + locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac' + } + + @transforms.add def setup_name(config, jobs): for job in jobs: dep = job['dependent-task'] if dep.attributes.get('nightly'): # Set the name to the same as the dep task, without kind name. # Label will get set automatically with this kinds name. job['name'] = job.get('name', @@ -227,62 +244,65 @@ def handle_keyed_by(config, jobs): for field in fields: resolve_keyed_by(item=job, field=field, item_name=job['name']) yield job @transforms.add def all_locales_attribute(config, jobs): for job in jobs: - locales = set(_parse_locales_file(job["locales-file"])) - # ja-JP-mac is a mac-only locale, but there are no - # mac builds being repacked, so just omit it unconditionally - locales = locales - set(("ja-JP-mac", )) - # Convert to mutable list. - locales = list(sorted(locales)) + locales_with_changesets = _parse_locales_file(job["locales-file"]) + locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets) + + locales = sorted(locales_with_changesets.keys()) attributes = job.setdefault('attributes', {}) attributes["all_locales"] = locales + attributes["all_locales_with_changesets"] = locales_with_changesets yield job @transforms.add def chunk_locales(config, jobs): """ Utilizes chunking for l10n stuff """ for job in jobs: chunks = job.get('chunks') - all_locales = job['attributes']['all_locales'] + locales_with_changesets = job['attributes']['all_locales_with_changesets'] if chunks: - if chunks > len(all_locales): + if chunks > len(locales_with_changesets): # Reduce chunks down to the number of locales - chunks = len(all_locales) + chunks = len(locales_with_changesets) for this_chunk in range(1, chunks + 1): chunked = copy.deepcopy(job) chunked['name'] = chunked['name'].replace( '/', '-{}/'.format(this_chunk), 1 ) chunked['mozharness']['options'] = chunked['mozharness'].get('options', []) - my_locales = [] - my_locales = chunkify(all_locales, this_chunk, chunks) + # chunkify doesn't work with dicts + locales_with_changesets_as_list = locales_with_changesets.items() + chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks) chunked['mozharness']['options'].extend([ - "locale={}".format(locale) for locale in my_locales - ]) + 'locale={}:{}'.format(locale, changeset) + for locale, changeset in chunked_locales + ]) chunked['attributes']['l10n_chunk'] = str(this_chunk) - chunked['attributes']['chunk_locales'] = my_locales + # strip revision + chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales] # add the chunk number to the TH symbol group, symbol = split_symbol( chunked.get('treeherder', {}).get('symbol', '')) symbol += str(this_chunk) chunked['treeherder']['symbol'] = join_symbol(group, symbol) yield chunked else: job['mozharness']['options'] = job['mozharness'].get('options', []) job['mozharness']['options'].extend([ - "locale={}".format(locale) for locale in all_locales - ]) + 'locale={}:{}'.format(locale, changeset) + for locale, changeset in locales_with_changesets.items() + ]) yield job @transforms.add def mh_config_replace_project(config, jobs): """ Replaces {project} in mh config entries with the current project """ # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot for job in jobs: