author | Carsten "Tomcat" Book <cbook@mozilla.com> |
Fri, 17 Feb 2017 15:52:10 +0100 | |
changeset 343603 | d226bf0c5b39384fec0a48da7a7d5a8da92b7ef5 |
parent 343602 | 4becea6f05dfe07f6fd49eb0080dc56f52280662 |
child 343604 | 17d5fa614caa8d11f8718b9cc7e602d1d07d4c28 |
push id | 31382 |
push user | kwierso@gmail.com |
push date | Fri, 17 Feb 2017 21:41:52 +0000 |
treeherder | mozilla-central@0930fdc4cf8e [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
bugs | 1337825 |
milestone | 54.0a1 |
backs out | c7c9f77121635f205cc35c0ad32004367558ab9f |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
taskcluster/docs/attributes.rst | file | annotate | diff | comparison | revisions | |
taskcluster/taskgraph/transforms/l10n.py | file | annotate | diff | comparison | revisions |
--- a/taskcluster/docs/attributes.rst +++ b/taskcluster/docs/attributes.rst @@ -124,21 +124,16 @@ Signals whether the task is part of a ni out nightly tasks from full task set at target stage. all_locales =========== For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list of relevant locales for the platform. -all_locales_with_changesets -=========================== - -Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``). - l10n_chunk ========== For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk number of the job. Note that this is a string! chunk_locales ============= For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py +++ b/taskcluster/taskgraph/transforms/l10n.py @@ -3,17 +3,16 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Do transforms specific to l10n kind """ from __future__ import absolute_import, print_function, unicode_literals import copy -import json from mozbuild.chunkify import chunkify from taskgraph.transforms.base import ( TransformSequence, ) from taskgraph.util.schema import ( validate_schema, optionally_keyed_by, @@ -132,34 +131,24 @@ l10n_description_schema = Schema({ transforms = TransformSequence() def _parse_locales_file(locales_file, platform=None): """ Parse the passed locales file for a list of locales. If platform is unset matches all platforms. """ locales = [] - - with open(locales_file, mode='r') as f: - if locales_file.endswith('json'): - all_locales = json.load(f) - # XXX Only single locales are fetched - locales = {locale: data['revision'] for locale, data in all_locales.items() if 'android' in data['platforms']} - else: - all_locales = f.read().split() - # 'default' is the hg revision at the top of hg repo, in this context - locales = {locale: 'default' for locale in all_locales} + if locales_file.endswith('json'): + # Release process uses .json for locale files sometimes. + raise NotImplementedError("Don't know how to parse a .json locales file") + else: + with open(locales_file, mode='r') as lf: + locales = lf.read().split() return locales -def _remove_ja_jp_mac_locale(locales): - # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, so just omit it unconditionally - return { - locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac' - } - @transforms.add def setup_name(config, jobs): for job in jobs: dep = job['dependent-task'] if dep.attributes.get('nightly'): # Set the name to the same as the dep task, without kind name. # Label will get set automatically with this kinds name. @@ -238,62 +227,62 @@ def handle_keyed_by(config, jobs): for field in fields: resolve_keyed_by(item=job, field=field, item_name=job['name']) yield job @transforms.add def all_locales_attribute(config, jobs): for job in jobs: - locales_with_changesets = _parse_locales_file(job["locales-file"]) - locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets) - - locales = sorted(locales_with_changesets.keys()) + locales = set(_parse_locales_file(job["locales-file"])) + # ja-JP-mac is a mac-only locale, but there are no + # mac builds being repacked, so just omit it unconditionally + locales = locales - set(("ja-JP-mac", )) + # Convert to mutable list. + locales = list(sorted(locales)) attributes = job.setdefault('attributes', {}) attributes["all_locales"] = locales - attributes["all_locales_with_changesets"] = locales_with_changesets yield job @transforms.add def chunk_locales(config, jobs): """ Utilizes chunking for l10n stuff """ for job in jobs: chunks = job.get('chunks') - locales_with_changesets = job['attributes']['all_locales_with_changesets'] + all_locales = job['attributes']['all_locales'] if chunks: - if chunks > len(locales_with_changesets): + if chunks > len(all_locales): # Reduce chunks down to the number of locales - chunks = len(locales_with_changesets) + chunks = len(all_locales) for this_chunk in range(1, chunks + 1): chunked = copy.deepcopy(job) chunked['name'] = chunked['name'].replace( '/', '-{}/'.format(this_chunk), 1 ) chunked['mozharness']['options'] = chunked['mozharness'].get('options', []) - # chunkify doesn't work with dicts - locales_with_changesets_as_list = locales_with_changesets.items() - chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks) + my_locales = [] + my_locales = chunkify(all_locales, this_chunk, chunks) chunked['mozharness']['options'].extend([ - 'locale={}:{}'.format(locale, changeset) for locale, changeset in chunked_locales - ]) + "locale={}".format(locale) for locale in my_locales + ]) chunked['attributes']['l10n_chunk'] = str(this_chunk) - chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales] # strip revision + chunked['attributes']['chunk_locales'] = my_locales # add the chunk number to the TH symbol group, symbol = split_symbol( chunked.get('treeherder', {}).get('symbol', '')) symbol += str(this_chunk) chunked['treeherder']['symbol'] = join_symbol(group, symbol) yield chunked else: job['mozharness']['options'] = job['mozharness'].get('options', []) job['mozharness']['options'].extend([ - 'locale={}:{}'.format(locale, changeset) for locale, changeset in locales_with_changesets.items() - ]) + "locale={}".format(locale) for locale in all_locales + ]) yield job @transforms.add def mh_config_replace_project(config, jobs): """ Replaces {project} in mh config entries with the current project """ # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot for job in jobs: