Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki
☠☠ backed out by d226bf0c5b39 ☠ ☠
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 14 Feb 2017 15:04:54 +0100
changeset 343597 c7c9f77121635f205cc35c0ad32004367558ab9f
parent 343596 72bc1a760dddb809e14101eacff80bb650323828
child 343598 162b85cf472d3e43359fcfc660c00b2f80c79239
push id31382
push userkwierso@gmail.com
push dateFri, 17 Feb 2017 21:41:52 +0000
treeherdermozilla-central@0930fdc4cf8e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1337825
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki MozReview-Commit-ID: 1DEIjFYAx74
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -124,16 +124,21 @@ Signals whether the task is part of a ni
 out nightly tasks from full task set at target stage.
 
 all_locales
 ===========
 
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list
 of relevant locales for the platform.
 
+all_locales_with_changesets
+===========================
+
+Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``).
+
 l10n_chunk
 ==========
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
 number of the job. Note that this is a string!
 
 chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Do transforms specific to l10n kind
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
+import json
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import (
     TransformSequence,
 )
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
@@ -131,24 +132,34 @@ l10n_description_schema = Schema({
 transforms = TransformSequence()
 
 
 def _parse_locales_file(locales_file, platform=None):
     """ Parse the passed locales file for a list of locales.
         If platform is unset matches all platforms.
     """
     locales = []
-    if locales_file.endswith('json'):
-        # Release process uses .json for locale files sometimes.
-        raise NotImplementedError("Don't know how to parse a .json locales file")
-    else:
-        with open(locales_file, mode='r') as lf:
-            locales = lf.read().split()
+
+    with open(locales_file, mode='r') as f:
+        if locales_file.endswith('json'):
+            all_locales = json.load(f)
+            # XXX Only single locales are fetched
+            locales = {locale: data['revision'] for locale, data in all_locales.items() if 'android' in data['platforms']}
+        else:
+            all_locales = f.read().split()
+            # 'default' is the hg revision at the top of hg repo, in this context
+            locales = {locale: 'default' for locale in all_locales}
     return locales
 
+def _remove_ja_jp_mac_locale(locales):
+    # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, so just omit it unconditionally
+    return {
+        locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac'
+    }
+
 
 @transforms.add
 def setup_name(config, jobs):
     for job in jobs:
         dep = job['dependent-task']
         if dep.attributes.get('nightly'):
             # Set the name to the same as the dep task, without kind name.
             # Label will get set automatically with this kinds name.
@@ -227,62 +238,62 @@ def handle_keyed_by(config, jobs):
         for field in fields:
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
         yield job
 
 
 @transforms.add
 def all_locales_attribute(config, jobs):
     for job in jobs:
-        locales = set(_parse_locales_file(job["locales-file"]))
-        # ja-JP-mac is a mac-only locale, but there are no
-        # mac builds being repacked, so just omit it unconditionally
-        locales = locales - set(("ja-JP-mac", ))
-        # Convert to mutable list.
-        locales = list(sorted(locales))
+        locales_with_changesets = _parse_locales_file(job["locales-file"])
+        locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets)
+
+        locales = sorted(locales_with_changesets.keys())
         attributes = job.setdefault('attributes', {})
         attributes["all_locales"] = locales
+        attributes["all_locales_with_changesets"] = locales_with_changesets
         yield job
 
 
 @transforms.add
 def chunk_locales(config, jobs):
     """ Utilizes chunking for l10n stuff """
     for job in jobs:
         chunks = job.get('chunks')
-        all_locales = job['attributes']['all_locales']
+        locales_with_changesets = job['attributes']['all_locales_with_changesets']
         if chunks:
-            if chunks > len(all_locales):
+            if chunks > len(locales_with_changesets):
                 # Reduce chunks down to the number of locales
-                chunks = len(all_locales)
+                chunks = len(locales_with_changesets)
             for this_chunk in range(1, chunks + 1):
                 chunked = copy.deepcopy(job)
                 chunked['name'] = chunked['name'].replace(
                     '/', '-{}/'.format(this_chunk), 1
                 )
                 chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
-                my_locales = []
-                my_locales = chunkify(all_locales, this_chunk, chunks)
+                # chunkify doesn't work with dicts
+                locales_with_changesets_as_list = locales_with_changesets.items()
+                chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
                 chunked['mozharness']['options'].extend([
-                    "locale={}".format(locale) for locale in my_locales
-                    ])
+                    'locale={}:{}'.format(locale, changeset) for locale, changeset in chunked_locales
+                ])
                 chunked['attributes']['l10n_chunk'] = str(this_chunk)
-                chunked['attributes']['chunk_locales'] = my_locales
+                chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]  # strip revision
 
                 # add the chunk number to the TH symbol
                 group, symbol = split_symbol(
                     chunked.get('treeherder', {}).get('symbol', ''))
                 symbol += str(this_chunk)
                 chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                 yield chunked
         else:
             job['mozharness']['options'] = job['mozharness'].get('options', [])
             job['mozharness']['options'].extend([
-                "locale={}".format(locale) for locale in all_locales
-                ])
+                'locale={}:{}'.format(locale, changeset) for locale, changeset in locales_with_changesets.items()
+            ])
             yield job
 
 
 @transforms.add
 def mh_config_replace_project(config, jobs):
     """ Replaces {project} in mh config entries with the current project """
     # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
     for job in jobs: