Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki a=release
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 17 Feb 2017 15:57:11 +0100
changeset 378601 0ef6ce3951925a1972396f61db87aedfb3c9004a
parent 378600 cc472627593465885a69a4c1c7e2fd8ad3787c48
child 378602 9fe7f79fe23117ff47f8b86ff84de45c42b9f68a
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki, release
bugs1337825
milestone53.0a2
Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki a=release MozReview-Commit-ID: 1DEIjFYAx74
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -124,16 +124,21 @@ Signals whether the task is part of a ni
 out nightly tasks from full task set at target stage.
 
 all_locales
 ===========
 
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list
 of relevant locales for the platform.
 
+all_locales_with_changesets
+===========================
+
+Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``).
+
 l10n_chunk
 ==========
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
 number of the job. Note that this is a string!
 
 chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Do transforms specific to l10n kind
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
+import json
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import (
     TransformSequence,
     resolve_keyed_by,
     optionally_keyed_by,
     validate_schema
 )
@@ -129,25 +130,41 @@ l10n_description_schema = Schema({
 transforms = TransformSequence()
 
 
 def _parse_locales_file(locales_file, platform=None):
     """ Parse the passed locales file for a list of locales.
         If platform is unset matches all platforms.
     """
     locales = []
-    if locales_file.endswith('json'):
-        # Release process uses .json for locale files sometimes.
-        raise NotImplementedError("Don't know how to parse a .json locales file")
-    else:
-        with open(locales_file, mode='r') as lf:
-            locales = lf.read().split()
+
+    with open(locales_file, mode='r') as f:
+        if locales_file.endswith('json'):
+            all_locales = json.load(f)
+            # XXX Only single locales are fetched
+            locales = {
+                locale: data['revision']
+                for locale, data in all_locales.items()
+                if 'android' in data['platforms']
+            }
+        else:
+            all_locales = f.read().split()
+            # 'default' is the hg revision at the top of hg repo, in this context
+            locales = {locale: 'default' for locale in all_locales}
     return locales
 
 
+def _remove_ja_jp_mac_locale(locales):
+    # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked,
+    # so just omit it unconditionally
+    return {
+        locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac'
+    }
+
+
 @transforms.add
 def setup_name(config, jobs):
     for job in jobs:
         dep = job['dependent-task']
         if dep.attributes.get('nightly'):
             # Set the name to the same as the dep task, without kind name.
             # Label will get set automatically with this kinds name.
             job['name'] = job.get('name',
@@ -225,62 +242,65 @@ def handle_keyed_by(config, jobs):
         for field in fields:
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
         yield job
 
 
 @transforms.add
 def all_locales_attribute(config, jobs):
     for job in jobs:
-        locales = set(_parse_locales_file(job["locales-file"]))
-        # ja-JP-mac is a mac-only locale, but there are no
-        # mac builds being repacked, so just omit it unconditionally
-        locales = locales - set(("ja-JP-mac", ))
-        # Convert to mutable list.
-        locales = list(sorted(locales))
+        locales_with_changesets = _parse_locales_file(job["locales-file"])
+        locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets)
+
+        locales = sorted(locales_with_changesets.keys())
         attributes = job.setdefault('attributes', {})
         attributes["all_locales"] = locales
+        attributes["all_locales_with_changesets"] = locales_with_changesets
         yield job
 
 
 @transforms.add
 def chunk_locales(config, jobs):
     """ Utilizes chunking for l10n stuff """
     for job in jobs:
         chunks = job.get('chunks')
-        all_locales = job['attributes']['all_locales']
+        locales_with_changesets = job['attributes']['all_locales_with_changesets']
         if chunks:
-            if chunks > len(all_locales):
+            if chunks > len(locales_with_changesets):
                 # Reduce chunks down to the number of locales
-                chunks = len(all_locales)
+                chunks = len(locales_with_changesets)
             for this_chunk in range(1, chunks + 1):
                 chunked = copy.deepcopy(job)
                 chunked['name'] = chunked['name'].replace(
                     '/', '-{}/'.format(this_chunk), 1
                 )
                 chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
-                my_locales = []
-                my_locales = chunkify(all_locales, this_chunk, chunks)
+                # chunkify doesn't work with dicts
+                locales_with_changesets_as_list = locales_with_changesets.items()
+                chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
                 chunked['mozharness']['options'].extend([
-                    "locale={}".format(locale) for locale in my_locales
-                    ])
+                    'locale={}:{}'.format(locale, changeset)
+                    for locale, changeset in chunked_locales
+                ])
                 chunked['attributes']['l10n_chunk'] = str(this_chunk)
-                chunked['attributes']['chunk_locales'] = my_locales
+                # strip revision
+                chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]
 
                 # add the chunk number to the TH symbol
                 group, symbol = split_symbol(
                     chunked.get('treeherder', {}).get('symbol', ''))
                 symbol += str(this_chunk)
                 chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                 yield chunked
         else:
             job['mozharness']['options'] = job['mozharness'].get('options', [])
             job['mozharness']['options'].extend([
-                "locale={}".format(locale) for locale in all_locales
-                ])
+                'locale={}:{}'.format(locale, changeset)
+                for locale, changeset in locales_with_changesets.items()
+            ])
             yield job
 
 
 @transforms.add
 def mh_config_replace_project(config, jobs):
     """ Replaces {project} in mh config entries with the current project """
     # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
     for job in jobs: