Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 17 Feb 2017 15:57:11 +0100
changeset 343604 17d5fa614caa8d11f8718b9cc7e602d1d07d4c28
parent 343603 d226bf0c5b39384fec0a48da7a7d5a8da92b7ef5
child 343605 e952a5cc4a140c1c4015d6a3311732f944a2b90f
push id31382
push userkwierso@gmail.com
push dateFri, 17 Feb 2017 21:41:52 +0000
treeherdermozilla-central@0930fdc4cf8e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1337825
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki MozReview-Commit-ID: 1DEIjFYAx74
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -124,16 +124,21 @@ Signals whether the task is part of a ni
 out nightly tasks from full task set at target stage.
 
 all_locales
 ===========
 
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list
 of relevant locales for the platform.
 
+all_locales_with_changesets
+===========================
+
+Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``).
+
 l10n_chunk
 ==========
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
 number of the job. Note that this is a string!
 
 chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Do transforms specific to l10n kind
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
+import json
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import (
     TransformSequence,
 )
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
@@ -131,25 +132,41 @@ l10n_description_schema = Schema({
 transforms = TransformSequence()
 
 
 def _parse_locales_file(locales_file, platform=None):
     """ Parse the passed locales file for a list of locales.
         If platform is unset matches all platforms.
     """
     locales = []
-    if locales_file.endswith('json'):
-        # Release process uses .json for locale files sometimes.
-        raise NotImplementedError("Don't know how to parse a .json locales file")
-    else:
-        with open(locales_file, mode='r') as lf:
-            locales = lf.read().split()
+
+    with open(locales_file, mode='r') as f:
+        if locales_file.endswith('json'):
+            all_locales = json.load(f)
+            # XXX Only single locales are fetched
+            locales = {
+                locale: data['revision']
+                for locale, data in all_locales.items()
+                if 'android' in data['platforms']
+            }
+        else:
+            all_locales = f.read().split()
+            # 'default' is the hg revision at the top of hg repo, in this context
+            locales = {locale: 'default' for locale in all_locales}
     return locales
 
 
+def _remove_ja_jp_mac_locale(locales):
+    # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked,
+    # so just omit it unconditionally
+    return {
+        locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac'
+    }
+
+
 @transforms.add
 def setup_name(config, jobs):
     for job in jobs:
         dep = job['dependent-task']
         if dep.attributes.get('nightly'):
             # Set the name to the same as the dep task, without kind name.
             # Label will get set automatically with this kinds name.
             job['name'] = job.get('name',
@@ -227,62 +244,65 @@ def handle_keyed_by(config, jobs):
         for field in fields:
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
         yield job
 
 
 @transforms.add
 def all_locales_attribute(config, jobs):
     for job in jobs:
-        locales = set(_parse_locales_file(job["locales-file"]))
-        # ja-JP-mac is a mac-only locale, but there are no
-        # mac builds being repacked, so just omit it unconditionally
-        locales = locales - set(("ja-JP-mac", ))
-        # Convert to mutable list.
-        locales = list(sorted(locales))
+        locales_with_changesets = _parse_locales_file(job["locales-file"])
+        locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets)
+
+        locales = sorted(locales_with_changesets.keys())
         attributes = job.setdefault('attributes', {})
         attributes["all_locales"] = locales
+        attributes["all_locales_with_changesets"] = locales_with_changesets
         yield job
 
 
 @transforms.add
 def chunk_locales(config, jobs):
     """ Utilizes chunking for l10n stuff """
     for job in jobs:
         chunks = job.get('chunks')
-        all_locales = job['attributes']['all_locales']
+        locales_with_changesets = job['attributes']['all_locales_with_changesets']
         if chunks:
-            if chunks > len(all_locales):
+            if chunks > len(locales_with_changesets):
                 # Reduce chunks down to the number of locales
-                chunks = len(all_locales)
+                chunks = len(locales_with_changesets)
             for this_chunk in range(1, chunks + 1):
                 chunked = copy.deepcopy(job)
                 chunked['name'] = chunked['name'].replace(
                     '/', '-{}/'.format(this_chunk), 1
                 )
                 chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
-                my_locales = []
-                my_locales = chunkify(all_locales, this_chunk, chunks)
+                # chunkify doesn't work with dicts
+                locales_with_changesets_as_list = locales_with_changesets.items()
+                chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
                 chunked['mozharness']['options'].extend([
-                    "locale={}".format(locale) for locale in my_locales
-                    ])
+                    'locale={}:{}'.format(locale, changeset)
+                    for locale, changeset in chunked_locales
+                ])
                 chunked['attributes']['l10n_chunk'] = str(this_chunk)
-                chunked['attributes']['chunk_locales'] = my_locales
+                # strip revision
+                chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]
 
                 # add the chunk number to the TH symbol
                 group, symbol = split_symbol(
                     chunked.get('treeherder', {}).get('symbol', ''))
                 symbol += str(this_chunk)
                 chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                 yield chunked
         else:
             job['mozharness']['options'] = job['mozharness'].get('options', [])
             job['mozharness']['options'].extend([
-                "locale={}".format(locale) for locale in all_locales
-                ])
+                'locale={}:{}'.format(locale, changeset)
+                for locale, changeset in locales_with_changesets.items()
+            ])
             yield job
 
 
 @transforms.add
 def mh_config_replace_project(config, jobs):
     """ Replaces {project} in mh config entries with the current project """
     # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
     for job in jobs: