Bug 1353680, update compare-locales to 7.2.1, r=flod
authorAxel Hecht <axel@pike.org>
Thu, 02 May 2019 10:48:32 +0000
changeset 531072 dce9826524bcd6a99add436fba85e119fe5e7c2d
parent 531071 f6903c331545b2aa793ea72f4a59c39ff2b0a13c
child 531073 af9d93e2536a6d5b9bc7beae0cab89179431f503
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersflod
bugs1353680
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1353680, update compare-locales to 7.2.1, r=flod Differential Revision: https://phabricator.services.mozilla.com/D29000
third_party/python/compare-locales/compare_locales/__init__.py
third_party/python/compare-locales/compare_locales/commands.py
third_party/python/compare-locales/compare_locales/compare/__init__.py
third_party/python/compare-locales/compare_locales/compare/content.py
third_party/python/compare-locales/compare_locales/compare/observer.py
third_party/python/compare-locales/compare_locales/lint/__init__.py
third_party/python/compare-locales/compare_locales/lint/cli.py
third_party/python/compare-locales/compare_locales/lint/linter.py
third_party/python/compare-locales/compare_locales/lint/util.py
third_party/python/compare-locales/compare_locales/parser/__init__.py
third_party/python/compare-locales/compare_locales/parser/android.py
third_party/python/compare-locales/compare_locales/parser/base.py
third_party/python/compare-locales/compare_locales/parser/fluent.py
third_party/python/compare-locales/compare_locales/paths/files.py
third_party/python/compare-locales/compare_locales/paths/ini.py
third_party/python/compare-locales/compare_locales/paths/project.py
third_party/python/compare-locales/compare_locales/plurals.py
third_party/python/compare-locales/compare_locales/tests/android/test_merge.py
third_party/python/compare-locales/compare_locales/tests/android/test_parser.py
third_party/python/compare-locales/compare_locales/tests/fluent/test_parser.py
third_party/python/compare-locales/compare_locales/tests/lint/__init__.py
third_party/python/compare-locales/compare_locales/tests/lint/test_linter.py
third_party/python/compare-locales/compare_locales/tests/lint/test_util.py
third_party/python/compare-locales/compare_locales/tests/paths/__init__.py
third_party/python/compare-locales/compare_locales/tests/paths/test_files.py
third_party/python/compare-locales/compare_locales/tests/paths/test_project.py
third_party/python/compare-locales/compare_locales/tests/test_apps.py
--- a/third_party/python/compare-locales/compare_locales/__init__.py
+++ b/third_party/python/compare-locales/compare_locales/__init__.py
@@ -1,1 +1,1 @@
-version = "7.0.0"
+version = "7.2.1"
--- a/third_party/python/compare-locales/compare_locales/commands.py
+++ b/third_party/python/compare-locales/compare_locales/commands.py
@@ -124,26 +124,30 @@ Be careful to specify the right merge di
             var, _, value = define.partition('=')
             config_env[var] = value
         for config_path in config_paths:
             if config_path.endswith('.toml'):
                 try:
                     config = TOMLParser().parse(config_path, env=config_env)
                 except ConfigNotFound as e:
                     self.parser.exit('config file %s not found' % e.filename)
-                if locales:
-                    config.set_locales(locales, deep=locales_deep)
+                if locales_deep:
+                    if not locales:
+                        # no explicit locales given, force all locales
+                        config.set_locales(config.all_locales, deep=True)
+                    else:
+                        config.set_locales(locales, deep=True)
                 configs.append(config)
             else:
-                app = EnumerateApp(
-                    config_path, l10n_base_dir, locales)
+                app = EnumerateApp(config_path, l10n_base_dir)
                 configs.append(app.asConfig())
         try:
             observers = compareProjects(
                 configs,
+                locales,
                 l10n_base_dir,
                 quiet=quiet,
                 merge_stage=merge, clobber_merge=clobber)
         except (OSError, IOError) as exc:
             print("FAIL: " + str(exc))
             self.parser.exit(2)
 
         if json is None or json != '-':
--- a/third_party/python/compare-locales/compare_locales/compare/__init__.py
+++ b/third_party/python/compare-locales/compare_locales/compare/__init__.py
@@ -21,38 +21,40 @@ from .utils import Tree, AddRemove
     'Observer', 'ObserverList',
     'AddRemove', 'Tree',
     'compareProjects',
 ]
 
 
 def compareProjects(
             project_configs,
+            locales,
             l10n_base_dir,
             stat_observer=None,
             merge_stage=None,
             clobber_merge=False,
             quiet=0,
         ):
-    locales = set()
+    all_locales = set(locales)
     comparer = ContentComparer(quiet)
     observers = comparer.observers
     for project in project_configs:
         # disable filter if we're in validation mode
-        if None in project.locales:
+        if None in locales:
             filter = None
         else:
             filter = project.filter
         observers.append(
             Observer(
                 quiet=quiet,
                 filter=filter,
             ))
-        locales.update(project.locales)
-    for locale in sorted(locales):
+        if not locales:
+            all_locales.update(project.all_locales)
+    for locale in sorted(all_locales):
         files = paths.ProjectFiles(locale, project_configs,
                                    mergebase=merge_stage)
         if merge_stage is not None:
             if clobber_merge:
                 mergematchers = set(_m.get('merge') for _m in files.matchers)
                 mergematchers.discard(None)
                 for matcher in mergematchers:
                     clobberdir = matcher.prefix
--- a/third_party/python/compare-locales/compare_locales/compare/content.py
+++ b/third_party/python/compare-locales/compare_locales/compare/content.py
@@ -190,21 +190,19 @@ class ContentComparer:
                 else:
                     # just report
                     report += 1
             elif action == 'add':
                 # obsolete entity or junk
                 if isinstance(l10n_entities[entity_id],
                               parser.Junk):
                     junk = l10n_entities[entity_id]
-                    params = (junk.val,) + junk.position() + junk.position(-1)
                     self.observers.notify(
                         'error', l10n,
-                        'Unparsed content "%s" from line %d column %d'
-                        ' to line %d column %d' % params
+                        junk.error_message()
                     )
                     if merge_file is not None:
                         skips.append(junk)
                 elif (
                     self.observers.notify('obsoleteEntity', l10n, entity_id)
                     != 'ignore'
                 ):
                     obsolete += 1
--- a/third_party/python/compare-locales/compare_locales/compare/observer.py
+++ b/third_party/python/compare-locales/compare_locales/compare/observer.py
@@ -153,21 +153,23 @@ class ObserverList(Observer):
 
     def serializeSummaries(self):
         summaries = {
             loc: []
             for loc in self.summary.keys()
         }
         for observer in self.observers:
             for loc, lst in summaries.items():
-                lst.append(observer.summary.get(loc))
+                # Not all locales are on all projects,
+                # default to empty summary
+                lst.append(observer.summary.get(loc, {}))
         if len(self.observers) > 1:
             # add ourselves if there's more than one project
             for loc, lst in summaries.items():
-                lst.append(self.summary.get(loc))
+                lst.append(self.summary.get(loc, {}))
         # normalize missing and missingInFiles -> missing
         for summarylist in summaries.values():
             for summary in summarylist:
                 if 'missingInFiles' in summary:
                     summary['missing'] = (
                         summary.get('missing', 0)
                         + summary.pop('missingInFiles')
                     )
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/lint/cli.py
@@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import argparse
+import os
+
+from compare_locales.lint.linter import L10nLinter
+from compare_locales.lint.util import (
+    default_reference_and_tests,
+    mirror_reference_and_tests,
+    l10n_base_reference_and_tests,
+)
+from compare_locales import mozpath
+from compare_locales import paths
+from compare_locales import parser
+from compare_locales import version
+
+
+epilog = '''\
+moz-l10n-lint checks for common mistakes in localizable files. It tests for
+duplicate entries, parsing errors, and the like. Optionally, it can compare
+the strings to an external reference with strings and warn if a string might
+need to get a new ID.
+'''
+
+
+def main():
+    p = argparse.ArgumentParser(
+        description='Validate localizable strings',
+        epilog=epilog,
+    )
+    p.add_argument('l10n_toml')
+    p.add_argument(
+        '--version', action='version', version='%(prog)s ' + version
+    )
+    p.add_argument('-W', action='store_true', help='error on warnings')
+    p.add_argument(
+        '--l10n-reference',
+        dest='l10n_reference',
+        metavar='PATH',
+        help='check for conflicts against an l10n-only reference repository '
+        'like gecko-strings',
+    )
+    p.add_argument(
+        '--reference-project',
+        dest='ref_project',
+        metavar='PATH',
+        help='check for conflicts against a reference project like '
+        'android-l10n',
+    )
+    args = p.parse_args()
+    if args.l10n_reference:
+        l10n_base, locale = \
+            os.path.split(os.path.abspath(args.l10n_reference))
+        if not locale or not os.path.isdir(args.l10n_reference):
+            p.error('Pass an existing l10n reference')
+    else:
+        l10n_base = '.'
+        locale = None
+    pc = paths.TOMLParser().parse(args.l10n_toml, env={'l10n_base': l10n_base})
+    if locale:
+        pc.set_locales([locale], deep=True)
+    files = paths.ProjectFiles(locale, [pc])
+    get_reference_and_tests = default_reference_and_tests
+    if args.l10n_reference:
+        get_reference_and_tests = l10n_base_reference_and_tests(files)
+    elif args.ref_project:
+        get_reference_and_tests = mirror_reference_and_tests(
+            files, args.ref_project
+        )
+    linter = L10nLinter()
+    results = linter.lint(
+        (f for f, _, _, _ in files.iter_reference() if parser.hasParser(f)),
+        get_reference_and_tests
+    )
+    rv = 0
+    if results:
+        rv = 1
+        if all(r['level'] == 'warning' for r in results) and not args.W:
+            rv = 0
+    for result in results:
+        print('{} ({}:{}): {}'.format(
+            mozpath.relpath(result['path'], '.'),
+            result.get('lineno', 0),
+            result.get('column', 0),
+            result['message']
+        ))
+    return rv
+
+
+if __name__ == '__main__':
+    main()
new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/lint/linter.py
@@ -0,0 +1,120 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from collections import Counter
+import os
+
+from compare_locales import parser, checks
+from compare_locales.paths import File, REFERENCE_LOCALE
+
+
+class L10nLinter(object):
+
+    def lint(self, files, get_reference_and_tests):
+        results = []
+        for path in files:
+            if not parser.hasParser(path):
+                continue
+            ref, extra_tests = get_reference_and_tests(path)
+            results.extend(self.lint_file(path, ref, extra_tests))
+        return results
+
+    def lint_file(self, path, ref, extra_tests):
+        file_parser = parser.getParser(path)
+        if os.path.isfile(ref):
+            file_parser.readFile(ref)
+            reference = file_parser.parse()
+        else:
+            reference = {}
+        file_parser.readFile(path)
+        current = file_parser.parse()
+        checker = checks.getChecker(
+            File(path, path, locale=REFERENCE_LOCALE),
+            extra_tests=extra_tests
+        )
+        if checker and checker.needs_reference:
+            checker.set_reference(current)
+        linter = EntityLinter(current, checker, reference)
+        for current_entity in current:
+            for result in linter.lint_entity(current_entity):
+                result['path'] = path
+                yield result
+
+
+class EntityLinter(object):
+    '''Factored out helper to run linters on a single entity.'''
+    def __init__(self, current, checker, reference):
+        self.key_count = Counter(entity.key for entity in current)
+        self.checker = checker
+        self.reference = reference
+
+    def lint_entity(self, current_entity):
+        res = self.handle_junk(current_entity)
+        if res:
+            yield res
+            return
+        for res in self.lint_full_entity(current_entity):
+            yield res
+        for res in self.lint_value(current_entity):
+            yield res
+
+    def lint_full_entity(self, current_entity):
+        '''Checks that go good or bad for a full entity,
+        without a particular spot inside the entity.
+        '''
+        lineno = col = None
+        if self.key_count[current_entity.key] > 1:
+            lineno, col = current_entity.position()
+            yield {
+                'lineno': lineno,
+                'column': col,
+                'level': 'error',
+                'message': 'Duplicate string with ID: {}'.format(
+                    current_entity.key
+                )
+            }
+
+        if current_entity.key in self.reference:
+            reference_entity = self.reference[current_entity.key]
+            if not current_entity.equals(reference_entity):
+                if lineno is None:
+                    lineno, col = current_entity.position()
+                msg = 'Changes to string require a new ID: {}'.format(
+                    current_entity.key
+                )
+                yield {
+                    'lineno': lineno,
+                    'column': col,
+                    'level': 'warning',
+                    'message': msg,
+                }
+
+    def lint_value(self, current_entity):
+        '''Checks that error on particular locations in the entity value.
+        '''
+        if self.checker:
+            for tp, pos, msg, cat in self.checker.check(
+                current_entity, current_entity
+            ):
+                lineno, col = current_entity.value_position(pos)
+                yield {
+                    'lineno': lineno,
+                    'column': col,
+                    'level': tp,
+                    'message': msg,
+                }
+
+    def handle_junk(self, current_entity):
+        if not isinstance(current_entity, parser.Junk):
+            return None
+
+        lineno, col = current_entity.position()
+        return {
+            'lineno': lineno,
+            'column': col,
+            'level': 'error',
+            'message': current_entity.error_message()
+        }
new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/lint/util.py
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from compare_locales import paths
+
+
+def default_reference_and_tests(path):
+    return None, None
+
+
+def mirror_reference_and_tests(files, basedir):
+    '''Get reference files to check for conflicts in android-l10n and friends.
+    '''
+    def get_reference_and_tests(path):
+        for matchers in files.matchers:
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            if matcher.match(path) is None:
+                continue
+            ref_matcher = paths.Matcher(matcher, root=basedir)
+            ref_path = matcher.sub(ref_matcher, path)
+            return ref_path, matchers.get('test')
+        return None, None
+    return get_reference_and_tests
+
+
+def l10n_base_reference_and_tests(files):
+    '''Get reference files to check for conflicts in gecko-strings and friends.
+    '''
+    def get_reference_and_tests(path):
+        match = files.match(path)
+        if match is None:
+            return None, None
+        ref, _, _, extra_tests = match
+        return ref, extra_tests
+    return get_reference_and_tests
--- a/third_party/python/compare-locales/compare_locales/parser/__init__.py
+++ b/third_party/python/compare-locales/compare_locales/parser/__init__.py
@@ -52,16 +52,23 @@ from .properties import (
 
 def getParser(path):
     for item in __constructors:
         if re.search(item[0], path):
             return item[1]
     raise UserWarning("Cannot find Parser")
 
 
+def hasParser(path):
+    try:
+        return bool(getParser(path))
+    except UserWarning:
+        return False
+
+
 __constructors = [
     ('strings.*\\.xml$', AndroidParser()),
     ('\\.dtd$', DTDParser()),
     ('\\.properties$', PropertiesParser()),
     ('\\.ini$', IniParser()),
     ('\\.inc$', DefinesParser()),
     ('\\.ftl$', FluentParser()),
     ('\\.pot?$', PoParser()),
--- a/third_party/python/compare-locales/compare_locales/parser/android.py
+++ b/third_party/python/compare-locales/compare_locales/parser/android.py
@@ -56,16 +56,19 @@ class AndroidEntity(Entity):
     @property
     def key(self):
         return self._key_literal
 
     @property
     def raw_val(self):
         return self._raw_val_literal
 
+    def position(self, offset=0):
+        return (0, offset)
+
     def value_position(self, offset=0):
         return (0, offset)
 
     def wrap(self, raw_val):
         clone = self.node.cloneNode(True)
         if clone.childNodes.length == 1:
             child = clone.childNodes[0]
         else:
@@ -173,32 +176,32 @@ class AndroidParser(Parser):
             return
         ctx = self.ctx
         contents = ctx.contents
         try:
             doc = minidom.parseString(contents.encode('utf-8'))
         except Exception:
             yield XMLJunk(contents)
             return
-        if doc.documentElement.nodeName != 'resources':
+        docElement = doc.documentElement
+        if docElement.nodeName != 'resources':
             yield XMLJunk(doc.toxml())
             return
-        root_children = doc.documentElement.childNodes
+        root_children = docElement.childNodes
         if not only_localizable:
-            attributes = ''.join(
-                ' {}="{}"'.format(attr_name, attr_value)
-                for attr_name, attr_value in
-                doc.documentElement.attributes.items()
-            )
             yield DocumentWrapper(
                 '<?xml?><resources>',
-                '<?xml version="1.0" encoding="utf-8"?>\n<resources{}>'.format(
-                    attributes
+                '<?xml version="1.0" encoding="utf-8"?>\n<resources'
+            )
+            for attr_name, attr_value in docElement.attributes.items():
+                yield DocumentWrapper(
+                    attr_name,
+                    ' {}="{}"'.format(attr_name, attr_value)
                 )
-            )
+            yield DocumentWrapper('>', '>')
         child_num = 0
         while child_num < len(root_children):
             node = root_children[child_num]
             if node.nodeType == Node.COMMENT_NODE:
                 current_comment, child_num = self.handleComment(
                     node, root_children, child_num
                 )
                 if child_num < len(root_children):
--- a/third_party/python/compare-locales/compare_locales/parser/base.py
+++ b/third_party/python/compare-locales/compare_locales/parser/base.py
@@ -261,16 +261,23 @@ class Junk(object):
     @property
     def raw_val(self):
         return self.all
 
     @property
     def val(self):
         return self.all
 
+    def error_message(self):
+        params = (self.val,) + self.position() + self.position(-1)
+        return (
+            'Unparsed content "%s" from line %d column %d'
+            ' to line %d column %d' % params
+        )
+
     def __repr__(self):
         return self.key
 
 
 class Whitespace(Entry):
     '''Entity-like object representing an empty file with whitespace,
     if allowed
     '''
--- a/third_party/python/compare-locales/compare_locales/parser/fluent.py
+++ b/third_party/python/compare-locales/compare_locales/parser/fluent.py
@@ -104,17 +104,23 @@ class FluentEntity(Entity):
         return self._word_count
 
     def equals(self, other):
         return self.entry.equals(
             other.entry, ignored_fields=self.ignored_fields)
 
     # In Fluent we treat entries as a whole.  FluentChecker reports errors at
     # offsets calculated from the beginning of the entry.
-    def value_position(self, offset=0):
+    def value_position(self, offset=None):
+        if offset is None:
+            # no offset given, use our value start or id end
+            if self.val_span:
+                offset = self.val_span[0] - self.span[0]
+            else:
+                offset = self.key_span[1] - self.span[0]
         return self.position(offset)
 
     @property
     def attributes(self):
         for attr_node in self.entry.attributes:
             yield FluentAttribute(self, attr_node)
 
     def unwrap(self):
--- a/third_party/python/compare-locales/compare_locales/paths/files.py
+++ b/third_party/python/compare-locales/compare_locales/paths/files.py
@@ -18,19 +18,23 @@ class ProjectFiles(object):
     both reference and locale for a reference self-test.
     '''
     def __init__(self, locale, projects, mergebase=None):
         self.locale = locale
         self.matchers = []
         self.mergebase = mergebase
         configs = []
         for project in projects:
+            # Only add this project if we're not in validation mode,
+            # and the given locale is enabled for the project.
+            if locale is not None and locale not in project.all_locales:
+                continue
             configs.extend(project.configs)
         for pc in configs:
-            if locale and locale not in pc.locales:
+            if locale and pc.locales is not None and locale not in pc.locales:
                 continue
             for paths in pc.paths:
                 if (
                     locale and
                     'locales' in paths and
                     locale not in paths['locales']
                 ):
                     continue
--- a/third_party/python/compare-locales/compare_locales/paths/ini.py
+++ b/third_party/python/compare-locales/compare_locales/paths/ini.py
@@ -162,24 +162,22 @@ class SourceTreeConfigParser(L10nConfigP
                                     **self.defaults)
         cp.loadConfigs()
         self.children.append(cp)
 
 
 class EnumerateApp(object):
     reference = 'en-US'
 
-    def __init__(self, inipath, l10nbase, locales=None):
+    def __init__(self, inipath, l10nbase):
         self.setupConfigParser(inipath)
         self.modules = defaultdict(dict)
         self.l10nbase = mozpath.abspath(l10nbase)
         self.filters = []
         self.addFilters(*self.config.getFilters())
-        self.locales = locales or self.config.allLocales()
-        self.locales.sort()
 
     def setupConfigParser(self, inipath):
         self.config = L10nConfigParser(inipath)
         self.config.loadConfigs()
 
     def addFilters(self, *args):
         self.filters += args
 
@@ -188,17 +186,17 @@ class EnumerateApp(object):
         # Set the path and root to None to just keep our paths as is.
         config = ProjectConfig(None)
         config.set_root('.')  # sets to None because path is None
         config.add_environment(l10n_base=self.l10nbase)
         self._config_for_ini(config, self.config)
         filters = self.config.getFilters()
         if filters:
             config.set_filter_py(filters[0])
-        config.locales += self.locales
+        config.set_locales(self.config.allLocales(), deep=True)
         return config
 
     def _config_for_ini(self, projectconfig, aConfig):
         for k, (basepath, module) in aConfig.dirsIter():
             paths = {
                 'module': module,
                 'reference': mozpath.normpath('%s/%s/locales/en-US/**' %
                                               (basepath, module)),
--- a/third_party/python/compare-locales/compare_locales/paths/project.py
+++ b/third_party/python/compare-locales/compare_locales/paths/project.py
@@ -20,17 +20,19 @@ class ProjectConfig(object):
         #  'reference': pattern,  # optional
         #  'locales': [],  # optional
         #  'test': [],  # optional
         # }
         self.path = path
         self.root = None
         self.paths = []
         self.rules = []
-        self.locales = []
+        self.locales = None
+        # cache for all_locales, as that's not in `filter`
+        self._all_locales = None
         self.environ = {}
         self.children = []
         self._cache = None
 
     def same(self, other):
         '''Equality test, ignoring locales.
         '''
         if other.__class__ is not self.__class__:
@@ -58,17 +60,17 @@ class ProjectConfig(object):
 
     def add_paths(self, *paths):
         '''Add path dictionaries to this config.
         The dictionaries must have a `l10n` key. For monolingual files,
         `reference` is also required.
         An optional key `test` is allowed to enable additional tests for this
         path pattern.
         '''
-
+        self._all_locales = None  # clear cache
         for d in paths:
             rv = {
                 'l10n': Matcher(d['l10n'], env=self.environ, root=self.root),
                 'module': d.get('module')
             }
             if 'reference' in d:
                 rv['reference'] = Matcher(
                     d['reference'], env=self.environ, root=self.root
@@ -104,38 +106,54 @@ class ProjectConfig(object):
         '''Add rules to filter on.
         Assert that there's no legacy filter.py code hooked up.
         '''
         assert self.filter_py is None
         for rule in rules:
             self.rules.extend(self._compile_rule(rule))
 
     def add_child(self, child):
+        self._all_locales = None  # clear cache
         self.children.append(child)
 
     def set_locales(self, locales, deep=False):
+        self._all_locales = None  # clear cache
         self.locales = locales
+        if not deep:
+            return
         for child in self.children:
-            if not child.locales or deep:
-                child.set_locales(locales, deep=deep)
-            else:
-                locs = [loc for loc in locales if loc in child.locales]
-                child.set_locales(locs)
+            child.set_locales(locales, deep=deep)
 
     @property
     def configs(self):
         'Recursively get all configs in this project and its children'
         yield self
         for child in self.children:
             for config in child.configs:
                 yield config
 
+    @property
+    def all_locales(self):
+        'Recursively get all locales in this project and its paths'
+        if self._all_locales is None:
+            all_locales = set()
+            for config in self.configs:
+                if config.locales is not None:
+                    all_locales.update(config.locales)
+                for paths in config.paths:
+                    if 'locales' in paths:
+                        all_locales.update(paths['locales'])
+            self._all_locales = sorted(all_locales)
+        return self._all_locales
+
     def filter(self, l10n_file, entity=None):
         '''Filter a localization file or entities within, according to
         this configuration file.'''
+        if l10n_file.locale not in self.all_locales:
+            return 'ignore'
         if self.filter_py is not None:
             return self.filter_py(l10n_file.module, l10n_file.file,
                                   entity=entity)
         rv = self._filter(l10n_file, entity=entity)
         if rv is None:
             return 'ignore'
         return rv
 
--- a/third_party/python/compare-locales/compare_locales/plurals.py
+++ b/third_party/python/compare-locales/compare_locales/plurals.py
@@ -65,16 +65,17 @@ CATEGORIES_BY_LOCALE = {
     'an': CATEGORIES_BY_INDEX[1],
     'ar': CATEGORIES_BY_INDEX[12],
     'arn': CATEGORIES_BY_INDEX[1],
     'as': CATEGORIES_BY_INDEX[1],
     'ast': CATEGORIES_BY_INDEX[1],
     'az': CATEGORIES_BY_INDEX[1],
     'be': CATEGORIES_BY_INDEX[7],
     'bg': CATEGORIES_BY_INDEX[1],
+    'bn': CATEGORIES_BY_INDEX[2],
     'bn-BD': CATEGORIES_BY_INDEX[2],
     'bn-IN': CATEGORIES_BY_INDEX[2],
     'br': CATEGORIES_BY_INDEX[16],
     'brx': CATEGORIES_BY_INDEX[1],
     'bs': CATEGORIES_BY_INDEX[19],
     'ca': CATEGORIES_BY_INDEX[1],
     'cak': CATEGORIES_BY_INDEX[1],
     'crh': CATEGORIES_BY_INDEX[1],
--- a/third_party/python/compare-locales/compare_locales/tests/android/test_merge.py
+++ b/third_party/python/compare-locales/compare_locales/tests/android/test_merge.py
@@ -51,8 +51,32 @@ class TestMerge(unittest.TestCase):
 <?xml version="1.0" encoding="utf-8"?>
 <resources>
   <!-- Bar -->
   <string name="bar">other value</string>
   <!-- Foo -->
   <string name="foo">value</string>
 </resources>
 ''')
+
+    def test_namespaces(self):
+        channels = (
+            b'''\
+<?xml version="1.0" encoding="utf-8"?>
+<resources xmlns:ns1="urn:ns1">
+    <string ns1:one="test">string</string>
+</resources>
+''',
+            b'''\
+<?xml version="1.0" encoding="utf-8"?>
+<resources xmlns:ns2="urn:ns2">
+    <string ns2:two="test">string</string>
+</resources>
+'''
+        )
+        self.assertEqual(
+            merge_channels(self.name, channels), b'''\
+<?xml version="1.0" encoding="utf-8"?>
+<resources xmlns:ns2="urn:ns2" xmlns:ns1="urn:ns1">
+    <string ns2:two="test">string</string>
+    <string ns1:one="test">string</string>
+</resources>
+''')
--- a/third_party/python/compare-locales/compare_locales/tests/android/test_parser.py
+++ b/third_party/python/compare-locales/compare_locales/tests/android/test_parser.py
@@ -34,16 +34,17 @@ class TestAndroidParser(ParserTestMixin,
 
   <string name="baz">so lonely</string>
 </resources>
 '''
         self._test(
             source,
             (
                 (DocumentWrapper, '<?xml'),
+                (DocumentWrapper, '>'),
                 (Whitespace, '\n  '),
                 ('foo', 'value', 'bar'),
                 (Whitespace, '\n'),
                 ('bar', 'multi-line comment', 'bar\nfoo'),
                 (Whitespace, '\n  '),
                 (Comment, 'standalone'),
                 (Whitespace, '\n  '),
                 ('baz', 'so lonely'),
@@ -74,16 +75,17 @@ class TestAndroidParser(ParserTestMixin,
   <string nomine="dom">value</string>
   <string name="last">value</string>
 </resources>
 '''
         self._test(
             source,
             (
                 (DocumentWrapper, '<?xml'),
+                (DocumentWrapper, '>'),
                 (Whitespace, '\n  '),
                 ('first', 'value'),
                 (Whitespace, '\n  '),
                 (Junk, '<non-string name="bad">'),
                 (Whitespace, '\n  '),
                 ('mid', 'value'),
                 (Whitespace, '\n  '),
                 (Junk, '<string nomine="dom">'),
@@ -110,16 +112,17 @@ class TestAndroidParser(ParserTestMixin,
   <string name="one"></string>
   <string name="two"/>
 </resources>
 '''
         self._test(
             source,
             (
                 (DocumentWrapper, '<?xml'),
+                (DocumentWrapper, '>'),
                 (Whitespace, '\n  '),
                 ('one', ''),
                 (Whitespace, '\n  '),
                 ('two', ''),
                 (Whitespace, '\n'),
                 (DocumentWrapper, '</resources>')
             )
         )
--- a/third_party/python/compare-locales/compare_locales/tests/fluent/test_parser.py
+++ b/third_party/python/compare-locales/compare_locales/tests/fluent/test_parser.py
@@ -108,40 +108,48 @@ abc =
 
         [abc] = list(self.parser)
         self.assertEqual(abc.key, 'abc')
         self.assertEqual(abc.raw_val, '    A\n    B\n    C')
         self.assertEqual(abc.all, 'abc =\n    A\n    B\n    C')
 
     def test_message_with_attribute(self):
         self.parser.readContents(b'''\
+
+
 abc = ABC
     .attr = Attr
 ''')
 
         [abc] = list(self.parser)
         self.assertEqual(abc.key, 'abc')
         self.assertEqual(abc.raw_val, 'ABC')
         self.assertEqual(abc.all, 'abc = ABC\n    .attr = Attr')
+        self.assertEqual(abc.position(), (3, 1))
+        self.assertEqual(abc.value_position(), (3, 7))
+        attr = list(abc.attributes)[0]
+        self.assertEqual(attr.value_position(), (4, 13))
 
     def test_message_with_attribute_and_no_value(self):
         self.parser.readContents(b'''\
 abc =
     .attr = Attr
 ''')
 
         [abc] = list(self.parser)
         self.assertEqual(abc.key, 'abc')
         self.assertEqual(abc.raw_val, None)
         self.assertEqual(abc.all, 'abc =\n    .attr = Attr')
         attributes = list(abc.attributes)
         self.assertEqual(len(attributes), 1)
         attr = attributes[0]
         self.assertEqual(attr.key, 'attr')
         self.assertEqual(attr.raw_val, 'Attr')
+        self.assertEqual(abc.value_position(), (1, 4))
+        self.assertEqual(attr.value_position(), (2, 13))
 
     def test_non_localizable(self):
         self.parser.readContents(b'''\
 ### Resource Comment
 
 foo = Foo
 
 ## Group Comment
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/tests/lint/test_linter.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from compare_locales.lint import linter
+from compare_locales.parser import base as parser
+
+
+class MockChecker(object):
+    def __init__(self, mocked):
+        self.results = mocked
+
+    def check(self, ent, ref):
+        for r in self.results:
+            yield r
+
+
+class EntityTest(unittest.TestCase):
+    def test_junk(self):
+        el = linter.EntityLinter([], None, {})
+        ctx = parser.Parser.Context('foo\nbar\n')
+        ent = parser.Junk(ctx, (4, 7))
+        res = el.handle_junk(ent)
+        self.assertIsNotNone(res)
+        self.assertEqual(res['lineno'], 2)
+        self.assertEqual(res['column'], 1)
+        ent = parser.LiteralEntity('one', 'two', 'one = two')
+        self.assertIsNone(el.handle_junk(ent))
+
+    def test_full_entity(self):
+        ctx = parser.Parser.Context('''\
+one = two
+two = three
+one = four
+''')
+        entities = [
+            parser.Entity(ctx, None, None, (0, 10), (0, 3), (6, 9)),
+            parser.Entity(ctx, None, None, (10, 22), (10, 13), (16, 21)),
+            parser.Entity(ctx, None, None, (22, 33), (22, 25), (28, 32)),
+        ]
+        self.assertEqual(
+            (entities[0].all, entities[0].key, entities[0].val),
+            ('one = two\n', 'one', 'two')
+        )
+        self.assertEqual(
+            (entities[1].all, entities[1].key, entities[1].val),
+            ('two = three\n', 'two', 'three')
+        )
+        self.assertEqual(
+            (entities[2].all, entities[2].key, entities[2].val),
+            ('one = four\n', 'one', 'four')
+        )
+        el = linter.EntityLinter(entities, None, {})
+        results = list(el.lint_full_entity(entities[1]))
+        self.assertListEqual(results, [])
+        results = list(el.lint_full_entity(entities[2]))
+        self.assertEqual(len(results), 1)
+        result = results[0]
+        self.assertEqual(result['level'], 'error')
+        self.assertEqual(result['lineno'], 3)
+        self.assertEqual(result['column'], 1)
+        # finally check for conflict
+        el.reference = {
+            'two': parser.LiteralEntity('two = other', 'two', 'other')
+        }
+        results = list(el.lint_full_entity(entities[1]))
+        self.assertEqual(len(results), 1)
+        result = results[0]
+        self.assertEqual(result['level'], 'warning')
+        self.assertEqual(result['lineno'], 2)
+        self.assertEqual(result['column'], 1)
+
+    def test_in_value(self):
+        ctx = parser.Parser.Context('''\
+one = two
+''')
+        entities = [
+            parser.Entity(ctx, None, None, (0, 10), (0, 3), (6, 9)),
+        ]
+        self.assertEqual(
+            (entities[0].all, entities[0].key, entities[0].val),
+            ('one = two\n', 'one', 'two')
+        )
+        checker = MockChecker([
+            ('error', 2, 'Incompatible resource types', 'android'),
+        ])
+        el = linter.EntityLinter(entities, checker, {})
+        results = list(el.lint_value(entities[0]))
+        self.assertEqual(len(results), 1)
+        result = results[0]
+        self.assertEqual(result['level'], 'error')
+        self.assertEqual(result['lineno'], 1)
+        self.assertEqual(result['column'], 9)
new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/tests/lint/test_util.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import unittest
+
+from compare_locales.lint import util
+from compare_locales.paths.project import ProjectConfig
+from compare_locales.paths.files import ProjectFiles
+from compare_locales import mozpath
+
+
+class MirrorReferenceTest(unittest.TestCase):
+    def test_empty(self):
+        files = ProjectFiles(None, [])
+        get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertIsNone(ref)
+        self.assertIsNone(tests)
+
+    def test_no_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': 'some/{locale}/file.ftl',
+        })
+        files = ProjectFiles(None, [pc])
+        get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl')
+        self.assertEqual(tests, set())
+
+    def test_with_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': 'some/{locale}/file.ftl',
+            'test': ['more_stuff'],
+        })
+        files = ProjectFiles(None, [pc])
+        get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl')
+        self.assertEqual(tests, {'more_stuff'})
+
+
+class L10nBaseReferenceTest(unittest.TestCase):
+    def test_empty(self):
+        files = ProjectFiles(None, [])
+        get_reference_and_tests = util.l10n_base_reference_and_tests(files)
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertIsNone(ref)
+        self.assertIsNone(tests)
+
+    def test_no_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_environment(l10n_base='l10n_orig')
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': '{l10n_base}/{locale}/some/file.ftl',
+        })
+        pc.set_locales(['gecko'], deep=True)
+        files = ProjectFiles('gecko', [pc])
+        get_reference_and_tests = util.l10n_base_reference_and_tests(files)
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(
+            mozpath.relpath(ref, 'l10n_orig/gecko'),
+            'some/file.ftl'
+        )
+        self.assertEqual(tests, set())
+
+    def test_with_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_environment(l10n_base='l10n_orig')
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': '{l10n_base}/{locale}/some/file.ftl',
+            'test': ['more_stuff'],
+        })
+        pc.set_locales(['gecko'], deep=True)
+        files = ProjectFiles('gecko', [pc])
+        get_reference_and_tests = util.l10n_base_reference_and_tests(files)
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(
+            mozpath.relpath(ref, 'l10n_orig/gecko'),
+            'some/file.ftl'
+        )
+        self.assertEqual(tests, {'more_stuff'})
--- a/third_party/python/compare-locales/compare_locales/tests/paths/__init__.py
+++ b/third_party/python/compare-locales/compare_locales/tests/paths/__init__.py
@@ -28,16 +28,17 @@ class SetupMixin(object):
         self.file = File(
             '/tmp/somedir/de/browser/one/two/file.ftl',
             'file.ftl',
             module='browser', locale='de')
         self.other_file = File(
             '/tmp/somedir/de/toolkit/two/one/file.ftl',
             'file.ftl',
             module='toolkit', locale='de')
+        self.cfg.set_locales(['de'])
 
 
 class MockNode(object):
     def __init__(self, name):
         self.name = name
         self.files = []
         self.dirs = {}
 
--- a/third_party/python/compare-locales/compare_locales/tests/paths/test_files.py
+++ b/third_party/python/compare-locales/compare_locales/tests/paths/test_files.py
@@ -15,17 +15,17 @@ from . import (
     Rooted,
 )
 
 
 class TestProjectPaths(Rooted, unittest.TestCase):
     def test_l10n_path(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.root)
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/*'
         })
         mocks = [
             self.path(leaf)
             for leaf in (
                 '/de/good.ftl',
                 '/de/not/subdir/bad.ftl',
@@ -60,17 +60,17 @@ class TestProjectPaths(Rooted, unittest.
              set()))
         # 'fr' is not in the locale list, should return no files
         files = MockProjectFiles(mocks, 'fr', [cfg])
         self.assertListEqual(list(files), [])
 
     def test_single_reference_path(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.path('/l10n'))
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/good.ftl',
             'reference': self.path('/reference/good.ftl')
         })
         mocks = [
             self.path('/reference/good.ftl'),
             self.path('/reference/not/subdir/bad.ftl'),
         ]
@@ -96,17 +96,17 @@ class TestProjectPaths(Rooted, unittest.
              self.path('/reference/good.ftl'),
              None,
              set()),
             )
 
     def test_reference_path(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.path('/l10n'))
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/*',
             'reference': self.path('/reference/*')
         })
         mocks = [
             self.path(leaf)
             for leaf in [
                 '/l10n/de/good.ftl',
@@ -170,17 +170,17 @@ class TestProjectPaths(Rooted, unittest.
              'merging/de/good.ftl', set()),
             )
         # 'fr' is not in the locale list, should return no files
         files = MockProjectFiles(mocks, 'fr', [cfg])
         self.assertListEqual(list(files), [])
 
     def test_partial_l10n(self):
         cfg = ProjectConfig(None)
-        cfg.locales.extend(['de', 'fr'])
+        cfg.set_locales(['de', 'fr'])
         cfg.add_paths({
             'l10n': self.path('/{locale}/major/*')
         }, {
             'l10n': self.path('/{locale}/minor/*'),
             'locales': ['de']
         })
         mocks = [
             self.path(leaf)
@@ -211,17 +211,17 @@ class TestProjectPaths(Rooted, unittest.
             [
                 (self.path('/fr/major/good.ftl'), None, None, set()),
             ])
         self.assertIsNone(files.match(self.path('/fr/minor/some.ftl')))
 
     def test_validation_mode(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.path('/l10n'))
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/*',
             'reference': self.path('/reference/*')
         })
         mocks = [
             self.path(leaf)
             for leaf in [
                 '/l10n/de/good.ftl',
--- a/third_party/python/compare-locales/compare_locales/tests/paths/test_project.py
+++ b/third_party/python/compare-locales/compare_locales/tests/paths/test_project.py
@@ -166,16 +166,42 @@ class TestConfigRules(SetupMixin, unitte
 
 class TestProjectConfig(unittest.TestCase):
     def test_children(self):
         pc = ProjectConfig(None)
         child = ProjectConfig(None)
         pc.add_child(child)
         self.assertListEqual([pc, child], list(pc.configs))
 
+    def test_locales_in_children(self):
+        pc = ProjectConfig(None)
+        child = ProjectConfig(None)
+        child.add_paths({
+            'l10n': '/tmp/somedir/{locale}/toolkit/**',
+        })
+        child.set_locales([])
+        pc.add_child(child)
+        self.assertListEqual(pc.all_locales, [])
+        pc.set_locales(['de', 'fr'])
+        self.assertListEqual(child.locales, [])
+        self.assertListEqual(pc.all_locales, ['de', 'fr'])
+
+    def test_locales_in_paths(self):
+        pc = ProjectConfig(None)
+        child = ProjectConfig(None)
+        child.add_paths({
+            'l10n': '/tmp/somedir/{locale}/toolkit/**',
+            'locales': ['it']
+        })
+        child.set_locales([])
+        pc.add_child(child)
+        self.assertListEqual(pc.all_locales, ['it'])
+        pc.set_locales(['de', 'fr'])
+        self.assertListEqual(pc.all_locales, ['de', 'fr', 'it'])
+
 
 class TestSameConfig(unittest.TestCase):
 
     def test_path(self):
         one = ProjectConfig('one.toml')
         one.set_locales(['ab'])
         self.assertTrue(one.same(ProjectConfig('one.toml')))
         self.assertFalse(one.same(ProjectConfig('two.toml')))
--- a/third_party/python/compare-locales/compare_locales/tests/test_apps.py
+++ b/third_party/python/compare-locales/compare_locales/tests/test_apps.py
@@ -76,17 +76,17 @@ class TestApp(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.stage)
 
     def test_app(self):
         'Test parsing a App'
         app = EnumerateApp(
             mozpath.join(self.stage, 'comm', 'mail', 'locales', 'l10n.ini'),
             mozpath.join(self.stage, 'l10n-central'))
-        self.assertListEqual(app.locales, ['af', 'de', 'fr'])
+        self.assertListEqual(app.config.allLocales(), ['af', 'de', 'fr'])
         self.assertEqual(len(app.config.children), 1)
         projectconfig = app.asConfig()
         self.assertListEqual(projectconfig.locales, ['af', 'de', 'fr'])
         files = ProjectFiles('de', [projectconfig])
         files = list(files)
         self.assertEqual(len(files), 3)
 
         l10nfile, reffile, mergefile, test = files[0]