Bug 1353680, create test to prevent bad content in localizable strings, r=ahal,flod
authorAxel Hecht <axel@pike.org>
Thu, 02 May 2019 10:50:43 +0000
changeset 531073 af9d93e2536a6d5b9bc7beae0cab89179431f503
parent 531072 dce9826524bcd6a99add436fba85e119fe5e7c2d
child 531074 827f19b7bf2b6586fd2e1876bffe1dbf9bb0482d
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal, flod
bugs1353680
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1353680, create test to prevent bad content in localizable strings, r=ahal,flod Differential Revision: https://phabricator.services.mozilla.com/D20465
taskcluster/ci/source-test/mozlint.yml
tools/lint/docs/index.rst
tools/lint/docs/linters/l10n.rst
tools/lint/l10n.yml
tools/lint/python/l10n_lint.py
--- a/taskcluster/ci/source-test/mozlint.yml
+++ b/taskcluster/ci/source-test/mozlint.yml
@@ -169,8 +169,23 @@ shellcheck:
     platform: lint/opt
     treeherder:
         symbol: shell
     run:
         mach: lint -l shellcheck -f treeherder -f json:/builds/worker/mozlint.json
     when:
         files-changed:
             - '**/*.sh'
+
+
+localization:
+    description: l10n tests for strings with errors and conflicts with cross-channel
+    platform: lint/opt
+    treeherder:
+        symbol: l1nt
+    run:
+        mach: lint -l l10n -f treeherder
+    when:
+        files-changed:
+            - '**/locales/en-US/**'
+            - '**/l10n.toml'
+            - 'third_party/python/compare-locales/**'
+            - 'third_party/python/fluent/**'
--- a/tools/lint/docs/index.rst
+++ b/tools/lint/docs/index.rst
@@ -23,15 +23,16 @@ like mach, mozreview and taskcluster.
 .. toctree::
   :caption: Linting User Guide
   :maxdepth: 2
 
   usage
   create
   linters/eslint
   linters/flake8
+  linters/l10n
 
 Indices and tables
 ==================
 
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
new file mode 100644
--- /dev/null
+++ b/tools/lint/docs/linters/l10n.rst
@@ -0,0 +1,39 @@
+L10n
+====
+
+The l10n linter checks for mistakes and problems in the localizable files.
+Most of the code lives inside the
+`compare-locales <https://pypi.org/project/compare-locales/>`_
+package, and is shipping as the ``moz-l10n-lint`` command.
+
+The linter checks for fundamental issues like parsing errors, but it also
+finds more subtle mistakes like duplicated messages. It also warns if you're
+trying to change a string without changing the ID, or to add a string that's
+still in use in a stable channel with a different value.
+
+The warnings on string ID changes get reported on phabricator, but they're
+not making the build fail. To find out when to change IDs and when not to,
+read the :doc:`Lifecycle & Workflow </intl/localization>` section in the
+localization documentation.
+
+Run Locally
+-----------
+
+The can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter l10n <file paths>
+
+Alternatively, omit the ``--linter l10n`` and run all configured linters, which
+will include the l10n linter.
+
+
+Updating the Reference
+----------------------
+
+The linter checks out the cross-channel localization files into your
+``.mozbuild`` state directory. By default this is updated automatically after
+48 hours. There might be new strings anyway, if you want to ensure an
+updated clone, remove the marker file in
+``~/.mozbuild/gecko-strings/.hg/l10n_pull_marker``.
new file mode 100644
--- /dev/null
+++ b/tools/lint/l10n.yml
@@ -0,0 +1,41 @@
+---
+l10n:
+    description: Localization linter
+    # list of include directories of both
+    # browser and mobile/android l10n.tomls
+    include:
+        - browser/branding/official/locales/en-US
+        - browser/extensions/formautofill/locales/en-US
+        - browser/extensions/fxmonitor/locales/en-US
+        - browser/extensions/report-site-issue/locales/en-US
+        - browser/locales/en-US
+        - devtools/client/locales/en-US
+        - devtools/shared/locales/en-US
+        - devtools/startup/locales/en-US
+        - dom/locales/en-US
+        - mobile/android/base/locales/en-US
+        - mobile/android/locales/en-US
+        - mobile/locales/en-US
+        - netwerk/locales/en-US
+        - security/manager/locales/en-US
+        - services/sync/locales/en-US
+        - toolkit/locales/en-US
+        - tools/lint/l10n.yml
+    # files not supported by compare-locales,
+    # and also not relevant to this linter
+    exclude:
+        - browser/locales/en-US/firefox-l10n.js
+        - mobile/android/locales/en-US/mobile-l10n.js
+        - toolkit/locales/en-US/chrome/global/intl.css
+    l10n_configs:
+        - browser/locales/l10n.toml
+        - mobile/android/locales/l10n.toml
+    type: external
+    payload: python.l10n_lint:lint
+    setup: python.l10n_lint:gecko_strings_setup
+    support-files:
+        - '**/l10n.toml'
+        - 'third_party/python/compare-locales/**'
+        - 'third_party/python/fluent/**'
+        - 'tools/lint/python/l10n_lint.py'
+        - 'tools/lint/l10n.yml'
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/l10n_lint.py
@@ -0,0 +1,157 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+from datetime import datetime, timedelta
+import os
+
+from mozboot import util as mb_util
+from mozlint import result, pathutils
+from mozpack import path as mozpath
+import mozversioncontrol.repoupdate
+
+from compare_locales.lint.linter import L10nLinter
+from compare_locales.lint.util import l10n_base_reference_and_tests
+from compare_locales import parser
+from compare_locales.paths import TOMLParser, ProjectFiles
+
+
+LOCALE = 'gecko-strings'
+
+
+PULL_AFTER = timedelta(days=2)
+
+
+def lint(paths, lintconfig, **lintargs):
+    l10n_base = mb_util.get_state_dir()
+    root = lintargs['root']
+    exclude = lintconfig.get('exclude')
+    extensions = lintconfig.get('extensions')
+
+    # Load l10n.toml configs
+    l10nconfigs = load_configs(lintconfig, root, l10n_base)
+
+    # Check include paths in l10n.yml if it's in our given paths
+    # Only the l10n.yml will show up here, but if the l10n.toml files
+    # change, we also get the l10n.yml as the toml files are listed as
+    # support files.
+    if lintconfig['path'] in paths:
+        results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
+        paths.remove(lintconfig['path'])
+    else:
+        results = []
+
+    all_files = []
+    for p in paths:
+        fp = pathutils.FilterPath(p)
+        if fp.isdir:
+            for _, fileobj in fp.finder:
+                all_files.append(fileobj.path)
+        if fp.isfile:
+            all_files.append(p)
+    # Filter again, our directories might have picked up files the
+    # explicitly excluded in the l10n.yml configuration.
+    # `browser/locales/en-US/firefox-l10n.js` is a good example.
+    all_files, _ = pathutils.filterpaths(
+        lintargs['root'], all_files, lintconfig['include'],
+        exclude=exclude, extensions=extensions
+    )
+    # These should be excluded in l10n.yml
+    skips = {p for p in all_files if not parser.hasParser(p)}
+    results.extend(
+        result.from_config(
+            lintconfig,
+            level='warning',
+            path=path,
+            message="file format not supported in compare-locales"
+            )
+        for path in skips
+    )
+    all_files = [p for p in all_files if p not in skips]
+    files = ProjectFiles(LOCALE, l10nconfigs)
+
+    get_reference_and_tests = l10n_base_reference_and_tests(files)
+
+    linter = MozL10nLinter(lintconfig)
+    results += linter.lint(all_files, get_reference_and_tests)
+    return results
+
+
+def gecko_strings_setup(**lint_args):
+    gs = mozpath.join(mb_util.get_state_dir(), LOCALE)
+    marker = mozpath.join(gs, '.hg', 'l10n_pull_marker')
+    try:
+        last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
+        skip_clone = datetime.now() < last_pull + PULL_AFTER
+    except OSError:
+        skip_clone = False
+    if skip_clone:
+        return
+    hg = mozversioncontrol.get_tool_path('hg')
+    mozversioncontrol.repoupdate.update_mercurial_repo(
+        hg,
+        'https://hg.mozilla.org/l10n/gecko-strings',
+        gs
+    )
+    with open(marker, 'w') as fh:
+        fh.flush()
+
+
+def load_configs(lintconfig, root, l10n_base):
+    '''Load l10n configuration files specified in the linter configuration.'''
+    configs = []
+    env = {
+        'l10n_base': l10n_base
+    }
+    for toml in lintconfig['l10n_configs']:
+        cfg = TOMLParser().parse(
+            mozpath.join(root, toml),
+            env=env,
+            ignore_missing_includes=True
+        )
+        cfg.set_locales([LOCALE], deep=True)
+        configs.append(cfg)
+    return configs
+
+
+def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
+    '''Check l10n.yml config against l10n.toml configs.'''
+    reference_paths = set(
+        mozpath.relpath(p['reference'].prefix, lintargs['root'])
+        for project in l10nconfigs
+        for config in project.configs
+        for p in config.paths
+    )
+    # Just check for directories
+    reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
+    missing_in_yml = [
+        refd for refd in reference_dirs if refd not in lintconfig['include']
+    ]
+    # These might be subdirectories in the config, though
+    missing_in_yml = [
+        d for d in missing_in_yml
+        if not any(d.startswith(parent + '/') for parent in lintconfig['include'])
+    ]
+    if missing_in_yml:
+        dirs = ', '.join(missing_in_yml)
+        return [result.from_config(
+            lintconfig, path=lintconfig['path'],
+            message='l10n.yml out of sync with l10n.toml, add: ' + dirs
+        )]
+    return []
+
+
+class MozL10nLinter(L10nLinter):
+    '''Subclass linter to generate the right result type.'''
+    def __init__(self, lintconfig):
+        super(MozL10nLinter, self).__init__()
+        self.lintconfig = lintconfig
+
+    def lint(self, files, get_reference_and_tests):
+        return [
+            result.from_config(self.lintconfig, **result_data)
+            for result_data in super(MozL10nLinter, self).lint(
+                files, get_reference_and_tests
+            )
+        ]