Bug 1594676, normalize Fluent files when validating migration tests, r=flod,stas
authorAxel Hecht <axel@pike.org>
Fri, 15 Nov 2019 17:37:34 +0000
changeset 502248 eebf1f2557be7ef8fd8558d07531dd1318e1eedf
parent 502247 493025785e6adfde44b1e2ad0a00b48b3ff4cb7b
child 502249 db1a18b908bfaf15485d1dd5430f67c46b5d65a8
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersflod, stas
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1594676, normalize Fluent files when validating migration tests, r=flod,stas There's a lot of formatting that is irrelevant for Fluent, and it's hard to diff usefully on the text file. Instead, normalize the Fluent files by parsing them, and serialzing them with Junk included. Differential Revision: https://phabricator.services.mozilla.com/D53202
--- a/python/l10n/test_fluent_migrations/fmt.py
+++ b/python/l10n/test_fluent_migrations/fmt.py
@@ -1,22 +1,26 @@
 from __future__ import absolute_import, print_function
+import codecs
+from difflib import unified_diff
 import logging
 import os
 import re
 import shutil
+import sys
 import hglib
 from mozboot.util import get_state_dir
 import mozpack.path as mozpath
 from compare_locales.merge import merge_channels
 from compare_locales.paths.files import ProjectFiles
 from compare_locales.paths.configparser import TOMLParser
 from fluent.migrate import validator
+from fluent.syntax import FluentParser, FluentSerializer
 def inspect_migration(path):
     '''Validate recipe and extract some metadata.
     return validator.Validator.validate(path)
@@ -37,16 +41,30 @@ def prepare_object_dir(cmd):
             ['hg', 'clone', 'https://hg.mozilla.org/l10n/gecko-strings'],
     return obj_dir
+def diff_resources(left_path, right_path):
+    parser = FluentParser(with_spans=False)
+    serializer = FluentSerializer(with_junk=True)
+    lines = []
+    for p in (left_path, right_path):
+        with codecs.open(p, encoding='utf-8') as fh:
+            res = parser.parse(fh.read())
+            lines.append(serializer.serialize(res).splitlines(True))
+    sys.stdout.writelines(
+        chunk.encode('utf-8')
+        for chunk in unified_diff(lines[0], lines[1], left_path, right_path)
+    )
 def test_migration(cmd, obj_dir, to_test, references):
     '''Test the given recipe.
     This creates a workdir by l10n-merging gecko-strings and the m-c source,
     to mimmic gecko-strings after the patch to test landed.
     It then runs the recipe with a gecko-strings clone as localization, both
     dry and wet.
     It inspects the generated commits, and shows a diff between the merged
@@ -118,21 +136,20 @@ def test_migration(cmd, obj_dir, to_test
     tip = client.tip().node
     if old_tip == tip:
         cmd.log(logging.WARN, 'fluent-migration-test', {
             'file': to_test,
         }, 'No migration applied for {file}')
         return rv
     for ref in references:
-        cmd.run_process([
-            'diff', '-u', '-B',
+        diff_resources(
             mozpath.join(work_dir, 'reference', ref),
             mozpath.join(work_dir, 'en-US', ref),
-        ], ensure_exit_code=False, line_handler=print)
+        )
     messages = [l.desc for l in client.log('::{} - ::{}'.format(tip, old_tip))]
     bug = re.search('[0-9]{5,}', migration_name).group()
     # Just check first message for bug number, they're all following the same pattern
     if bug not in messages[0]:
         rv = 1
         cmd.log(logging.ERROR, 'fluent-migration-test', {
             'file': to_test,
         }, 'Missing or wrong bug number for {file}')