Bug 1575135 - Make configure complain on non-unicode strings. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 20 Aug 2019 21:31:34 +0000
changeset 489123 b651185f9573fcb05a1c25d7a80cf203d249cc67
parent 489122 7e4b8df1e96a286ce22f2c2aff54ce2bcc445af4
child 489124 9c83333374ef9eeea424ed2ce3cc0496eb872b59
push id36465
push userdvarga@mozilla.com
push dateWed, 21 Aug 2019 16:47:43 +0000
treeherdermozilla-central@4ab60925635c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1575135
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1575135 - Make configure complain on non-unicode strings. r=nalexander Make it a hard error when the sandbox returns non-unicode strings. This should help quickly catch any remaining non-unicode string that are not caught by automation. Differential Revision: https://phabricator.services.mozilla.com/D42607
configure.py
python/mozbuild/mozbuild/test/frontend/test_sandbox.py
--- a/configure.py
+++ b/configure.py
@@ -5,16 +5,17 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import codecs
 import itertools
 import logging
 import os
 import sys
 import textwrap
+from collections import Iterable
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(base_dir, 'python', 'mozbuild'))
 sys.path.insert(0, os.path.join(base_dir, 'third_party', 'python', 'six'))
 from mozbuild.configure import (
     ConfigureSandbox,
     TRACE,
@@ -22,16 +23,17 @@ from mozbuild.configure import (
 from mozbuild.pythonutil import iter_modules_in_path
 from mozbuild.backend.configenvironment import PartialConfigEnvironment
 from mozbuild.util import (
     indented_repr,
     encode,
     system_encoding,
 )
 import mozpack.path as mozpath
+import six
 
 
 def main(argv):
     config = {}
 
     sandbox = ConfigureSandbox(config, os.environ, argv)
 
     if os.environ.get('MOZ_CONFIGURE_TRACE'):
@@ -40,16 +42,37 @@ def main(argv):
     sandbox.run(os.path.join(os.path.dirname(__file__), 'moz.configure'))
 
     if sandbox._help:
         return 0
 
     return config_status(config)
 
 
+def check_unicode(obj):
+    '''Recursively check that all strings in the object are unicode strings.'''
+    if isinstance(obj, dict):
+        result = True
+        for k, v in six.iteritems(obj):
+            if not check_unicode(k):
+                print("%s key is not unicode." % k, file=sys.stderr)
+                result = False
+            elif not check_unicode(v):
+                print("%s value is not unicode." % k, file=sys.stderr)
+                result = False
+        return result
+    if isinstance(obj, bytes):
+        return False
+    if isinstance(obj, six.text_type):
+        return True
+    if isinstance(obj, Iterable):
+        return all(check_unicode(o) for o in obj)
+    return True
+
+
 def config_status(config):
     # Sanitize config data to feed config.status
     # Ideally, all the backend and frontend code would handle the booleans, but
     # there are so many things involved, that it's easier to keep config.status
     # untouched for now.
     def sanitized_bools(v):
         if v is True:
             return '1'
@@ -66,16 +89,21 @@ def config_status(config):
     sanitized_config['defines'] = {
         k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
     }
     sanitized_config['non_global_defines'] = config['non_global_defines']
     sanitized_config['topsrcdir'] = config['TOPSRCDIR']
     sanitized_config['topobjdir'] = config['TOPOBJDIR']
     sanitized_config['mozconfig'] = config.get('MOZCONFIG')
 
+    if not check_unicode(sanitized_config):
+        print("Configuration should be all unicode.", file=sys.stderr)
+        print("Please file a bug for the above.", file=sys.stderr)
+        sys.exit(1)
+
     # Create config.status. Eventually, we'll want to just do the work it does
     # here, when we're able to skip configure tests/use cached results/not rely
     # on autoconf.
     logging.getLogger('moz.configure').info('Creating config.status')
     with codecs.open('config.status', 'w', system_encoding) as fh:
         fh.write(textwrap.dedent('''\
             #!%(python)s
             # coding=%(encoding)s
--- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
@@ -329,27 +329,16 @@ class TestMozbuildSandbox(unittest.TestC
         sandbox = self.sandbox()
         sandbox._context.add_source(sandbox.normalize_path('moz.build'))
 
         sandbox.exec_source('CONFIGURE_SUBST_FILES += ["bar", "foo"]')
         self.assertEqual(sandbox['CONFIGURE_SUBST_FILES'], ['bar', 'foo'])
         for item in sandbox['CONFIGURE_SUBST_FILES']:
             self.assertIsInstance(item, SourcePath)
 
-    def test_invalid_utf8_substs(self):
-        """Ensure invalid UTF-8 in substs is converted with an error."""
-
-        # This is really mbcs. It's a bunch of invalid UTF-8.
-        config = MockConfig(extra_substs={'BAD_UTF8': b'\x83\x81\x83\x82\x3A'})
-
-        sandbox = MozbuildSandbox(Context(VARIABLES, config))
-
-        self.assertEqual(sandbox['CONFIG']['BAD_UTF8'],
-                         u'\ufffd\ufffd\ufffd\ufffd:')
-
     def test_invalid_exports_set_base(self):
         sandbox = self.sandbox()
 
         with self.assertRaises(SandboxExecutionError) as se:
             sandbox.exec_source('EXPORTS = "foo.h"')
 
         self.assertEqual(se.exception.exc_type, ValueError)