Bug 1274090 - Attempt to convert str objects containing non-ascii characters in Python configure rather than failing outright. r=glandium,gps
authorChris Manchester <cmanchester@mozilla.com>
Tue, 31 May 2016 10:34:05 -0700
changeset 340823 aeee5ac6aeb92823720dd54dca6240dccadd8e42
parent 340822 7383d49bc7c8e9b1683d4dbfbe9642e112b9f43c
child 340824 7fc7692916d6e1e8067d8443f71f08bc796d29ea
push id1183
push userraliiev@mozilla.com
push dateMon, 05 Sep 2016 20:01:49 +0000
treeherdermozilla-release@3148731bed45 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, gps
bugs1274090
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1274090 - Attempt to convert str objects containing non-ascii characters in Python configure rather than failing outright. r=glandium,gps MozReview-Commit-ID: CxFqFXS7Dh9
python/mozbuild/mozbuild/configure/__init__.py
python/mozbuild/mozbuild/configure/util.py
python/mozbuild/mozbuild/test/configure/data/subprocess.configure
python/mozbuild/mozbuild/test/configure/test_util.py
--- a/python/mozbuild/mozbuild/configure/__init__.py
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -20,16 +20,17 @@ from mozbuild.configure.options import (
     NegativeOptionValue,
     Option,
     OptionValue,
     PositiveOptionValue,
 )
 from mozbuild.configure.help import HelpFormatter
 from mozbuild.configure.util import (
     ConfigureOutputHandler,
+    getpreferredencoding,
     LineIO,
 )
 from mozbuild.util import (
     exec_,
     memoize,
     ReadOnlyDict,
     ReadOnlyNamespace,
 )
@@ -152,18 +153,34 @@ class ConfigureSandbox(dict):
 
         else:
             assert isinstance(logger, logging.Logger)
             moz_logger = None
             @contextmanager
             def queue_debug():
                 yield
 
+        # Some callers will manage to log a bytestring with characters in it
+        # that can't be converted to ascii. Make our log methods robust to this
+        # by detecting the encoding that a producer is likely to have used.
+        encoding = getpreferredencoding()
+        def wrapped_log_method(logger, key):
+            method = getattr(logger, key)
+            if not encoding:
+                return method
+            def wrapped(*args, **kwargs):
+                out_args = [
+                    arg.decode(encoding) if isinstance(arg, str) else arg
+                    for arg in args
+                ]
+                return method(*out_args, **kwargs)
+            return wrapped
+
         log_namespace = {
-            k: getattr(logger, k)
+            k: wrapped_log_method(logger, k)
             for k in ('debug', 'info', 'warning', 'error')
         }
         log_namespace['queue_debug'] = queue_debug
         self.log_impl = ReadOnlyNamespace(**log_namespace)
 
         self._help = None
         self._help_option = self.option_impl('--help',
                                              help='print this message')
--- a/python/mozbuild/mozbuild/configure/util.py
+++ b/python/mozbuild/mozbuild/configure/util.py
@@ -9,16 +9,30 @@ import itertools
 import locale
 import logging
 import os
 import sys
 from collections import deque
 from contextlib import contextmanager
 from distutils.version import LooseVersion
 
+def getpreferredencoding():
+    # locale._parse_localename makes locale.getpreferredencoding
+    # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or
+    # 'ANSI_X3.4-1968' when it uses nl_langinfo.
+    encoding = None
+    try:
+        encoding = locale.getpreferredencoding()
+    except ValueError:
+        # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and
+        # that throws off locale._parse_localename, which ends up
+        # being used on e.g. homebrew python.
+        if os.environ.get('LC_ALL', '').upper() == 'UTF-8':
+            encoding = 'utf-8'
+    return encoding
 
 class Version(LooseVersion):
     '''A simple subclass of distutils.version.LooseVersion.
     Adds attributes for `major`, `minor`, `patch` for the first three
     version components so users can easily pull out major/minor
     versions, like:
 
     v = Version('1.2b')
@@ -65,29 +79,17 @@ class ConfigureOutputHandler(logging.Han
         # ascii, which blatantly fails when trying to print out non-ascii.
         def fix_encoding(fh):
             try:
                 isatty = fh.isatty()
             except AttributeError:
                 isatty = True
 
             if not isatty:
-                encoding = None
-                try:
-                    encoding = locale.getpreferredencoding()
-                except ValueError:
-                    # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and
-                    # that throws off locale._parse_localename, which ends up
-                    # being used on e.g. homebrew python.
-                    if os.environ.get('LC_ALL', '').upper() == 'UTF-8':
-                        encoding = 'utf-8'
-
-                # locale._parse_localename makes locale.getpreferredencoding
-                # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or
-                # 'ANSI_X3.4-1968' when it uses nl_langinfo.
+                encoding = getpreferredencoding()
                 if encoding:
                     return codecs.getwriter(encoding)(fh)
             return fh
 
         self._stdout = fix_encoding(stdout)
         self._stderr = fix_encoding(stderr) if stdout != stderr else self._stdout
         try:
             fd1 = self._stdout.fileno()
@@ -190,18 +192,21 @@ class ConfigureOutputHandler(logging.Han
 
 class LineIO(object):
     '''File-like class that sends each line of the written data to a callback
     (without carriage returns).
     '''
     def __init__(self, callback):
         self._callback = callback
         self._buf = ''
+        self._encoding = getpreferredencoding()
 
     def write(self, buf):
+        if self._encoding and isinstance(buf, str):
+            buf = buf.decode(self._encoding)
         lines = buf.splitlines()
         if not lines:
             return
         if self._buf:
             lines[0] = self._buf + lines[0]
             self._buf = ''
         if not buf.endswith('\n'):
             self._buf = lines.pop()
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/configure/data/subprocess.configure
@@ -0,0 +1,23 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+@depends('--help')
+@imports('codecs')
+@imports(_from='mozbuild.configure.util', _import='getpreferredencoding')
+@imports('os')
+@imports(_from='__builtin__', _import='open')
+def dies_when_logging(_):
+    test_file = 'test.txt'
+    quote_char = "'"
+    if getpreferredencoding().lower() == 'utf-8':
+        quote_char = '\u00B4'.encode('utf-8')
+    try:
+        with open(test_file, 'w+') as fh:
+            fh.write(quote_char)
+        out = check_cmd_output('cat', 'test.txt')
+        log.info(out)
+    finally:
+        os.remove(test_file)
--- a/python/mozbuild/mozbuild/test/configure/test_util.py
+++ b/python/mozbuild/mozbuild/test/configure/test_util.py
@@ -13,19 +13,25 @@ import sys
 
 from StringIO import StringIO
 
 from mozunit import main
 from mozpack import path as mozpath
 
 from mozbuild.configure.util import (
     ConfigureOutputHandler,
+    getpreferredencoding,
     LineIO,
     Version,
 )
+
+from mozbuild.configure import (
+    ConfigureSandbox,
+)
+
 from mozbuild.util import exec_
 
 from buildconfig import topsrcdir
 from common import ConfigureTestSandbox
 
 
 class TestConfigureOutputHandler(unittest.TestCase):
     def test_separation(self):
@@ -406,16 +412,40 @@ class TestLineIO(unittest.TestCase):
         with LineIO(lambda l: lines.append(l)) as l:
             l.write('a\nb\nc')
 
             self.assertEqual(lines, ['a', 'b'])
 
         self.assertEqual(lines, ['a', 'b', 'c'])
 
 
+class TestLogSubprocessOutput(unittest.TestCase):
+
+    def test_non_ascii_subprocess_output(self):
+        out = StringIO()
+        sandbox = ConfigureSandbox({}, {}, [], out, out)
+
+        sandbox.include_file(mozpath.join(topsrcdir, 'build',
+                             'moz.configure', 'util.configure'))
+        sandbox.include_file(mozpath.join(topsrcdir, 'python', 'mozbuild',
+                                          'mozbuild', 'test', 'configure',
+                                          'data', 'subprocess.configure'))
+        status = 0
+        try:
+            sandbox.run()
+        except SystemExit as e:
+            status = e.code
+
+        self.assertEquals(status, 0)
+        quote_char = "'"
+        if getpreferredencoding().lower() == 'utf-8':
+            quote_char = '\u00B4'.encode('utf-8')
+        self.assertEquals(out.getvalue().strip(), quote_char)
+
+
 class TestVersion(unittest.TestCase):
     def test_version_simple(self):
         v = Version('1')
         self.assertEqual(v, '1')
         self.assertLess(v, '2')
         self.assertGreater(v, '0.5')
         self.assertEqual(v.major, 1)
         self.assertEqual(v.minor, 0)