Bug 794713 - Convert process output to Unicode; handle Unicode in logger properly; r=glandium
authorGregory Szorc <gps@mozilla.com>
Fri, 28 Sep 2012 10:29:32 -0700
changeset 108517 42632e7878f306b36b80fbdd9287d41955a153b0
parent 108516 b9ec9ad2e46e6c41dc7971a9635aaaf61a71043d
child 108518 ed5f377741045a9caf858d4edff4685b50f11961
push id82
push usershu@rfrn.org
push dateFri, 05 Oct 2012 13:20:22 +0000
reviewersglandium
bugs794713
milestone18.0a1
Bug 794713 - Convert process output to Unicode; handle Unicode in logger properly; r=glandium DONTBUILD (NPOTB)
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/logger.py
python/mozbuild/mozbuild/test/test_logger.py
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -4,16 +4,17 @@
 
 from __future__ import unicode_literals
 
 import logging
 import multiprocessing
 import os
 import pymake.parser
 import shlex
+import sys
 import subprocess
 import which
 
 from mozprocess.processhandler import ProcessHandlerMixin
 from pymake.data import Makefile
 from tempfile import TemporaryFile
 
 from mozbuild.config import ConfigProvider
@@ -299,16 +300,20 @@ class MozbuildObject(object):
         within a UNIX environment. Basically, if we are on Windows, it will
         execute the command via an appropriate UNIX-like shell.
         """
         args = self._normalize_command(args, require_unix_environment)
 
         self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
 
         def handleLine(line):
+            # Converts str to unicode on Python 2 and bytes to str on Python 3.
+            if isinstance(line, bytes):
+                line = line.decode(sys.stdout.encoding)
+
             if line_handler:
                 line_handler(line)
 
             if not log_name:
                 return
 
             self.log(log_level, log_name, {'line': line.strip()}, '{line}')
 
--- a/python/mozbuild/mozbuild/logger.py
+++ b/python/mozbuild/mozbuild/logger.py
@@ -92,17 +92,17 @@ class StructuredTerminalFormatter(Struct
 
         result = s
 
         if s.startswith('TEST-PASS'):
             result = self.terminal.green(s[0:9]) + s[9:]
         elif s.startswith('TEST-UNEXPECTED'):
             result = self.terminal.red(s[0:20]) + s[21:]
 
-        return result.decode('UTF-8', 'ignore')
+        return result
 
 
 class LoggingManager(object):
     """Holds and controls global logging state.
 
     A mozbuild application should instantiate one of these and configure it
     as needed.
 
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/test_logger.py
@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import time
+import unittest
+
+from mozbuild.logger import StructuredHumanFormatter
+
+
+class DummyLogger(logging.Logger):
+    def __init__(self, cb):
+        logging.Logger.__init__(self, 'test')
+
+        self._cb = cb
+
+    def handle(self, record):
+        self._cb(record)
+
+
+class TestStructuredHumanFormatter(unittest.TestCase):
+    def test_non_ascii_logging(self):
+        # Ensures the formatter doesn't choke when non-ASCII characters are
+        # present in printed parameters.
+        formatter = StructuredHumanFormatter(time.time())
+
+        def on_record(record):
+            result = formatter.format(record)
+            relevant = result[5:]
+
+            self.assertEqual(relevant, 'Test: s\xe9curit\xe9')
+
+        logger = DummyLogger(on_record)
+
+        value = 's\xe9curit\xe9'
+
+        logger.log(logging.INFO, 'Test: {utf}',
+            extra={'action': 'action', 'params': {'utf': value}})
+
+
+