Bug 1077381 - Encode generate_browsersearch verbose output as utf-8. r=gps
authorNick Alexander <nalexander@mozilla.com>
Mon, 06 Oct 2014 11:32:24 -0700
changeset 208961 d02d1b622a7a73e1e404cd1b965e18f99fb3311d
parent 208960 9b6a0ae120b33d59fede82961fdf6430dfbecaf7
child 208962 0990d45d42a6063202f6ee84dde24f87b10be494
push id9181
push usernalexander@mozilla.com
push dateMon, 06 Oct 2014 18:33:10 +0000
treeherderfx-team@0990d45d42a6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1077381
milestone35.0a1
Bug 1077381 - Encode generate_browsersearch verbose output as utf-8. r=gps We need codecs for the print function and unicode_literals to let the format() string stay undecorated.
python/moz.build
python/mozbuild/mozbuild/action/generate_browsersearch.py
python/mozbuild/mozbuild/test/action/data/invalid/region.properties
python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties
python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py
--- a/python/moz.build
+++ b/python/moz.build
@@ -18,16 +18,17 @@ PYTHON_UNIT_TESTS += [
     'mach/mach/test/test_config.py',
     'mach/mach/test/test_entry_point.py',
     'mach/mach/test/test_error_output.py',
     'mach/mach/test/test_logger.py',
     'mozbuild/dumbmake/test/__init__.py',
     'mozbuild/dumbmake/test/test_dumbmake.py',
     'mozbuild/mozbuild/test/__init__.py',
     'mozbuild/mozbuild/test/action/test_buildlist.py',
+    'mozbuild/mozbuild/test/action/test_generate_browsersearch.py',
     'mozbuild/mozbuild/test/backend/__init__.py',
     'mozbuild/mozbuild/test/backend/common.py',
     'mozbuild/mozbuild/test/backend/test_android_eclipse.py',
     'mozbuild/mozbuild/test/backend/test_configenvironment.py',
     'mozbuild/mozbuild/test/backend/test_recursivemake.py',
     'mozbuild/mozbuild/test/backend/test_visualstudio.py',
     'mozbuild/mozbuild/test/common.py',
     'mozbuild/mozbuild/test/compilation/__init__.py',
--- a/python/mozbuild/mozbuild/action/generate_browsersearch.py
+++ b/python/mozbuild/mozbuild/action/generate_browsersearch.py
@@ -17,19 +17,23 @@ 2. Read the default search plugin from t
 3. Read the list of search plugins from the 'browser.search.order.INDEX'
 properties with values identifying particular search plugins by name.
 
 4. Generate a JSON representation of 2. and 3., and write the result to
 browsersearch.json in the locale-specific raw resource directory
 e.g. raw/browsersearch.json, raw-pt-rBR/browsersearch.json.
 '''
 
-from __future__ import print_function
+from __future__ import (
+    print_function,
+    unicode_literals,
+)
 
 import argparse
+import codecs
 import json
 import re
 import sys
 import os
 
 from mozbuild.dotproperties import (
     DotProperties,
 )
@@ -67,18 +71,19 @@ def main(args):
 
     # Use reversed order so that the first srcdir has higher priority to override keys.
     properties = merge_properties('region.properties', reversed(opts.srcdir))
 
     default = properties.get('browser.search.defaultenginename')
     engines = properties.get_list('browser.search.order')
 
     if opts.verbose:
-        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines))
-        print("Default engine is '{default}'.".format(default=default))
+        writer = codecs.getwriter('utf-8')(sys.stdout)
+        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines), file=writer)
+        print("Default engine is '{default}'.".format(default=default), file=writer)
 
     browsersearch = {}
     browsersearch['default'] = default
     browsersearch['engines'] = engines
 
     # FileAvoidWrite creates its parent directories.
     output = os.path.abspath(opts.output)
     fh = FileAvoidWrite(output)
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/action/data/invalid/region.properties
@@ -0,0 +1,12 @@
+# A region.properties file with invalid unicode byte sequences.  The
+# sequences were cribbed from Markus Kuhn's "UTF-8 decoder capability
+# and stress test", available at
+# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+
+# 3.5  Impossible bytes                                                         |
+#                                                                               |
+# The following two bytes cannot appear in a correct UTF-8 string               |
+#                                                                               |
+# 3.5.1  fe = ""                                                               |
+# 3.5.2  ff = ""                                                               |
+# 3.5.3  fe fe ff ff = ""                                                   |
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Default search engine
+browser.search.defaultenginename=百度
+
+# Search engine order (order displayed in the search bar dropdown)s
+browser.search.order.1=百度
+browser.search.order.2=Google
+
+# This is the default set of web based feed handlers shown in the reader
+# selection UI
+browser.contentHandlers.types.0.title=Bloglines
+browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s
+
+# increment this number when anything gets changed in the list below.  This will
+# cause Firefox to re-read these prefs and inject any new handlers into the
+# profile database.  Note that "new" is defined as "has a different URL"; this
+# means that it's not possible to update the name of existing handler, so
+# don't make any spelling errors here.
+gecko.handlerService.defaultHandlersVersion=3
+
+# The default set of protocol handlers for webcal:
+gecko.handlerService.schemes.webcal.0.name=30 Boxes
+gecko.handlerService.schemes.webcal.0.uriTemplate=https://30boxes.com/external/widget?refer=ff&url=%s
+
+# The default set of protocol handlers for mailto:
+gecko.handlerService.schemes.mailto.0.name=Yahoo! 邮件
+gecko.handlerService.schemes.mailto.0.uriTemplate=https://compose.mail.yahoo.com/?To=%s
+gecko.handlerService.schemes.mailto.1.name=Gmail
+gecko.handlerService.schemes.mailto.1.uriTemplate=https://mail.google.com/mail/?extsrc=mailto&url=%s
+
+# This is the default set of web based feed handlers shown in the reader
+# selection UI
+browser.contentHandlers.types.0.title=My Yahoo!
+browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+from __future__ import unicode_literals
+
+import json
+import os
+import unittest
+
+import mozunit
+
+import mozbuild.action.generate_browsersearch as generate_browsersearch
+
+from mozfile.mozfile import (
+    NamedTemporaryFile,
+    TemporaryDirectory,
+)
+
+import mozpack.path as mozpath
+
+
+test_data_path = mozpath.abspath(mozpath.dirname(__file__))
+test_data_path = mozpath.join(test_data_path, 'data')
+
+
+class TestGenerateBrowserSearch(unittest.TestCase):
+    """
+    Unit tests for generate_browsersearch.py.
+    """
+
+    def _test_one(self, name):
+        with TemporaryDirectory() as tmpdir:
+            with NamedTemporaryFile(mode='rw') as temp:
+                srcdir = os.path.join(test_data_path, name)
+
+                generate_browsersearch.main([
+                    '--verbose',
+                    '--srcdir', srcdir,
+                    temp.name])
+                return json.load(temp)
+
+    def test_valid_unicode(self):
+        o = self._test_one('valid-zh-CN')
+        self.assertEquals(o['default'], '百度')
+        self.assertEquals(o['engines'], ['百度', 'Google'])
+
+    def test_invalid_unicode(self):
+        with self.assertRaises(UnicodeDecodeError):
+            self._test_one('invalid')
+
+
+if __name__ == '__main__':
+    mozunit.main()