Bug 1077381 - Encode generate_browsersearch verbose output as utf-8. r=gps
authorNick Alexander <nalexander@mozilla.com>
Mon, 06 Oct 2014 11:32:24 -0700
changeset 232274 d02d1b622a7a73e1e404cd1b965e18f99fb3311d
parent 232273 9b6a0ae120b33d59fede82961fdf6430dfbecaf7
child 232275 0990d45d42a6063202f6ee84dde24f87b10be494
push id4187
push userbhearsum@mozilla.com
push dateFri, 28 Nov 2014 15:29:12 +0000
treeherdermozilla-beta@f23cc6a30c11 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1077381
milestone35.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1077381 - Encode generate_browsersearch verbose output as utf-8. r=gps We need codecs for the print function and unicode_literals to let the format() string stay undecorated.
python/moz.build
python/mozbuild/mozbuild/action/generate_browsersearch.py
python/mozbuild/mozbuild/test/action/data/invalid/region.properties
python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties
python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py
--- a/python/moz.build
+++ b/python/moz.build
@@ -18,16 +18,17 @@ PYTHON_UNIT_TESTS += [
     'mach/mach/test/test_config.py',
     'mach/mach/test/test_entry_point.py',
     'mach/mach/test/test_error_output.py',
     'mach/mach/test/test_logger.py',
     'mozbuild/dumbmake/test/__init__.py',
     'mozbuild/dumbmake/test/test_dumbmake.py',
     'mozbuild/mozbuild/test/__init__.py',
     'mozbuild/mozbuild/test/action/test_buildlist.py',
+    'mozbuild/mozbuild/test/action/test_generate_browsersearch.py',
     'mozbuild/mozbuild/test/backend/__init__.py',
     'mozbuild/mozbuild/test/backend/common.py',
     'mozbuild/mozbuild/test/backend/test_android_eclipse.py',
     'mozbuild/mozbuild/test/backend/test_configenvironment.py',
     'mozbuild/mozbuild/test/backend/test_recursivemake.py',
     'mozbuild/mozbuild/test/backend/test_visualstudio.py',
     'mozbuild/mozbuild/test/common.py',
     'mozbuild/mozbuild/test/compilation/__init__.py',
--- a/python/mozbuild/mozbuild/action/generate_browsersearch.py
+++ b/python/mozbuild/mozbuild/action/generate_browsersearch.py
@@ -17,19 +17,23 @@ 2. Read the default search plugin from t
 3. Read the list of search plugins from the 'browser.search.order.INDEX'
 properties with values identifying particular search plugins by name.
 
 4. Generate a JSON representation of 2. and 3., and write the result to
 browsersearch.json in the locale-specific raw resource directory
 e.g. raw/browsersearch.json, raw-pt-rBR/browsersearch.json.
 '''
 
-from __future__ import print_function
+from __future__ import (
+    print_function,
+    unicode_literals,
+)
 
 import argparse
+import codecs
 import json
 import re
 import sys
 import os
 
 from mozbuild.dotproperties import (
     DotProperties,
 )
@@ -67,18 +71,19 @@ def main(args):
 
     # Use reversed order so that the first srcdir has higher priority to override keys.
     properties = merge_properties('region.properties', reversed(opts.srcdir))
 
     default = properties.get('browser.search.defaultenginename')
     engines = properties.get_list('browser.search.order')
 
     if opts.verbose:
-        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines))
-        print("Default engine is '{default}'.".format(default=default))
+        writer = codecs.getwriter('utf-8')(sys.stdout)
+        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines), file=writer)
+        print("Default engine is '{default}'.".format(default=default), file=writer)
 
     browsersearch = {}
     browsersearch['default'] = default
     browsersearch['engines'] = engines
 
     # FileAvoidWrite creates its parent directories.
     output = os.path.abspath(opts.output)
     fh = FileAvoidWrite(output)
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/action/data/invalid/region.properties
@@ -0,0 +1,12 @@
+# A region.properties file with invalid unicode byte sequences.  The
+# sequences were cribbed from Markus Kuhn's "UTF-8 decoder capability
+# and stress test", available at
+# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+
+# 3.5  Impossible bytes                                                         |
+#                                                                               |
+# The following two bytes cannot appear in a correct UTF-8 string               |
+#                                                                               |
+# 3.5.1  fe = ""                                                               |
+# 3.5.2  ff = ""                                                               |
+# 3.5.3  fe fe ff ff = ""                                                   |
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Default search engine
+browser.search.defaultenginename=百度
+
+# Search engine order (order displayed in the search bar dropdown)s
+browser.search.order.1=百度
+browser.search.order.2=Google
+
+# This is the default set of web based feed handlers shown in the reader
+# selection UI
+browser.contentHandlers.types.0.title=Bloglines
+browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s
+
+# increment this number when anything gets changed in the list below.  This will
+# cause Firefox to re-read these prefs and inject any new handlers into the
+# profile database.  Note that "new" is defined as "has a different URL"; this
+# means that it's not possible to update the name of existing handler, so
+# don't make any spelling errors here.
+gecko.handlerService.defaultHandlersVersion=3
+
+# The default set of protocol handlers for webcal:
+gecko.handlerService.schemes.webcal.0.name=30 Boxes
+gecko.handlerService.schemes.webcal.0.uriTemplate=https://30boxes.com/external/widget?refer=ff&url=%s
+
+# The default set of protocol handlers for mailto:
+gecko.handlerService.schemes.mailto.0.name=Yahoo! 邮件
+gecko.handlerService.schemes.mailto.0.uriTemplate=https://compose.mail.yahoo.com/?To=%s
+gecko.handlerService.schemes.mailto.1.name=Gmail
+gecko.handlerService.schemes.mailto.1.uriTemplate=https://mail.google.com/mail/?extsrc=mailto&url=%s
+
+# This is the default set of web based feed handlers shown in the reader
+# selection UI
+browser.contentHandlers.types.0.title=My Yahoo!
+browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+from __future__ import unicode_literals
+
+import json
+import os
+import unittest
+
+import mozunit
+
+import mozbuild.action.generate_browsersearch as generate_browsersearch
+
+from mozfile.mozfile import (
+    NamedTemporaryFile,
+    TemporaryDirectory,
+)
+
+import mozpack.path as mozpath
+
+
+test_data_path = mozpath.abspath(mozpath.dirname(__file__))
+test_data_path = mozpath.join(test_data_path, 'data')
+
+
+class TestGenerateBrowserSearch(unittest.TestCase):
+    """
+    Unit tests for generate_browsersearch.py.
+    """
+
+    def _test_one(self, name):
+        with TemporaryDirectory() as tmpdir:
+            with NamedTemporaryFile(mode='rw') as temp:
+                srcdir = os.path.join(test_data_path, name)
+
+                generate_browsersearch.main([
+                    '--verbose',
+                    '--srcdir', srcdir,
+                    temp.name])
+                return json.load(temp)
+
+    def test_valid_unicode(self):
+        o = self._test_one('valid-zh-CN')
+        self.assertEquals(o['default'], '百度')
+        self.assertEquals(o['engines'], ['百度', 'Google'])
+
+    def test_invalid_unicode(self):
+        with self.assertRaises(UnicodeDecodeError):
+            self._test_one('invalid')
+
+
+if __name__ == '__main__':
+    mozunit.main()