Bug 1389099 - Properly encode multi-byte translations in CSS r=kmag
authorRob Wu <rob@robwu.nl>
Tue, 08 May 2018 22:35:22 +0200
changeset 418787 fb806e16195a
parent 418786 94304ecaaadf
child 418788 8ad87d070ec9
push id34013
push userdluca@mozilla.com
push date2018-05-18 09:56 +0000
treeherdermozilla-central@11ee70f24ea5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmag
bugs1389099
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1389099 - Properly encode multi-byte translations in CSS r=kmag The localization filter was not unicode-aware because convertToStream assigns the output to a nsIStringInputStream (which takes 8-bit chars). The input was read as a 8-bit string, but after localization it can contain wide strings if a translation has a multi-byte character. To fix this, the input stream is now first read as a UTF-8 string, then localized, and finally exported via a nsIArrayBufferInputStream.. MozReview-Commit-ID: LjCxczIFKCR
toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
toolkit/components/extensions/test/xpcshell/test_locale_converter.js
toolkit/components/utils/simpleServices.js
--- a/toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js
@@ -15,16 +15,20 @@ const {
   promiseShutdownManager,
   promiseStartupManager,
 } = AddonTestUtils;
 
 AddonTestUtils.init(this);
 
 createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "42");
 
+// Some multibyte characters. This sample was taken from the encoding/api-basics.html web platform test.
+const MULTIBYTE_STRING = "z\xA2\u6C34\uD834\uDD1E\uF8FF\uDBFF\uDFFD\uFFFE";
+let getCSS = (a, b) => `a { content: '${a}'; } b { content: '${b}'; }`;
+
 let extensionData = {
   background: function() {
     function backgroundFetch(url) {
       return new Promise((resolve, reject) => {
         let xhr = new XMLHttpRequest();
         xhr.overrideMimeType("text/plain");
         xhr.open("GET", url);
         xhr.onload = () => { resolve(xhr.responseText); };
@@ -37,27 +41,27 @@ let extensionData = {
       browser.test.assertEq("body { max-width: 42px; }", results[0], "CSS file localized");
       browser.test.assertEq("body { max-width: 42px; }", results[1], "CSS file localized");
 
       browser.test.assertEq("body { __MSG_foo__; }", results[2], "Text file not localized");
 
       browser.test.notifyPass("i18n-css");
     });
 
-    browser.test.sendMessage("ready", browser.runtime.getURL("foo.css"));
+    browser.test.sendMessage("ready", browser.runtime.getURL("/"));
   },
 
   manifest: {
     "applications": {
       "gecko": {
         "id": "i18n_css@mochi.test",
       },
     },
 
-    "web_accessible_resources": ["foo.css", "foo.txt", "locale.css"],
+    "web_accessible_resources": ["foo.css", "foo.txt", "locale.css", "multibyte.css"],
 
     "content_scripts": [
       {
         "matches": ["http://*/*/file_sample.html"],
         "css": ["foo.css"],
         "run_at": "document_start",
       },
       {
@@ -70,66 +74,71 @@ let extensionData = {
   },
 
   files: {
     "_locales/en/messages.json": JSON.stringify({
       "foo": {
         "message": "max-width: 42px",
         "description": "foo",
       },
+      "multibyteKey": {
+        "message": MULTIBYTE_STRING,
+      },
     }),
 
     "content.js": function() {
       let style = getComputedStyle(document.body);
       browser.test.sendMessage("content-maxWidth", style.maxWidth);
     },
 
     "foo.css": "body { __MSG_foo__; }",
     "bar.CsS": "body { __MSG_foo__; }",
     "foo.txt": "body { __MSG_foo__; }",
     "locale.css": '* { content: "__MSG_@@ui_locale__ __MSG_@@bidi_dir__ __MSG_@@bidi_reversed_dir__ __MSG_@@bidi_start_edge__ __MSG_@@bidi_end_edge__" }',
+    "multibyte.css": getCSS("__MSG_multibyteKey__", MULTIBYTE_STRING),
   },
 };
 
 async function test_i18n_css(options = {}) {
   extensionData.useAddonManager = options.useAddonManager;
   let extension = ExtensionTestUtils.loadExtension(extensionData);
 
   await extension.startup();
-  let cssURL = await extension.awaitMessage("ready");
+  let baseURL = await extension.awaitMessage("ready");
 
   let contentPage = await ExtensionTestUtils.loadContentPage(`${BASE_URL}/file_sample.html`);
 
-  let css = await contentPage.fetch(cssURL);
+  let css = await contentPage.fetch(baseURL + "foo.css");
 
   equal(css, "body { max-width: 42px; }", "CSS file localized in mochitest scope");
 
   let maxWidth = await extension.awaitMessage("content-maxWidth");
 
   equal(maxWidth, "42px", "stylesheet correctly applied");
 
-  cssURL = cssURL.replace(/foo.css$/, "locale.css");
+  css = await contentPage.fetch(baseURL + "locale.css");
+  equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
 
-  css = await contentPage.fetch(cssURL);
-  equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
+  css = await contentPage.fetch(baseURL + "multibyte.css");
+  equal(css, getCSS(MULTIBYTE_STRING, MULTIBYTE_STRING), "CSS file contains multibyte string");
 
   await contentPage.close();
 
   // We don't currently have a good way to mock this.
   if (false) {
     const DIR = "intl.uidirection";
 
     // We don't wind up actually switching the chrome registry locale, since we
     // don't have a chrome package for Hebrew. So just override it, and force
     // RTL directionality.
     const origReqLocales = Services.locale.getRequestedLocales();
     Services.locale.setRequestedLocales(["he"]);
     Preferences.set(DIR, 1);
 
-    css = await fetch(cssURL);
+    css = await fetch(baseURL + "locale.css");
     equal(css, '* { content: "he rtl ltr right left" }', "CSS file localized in mochitest scope");
 
     Services.locale.setRequestedLocales(origReqLocales);
     Preferences.reset(DIR);
   }
 
   await extension.awaitFinish("i18n-css");
   await extension.unload();
--- a/toolkit/components/extensions/test/xpcshell/test_locale_converter.js
+++ b/toolkit/components/extensions/test/xpcshell/test_locale_converter.js
@@ -124,10 +124,10 @@ add_task(async function testInvalidUUID(
   }, expectInvalidContextException);
 });
 
 
 // Test that an empty stream does not throw an NS_ERROR_ILLEGAL_VALUE.
 add_task(async function testEmptyStream() {
   let stream = StringStream("");
   let resultStream = convService.convert(stream, FROM_TYPE, TO_TYPE, URI);
-  equal(resultStream.data, "");
+  equal(resultStream.available(), 0, "Size of output stream should match size of input stream");
 });
--- a/toolkit/components/utils/simpleServices.js
+++ b/toolkit/components/utils/simpleServices.js
@@ -20,16 +20,19 @@ ChromeUtils.defineModuleGetter(this, "Ne
                                "resource://gre/modules/NetUtil.jsm");
 ChromeUtils.defineModuleGetter(this, "Services",
                                "resource://gre/modules/Services.jsm");
 
 XPCOMUtils.defineLazyServiceGetter(this, "catMan", "@mozilla.org/categorymanager;1",
                                    "nsICategoryManager");
 XPCOMUtils.defineLazyServiceGetter(this, "streamConv", "@mozilla.org/streamConverters;1",
                                    "nsIStreamConverterService");
+const ArrayBufferInputStream = Components.Constructor(
+  "@mozilla.org/io/arraybuffer-input-stream;1",
+  "nsIArrayBufferInputStream", "setData");
 
 /*
  * This class provides a stream filter for locale messages in CSS files served
  * by the moz-extension: protocol handler.
  *
  * See SubstituteChannel in netwerk/protocol/res/ExtensionProtocolHandler.cpp
  * for usage.
  */
@@ -63,56 +66,56 @@ AddonLocalizationConverter.prototype = {
     let addon = WebExtensionPolicy.getByURI(uri);
     if (!addon) {
       throw new Components.Exception("Invalid context", Cr.NS_ERROR_INVALID_ARG);
     }
     return addon;
   },
 
   convertToStream(aAddon, aString) {
-    let stream = Cc["@mozilla.org/io/string-input-stream;1"]
-      .createInstance(Ci.nsIStringInputStream);
-
-    stream.data = aAddon.localize(aString);
-    return stream;
+    aString = aAddon.localize(aString);
+    let bytes = new TextEncoder().encode(aString).buffer;
+    return new ArrayBufferInputStream(bytes, 0, bytes.byteLength);
   },
 
   convert(aStream, aFromType, aToType, aContext) {
     this.checkTypes(aFromType, aToType);
     let addon = this.getAddon(aContext);
 
-    let string = (
-      aStream.available() ?
-      NetUtil.readInputStreamToString(aStream, aStream.available()) : ""
-    );
+    let count = aStream.available();
+    let string = count ?
+      new TextDecoder().decode(NetUtil.readInputStream(aStream, count)) : "";
     return this.convertToStream(addon, string);
   },
 
   asyncConvertData(aFromType, aToType, aListener, aContext) {
     this.checkTypes(aFromType, aToType);
     this.addon = this.getAddon(aContext);
     this.listener = aListener;
   },
 
   onStartRequest(aRequest, aContext) {
     this.parts = [];
+    this.decoder = new TextDecoder();
   },
 
   onDataAvailable(aRequest, aContext, aInputStream, aOffset, aCount) {
-    this.parts.push(NetUtil.readInputStreamToString(aInputStream, aCount));
+    let bytes = NetUtil.readInputStream(aInputStream, aCount);
+    this.parts.push(this.decoder.decode(bytes, {stream: true}));
   },
 
   onStopRequest(aRequest, aContext, aStatusCode) {
     try {
       this.listener.onStartRequest(aRequest, null);
       if (Components.isSuccessCode(aStatusCode)) {
+        this.parts.push(this.decoder.decode());
         let string = this.parts.join("");
         let stream = this.convertToStream(this.addon, string);
 
-        this.listener.onDataAvailable(aRequest, null, stream, 0, stream.data.length);
+        this.listener.onDataAvailable(aRequest, null, stream, 0, stream.available());
       }
     } catch (e) {
       aStatusCode = e.result || Cr.NS_ERROR_FAILURE;
     }
     this.listener.onStopRequest(aRequest, null, aStatusCode);
   },
 };