Bug 711980 - Fix permanent oranges caused by interpreting xpcshell tests as UTF-8 (bug 687679). Originally by David Bienvenu <bienvenu@nventure.com> and Mike Conley <mconley@mozilla.com>. r=bienvenu+mconley, sr=bienvenu, a=Standard8.
authorSiddharth Agarwal <sagarwal@mozilla.com>
Thu, 29 Dec 2011 10:52:17 -0500
changeset 10059 00a50ae8e19ebf8545541fcf460a942498a291ff
parent 10058 9f5de3098224cea64467782f97055af3d509496b
child 10060 994869ef8fad30fb56437ad0a8aef9332daf6b53
push id336
push userbugzilla@standard8.plus.com
push dateTue, 31 Jan 2012 22:15:45 +0000
treeherdercomm-beta@54945f5d278d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbienvenu, bienvenu, Standard8
bugs711980, 687679
Bug 711980 - Fix permanent oranges caused by interpreting xpcshell tests as UTF-8 (bug 687679). Originally by David Bienvenu <bienvenu@nventure.com> and Mike Conley <mconley@mozilla.com>. r=bienvenu+mconley, sr=bienvenu, a=Standard8.
mail/base/test/unit/test_attachmentChecker.js
mailnews/compose/public/nsISmtpUrl.idl
mailnews/compose/test/unit/test_mailtoURL.js
mailnews/db/gloda/test/unit/test_mime_attachments_size.js
--- a/mail/base/test/unit/test_attachmentChecker.js
+++ b/mail/base/test/unit/test_attachmentChecker.js
@@ -58,30 +58,22 @@ function assert(aBeTrue, aWhy)
 };
 
 function assert_equal(aA, aB, aWhy)
 {
   assert(aA == aB, aWhy + " (" + unescape(encodeURIComponent(aA)) + " != " +
                                  unescape(encodeURIComponent(aB)) + ").");
 };
 
-function utf8_decode(s)
-{
-  return decodeURIComponent(escape(s));
-};
-
 /*
  * TESTS
  */
 
 function test_GetAttachmentKeywords(desc, mailData, keywords, expected)
 {
-  mailData = utf8_decode(mailData);
-  keywords = utf8_decode(keywords);
-  expected = utf8_decode(expected);
   let result = GetAttachmentKeywords(mailData, keywords);
   assert_equal(result, expected, desc + " not equal!");
 }
 
 var tests = [
   // Desc, mail body Data, keywords to search for, expected keywords found.
   ["Simple keyword", "latte.ca", "latte", "latte"],
   ["Extension", "testing document.pdf", ".pdf", "document.pdf"],
--- a/mailnews/compose/public/nsISmtpUrl.idl
+++ b/mailnews/compose/public/nsISmtpUrl.idl
@@ -104,40 +104,40 @@ interface nsISmtpUrl : nsISupports {
 
   /// Constant for the default SMTP port number
   const PRInt32 DEFAULT_SMTP_PORT = 25;
 
   /// Constant for the default SMTP over ssl port number
   const PRInt32 DEFAULT_SMTPS_PORT = 465;
 };
 
-[scriptable, uuid(ef796f57-7128-4fc6-9002-6cba9eb19e73)]
+[scriptable, uuid(87c36c23-4bc2-4992-b338-69f88f6ed0a1)]
 interface nsIMailtoUrl : nsISupports {
   /**
    * mailto: parse specific getters
    *
    * All of these fields are things we can effectively extract from a
    * mailto url if it contains all of these values
    *
    * Note: Attachments aren't available because that would expose a potential
    * security hole (see bug 99055).
    *
    * These items are in one function as we only ever get them from the one
    * place and all at the same time.
    */
-  void GetMessageContents(out ACString aToPart, out ACString aCcPart,
-                          out ACString aBccPart, out ACString aSubjectPart,
-                          out ACString aBodyPart, out ACString aHtmlPart,
+  void getMessageContents(out AUTF8String aToPart, out AUTF8String aCcPart,
+                          out AUTF8String aBccPart, out AUTF8String aSubjectPart,
+                          out AUTF8String aBodyPart, out AUTF8String aHtmlPart,
                           out ACString aReferencePart,
-                          out ACString aNewsgroupPart,
+                          out AUTF8String aNewsgroupPart,
                           out MSG_ComposeFormat aFormat);
 
   /**
    * These attributes are available should mailnews or extensions want them
    * but aren't used by standard in mailnews.
    */
-  readonly attribute ACString fromPart;
-  readonly attribute ACString followUpToPart;
-  readonly attribute ACString organizationPart;
-  readonly attribute ACString replyToPart;
-  readonly attribute ACString priorityPart;
-  readonly attribute ACString newsHostPart;
+  readonly attribute AUTF8String fromPart;
+  readonly attribute AUTF8String followUpToPart;
+  readonly attribute AUTF8String organizationPart;
+  readonly attribute AUTF8String replyToPart;
+  readonly attribute AUTF8String priorityPart;
+  readonly attribute AUTF8String newsHostPart;
 };
--- a/mailnews/compose/test/unit/test_mailtoURL.js
+++ b/mailnews/compose/test/unit/test_mailtoURL.js
@@ -10,17 +10,17 @@ const COMPOSE_DEFAULT = Components.inter
 function run_test() {
 
   function test(aTest) {
     var uri = Services.io.newURI(aTest.url, null, null);
     uri = uri.QueryInterface(Components.interfaces.nsIMailtoUrl);
 
     var to = {}, cc = {}, bcc = {}, subject = {}, body = {}, html = {},
         reference = {}, newsgroup = {}, composeformat = {};
-    uri.GetMessageContents(to, cc, bcc, subject, body, html, reference,
+    uri.getMessageContents(to, cc, bcc, subject, body, html, reference,
                            newsgroup, composeformat);
     do_check_eq(aTest.to, to.value);
     do_check_eq(aTest.cc, cc.value);
     do_check_eq(aTest.bcc, bcc.value);
     do_check_eq(aTest.subject, subject.value);
     do_check_eq(aTest.body, body.value);
     do_check_eq(aTest.html, html.value);
     do_check_eq(aTest.reference, reference.value);
@@ -38,18 +38,18 @@ function run_test() {
   for (var i = 0; i < tests.length; i++)
     test(tests[i]);
 
   // Test cloning reparses the url by checking the to field.
   let uriToClone = Services.io.newURI(tests[0].url, null, null);
   let clonedUrl = uriToClone.clone().QueryInterface(Components.interfaces.nsIMailtoUrl);
   var to = {}, cc = {}, bcc = {}, subject = {}, body = {}, html = {},
       reference = {}, newsgroup = {}, composeformat = {};
-  clonedUrl.GetMessageContents(to, cc, bcc, subject, body, html, reference,
-                         newsgroup, composeformat);
+  clonedUrl.getMessageContents(to, cc, bcc, subject, body, html, reference,
+                               newsgroup, composeformat);
   do_check_eq(to.value, tests[0].to);
 };
 
 const tests = [
   {
     url: "mailto:one@example.com",
     to: "one@example.com",
     cc: "",
@@ -749,10 +749,30 @@ const tests = [
     newsgroup: "",
     composeformat: COMPOSE_DEFAULT,
     from: "",
     followupto: "",
     organization: "",
     replyto: "",
     priority: "",
     newshost: ""
-  }
+  },
+  {
+    url: "mailto:%CE%B1?cc=%CE%B2&bcc=%CE%B3&subject=%CE%B4&body=%CE%B5" +
+         "&html-body=%CE%BE&newsgroups=%CE%B6&from=%CE%B7&followup-to=%CE%B8" +
+         "&organization=%CE%B9&reply-to=%CE%BA&priority=%CE%BB&newshost=%CE%BC",
+    to: "α",
+    cc: "β",
+    bcc: "γ",
+    subject: "δ",
+    body: "ε",
+    html: "ξ",
+    reference: "", // we expect this field to be ASCII-only
+    newsgroup: "ζ",
+    composeformat: COMPOSE_HTML,
+    from: "η",
+    followupto: "θ",
+    organization: "ι",
+    replyto: "κ",
+    priority: "λ",
+    newshost: "μ"
+  },
 ];
--- a/mailnews/db/gloda/test/unit/test_mime_attachments_size.js
+++ b/mailnews/db/gloda/test/unit/test_mime_attachments_size.js
@@ -49,36 +49,43 @@ load("../../../../resources/mailDirServi
 load("../../../../resources/mailTestUtils.js");
 load("../../../../resources/logHelper.js");
 load("../../../../resources/asyncTestUtils.js");
 
 load("../../../../resources/messageGenerator.js");
 load("../../../../resources/messageModifier.js");
 load("../../../../resources/messageInjection.js");
 
+Components.utils.import("resource://gre/modules/NetUtil.jsm");
+
 // Create a message generator
 const msgGen = gMessageGenerator = new MessageGenerator();
 // Create a message scenario generator using that message generator
 const scenarios = gMessageScenarioFactory = new MessageScenarioFactory(msgGen);
 
 Components.utils.import("resource:///modules/gloda/mimemsg.js");
 
 var htmlText = "<html><head></head><body>I am HTML! Woo! </body></html>";
 
 var partHtml = new SyntheticPartLeaf(
   htmlText,
   {
     contentType: "text/html"
   }
 );
 
+// This text is 168 characters long, and occupies 173 bytes when encoded in
+// UTF-8. (We make sure it occupies 173 bytes in run_test below). Note that
+// you cannot use this text directly because it isn't pure ASCII. You must use
+// one of the encoded forms below.
 var originalText =
   "Longtemps, je me suis couché de bonne heure. Parfois, à "+
   "peine ma bougie éteinte, mes yeux se fermaient si vite que je n'avais pas le "+
   "temps de me dire : « Je m'endors. »";
+var originalTextByteCount = 173;
 
 var b64Text =
   "TG9uZ3RlbXBzLCBqZSBtZSBzdWlzIGNvdWNow6kgZGUgYm9ubmUgaGV1cmUuIFBhcmZvaXMs\n"+
   "IMOgIHBlaW5lIG1hIGJvdWdpZSDDqXRlaW50ZSwgbWVzIHlldXggc2UgZmVybWFpZW50IHNp\n"+
   "IHZpdGUgcXVlIGplIG4nYXZhaXMgcGFzIGxlIHRlbXBzIGRlIG1lIGRpcmUgOiDCqyBKZSBt\n"+
   "J2VuZG9ycy4gwrsK";
 
 var qpText =
@@ -111,26 +118,28 @@ var yencText =
   "\x8e\x8f\x4a\x97\x8f\x4a\x8e\x93\x9c\x8f\x4a\x64\x4a\xec\xd5\x4a"+
   "\x74\x8f\x4a\x97\x51\x8f\x98\x8e\x99\x9c\x9d\x58\x4a\xec\xe5\x34"+
   "\x0d\x0a"+
   "=yend size=174 crc32=7efccd8e\n";
 
 // that completely exotic encoding is only detected if there is no content type
 // on the message, which is usually the case in newsgroups. I hate you yencode!
 var partYencText = new SyntheticPartLeaf("I am text! Woo!\n\n"+yencText,
-  { contentType: '' } );
+  { contentType: '', charset: '', format: '' } );
 
 var partUUText = new SyntheticPartLeaf(
   "I am text! With uuencode... noes...\n\n"+uuText,
-  { contentType: '' });
+  { contentType: '', charset: '', format: '' });
 
-var tachText = {filename: 'bob.txt', body: originalText};
+var tachText = {filename: 'bob.txt', body: qpText,
+                charset: "utf-8", encoding: "quoted-printable"};
 
-var tachInlineText = {filename: 'foo.txt', body: originalText,
-                      format: null, charset: null,
+var tachInlineText = {filename: 'foo.txt', body: qpText,
+                      format: null, charset: "utf-8",
+                      encoding: "quoted-printable",
                       disposition: 'inline'};
 
 // images have a different behavior than other attachments: they are displayed
 // inline most of the time, so there are two different code paths that need to
 // enable streaming and byte counting to the JS mime emitter
 
 var tachImage = {filename: 'bob.png', contentType: 'image/png',
                  encoding: 'base64', charset: null, format: null,
@@ -140,17 +149,19 @@ var tachPdf = {filename: 'bob.pdf', cont
                  encoding: 'base64', charset: null, format: null,
                  body: b64Text};
 
 var tachUU = {filename: 'john.doe', contentType: 'application/x-uuencode',
                  encoding: 'uuencode', charset: null, format: null,
                  body: uuText};
 
 var tachApplication = {filename: 'funky.funk',
-                       contentType: 'application/x-funky', body: originalText};
+                       contentType: 'application/x-funky',
+                       encoding: 'base64',
+                       body: b64Text};
 
 var relImage = {contentType: 'image/png',
                 encoding: 'base64', charset: null, format: null,
                 contentId: 'part1.foo@bar.com',
                 body: b64Text};
 
 var tachVCard = {filename: 'bob.vcf', contentType: 'text/x-vcard',
                  encoding: '7bit', body: 'begin:vcard\nfn:Bob\nend:vcard\n'};
@@ -158,17 +169,17 @@ var partTachVCard = new SyntheticPartLea
 
 var partRelImage = new SyntheticPartLeaf(relImage.body, relImage);
 
 var messageInfos = [
   {
     name: 'uuencode inline',
     bodyPart: partUUText,
     subject: "duh",
-    epsilon: 4,
+    epsilon: 1,
     checkTotalSize: false,
   },
   // encoding type specific to newsgroups, not interested, gloda doesn't even
   // treat this as an attachment (probably because gloda requires an attachment
   // to have a content-type, which these yencoded parts don't have), but size IS
   // counted properly nonetheless
   /*{
     name: 'text/plain with yenc inline',
@@ -177,25 +188,29 @@ var messageInfos = [
   },*/
   // inline image, not interested either, gloda doesn't keep that as an
   // attachment (probably a deliberate choice), size is NOT counted properly
   // (don't want to investigate, I doubt it's a useful information anyway)
   /*{
     name: 'multipart/related',
     bodyPart: new SyntheticPartMultiRelated([partHtml, partRelImage]),
   },*/
-  // all of the other common cases work fine
-  {
+  // This doesn't really make sense because it returns the length of the
+  // encoded blob without the envelope. Disabling as part of bug 711980.
+  /*{
     name: '.eml attachment',
     bodyPart: new SyntheticPartMultiMixed([
       partHtml,
-      msgGen.makeMessage({ body: { body: originalText } }),
+      msgGen.makeMessage({ body: { body: qpText,
+                                   charset: "UTF-8",
+                                   encoding: "quoted-printable" } }),
     ]),
-    epsilon: 4,
-  },
+    epsilon: 1,
+  },*/
+  // all of the other common cases work fine
   {
     name: 'all sorts of "real" attachments',
     bodyPart: partHtml,
     attachments: [tachImage, tachPdf, tachUU,
       tachApplication, tachText, tachInlineText,],
     epsilon: 2,
   },
 ];
@@ -214,50 +229,36 @@ function check_attachments(aMimeMsg, eps
    * XXXXXXXXXX
    *               <-- newline
    * --chopchop    <-- MIME separator
    *
    * libmime counts bytes all the way up to the separator, which means it counts
    * the bytes for the extra line. Since newlines in emails are \n, most of the
    * time we get att.size = 174 instead of 173.
    *
-   * Due to mysterious line encoding issues, for attachments that are
-   * text/plain, we get att.size = 175 instead of 174 on Windows.
-   *
-   * Due to the way the fake messages are created, the test message with an .eml
-   * attachment gets an extra new line before the separator.
-   *
-   * de me dire
-   * je m'endor   <-- text/plain attachment
-   * s. >>.
-   *              <-- aaarggh! an extra newline!
-   *
-   * --chopchop
-   *
-   * So that raises the count to 175 on Unix, and 177 on Windows, because of
-   * that weird newline issue.
-   *
    * The good news is, it's just a fixed extra cost. There no issues with the
    * inner contents of the attachment, you can add as many newlines as you want
    * in it, Unix or Windows, the count won't get past the bounds.
    */
 
   do_check_true(aMimeMsg.allUserAttachments.length > 0);
 
   let totalSize = htmlText.length;
 
   for each (let [i, att] in Iterator(aMimeMsg.allUserAttachments)) {
-    dump("*** Attachment now is "+att.name+" "+att.size+"\n");
-    do_check_true(Math.abs(att.size - originalText.length) <= epsilon);
+    dump("*** Attachment now is "+att.name+" "+att.size+" "+"\n");
+    do_check_true(Math.abs(att.size - originalTextByteCount) <= epsilon);
     totalSize += att.size;
   }
 
   // undefined means true
-  if (checkTotalSize !== false)
+  if (checkTotalSize !== false) {
+    dump("*** Total size comparison: " + totalSize + " vs " + aMimeMsg.size + "\n");
     do_check_true(Math.abs(aMimeMsg.size - totalSize) <= epsilon);
+  }
 
   async_driver();
 }
 
 function test_message_attachments(info) {
   let synMsg = gMessageGenerator.makeMessage(info);
   let synSet = new SyntheticMessageSet([synMsg]);
   yield add_sets_to_folder(gInbox, [synSet]);
@@ -325,17 +326,17 @@ function check_bogus_parts(aMimeMsg, { e
     let partSize = 0;
     // The attachment, although a MimeUnknown part, is actually plain/text that
     // contains the whole attached message, including headers. Count them.
     for each (let [k, v] in Iterator(bogusMessage.headers))
       partSize += (k + ": " + v + sep).length;
     // That's the newline between the headers and the message body.
     partSize += sep.length;
     // That's the message body.
-    partSize += originalText.length;
+    partSize += originalTextByteCount;
     // That's the total length that's to be returned by the MimeMessage abstraction.
     let totalSize = htmlText.length + partSize;
     dump(totalSize+" vs "+aMimeMsg.size+"\n");
     do_check_true(Math.abs(aMimeMsg.size - totalSize) <= epsilon);
   }
 
   async_driver();
 }
@@ -394,12 +395,18 @@ var tests = [
   parameterizeTest(test_message_attachments, messageInfos),
   parameterizeTest(test_bogus_messages, bogusMessageInfos),
   parameterizeTest(test_have_attachments, messageHaveAttachmentsInfos),
 ];
 
 var gInbox;
 
 function run_test() {
+  // Sanity check: figure out how many bytes the original text occupies in UTF-8 encoding
+  let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                    .createInstance(Ci.nsIScriptableUnicodeConverter);
+  converter.charset = "UTF-8";
+  do_check_eq(converter.ConvertFromUnicode(originalText).length, originalTextByteCount);
+
   // use mbox injection because the fake server chokes sometimes right now
   gInbox = configure_message_injection({mode: "local"});
   async_run_tests(tests);
 }