Bug 561568 - Mails auto sent by "Reply with template" encode non-ASCII characters (in mail body) incorrectly. r=rkent
authorMagnus Melin <mkmelin+mozilla@iki.fi>
Fri, 04 Mar 2016 23:10:26 +0200
changeset 19048 afc38db4e0efe0037b21b0f2c6e49e66caea9a16
parent 19047 7b5667355f553e941f8829ac9e4b21f185a18494
child 19049 efbcfcec00d35ff62239d67d58a554b3f6c082b8
push id11695
push usermkmelin@iki.fi
push dateFri, 04 Mar 2016 21:10:52 +0000
treeherdercomm-central@afc38db4e0ef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrkent
bugs561568
Bug 561568 - Mails auto sent by "Reply with template" encode non-ASCII characters (in mail body) incorrectly. r=rkent
mailnews/compose/src/nsMsgCompose.cpp
mailnews/compose/src/nsMsgComposeService.cpp
mailnews/compose/test/unit/test_autoReply.js
mailnews/test/data/template-latin1
mailnews/test/data/template-utf8
--- a/mailnews/compose/src/nsMsgCompose.cpp
+++ b/mailnews/compose/src/nsMsgCompose.cpp
@@ -1256,17 +1256,16 @@ nsMsgCompose::SendMsgToServer(MSG_Delive
   if (NS_FAILED(rv))
     NotifyStateListeners(nsIMsgComposeNotificationType::ComposeProcessDone, rv);
 
   return rv;
 }
 
 NS_IMETHODIMP nsMsgCompose::SendMsg(MSG_DeliverMode deliverMode, nsIMsgIdentity *identity, const char *accountKey, nsIMsgWindow *aMsgWindow, nsIMsgProgress *progress)
 {
-
   NS_ENSURE_TRUE(m_compFields, NS_ERROR_NOT_INITIALIZED);
   nsresult rv = NS_OK;
   nsCOMPtr<nsIPrompt> prompt;
 
   // i'm assuming the compose window is still up at this point...
   if (m_window) {
     nsCOMPtr<nsPIDOMWindowOuter> window = nsPIDOMWindowOuter::From(m_window);
     window->GetPrompter(getter_AddRefs(prompt));
@@ -2031,34 +2030,36 @@ nsresult nsMsgCompose::CreateMessage(con
       nsCString decodedCString;
 
       if (!charsetOverride && charset.IsEmpty())
       {
         rv = msgHdr->GetCharset(getter_Copies(charset));
         if (NS_FAILED(rv)) return rv;
       }
 
-      // save the charset of a message being replied to because
-      // we need to use it when decoding RFC-2047-encoded author name
-      // with |charsetOverride|.
-      nsAutoCString originCharset(charset);
-
       bool replyInDefault = false;
       prefs->GetBoolPref("mailnews.reply_in_default_charset",
                           &replyInDefault);
       // Use send_default_charset if reply_in_default_charset is on.
       if (replyInDefault)
       {
         nsString str;
         NS_GetLocalizedUnicharPreferenceWithDefault(prefs, "mailnews.send_default_charset",
                                                     EmptyString(), str);
         if (!str.IsEmpty())
           LossyCopyUTF16toASCII(str, charset);
       }
 
+      // ReplyWithTemplate needs to always use the charset of the template,
+      // nothing else. That is passed in through the compFields.
+      if (type == nsIMsgCompType::ReplyWithTemplate) {
+         rv = compFields->GetCharacterSet(getter_Copies(charset));
+         NS_ENSURE_SUCCESS(rv,rv);
+      }
+
       // No matter what, we should block x-windows-949 (our internal name)
       // from being used for outgoing emails (bug 234958)
       if (charset.Equals("x-windows-949",
             nsCaseInsensitiveCStringComparator()))
         charset = "EUC-KR";
 
       // get an original charset, used for a label, UTF-8 is used for the internal processing
       if (isFirstPass && !charset.IsEmpty())
--- a/mailnews/compose/src/nsMsgComposeService.cpp
+++ b/mailnews/compose/src/nsMsgComposeService.cpp
@@ -954,17 +954,22 @@ NS_IMETHODIMP nsMsgTemplateReplyHelper::
     subject.Append(NS_LITERAL_STRING(" (was: "));
     subject.Append(replySubject);
     subject.Append(NS_LITERAL_STRING(")"));
   }
 
   compFields->SetSubject(subject);
   compFields->SetRawHeader("Auto-Submitted", NS_LITERAL_CSTRING("auto-replied"), nullptr);
 
-  CopyASCIItoUTF16(mTemplateBody, body);
+  nsCString charset;
+  rv = mTemplateHdr->GetCharset(getter_Copies(charset));
+  NS_ENSURE_SUCCESS(rv, rv);
+  compFields->SetCharacterSet(charset.get());
+  rv = nsMsgI18NConvertToUnicode(charset.get(), mTemplateBody, body);
+  NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "couldn't convert templ body to unicode");
   compFields->SetBody(body);
 
   nsCString msgUri;
   nsCOMPtr <nsIMsgFolder> folder;
   mHdrToReplyTo->GetFolder(getter_AddRefs(folder));
   folder->GetUriForMsg(mHdrToReplyTo, msgUri);
   // populate the compose params
   pMsgComposeParams->SetType(nsIMsgCompType::ReplyWithTemplate);
--- a/mailnews/compose/test/unit/test_autoReply.js
+++ b/mailnews/compose/test/unit/test_autoReply.js
@@ -16,29 +16,33 @@ load("../../../resources/logHelper.js");
 
 const kSender = "from@foo.invalid";
 
 var gIncomingMailFile = do_get_file("../../../data/bugmail10"); // mail to reply to
 // reply-filter-testmail: mail to reply to (but not really)
 var gIncomingMailFile2 = do_get_file("../../../data/reply-filter-testmail");
 // mail to reply to (but not really, no from)
 var gIncomingMailFile3 = do_get_file("../../../data/mail-without-from");
-var gTemplateMailFile = do_get_file("../../../data/draft1"); // template
+var gTemplateMailFile = do_get_file("../../../data/template-latin1"); // template
+var gTemplateMailFile2 = do_get_file("../../../data/template-utf8"); // template2
 var gTemplateFolder;
 
 var gServer;
 
 function run_test() {
   localAccountUtils.loadLocalMailAccount();
   gTemplateFolder = localAccountUtils.rootFolder
                                      .createLocalSubfolder("Templates");
 
   gServer = setupServerDaemon();
   gServer.start();
 
+  // encodeMimePartIIStr_UTF8 complains about latin1 encoding...
+  Services.prefs.setBoolPref("devtools.errorconsole.deprecation_warnings", false);
+
   run_next_test();
 }
 
 add_task(function* copy_gIncomingMailFile() {
   let promiseCopyListener = new PromiseTestUtils.PromiseCopyListener();
   // Copy gIncomingMailFile into the Inbox.
   MailServices.copy.CopyFileMessage(gIncomingMailFile,
     localAccountUtils.inboxFolder, null, false, 0, "",
@@ -68,37 +72,56 @@ add_task(function* copy_gTemplateMailFil
   let promiseCopyListener = new PromiseTestUtils.PromiseCopyListener();
   // Copy gTemplateMailFile into the Templates folder.
   MailServices.copy.CopyFileMessage(gTemplateMailFile,
     gTemplateFolder, null, true, 0, "",
     promiseCopyListener, null);
   yield promiseCopyListener.promise;
 });
 
+add_task(function* copy_gTemplateMailFile2() {
+  let promiseCopyListener = new PromiseTestUtils.PromiseCopyListener();
+  // Copy gTemplateMailFile2 into the Templates folder.
+  MailServices.copy.CopyFileMessage(gTemplateMailFile2,
+    gTemplateFolder, null, true, 0, "",
+    promiseCopyListener, null);
+  yield promiseCopyListener.promise;
+});
+
 /// Test that a reply is NOT sent when the message is not addressed to "me".
-add_task(function testReplyingToUnadressedFails() {
+add_task(function testReplyingToUnaddressedFails() {
   try {
     testReply(0); // mail 0 is not to us!
     do_throw("Replied to a message not addressed to us!");
   }
   catch (e) {
     if (e.result != Components.results.NS_ERROR_ABORT)
       throw e;
     // Ok! We didn't reply to the message not specifically addressed to
     // us (from@foo.invalid).
   }
 });
 
 /// Test that a reply is sent when the message is addressed to "me".
-add_task(function testReplyingToAdressedWorks() {
+add_task(function testReplyingToAdressedWorksLatin1() {
   try {
-    testReply(1); // mail 1 is adressed to us
+    testReply(1); // mail 1 is addressed to us, using template-latin1
   }
   catch (e) {
-    do_throw("Didn't reply to a message addressed to us! "  + e);
+    do_throw("Didn't reply properly to a message addressed to us! "  + e);
+  }
+});
+
+/// Test that a reply is sent when the message is addressed to "me".
+add_task(function testReplyingToAdressedWorksUTF8() {
+  try {
+    testReply(1, 1); // mail 1 is addressed to us, template-utf8
+  }
+  catch (e) {
+    do_throw("Didn't reply properly to a message addressed to us! "  + e);
   }
 });
 
 /// Test that a reply is NOT even tried when the message has no From.
 add_task(function testReplyingToMailWithNoFrom() {
   try {
     testReply(2); // mail 2 has no From
     do_throw("Shouldn't even have tried to reply reply to the message " +
@@ -106,41 +129,64 @@ add_task(function testReplyingToMailWith
   }
   catch (e) {
     if (e.result != Components.results.NS_ERROR_FAILURE)
       throw e;
   }
 });
 
 /// Test reply with template.
-function testReply(aHrdIdx) {
+function testReply(aHrdIdx, aTemplateHdrIdx = 0) {
   let smtpServer = getBasicSmtpServer();
   smtpServer.port = gServer.port;
 
   let identity = getSmtpIdentity(kSender, smtpServer);
   localAccountUtils.msgAccount.addIdentity(identity);
 
   let msgHdr = mailTestUtils.getMsgHdrN(localAccountUtils.inboxFolder, aHrdIdx);
   do_print("Msg#" + aHrdIdx +  " author=" + msgHdr.author + ", recipients=" +
            msgHdr.recipients);
-  let templateHdr = mailTestUtils.getMsgHdrN(gTemplateFolder, 0);
+  let templateHdr = mailTestUtils.getMsgHdrN(gTemplateFolder, aTemplateHdrIdx);
 
   // See <method name="getTemplates"> in searchWidgets.xml
   let msgTemplateUri = gTemplateFolder.URI +
                        "?messageId=" + templateHdr.messageId +
                        "&subject=" + templateHdr.mime2DecodedSubject;
   MailServices.compose.replyWithTemplate(msgHdr, msgTemplateUri, null,
     localAccountUtils.incomingServer);
   gServer.performTest();
-  let headers = MimeParser.extractHeaders(gServer._daemon.post);
+
+  let headers, body;
+  [headers, body] = MimeParser.extractHeadersAndBody(gServer._daemon.post);
+  //dump("xxxmagnus gServer._daemon.post=" + gServer._daemon.post + "\n");
   do_check_true(headers.get("Subject").startsWith("Auto: "));
   do_check_eq(headers.get("Auto-submitted"), "auto-replied");
   do_check_eq(headers.get("In-Reply-To"), "<" + msgHdr.messageId + ">");
   do_check_eq(headers.get("References"), "<" + msgHdr.messageId + ">");
-
+  // XXX: something's wrong with how the fake server gets the data.
+  // The text gets converted to UTF-8 (regardless of what it is) at some point.
+  // Suspect a bug with how BinaryInputStream handles the strings.
+  if (templateHdr.Charset == "windows-1252") {
+    // XXX: should really check for "��� åäö"
+    if (!body.includes("åäö xlatin1")) { // template-latin1 contains this
+      do_throw("latin1 body didn't go through!  hdr msgid=" +
+               templateHdr.messageId + ", msgbody=" + body);
+    }
+  } else if (templateHdr.Charset == "utf-8") {
+    // XXX: should really check for "åäö xutf8"
+    if (!body.includes("åäö xutf8")) { // template-utf8 contains this
+      do_throw("utf8 body didn't go through! hdr msgid=" +
+               templateHdr.messageId + ", msgbody=" + body);
+    }
+  } else if (templateHdr.Charset) {
+    do_throw("unexpected msg charset: " + templateHdr.Charset + ", hdr msgid=" +
+             templateHdr.messageId);
+  } else {
+    do_throw("didn't find a msg charset! hdr msgid=" + templateHdr.messageId);
+  }
   gServer.resetTest();
 }
 
 add_task(function teardown() {
   // fake server cleanup
   gServer.stop();
 });
 
new file mode 100644
--- /dev/null
+++ b/mailnews/test/data/template-latin1
@@ -0,0 +1,30 @@
+From - Sun 24 Jan 2016 21:57:32
+X-Mozilla-Status: 0000
+X-Mozilla-Status2: 00000000
+X-Mozilla-Keys:                                                                                 
+FCC: mailbox://nobody@Local%20Folders/Sent
+BCC: Some User <u1@example.com>, Another Person <u2@example.com>
+X-Identity-Key: id2
+From: Some User <example@example.com>
+Subject: =?UTF-8?B?bGF0aW4xIHRlbXBsYXRlIMOlw6TDtg==?=
+Message-ID: <8fd7dc38-5b3b-c1a3-2003-4d33dd1c70bf@example.com>
+Date: Sun, 24 Jan 2016 21:57:32 +0200
+X-Mozilla-Draft-Info: internal/draft; vcard=0; receipt=0; DSN=0; uuencode=0;
+ attachmentreminder=0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:46.0) Gecko/20100101
+ Thunderbird/46.0a1
+MIME-Version: 1.0
+Content-Type: text/html; charset=windows-1252
+Content-Transfer-Encoding: 8bit
+
+<html>
+  <head>
+    <meta content="text/html; charset=windows-1252"
+      http-equiv="Content-Type">
+  </head>
+  <body bgcolor="#FFFFFF" text="#000000">
+    html  xlatin1<br>
+  </body>
+</html>
+
+
new file mode 100644
--- /dev/null
+++ b/mailnews/test/data/template-utf8
@@ -0,0 +1,33 @@
+From - Sat 23 Jan 2016 22:06:43 +0200
+X-Mozilla-Status: 0000
+X-Mozilla-Status2: 00000000
+X-Mozilla-Keys:                                                                                 
+FCC: mailbox://nobody@Local%20Folders/Sent
+BCC: Some User <u1@example.com>, Another Person <u2@example.com>
+X-Identity-Key: id2
+From: Some User <example@example.com>
+Subject: utf-8 test
+Message-ID: <ee022a93-1fbf-7086-24c7-a61939ea52ce@example.com>
+Date: Sat, 23 Jan 2016 22:06:43 +0200
+X-Mozilla-Draft-Info: internal/draft; vcard=1; receipt=0; DSN=0; uuencode=0;
+ attachmentreminder=0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:46.0) Gecko/20100101
+ Thunderbird/46.0a1
+MIME-Version: 1.0
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+<html>
+  <head>
+
+    <meta http-equiv="content-type" content="text/html; charset=utf-8">
+  </head>
+  <body bgcolor="#FFFFFF" text="#000000">
+    <p>åäö xutf8<br>
+    </p>
+    <br>
+    <br>
+  </body>
+</html>
+
+