Bug 583677 - Fix so custom tags (keywords) are visble to all users. r=jorgk
authorGene Smith <gds@chartertn.net>
Tue, 18 Dec 2018 15:55:57 -0500
changeset 33173 2d9043361668
parent 33172 e2b8bf66d263
child 33174 d1dec28a7647
push id2368
push userclokep@gmail.com
push dateMon, 28 Jan 2019 21:12:50 +0000
treeherdercomm-beta@56d23c07d815 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjorgk
bugs583677
Bug 583677 - Fix so custom tags (keywords) are visble to all users. r=jorgk See Bug 583677 comment 85 for basic description of the fix.
mailnews/imap/public/nsIImapFlagAndUidState.idl
mailnews/imap/src/nsImapFlagAndUidState.cpp
mailnews/imap/src/nsImapFlagAndUidState.h
mailnews/imap/src/nsImapMailFolder.cpp
mailnews/imap/src/nsImapMailFolder.h
mailnews/imap/src/nsImapServerResponseParser.cpp
mailnews/imap/src/nsImapServerResponseParser.h
--- a/mailnews/imap/public/nsIImapFlagAndUidState.idl
+++ b/mailnews/imap/public/nsIImapFlagAndUidState.idl
@@ -65,10 +65,12 @@ interface nsIImapFlagAndUidState : nsISu
    * Gets the custom attributes from the hash table where they were stored earlier
    * them.
    * @param aUid   UID of the associated msg
    * @param aCustomAttributeName   Name of the custom attribute value
    * @param aCustomAttributeValue   Value of the attribute,
    */
   ACString getCustomAttribute(in unsigned long aUid,
                               in ACString aCustomAttributeName);
+  void setOtherKeywords(in unsigned short index, in AUTF8String otherKeyword);
+  AUTF8String getOtherKeywords(in unsigned short index);
 };
 
--- a/mailnews/imap/src/nsImapFlagAndUidState.cpp
+++ b/mailnews/imap/src/nsImapFlagAndUidState.cpp
@@ -105,16 +105,37 @@ nsImapFlagAndUidState::OrSupportedUserFl
 NS_IMETHODIMP
 nsImapFlagAndUidState::GetSupportedUserFlags(uint16_t *aFlags)
 {
   NS_ENSURE_ARG_POINTER(aFlags);
   *aFlags = fSupportedUserFlags;
   return NS_OK;
 }
 
+NS_IMETHODIMP
+nsImapFlagAndUidState::SetOtherKeywords(uint16_t index, const nsACString &otherKeyword)
+{
+  if (index == 0)
+    fOtherKeywords.Clear();
+  nsAutoCString flag(otherKeyword);
+  ToLowerCase(flag);
+  fOtherKeywords.AppendElement(flag);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsImapFlagAndUidState::GetOtherKeywords(uint16_t index, nsACString &aKeyword)
+{
+  if (index < fOtherKeywords.Length())
+    aKeyword = fOtherKeywords[index];
+  else
+    aKeyword = EmptyCString();
+  return NS_OK;
+}
+
 // we need to reset our flags, (re-read all) but chances are the memory allocation needed will be
 // very close to what we were already using
 
 NS_IMETHODIMP nsImapFlagAndUidState::Reset()
 {
   PR_CEnterMonitor(this);
   fNumberDeleted = 0;
   m_customFlagsHash.Clear();
--- a/mailnews/imap/src/nsImapFlagAndUidState.h
+++ b/mailnews/imap/src/nsImapFlagAndUidState.h
@@ -46,12 +46,14 @@ private:
     nsDataHashtable<nsUint32HashKey, nsCString> m_customFlagsHash;
     // Hash table, mapping UID+customAttributeName to customAttributeValue.
     nsDataHashtable<nsCStringHashKey, nsCString> m_customAttributesHash;
     uint16_t                fSupportedUserFlags;
     int32_t                 fNumberDeleted;
     bool                    fPartialUIDFetch;
     uint32_t                fNumAdded;
     bool                    fStartCapture;
+    // Keywords (aka, tags) in FLAGS response to SELECT defined by other clients
+    nsTArray<nsCString>     fOtherKeywords;
     mozilla::Mutex mLock;
 };
 
 #endif
--- a/mailnews/imap/src/nsImapMailFolder.cpp
+++ b/mailnews/imap/src/nsImapMailFolder.cpp
@@ -95,16 +95,17 @@
 
 static NS_DEFINE_CID(kRDFServiceCID, NS_RDFSERVICE_CID);
 static NS_DEFINE_CID(kParseMailMsgStateCID, NS_PARSEMAILMSGSTATE_CID);
 static NS_DEFINE_CID(kCImapHostSessionList, NS_IIMAPHOSTSESSIONLIST_CID);
 
 extern mozilla::LazyLogModule gAutoSyncLog; // defined in nsAutoSyncManager.cpp
 extern mozilla::LazyLogModule IMAP;         // defined in nsImapProtocol.cpp
 extern mozilla::LazyLogModule IMAP_CS;      // For CONDSTORE, defined in nsImapProtocol.cpp
+mozilla::LazyLogModule IMAP_KW("IMAP_KW");  // for logging keyword (tag) processing
 
 #define MAILNEWS_CUSTOM_HEADERS "mailnews.customHeaders"
 
 /*
     Copies the contents of srcDir into destDir.
     destDir will be created if it doesn't exist.
 */
 
@@ -4435,17 +4436,17 @@ void nsImapMailFolder::TweakHeaderFlags(
         // we need to set label attribute on header because the dbview code
         // does msgHdr->GetLabel when asked to paint a row
         tweakMe->SetLabel((imap_flags & kImapMsgLabelFlags) >> 9);
         newFlags |= (imap_flags & kImapMsgLabelFlags) << 16;
       }
       if (newFlags)
         tweakMe->OrFlags(newFlags, &dbHdrFlags);
       if (!customFlags.IsEmpty())
-        (void) HandleCustomFlags(m_curMsgUid, tweakMe, userFlags, customFlags);
+        (void) HandleCustomFlags(m_curMsgUid, tweakMe, userFlags, customFlags, nullptr);
     }
   }
 }
 
 NS_IMETHODIMP
 nsImapMailFolder::SetupMsgWriteStream(nsIFile * aFile, bool addDummyEnvelope)
 {
   nsresult rv;
@@ -4746,17 +4747,18 @@ NS_IMETHODIMP
 nsImapMailFolder::BeginMessageUpload()
 {
   return NS_ERROR_FAILURE;
 }
 
 nsresult nsImapMailFolder::HandleCustomFlags(nsMsgKey uidOfMessage,
                                              nsIMsgDBHdr *dbHdr,
                                              uint16_t userFlags,
-                                             nsCString &keywords)
+                                             nsCString &keywords,
+                                             nsIImapFlagAndUidState *flagState)
 {
   nsresult rv = GetDatabase();
   NS_ENSURE_SUCCESS(rv, rv);
 
   ToLowerCase(keywords);
   bool messageClassified = true;
   // Mac Mail uses "NotJunk"
   if (keywords.Find("NonJunk", /* ignoreCase = */ true) != kNotFound ||
@@ -4780,16 +4782,81 @@ nsresult nsImapMailFolder::HandleCustomF
   if (messageClassified)
   {
     // only set the junkscore origin if it wasn't set before.
     nsCString existingProperty;
     dbHdr->GetStringProperty("junkscoreorigin", getter_Copies(existingProperty));
     if (existingProperty.IsEmpty())
       dbHdr->SetStringProperty("junkscoreorigin", "imapflag");
   }
+
+  if (flagState && !(userFlags & kImapMsgSupportUserFlag))
+  {
+    nsCString localKeywords;
+    if (!(userFlags & kImapMsgSupportUserFlag))
+    {
+      dbHdr->GetStringProperty("keywords", getter_Copies(localKeywords));
+      MOZ_LOG(IMAP_KW, mozilla::LogLevel::Debug, ("UID=%" PRIu32 ", localKeywords=|%s| rcvdKeyword=|%s|",
+              uidOfMessage, localKeywords.get(), keywords.get()));
+    }
+    nsTArray<nsCString> localKeywordArray;
+    nsTArray<nsCString> rcvdKeywordArray;
+    ParseString(localKeywords, ' ', localKeywordArray);
+    ParseString(keywords, ' ', rcvdKeywordArray);
+
+    nsAutoCString mozLogDefinedKWs;
+    if (MOZ_LOG_TEST(IMAP_KW, mozilla::LogLevel::Debug))
+      mozLogDefinedKWs.AppendLiteral("Defined keywords = |");
+    uint32_t i = 0;
+    while (true)
+    {
+      nsAutoCString definedKeyword;
+      flagState->GetOtherKeywords(i++, definedKeyword);
+      if (definedKeyword.IsEmpty())
+      {
+        if (MOZ_LOG_TEST(IMAP_KW, mozilla::LogLevel::Debug))
+        {
+          mozLogDefinedKWs.Append('|');
+          MOZ_LOG(IMAP_KW, mozilla::LogLevel::Debug, ("%s", mozLogDefinedKWs.get()));
+        }
+        break;
+      }
+
+      if (MOZ_LOG_TEST(IMAP_KW, mozilla::LogLevel::Debug))
+      {
+        mozLogDefinedKWs.Append(definedKeyword.get());
+        mozLogDefinedKWs.Append(' ');
+      }
+
+      bool inLocal = localKeywordArray.Contains(definedKeyword);
+      bool inRcvd = rcvdKeywordArray.Contains(definedKeyword);
+      if (inLocal && inRcvd)
+        rcvdKeywordArray.RemoveElement(definedKeyword);
+      if (inLocal && !inRcvd)
+        localKeywordArray.RemoveElement(definedKeyword);
+    }
+    // Combine local and rcvd keyword arrays into a single string
+    // so it can be passed to SetStringProperty(). If element of
+    // local already in rcvd, avoid duplicates in combined string.
+    nsAutoCString combinedKeywords;
+    for (i = 0; i < localKeywordArray.Length(); i++)
+    {
+      if (!rcvdKeywordArray.Contains(localKeywordArray[i]))
+      {
+        combinedKeywords.Append(localKeywordArray[i]);
+        combinedKeywords.Append(' ');
+      }
+    }
+    for (i = 0; i < rcvdKeywordArray.Length(); i++)
+    {
+      combinedKeywords.Append(rcvdKeywordArray[i]);
+      combinedKeywords.Append(' ');
+    }
+    return dbHdr->SetStringProperty("keywords", combinedKeywords.get());
+  }
   return (userFlags & kImapMsgSupportUserFlag) ?
           dbHdr->SetStringProperty("keywords", keywords.get()) : NS_OK;
 }
 
 // synchronize the message flags in the database with the server flags
 nsresult nsImapMailFolder::SyncFlags(nsIImapFlagAndUidState *flagState)
 {
   nsresult rv = GetDatabase(); // we need a database for this
@@ -4825,17 +4892,17 @@ nsresult nsImapMailFolder::SyncFlags(nsI
       continue;
 
     rv = mDatabase->GetMsgHdrForKey(uidOfMessage, getter_AddRefs(dbHdr));
     if (NS_SUCCEEDED(dbHdr->GetMessageSize(&messageSize)))
       newFolderSize += messageSize;
 
     nsCString keywords;
     if (NS_SUCCEEDED(flagState->GetCustomFlags(uidOfMessage, getter_Copies(keywords))))
-        HandleCustomFlags(uidOfMessage, dbHdr, supportedUserFlags, keywords);
+        HandleCustomFlags(uidOfMessage, dbHdr, supportedUserFlags, keywords, flagState);
 
     NotifyMessageFlagsFromHdr(dbHdr, uidOfMessage, flags);
   }
   if (!partialUIDFetch && newFolderSize != mFolderSize)
   {
     int64_t oldFolderSize = mFolderSize;
     mFolderSize = newFolderSize;
     NotifyIntPropertyChanged(kFolderSize, oldFolderSize, mFolderSize);
@@ -4930,17 +4997,17 @@ nsImapMailFolder::NotifyMessageFlags(uin
       return rv;
     rv = mDatabase->GetMsgHdrForKey(aMsgKey, getter_AddRefs(dbHdr));
     if (NS_SUCCEEDED(rv) && dbHdr)
     {
       uint32_t supportedUserFlags;
       GetSupportedUserFlags(&supportedUserFlags);
       NotifyMessageFlagsFromHdr(dbHdr, aMsgKey, aFlags);
       nsCString keywords(aKeywords);
-      HandleCustomFlags(aMsgKey, dbHdr, supportedUserFlags, keywords);
+      HandleCustomFlags(aMsgKey, dbHdr, supportedUserFlags, keywords, nullptr);
     }
   }
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsImapMailFolder::NotifyMessageDeleted(const char * onlineFolderName, bool deleteAllMsgs, const char * msgIdString)
 {
--- a/mailnews/imap/src/nsImapMailFolder.h
+++ b/mailnews/imap/src/nsImapMailFolder.h
@@ -349,17 +349,18 @@ protected:
     &keysToFetch, uint32_t &numNewUnread, nsIImapFlagAndUidState *flagState);
   void FindKeysToDelete(const nsTArray<nsMsgKey> &existingKeys, nsTArray<nsMsgKey>
     &keysToFetch, nsIImapFlagAndUidState *flagState, uint32_t boxFlags);
   void PrepareToAddHeadersToMailDB(nsIImapProtocol* aProtocol);
   void TweakHeaderFlags(nsIImapProtocol* aProtocol, nsIMsgDBHdr *tweakMe);
 
   nsresult SyncFlags(nsIImapFlagAndUidState *flagState);
   nsresult HandleCustomFlags(nsMsgKey uidOfMessage, nsIMsgDBHdr *dbHdr,
-                             uint16_t userFlags, nsCString& keywords);
+                             uint16_t userFlags, nsCString& keywords,
+                             nsIImapFlagAndUidState *flagState);
   nsresult NotifyMessageFlagsFromHdr(nsIMsgDBHdr *dbHdr, nsMsgKey msgKey,
                                      uint32_t flags);
 
   nsresult SetupHeaderParseStream(uint32_t size, const nsACString& content_type, nsIMailboxSpec *boxSpec);
   nsresult  ParseAdoptedHeaderLine(const char *messageLine, nsMsgKey msgKey);
   nsresult  NormalEndHeaderParseStream(nsIImapProtocol *aProtocol, nsIImapUrl *imapUrl);
 
   void EndOfflineDownload();
--- a/mailnews/imap/src/nsImapServerResponseParser.cpp
+++ b/mailnews/imap/src/nsImapServerResponseParser.cpp
@@ -777,17 +777,17 @@ void nsImapServerResponseParser::mailbox
 {
   if (!PL_strcasecmp(fNextToken, "FLAGS"))
   {
     // this handles the case where we got the permanent flags response
     // before the flags response, in which case, we want to ignore these flags.
     if (fGotPermanentFlags)
       skip_to_CRLF();
     else
-      parse_folder_flags();
+      parse_folder_flags(true);
   }
   else if (!PL_strcasecmp(fNextToken, "LIST") ||
            !PL_strcasecmp(fNextToken, "XLIST"))
   {
     AdvanceToNextToken();
     if (ContinueParse())
       mailbox_list(false);
   }
@@ -1845,56 +1845,84 @@ void nsImapServerResponseParser::text_mi
  text            ::= 1*TEXT_CHAR
 
 */
 void nsImapServerResponseParser::text()
 {
   skip_to_CRLF();
 }
 
-void nsImapServerResponseParser::parse_folder_flags()
+void nsImapServerResponseParser::parse_folder_flags(bool calledForFlags)
 {
   uint16_t labelFlags = 0;
+  bool storeUserFlags = !(fSupportsUserDefinedFlags & kImapMsgSupportUserFlag) &&
+                        calledForFlags && fFlagState;
+  uint16_t numOtherKeywords = 0;
 
   do
   {
     AdvanceToNextToken();
     if (*fNextToken == '(')
       fNextToken++;
-    if (!PL_strncasecmp(fNextToken, "$MDNSent", 8))
-      fSupportsUserDefinedFlags |= kImapMsgSupportMDNSentFlag;
-    else if (!PL_strncasecmp(fNextToken, "$Forwarded", 10))
-      fSupportsUserDefinedFlags |= kImapMsgSupportForwardedFlag;
-    else if (!PL_strncasecmp(fNextToken, "\\Seen", 5))
+    if (!PL_strncasecmp(fNextToken, "\\Seen", 5))
       fSettablePermanentFlags |= kImapMsgSeenFlag;
     else if (!PL_strncasecmp(fNextToken, "\\Answered", 9))
       fSettablePermanentFlags |= kImapMsgAnsweredFlag;
     else if (!PL_strncasecmp(fNextToken, "\\Flagged", 8))
       fSettablePermanentFlags |= kImapMsgFlaggedFlag;
     else if (!PL_strncasecmp(fNextToken, "\\Deleted", 8))
       fSettablePermanentFlags |= kImapMsgDeletedFlag;
     else if (!PL_strncasecmp(fNextToken, "\\Draft", 6))
       fSettablePermanentFlags |= kImapMsgDraftFlag;
-    else if (!PL_strncasecmp(fNextToken, "$Label1", 7))
-      labelFlags |= 1;
-    else if (!PL_strncasecmp(fNextToken, "$Label2", 7))
-      labelFlags |= 2;
-    else if (!PL_strncasecmp(fNextToken, "$Label3", 7))
-      labelFlags |= 4;
-    else if (!PL_strncasecmp(fNextToken, "$Label4", 7))
-      labelFlags |= 8;
-    else if (!PL_strncasecmp(fNextToken, "$Label5", 7))
-      labelFlags |= 16;
     else if (!PL_strncasecmp(fNextToken, "\\*", 2))
     {
+      // User defined and special keywords (tags) can be defined and set for
+      // mailbox. Should only occur in PERMANENTFLAGS response.
       fSupportsUserDefinedFlags |= kImapMsgSupportUserFlag;
       fSupportsUserDefinedFlags |= kImapMsgSupportForwardedFlag;
       fSupportsUserDefinedFlags |= kImapMsgSupportMDNSentFlag;
       fSupportsUserDefinedFlags |= kImapMsgLabelFlags;
     }
+    else
+    {
+      // Treat special and built-in $LabelX's as user defined if a
+      // save occurs below.
+      if (!PL_strncasecmp(fNextToken, "$MDNSent", 8))
+        fSupportsUserDefinedFlags |= kImapMsgSupportMDNSentFlag;
+      else if (!PL_strncasecmp(fNextToken, "$Forwarded", 10))
+        fSupportsUserDefinedFlags |= kImapMsgSupportForwardedFlag;
+      else if (!PL_strncasecmp(fNextToken, "$Label1", 7))
+        labelFlags |= 1;
+      else if (!PL_strncasecmp(fNextToken, "$Label2", 7))
+        labelFlags |= 2;
+      else if (!PL_strncasecmp(fNextToken, "$Label3", 7))
+        labelFlags |= 4;
+      else if (!PL_strncasecmp(fNextToken, "$Label4", 7))
+        labelFlags |= 8;
+      else if (!PL_strncasecmp(fNextToken, "$Label5", 7))
+        labelFlags |= 16;
+
+      // Save user keywords defined for mailbox, usually by other clients.
+      // But only do this for FLAGS response, not PERMANENTFLAGS response
+      // and if '\*' has not appeared in a PERMANENTFLAGS response.
+      if (storeUserFlags && *fNextToken != '\r')
+      {
+        if (*(fNextToken + strlen(fNextToken) - 1) != ')')
+        {
+          // Token doesn't end in ')' so save it as is.
+          fFlagState->SetOtherKeywords(numOtherKeywords++, nsDependentCString(fNextToken));
+        }
+        else
+        {
+          // Token ends in ')' so end of list. Change ')' to null and save.
+          fFlagState->SetOtherKeywords(numOtherKeywords++,
+                                       nsDependentCSubstring(fNextToken, strlen(fNextToken) - 1));
+        }
+      }
+    }
   } while (!fAtEndOfLine && ContinueParse());
 
   if (labelFlags == 31)
     fSupportsUserDefinedFlags |= kImapMsgLabelFlags;
 
   if (fFlagState)
     fFlagState->OrSupportedUserFlags(fSupportsUserDefinedFlags);
 }
@@ -1942,17 +1970,17 @@ void nsImapServerResponseParser::resp_te
     {
       skip_to_CRLF();
     }
     else if (!PL_strcasecmp(fNextToken,"PERMANENTFLAGS"))
     {
       uint32_t saveSettableFlags = fSettablePermanentFlags;
       fSupportsUserDefinedFlags = 0;  // assume no unless told
       fSettablePermanentFlags = 0;            // assume none, unless told otherwise.
-      parse_folder_flags();
+      parse_folder_flags(false);
       // if the server tells us there are no permanent flags, we're
       // just going to pretend that the FLAGS response flags, if any, are
       // permanent in case the server is broken. This will allow us
       // to store delete and seen flag changes - if they're not permanent,
       // they're not permanent, but at least we'll try to set them.
       if (!fSettablePermanentFlags)
         fSettablePermanentFlags = saveSettableFlags;
       fGotPermanentFlags = true;
--- a/mailnews/imap/src/nsImapServerResponseParser.h
+++ b/mailnews/imap/src/nsImapServerResponseParser.h
@@ -132,17 +132,17 @@ protected:
   virtual void    internal_date();
   virtual nsresult BeginMessageDownload(const char *content_type);
 
   virtual void    response_data();
   virtual void    resp_text();
   virtual void    resp_cond_state(bool isTagged);
   virtual void    text_mime2();
   virtual void    text();
-  virtual void    parse_folder_flags();
+  virtual void    parse_folder_flags(bool calledForFlags);
   virtual void    enable_data();
   virtual void    language_data();
   virtual void    authChallengeResponse_data();
   virtual void    resp_text_code();
   virtual void    response_done();
   virtual void    response_tagged();
   virtual void    response_fatal();
   virtual void    resp_cond_bye();