Bug 1250723, fix crash and improve error handling when nsMsgKey hits max value, r=aceman a=rkent
authorR Kent James <rkent@caspia.com>
Wed, 16 Mar 2016 10:42:38 -0700
changeset 26816 4cc13f81190455bf93541058a0b01fb3417c3f41
parent 26815 d4bf9b03d79d927bfdb12858bde7493d47f8cc33
child 26817 e7e79f9aaf798b4221e2ad8568d20b8fa6d17cec
push id1850
push userclokep@gmail.com
push dateWed, 08 Mar 2017 19:29:12 +0000
treeherdercomm-esr52@028df196b2d9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaceman, rkent
bugs1250723
Bug 1250723, fix crash and improve error handling when nsMsgKey hits max value, r=aceman a=rkent
db/mork/src/morkRowSpace.cpp
mailnews/base/util/nsMsgDBFolder.cpp
mailnews/db/msgdb/src/nsMsgDatabase.cpp
mailnews/db/msgdb/src/nsMsgThread.cpp
mailnews/imap/src/nsImapUndoTxn.cpp
mailnews/local/src/nsMsgBrkMBoxStore.cpp
mailnews/local/src/nsMsgLocalStoreUtils.cpp
mailnews/local/src/nsMsgLocalStoreUtils.h
mailnews/local/src/nsMsgMaildirStore.cpp
mailnews/local/src/nsParseMailbox.cpp
--- a/db/mork/src/morkRowSpace.cpp
+++ b/db/mork/src/morkRowSpace.cpp
@@ -375,38 +375,41 @@ morkRowSpace::MakeNewTableId(morkEnv* ev
       ++id;
     }
   }
   
   mRowSpace_NextTableId = id + 1;
   return outTid;
 }
 
+#define MAX_AUTO_ID (morkRow_kMinusOneRid - 2)
 mork_rid
 morkRowSpace::MakeNewRowId(morkEnv* ev)
 {
   mork_rid outRid = 0;
   mork_rid id = mRowSpace_NextRowId;
   mork_num count = 9; // try up to eight times
   mdbOid oid;
   oid.mOid_Scope = this->SpaceScope();
-  
-  while ( !outRid && --count ) // still trying to find an unused row ID?
+
+  // still trying to find an unused row ID?
+  while (!outRid && --count && id <= MAX_AUTO_ID)
   {
     oid.mOid_Id = id;
     if ( !mRowSpace_Rows.GetOid(ev, &oid) )
       outRid = id;
     else
     {
       MORK_ASSERT(morkBool_kFalse); // alert developer about ID problems
       ++id;
     }
   }
   
-  mRowSpace_NextRowId = id + 1;
+  if (id < MAX_AUTO_ID)
+    mRowSpace_NextRowId = id + 1;
   return outRid;
 }
 
 morkAtomRowMap*
 morkRowSpace::make_index(morkEnv* ev, mork_column inCol)
 {
   morkAtomRowMap* outMap = 0;
   nsIMdbHeap* heap = mRowSpace_SlotHeap;
--- a/mailnews/base/util/nsMsgDBFolder.cpp
+++ b/mailnews/base/util/nsMsgDBFolder.cpp
@@ -2142,17 +2142,20 @@ nsMsgDBFolder::SetDBTransferInfo(nsIDBFo
   NS_ENSURE_ARG(aTransferInfo);
   nsCOMPtr <nsIDBFolderInfo> dbFolderInfo;
   nsCOMPtr <nsIMsgDatabase> db;
   GetMsgDatabase(getter_AddRefs(db));
   if (db)
   {
     db->GetDBFolderInfo(getter_AddRefs(dbFolderInfo));
     if(dbFolderInfo)
+    {
       dbFolderInfo->InitFromTransferInfo(aTransferInfo);
+      dbFolderInfo->SetBooleanProperty("forceReparse", false);
+    }
     db->SetSummaryValid(true);
   }
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsMsgDBFolder::GetStringProperty(const char *propertyName, nsACString& propertyValue)
 {
--- a/mailnews/db/msgdb/src/nsMsgDatabase.cpp
+++ b/mailnews/db/msgdb/src/nsMsgDatabase.cpp
@@ -58,16 +58,17 @@ using namespace mozilla;
 const int32_t kMaxHdrsInCache = 512;
 
 // special keys
 static const nsMsgKey kAllMsgHdrsTableKey = 1;
 static const nsMsgKey kTableKeyForThreadOne = 0xfffffffe;
 static const nsMsgKey kAllThreadsTableKey = 0xfffffffd;
 static const nsMsgKey kFirstPseudoKey = 0xfffffff0;
 static const nsMsgKey kIdStartOfFake = 0xffffff80;
+static const nsMsgKey kForceReparseKey = 0xfffffff0;
 
 static PRLogModuleInfo* DBLog;
 
 PRTime nsMsgDatabase::gLastUseTime;
 
 NS_IMPL_ISUPPORTS(nsMsgDBService, nsIMsgDBService)
 
 nsMsgDBService::nsMsgDBService()
@@ -805,16 +806,19 @@ nsresult nsMsgDatabase::RemoveHdrFromUse
   }
   return NS_OK;
 }
 
 
 nsresult
 nsMsgDatabase::CreateMsgHdr(nsIMdbRow* hdrRow, nsMsgKey key, nsIMsgDBHdr* *result)
 {
+  NS_ENSURE_ARG_POINTER(hdrRow);
+  NS_ENSURE_ARG_POINTER(result);
+
   nsresult rv = GetHdrFromUseCache(key, result);
   if (NS_SUCCEEDED(rv) && *result)
   {
     hdrRow->Release();
     return rv;
   }
 
   nsMsgHdr *msgHdr = new nsMsgHdr(this, hdrRow);
@@ -1250,16 +1254,27 @@ nsresult nsMsgDatabase::CheckForErrors(n
         if (!valid)
           err = NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE;
       }
       // compare current version of db versus filed out version info.
       uint32_t version;
       m_dbFolderInfo->GetVersion(&version);
       if (GetCurVersion() != version)
         err = NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE;
+
+      // Check if we should force a reparse because, for example, we have
+      // reached the key limit.
+      bool forceReparse;
+      m_dbFolderInfo->GetBooleanProperty("forceReparse", false, &forceReparse);
+      if (forceReparse)
+      {
+        NS_WARNING("Forcing a reparse presumably because key limit reached");
+        err = NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE;
+      }
+
     }
     if (NS_FAILED(err) && !m_leaveInvalidDB)
       deleteInvalidDB = true;
   }
   else
   {
     err = NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE;
     deleteInvalidDB = true;
@@ -1613,16 +1628,17 @@ nsresult nsMsgDatabase::InitNewDB()
   if (NS_SUCCEEDED(err))
   {
     nsDBFolderInfo *dbFolderInfo = new nsDBFolderInfo(this);
     if (dbFolderInfo)
     {
       NS_ADDREF(dbFolderInfo);
       err = dbFolderInfo->AddToNewMDB();
       dbFolderInfo->SetVersion(GetCurVersion());
+      dbFolderInfo->SetBooleanProperty("forceReparse", false);
       dbFolderInfo->SetBooleanProperty(kFixedBadRefThreadingProp, true);
       nsIMdbStore *store = GetStore();
       // create the unique table for the dbFolderInfo.
       struct mdbOid allMsgHdrsTableOID;
       struct mdbOid allThreadsTableOID;
       if (!store)
         return NS_ERROR_NULL_POINTER;
 
@@ -3412,17 +3428,17 @@ nsMsgDatabase::EnumerateMessagesWithFlag
     return NS_ERROR_OUT_OF_MEMORY;
   NS_ADDREF(*result = e);
   return NS_OK;
 }
 
 NS_IMETHODIMP nsMsgDatabase::CreateNewHdr(nsMsgKey key, nsIMsgDBHdr **pnewHdr)
 {
   nsresult  err = NS_OK;
-  nsIMdbRow    *hdrRow;
+  nsIMdbRow    *hdrRow = nullptr;
   struct mdbOid allMsgHdrsTableOID;
 
   if (!pnewHdr || !m_mdbAllMsgHeadersTable || !m_mdbStore)
     return NS_ERROR_NULL_POINTER;
 
   if (key != nsMsgKey_None)
   {
   allMsgHdrsTableOID.mOid_Scope = m_hdrRowScopeToken;
@@ -3437,16 +3453,38 @@ NS_IMETHODIMP nsMsgDatabase::CreateNewHd
     // Mork will assign an ID to the new row, generally the next available ID.
     err  = m_mdbStore->NewRow(GetEnv(), m_hdrRowScopeToken, &hdrRow);
     if (hdrRow)
     {
       struct mdbOid oid;
       hdrRow->GetOid(GetEnv(), &oid);
       key = oid.mOid_Id;
     }
+    else
+    {
+      // We failed to create a new row. That can happen if we run out of keys,
+      // which will force a reparse.
+      RefPtr<nsMsgKeyArray> keys = new nsMsgKeyArray;
+      if (NS_SUCCEEDED(ListAllKeys(keys)))
+      {
+        uint32_t numKeys;
+        keys->GetLength(&numKeys);
+        for (uint32_t i = 0; i < numKeys; i++)
+        {
+          if (keys->m_keys[i] >= kForceReparseKey)
+          {
+            // Force a reparse.
+            if (m_dbFolderInfo)
+              m_dbFolderInfo->SetBooleanProperty("forceReparse", true);
+            break;
+          }
+        }
+      }
+      err = NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE;
+    }
   }
   if (NS_FAILED(err))
     return err;
   err = CreateMsgHdr(hdrRow, key, pnewHdr);
   return err;
 }
 
 NS_IMETHODIMP nsMsgDatabase::AddNewHdrToDB(nsIMsgDBHdr *newHdr, bool notify)
--- a/mailnews/db/msgdb/src/nsMsgThread.cpp
+++ b/mailnews/db/msgdb/src/nsMsgThread.cpp
@@ -206,16 +206,17 @@ NS_IMETHODIMP nsMsgThread::AddChild(nsIM
   nsresult rv = NS_OK;
   nsMsgHdr* hdr = static_cast<nsMsgHdr*>(child);          // closed system, cast ok
   uint32_t newHdrFlags = 0;
   uint32_t msgDate;
   nsMsgKey newHdrKey = 0;
   bool parentKeyNeedsSetting = true;
 
   nsIMdbRow *hdrRow = hdr->GetMDBRow();
+  NS_ENSURE_STATE(hdrRow);
   hdr->GetRawFlags(&newHdrFlags);
   hdr->GetMessageKey(&newHdrKey);
   hdr->GetDateInSeconds(&msgDate);
   if (msgDate > m_newestMsgDate)
     SetNewestMsgDate(msgDate);
 
   if (newHdrFlags & nsMsgMessageFlags::Watched)
     SetFlags(m_flags | nsMsgMessageFlags::Watched);
--- a/mailnews/imap/src/nsImapUndoTxn.cpp
+++ b/mailnews/imap/src/nsImapUndoTxn.cpp
@@ -43,38 +43,39 @@ nsImapMoveCopyMsgTxn::Init(nsIMsgFolder*
   NS_ENSURE_SUCCESS(rv, rv);
   uint32_t i, count = m_srcKeyArray.Length();
   nsCOMPtr<nsIMsgDBHdr> srcHdr;
   nsCOMPtr<nsIMsgDBHdr> copySrcHdr;
   nsCString messageId;
 
   for (i = 0; i < count; i++)
   {
-    nsMsgKey pseudoKey;
     rv = srcDB->GetMsgHdrForKey(m_srcKeyArray[i],
       getter_AddRefs(srcHdr));
     if (NS_SUCCEEDED(rv))
     {
       // ** jt -- only do this for mailbox protocol
       if (MsgLowerCaseEqualsLiteral(protocolType, "mailbox"))
       {
         m_srcIsPop3 = true;
         uint32_t msgSize;
         rv = srcHdr->GetMessageSize(&msgSize);
         if (NS_SUCCEEDED(rv))
           m_srcSizeArray.AppendElement(msgSize);
         if (isMove)
         {
-          srcDB->GetNextPseudoMsgKey(&pseudoKey);
-          pseudoKey--;
+          rv = srcDB->CopyHdrFromExistingHdr(nsMsgKey_None, srcHdr, false,
+                                             getter_AddRefs(copySrcHdr));
+          nsMsgKey pseudoKey = nsMsgKey_None;
+          if (NS_SUCCEEDED(rv))
+          {
+            copySrcHdr->GetMessageKey(&pseudoKey);
+            m_srcHdrs.AppendObject(copySrcHdr);
+          }
           m_dupKeyArray[i] = pseudoKey;
-          rv = srcDB->CopyHdrFromExistingHdr(pseudoKey, srcHdr, false,
-                                             getter_AddRefs(copySrcHdr));
-          if (NS_SUCCEEDED(rv)) 
-            m_srcHdrs.AppendObject(copySrcHdr);
         }
       }
       srcHdr->GetMessageId(getter_Copies(messageId));
       m_srcMessageIds.AppendElement(messageId);
     }
   }
   return nsMsgTxn::Init();
 }
@@ -496,24 +497,40 @@ nsImapOfflineTxn::nsImapOfflineTxn(nsIMs
     nsCOMPtr<nsIMsgDatabase> srcDB;
     nsCOMPtr<nsIDBFolderInfo> folderInfo;
 
     nsresult rv = srcFolder->GetDBFolderInfoAndDB(getter_AddRefs(folderInfo), getter_AddRefs(srcDB));
     if (NS_SUCCEEDED(rv) && srcDB)
     {
       nsMsgKey pseudoKey;
       nsCOMPtr <nsIMsgDBHdr> copySrcHdr;
+
+      // Imap protocols have conflated key/UUID so we cannot use
+      // auto key with them.
+      nsCString protocolType;
+      srcFolder->GetURI(protocolType);
+      protocolType.SetLength(protocolType.FindChar(':'));
       for (int32_t i = 0; i < srcHdrs.Count(); i++)
       {
-        srcDB->GetNextPseudoMsgKey(&pseudoKey);
-        pseudoKey--;
-        m_dupKeyArray[i] = pseudoKey;
+        if (protocolType.EqualsLiteral("imap"))
+        {
+          srcDB->GetNextPseudoMsgKey(&pseudoKey);
+          pseudoKey--;
+        }
+        else
+        {
+          pseudoKey = nsMsgKey_None;
+        }
         rv = srcDB->CopyHdrFromExistingHdr(pseudoKey, srcHdrs[i], false, getter_AddRefs(copySrcHdr));
         if (NS_SUCCEEDED(rv))
+        {
+          copySrcHdr->GetMessageKey(&pseudoKey);
           m_srcHdrs.AppendObject(copySrcHdr);
+        }
+        m_dupKeyArray[i] = pseudoKey;
       }
     }
   }
   else
     m_srcHdrs.AppendObjects(srcHdrs);
 }
 
 nsImapOfflineTxn::~nsImapOfflineTxn()
--- a/mailnews/local/src/nsMsgBrkMBoxStore.cpp
+++ b/mailnews/local/src/nsMsgBrkMBoxStore.cpp
@@ -793,17 +793,16 @@ NS_IMETHODIMP nsMsgBrkMBoxStore::Compact
 }
 
 NS_IMETHODIMP nsMsgBrkMBoxStore::RebuildIndex(nsIMsgFolder *aFolder,
                                               nsIMsgDatabase *aMsgDB,
                                               nsIMsgWindow *aMsgWindow,
                                               nsIUrlListener *aListener)
 {
   NS_ENSURE_ARG_POINTER(aFolder);
-  // We don't use aMsgDB, but I think nsMsgMailDirStore needs it.
   nsCOMPtr<nsIFile> pathFile;
   nsresult rv = aFolder->GetFilePath(getter_AddRefs(pathFile));
   if (NS_FAILED(rv))
     return rv;
 
   bool isLocked;
   aFolder->GetLocked(&isLocked);
   if (isLocked)
@@ -816,18 +815,21 @@ NS_IMETHODIMP nsMsgBrkMBoxStore::Rebuild
     do_GetService(NS_MAILBOXSERVICE_CONTRACTID1, &rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
   RefPtr<nsMsgMailboxParser> parser = new nsMsgMailboxParser(aFolder);
   NS_ENSURE_TRUE(parser, NS_ERROR_OUT_OF_MEMORY);
   rv = parser->Init();
   NS_ENSURE_SUCCESS(rv, rv);
 
-  return mailboxService->ParseMailbox(aMsgWindow, pathFile, parser, aListener,
-                                      nullptr);
+  rv = mailboxService->ParseMailbox(aMsgWindow, pathFile, parser, aListener,
+                                    nullptr);
+  if (NS_SUCCEEDED(rv))
+    ResetForceReparse(aMsgDB);
+  return rv;
 }
 
 nsresult
 nsMsgBrkMBoxStore::GetOutputStream(nsIArray *aHdrArray,
                                    nsCOMPtr<nsIOutputStream> &outputStream,
                                    nsCOMPtr<nsISeekableStream> &seekableStream,
                                    int64_t &restorePos)
 {
--- a/mailnews/local/src/nsMsgLocalStoreUtils.cpp
+++ b/mailnews/local/src/nsMsgLocalStoreUtils.cpp
@@ -1,16 +1,18 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
  
 #include "msgCore.h"    // precompiled header...
 #include "nsMsgLocalStoreUtils.h"
 #include "nsIFile.h"
+#include "nsIDBFolderInfo.h"
+#include "nsIMsgDatabase.h"
 #include "prprf.h"
 
 #define EXTRA_SAFETY_SPACE 0x400000 // (4MiB)
 
 nsMsgLocalStoreUtils::nsMsgLocalStoreUtils()
 {
 }
 
@@ -316,8 +318,25 @@ nsMsgLocalStoreUtils::DiskSpaceAvailable
     //
     // We'll leave a debug message to warn people.
 #ifdef DEBUG
     printf("Call to GetDiskSpaceAvailable FAILED! \n");
 #endif
     return true;
   }
 }
+
+/**
+ * Resets forceReparse in the database.
+ *
+ * @param aMsgDb The database to reset.
+ */
+void
+nsMsgLocalStoreUtils::ResetForceReparse(nsIMsgDatabase *aMsgDB)
+{
+  if (aMsgDB)
+  {
+    nsCOMPtr<nsIDBFolderInfo> folderInfo;
+    aMsgDB->GetDBFolderInfo(getter_AddRefs(folderInfo));
+    if (folderInfo)
+      folderInfo->SetBooleanProperty("forceReparse", false);
+  }
+}
--- a/mailnews/local/src/nsMsgLocalStoreUtils.h
+++ b/mailnews/local/src/nsMsgLocalStoreUtils.h
@@ -32,16 +32,17 @@ public:
   static void ChangeKeywordsHelper(nsIMsgDBHdr *message,
                             uint64_t desiredOffset,
                             nsLineBuffer<char> *lineBuffer,
                             nsTArray<nsCString> &keywordArray,
                             bool aAdd,
                             nsIOutputStream *outputStream,
                             nsISeekableStream *seekableStream,
                             nsIInputStream *inputStream);
+  static void ResetForceReparse(nsIMsgDatabase *aMsgDB);
 
   nsresult UpdateFolderFlag(nsIMsgDBHdr *mailHdr, bool bSet,
                             nsMsgMessageFlagType flag,
                             nsIOutputStream *fileStream);
   bool DiskSpaceAvailableInStore(nsIFile *aFile,
                                  uint64_t aSpaceRequested);
 };
 
--- a/mailnews/local/src/nsMsgMaildirStore.cpp
+++ b/mailnews/local/src/nsMsgMaildirStore.cpp
@@ -1290,16 +1290,17 @@ NS_IMETHODIMP nsMsgMaildirStore::Rebuild
   rv = path->GetDirectoryEntries(getter_AddRefs(directoryEnumerator));
   NS_ENSURE_SUCCESS(rv, rv);
 
   MaildirStoreParser *fileParser = new MaildirStoreParser(aFolder, aMsgDB,
                                                           directoryEnumerator,
                                                           aListener);
   NS_ENSURE_TRUE(fileParser, NS_ERROR_OUT_OF_MEMORY);
   fileParser->StartTimer();
+  ResetForceReparse(aMsgDB);
   return NS_OK;
 }
 
 NS_IMETHODIMP nsMsgMaildirStore::ChangeFlags(nsIArray *aHdrArray,
                                              uint32_t aFlags,
                                              bool aSet)
 {
   NS_ENSURE_ARG_POINTER(aHdrArray);
--- a/mailnews/local/src/nsParseMailbox.cpp
+++ b/mailnews/local/src/nsParseMailbox.cpp
@@ -932,18 +932,22 @@ nsresult nsParseMailMessageState::StartN
 
 /* largely lifted from mimehtml.c, which does similar parsing, sigh...
 */
 nsresult nsParseMailMessageState::ParseHeaders ()
 {
   char *buf = m_headers.GetBuffer();
   uint32_t buf_length = m_headers.GetBufferPos();
   char *buf_end = buf + buf_length;
-  MOZ_ASSERT(buf_length > 1 && (buf[buf_length - 1] == '\r' ||
-    buf[buf_length - 1] == '\n'), "Header text should always end in a newline");
+  if (!(buf_length > 1 && (buf[buf_length - 1] == '\r' ||
+        buf[buf_length - 1] == '\n')))
+  {
+    NS_WARNING("Header text should always end in a newline");
+    return NS_ERROR_UNEXPECTED;
+  }
   while (buf < buf_end)
   {
     char *colon = PL_strnchr(buf, ':', buf_end - buf);
     char *end;
     char *value = 0;
     struct message_header *header = 0;
     struct message_header receivedBy;