Bug 449768 - "Reindexing should save message metadata (junk-related, tags, etc.)" [r+sr=bienvenu]
authorKent James <kent@caspia.com>
Thu, 30 Oct 2008 15:38:30 +0000
changeset 772 4fd3dba65a6e6b89fab64d25d5e8958d98ab545b
parent 771 dc4edb5fe229fe1c743e4f421b2e55d2e526e068
child 773 0a9821e4db57bda91775e57961614f857c545e37
push idunknown
push userunknown
push dateunknown
bugs449768
Bug 449768 - "Reindexing should save message metadata (junk-related, tags, etc.)" [r+sr=bienvenu]
mail/base/content/widgetglue.js
mailnews/base/public/nsIMsgFolder.idl
mailnews/base/resources/content/widgetglue.js
mailnews/base/util/nsMsgDBFolder.cpp
mailnews/base/util/nsMsgDBFolder.h
mailnews/db/msgdb/src/nsMsgDatabase.cpp
mailnews/imap/src/nsImapMailFolder.cpp
mailnews/local/public/nsIMsgParseMailMsgState.idl
mailnews/local/src/nsLocalMailFolder.cpp
mailnews/local/src/nsParseMailbox.cpp
mailnews/local/src/nsParseMailbox.h
--- a/mail/base/content/widgetglue.js
+++ b/mail/base/content/widgetglue.js
@@ -232,17 +232,23 @@ function RebuildSummaryFile(msgFolder)
 {
   if (msgFolder.locked)
   {
     msgFolder.throwAlertMsg("operationFailedFolderBusy", msgWindow);
     return;
   }
   var msgDB = msgFolder.getMsgDatabase(msgWindow);
   msgDB.summaryValid = false;
-  msgFolder.ForceDBClosed();
+  try {
+    msgFolder.closeAndBackupFolderDB("");
+  }
+  catch(e) {
+    // In a failure, proceed anyway since we're dealing with problems
+    msgFolder.ForceDBClosed();
+  }
   // these two lines will cause the thread pane to get reloaded
   // when the download/reparse is finished. Only do this
   // if the selected folder is loaded (i.e., not thru the
   // context menu on a non-loaded folder).
   if (msgFolder == GetLoadedMsgFolder())
   {
     gRerootOnFolderLoad = true;
     gCurrentFolderToReroot = msgFolder.URI;
--- a/mailnews/base/public/nsIMsgFolder.idl
+++ b/mailnews/base/public/nsIMsgFolder.idl
@@ -62,18 +62,17 @@ interface nsILocalFile;
 interface nsIMsgIdentity;
 interface nsIArray;
 interface nsIMutableArray;
 
 typedef long nsMsgBiffState;
 
 // enumerated type for determining if a message has been replied to, forwarded, etc.
 typedef long nsMsgDispositionState;
-
-[scriptable, uuid(294b98e9-dbdd-4f7f-9d81-5c17073a32c3)]
+[scriptable, uuid(EAE4729C-4CC1-4167-8E00-0C159DA05FD2)]
 interface nsIMsgFolder : nsISupports {
 
   const nsMsgBiffState nsMsgBiffState_NewMail = 0; // User has new mail waiting.
   const nsMsgBiffState nsMsgBiffState_NoMail =  1; // No new mail is waiting.
   const nsMsgBiffState nsMsgBiffState_Unknown = 2; // We dunno whether there is new mail.
 
   nsISimpleEnumerator getMessages(in nsIMsgWindow aMsgWindow);
 
@@ -134,16 +133,25 @@ interface nsIMsgFolder : nsISupports {
   /**
    * function to get the filter list on folder's server
    * (or in the case of news, the filter list for this newsgroup)'
    */
   nsIMsgFilterList getFilterList(in nsIMsgWindow msgWindow);
   void setFilterList(in nsIMsgFilterList filterList);
 
   void ForceDBClosed ();
+  /**
+   * Close and backup a folder database prior to reparsing
+   *
+   * @param  newName  New name of the corresponding message folder.
+   *                  Used in rename to set the file name to match the renamed
+   *                  folder. Set to empty to use the existing folder name.
+   */
+  void closeAndBackupFolderDB(in ACString newName);
+
   void Delete ();
 
   void deleteSubFolders(in nsIArray folders, in nsIMsgWindow msgWindow);
   void propagateDelete(in nsIMsgFolder folder, in boolean deleteStorage,
                        in nsIMsgWindow msgWindow);
   void recursiveDelete(in boolean deleteStorage, in nsIMsgWindow msgWindow);
 
   void createSubfolder(in AString folderName, in nsIMsgWindow msgWindow);
@@ -407,16 +415,31 @@ interface nsIMsgFolder : nsISupports {
 
   void markMessagesRead(in nsIArray messages, in boolean markRead);
   void markAllMessagesRead();
   void markMessagesFlagged(in nsIArray messages, in boolean markFlagged);
   void markThreadRead(in nsIMsgThread thread);
   void setLabelForMessages(in nsIArray messages, in nsMsgLabelValue label);
   nsIMsgDatabase getMsgDatabase(in nsIMsgWindow msgWindow);
   void setMsgDatabase (in nsIMsgDatabase msgDatabase);
+  /**
+   * Get the backup message database, used in reparsing. This database must
+   * be created first using closeAndBackupFolderDB()
+   *
+   * @return   backup message database
+   */
+  nsIMsgDatabase getBackupMsgDatabase();
+  /**
+   * Remove the backup message database file
+   */
+  void removeBackupMsgDatabase();
+  /**
+   * Open the backup message database file
+   */
+  void openBackupMsgDatabase();
   nsIMsgDatabase getDBFolderInfoAndDB(out nsIDBFolderInfo folderInfo);
   nsIMsgDBHdr GetMessageHeader(in nsMsgKey msgKey);
 
   readonly attribute boolean supportsOffline;
   boolean shouldStoreMsgOffline(in nsMsgKey msgKey);
   boolean hasMsgOffline(in nsMsgKey msgKey);
 
   nsIInputStream getOfflineFileStream(in nsMsgKey msgKey, out PRUint32 offset,
--- a/mailnews/base/resources/content/widgetglue.js
+++ b/mailnews/base/resources/content/widgetglue.js
@@ -263,17 +263,24 @@ function RebuildSummaryFile(msgFolder)
 {
   if (msgFolder.locked)
   {
     msgFolder.throwAlertMsg("operationFailedFolderBusy", msgWindow);
     return;
   }
   var msgDB = msgFolder.getMsgDatabase(msgWindow);
   msgDB.summaryValid = false;
-  msgFolder.ForceDBClosed();
+  try {
+    msgFolder.closeAndBackupFolderDB("");
+  }
+  catch(e) {
+    // In a failure, proceed anyway since we're dealing with problems
+    msgFolder.ForceDBClosed();
+  }
+
   // these two lines will cause the thread pane to get reloaded
   // when the download/reparse are finised.
   gRerootOnFolderLoad = true;
   gCurrentFolderToReroot = msgFolder.URI;
   msgFolder.updateFolder(msgWindow);
 }
 
 function FolderProperties(name, oldName, uri)
--- a/mailnews/base/util/nsMsgDBFolder.cpp
+++ b/mailnews/base/util/nsMsgDBFolder.cpp
@@ -86,16 +86,17 @@
 #include "nsIMIMEHeaderParam.h"
 #include "plbase64.h"
 #include "nsArrayEnumerator.h"
 #include <time.h>
 #include "nsIMsgFolderNotificationService.h"
 #include "nsIMutableArray.h"
 #include "nsArrayUtils.h"
 #include "nsIMimeHeaders.h"
+#include "nsDirectoryServiceDefs.h"
 
 #define oneHour 3600000000U
 #include "nsMsgUtils.h"
 
 static PRTime gtimeOfLastPurgeCheck;    //variable to know when to check for purge_threshhold
 
 #define PREF_MAIL_PROMPT_PURGE_THRESHOLD "mail.prompt_purge_threshhold"
 #define PREF_MAIL_PURGE_THRESHOLD "mail.purge_threshhold"
@@ -218,17 +219,21 @@ nsMsgDBFolder::~nsMsgDBFolder(void)
 
 NS_IMETHODIMP nsMsgDBFolder::Shutdown(PRBool shutdownChildren)
 {
   if(mDatabase)
   {
     mDatabase->RemoveListener(this);
     mDatabase->Close(PR_TRUE);
     mDatabase = nsnull;
-
+    if (mBackupDatabase)
+    {
+      mBackupDatabase->ForceClosed();
+      mBackupDatabase = nsnull;
+    }
   }
 
   if(shutdownChildren)
   {
     PRInt32 count = mSubFolders.Count();
 
     for (PRInt32 i = 0; i < count; i++)
       mSubFolders[i]->Shutdown(PR_TRUE);
@@ -259,16 +264,134 @@ NS_IMETHODIMP nsMsgDBFolder::ForceDBClos
   {
     nsCOMPtr<nsIMsgDatabase> mailDBFactory = do_CreateInstance(kCMailDB);
     if (mailDBFactory)
       mailDBFactory->ForceFolderDBClosed(this);
   }
   return NS_OK;
 }
 
+NS_IMETHODIMP nsMsgDBFolder::CloseAndBackupFolderDB(const nsACString& newName)
+{
+  ForceDBClosed();
+
+  // We only support backup for mail at the moment
+  if ( !(mFlags & nsMsgFolderFlags::Mail))
+    return NS_OK;
+
+  nsCOMPtr<nsILocalFile> folderPath;
+  nsresult rv = GetFilePath(getter_AddRefs(folderPath));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> dbFile;
+  rv = GetSummaryFileLocation(folderPath, getter_AddRefs(dbFile));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> backupDir;
+  rv = CreateBackupDirectory(getter_AddRefs(backupDir));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> backupDBFile;
+  rv = GetBackupSummaryFile(getter_AddRefs(backupDBFile), newName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  if (mBackupDatabase)
+  {
+    mBackupDatabase->ForceClosed();
+    mBackupDatabase = nsnull;
+  }
+
+  backupDBFile->Remove(PR_FALSE);
+  PRBool backupExists;
+  backupDBFile->Exists(&backupExists);
+  NS_ASSERTION(!backupExists, "Couldn't delete database backup");
+  if (backupExists)
+    return NS_ERROR_FAILURE;
+
+  if (!newName.IsEmpty())
+  {
+    nsCAutoString backupName;
+    rv = backupDBFile->GetNativeLeafName(backupName);
+    NS_ENSURE_SUCCESS(rv, rv);
+    return dbFile->CopyToNative(backupDir, backupName);
+  }
+  else
+    return dbFile->CopyToNative(backupDir, EmptyCString());
+}
+
+NS_IMETHODIMP nsMsgDBFolder::OpenBackupMsgDatabase()
+{
+  if (mBackupDatabase)
+    return NS_OK;
+  nsCOMPtr<nsILocalFile> folderPath;
+  nsresult rv = GetFilePath(getter_AddRefs(folderPath));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoString folderName;
+  rv = folderPath->GetLeafName(folderName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> backupDir;
+  rv = CreateBackupDirectory(getter_AddRefs(backupDir));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // We use a dummy message folder file so we can use
+  // GetSummaryFileLocation to get the db file name
+  nsCOMPtr<nsILocalFile> backupDBDummyFolder;
+  rv = CreateBackupDirectory(getter_AddRefs(backupDBDummyFolder));
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = backupDBDummyFolder->Append(folderName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  mBackupDatabase = do_CreateInstance(NS_MAILBOXDB_CONTRACTID, &rv);
+  nsCOMPtr<nsIMsgDBService> msgDBService =
+      do_GetService(NS_MSGDB_SERVICE_CONTRACTID, &rv);
+  rv = msgDBService->OpenMailDBFromFile(
+      backupDBDummyFolder, PR_FALSE, PR_TRUE, getter_AddRefs(mBackupDatabase));
+
+  if (rv == NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE)
+    // this is normal in reparsing
+    rv = NS_OK;
+  return rv;
+}
+
+NS_IMETHODIMP nsMsgDBFolder::RemoveBackupMsgDatabase()
+{
+  nsCOMPtr<nsILocalFile> folderPath;
+  nsresult rv = GetFilePath(getter_AddRefs(folderPath));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoString folderName;
+  rv = folderPath->GetLeafName(folderName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> backupDir;
+  rv = CreateBackupDirectory(getter_AddRefs(backupDir));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // We use a dummy message folder file so we can use
+  // GetSummaryFileLocation to get the db file name
+  nsCOMPtr<nsILocalFile> backupDBDummyFolder;
+  rv = CreateBackupDirectory(getter_AddRefs(backupDBDummyFolder));
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = backupDBDummyFolder->Append(folderName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> backupDBFile;
+  rv = GetSummaryFileLocation(backupDBDummyFolder, getter_AddRefs(backupDBFile));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  if (mBackupDatabase)
+  {
+    mBackupDatabase->ForceClosed();
+    mBackupDatabase = nsnull;
+  }
+
+  return backupDBFile->Remove(PR_FALSE);
+}  
 
 NS_IMETHODIMP nsMsgDBFolder::StartFolderLoading(void)
 {
   if(mDatabase)
     mDatabase->RemoveListener(this);
   mAddListener = PR_FALSE;
   return NS_OK;
 }
@@ -775,16 +898,29 @@ nsMsgDBFolder::SetMsgDatabase(nsIMsgData
   mDatabase = aMsgDatabase;
 
   if (aMsgDatabase)
     aMsgDatabase->AddListener(this);
   return NS_OK;
 }
 
 NS_IMETHODIMP
+nsMsgDBFolder::GetBackupMsgDatabase(nsIMsgDatabase** aMsgDatabase)
+{
+  NS_ENSURE_ARG_POINTER(aMsgDatabase);
+  nsresult rv = OpenBackupMsgDatabase();
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (!mBackupDatabase)
+    return NS_ERROR_FAILURE;
+
+  NS_ADDREF(*aMsgDatabase = mBackupDatabase);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
 nsMsgDBFolder::GetDBFolderInfoAndDB(nsIDBFolderInfo **folderInfo, nsIMsgDatabase **database)
 {
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
 NS_IMETHODIMP
 nsMsgDBFolder::OnReadChanged(nsIDBChangeListener * aInstigator)
 {
@@ -3114,16 +3250,86 @@ nsresult nsMsgDBFolder::CreateDirectoryF
       rv = pathExists ? NS_MSG_COULD_NOT_CREATE_DIRECTORY : path->Create(nsIFile::DIRECTORY_TYPE, 0700);
     }
   }
   if (NS_SUCCEEDED(rv))
     path.swap(*resultFile);
   return rv;
 }
 
+/* Finds the backup directory associated with this folder, stored on the temp
+   drive. If that path doesn't currently exist then it will create it. Path is
+   strictly an out parameter.
+  */
+nsresult nsMsgDBFolder::CreateBackupDirectory(nsILocalFile **resultFile)
+{
+  nsCOMPtr<nsIFile> pathIFile;
+  nsresult rv = NS_GetSpecialDirectory(NS_OS_TEMP_DIR,
+                                       getter_AddRefs(pathIFile));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> path = do_QueryInterface(pathIFile, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = path->Append(NS_LITERAL_STRING("MozillaMailnews"));
+  PRBool pathIsDirectory;
+  path->IsDirectory(&pathIsDirectory);
+
+  // If that doesn't exist, then we have to create this directory
+  if (!pathIsDirectory)
+  {
+    PRBool pathExists;
+    path->Exists(&pathExists);
+    // If for some reason there's a file with the directory separator
+    // then we are going to fail.
+    rv = pathExists ? NS_MSG_COULD_NOT_CREATE_DIRECTORY :
+                      path->Create(nsIFile::DIRECTORY_TYPE, 0700);
+  }
+  if (NS_SUCCEEDED(rv))
+    path.swap(*resultFile);
+  return rv;
+}
+
+nsresult nsMsgDBFolder::GetBackupSummaryFile(nsILocalFile **aBackupFile, const nsACString& newName)
+{
+  nsCOMPtr<nsILocalFile> backupDir;
+  nsresult rv = CreateBackupDirectory(getter_AddRefs(backupDir));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // We use a dummy message folder file so we can use
+  // GetSummaryFileLocation to get the db file name
+  nsCOMPtr<nsILocalFile> backupDBDummyFolder;
+  rv = CreateBackupDirectory(getter_AddRefs(backupDBDummyFolder));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  if (!newName.IsEmpty())
+  {
+    rv = backupDBDummyFolder->AppendNative(newName);
+  }
+  else // if newName is null, use the folder name
+  {
+    nsCOMPtr<nsILocalFile> folderPath;
+    rv = GetFilePath(getter_AddRefs(folderPath));
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    nsCAutoString folderName;
+    rv = folderPath->GetNativeLeafName(folderName);
+    NS_ENSURE_SUCCESS(rv, rv);
+    rv = backupDBDummyFolder->AppendNative(folderName);
+  }
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsILocalFile> backupDBFile;
+  rv = GetSummaryFileLocation(backupDBDummyFolder, getter_AddRefs(backupDBFile));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  backupDBFile.swap(*aBackupFile);
+  return NS_OK;
+}
+
 NS_IMETHODIMP nsMsgDBFolder::Rename(const nsAString& aNewName, nsIMsgWindow *msgWindow)
 {
   nsCOMPtr<nsILocalFile> oldPathFile;
   nsCOMPtr<nsIAtom> folderRenameAtom;
   nsresult rv = GetFilePath(getter_AddRefs(oldPathFile));
   if (NS_FAILED(rv))
     return rv;
   nsCOMPtr<nsIMsgFolder> parentFolder;
--- a/mailnews/base/util/nsMsgDBFolder.h
+++ b/mailnews/base/util/nsMsgDBFolder.h
@@ -97,16 +97,18 @@ public:
   // struct used by the FE
   PRInt32 GetNumPendingUnread();
   PRInt32 GetNumPendingTotalMessages();
 
   void ChangeNumPendingUnread(PRInt32 delta);
   void ChangeNumPendingTotalMessages(PRInt32 delta);
 
   nsresult CreateDirectoryForFolder(nsILocalFile **result);
+  nsresult CreateBackupDirectory(nsILocalFile **result);
+  nsresult GetBackupSummaryFile(nsILocalFile **result, const nsACString& newName);
   nsresult GetMsgPreviewTextFromStream(nsIMsgDBHdr *msgHdr, nsIInputStream *stream);
 protected:
   
   // this is a little helper function that is not part of the public interface. 
   // we use it to get the IID of the incoming server for the derived folder.
   // w/out a function like this we would have to implement GetServer in each
   // derived folder class.
   virtual void GetIncomingServerType(nsCString& serverType) = 0;
@@ -160,16 +162,17 @@ protected:
   nsresult CloseDBIfFolderNotOpen();
 
   virtual nsresult SpamFilterClassifyMessage(const char *aURI, nsIMsgWindow *aMsgWindow, nsIJunkMailPlugin *aJunkMailPlugin);
   virtual nsresult SpamFilterClassifyMessages(const char **aURIArray, PRUint32 aURICount, nsIMsgWindow *aMsgWindow, nsIJunkMailPlugin *aJunkMailPlugin);
   void    SetMRUTime();
 
 protected:
   nsCOMPtr<nsIMsgDatabase> mDatabase;
+  nsCOMPtr<nsIMsgDatabase> mBackupDatabase;
   nsCString mCharset;
   PRBool mCharsetOverride;
   PRBool mAddListener;
   PRBool mNewMessages;
   PRBool mGettingNewMessages;
   nsMsgKey mLastMessageLoaded;
 
   nsCOMPtr <nsIMsgDBHdr> m_offlineHeader;
--- a/mailnews/db/msgdb/src/nsMsgDatabase.cpp
+++ b/mailnews/db/msgdb/src/nsMsgDatabase.cpp
@@ -1216,17 +1216,17 @@ NS_IMETHODIMP nsMsgDatabase::ForceClosed
   // make sure someone has a reference so object won't get deleted out from under us.
   AddRef();
   NotifyAnnouncerGoingAway();
   // make sure dbFolderInfo isn't holding onto mork stuff because mork db is going away
   if (m_dbFolderInfo)
     m_dbFolderInfo->ReleaseExternalReferences();
   NS_IF_RELEASE(m_dbFolderInfo);
 
-  err = CloseMDB(PR_FALSE);  // since we're about to delete it, no need to commit.
+  err = CloseMDB(PR_TRUE);  // Backup DB will try to recover info, so commit
   ClearCachedObjects(PR_TRUE);
   if (m_mdbAllMsgHeadersTable)
   {
     m_mdbAllMsgHeadersTable->Release();
     m_mdbAllMsgHeadersTable = nsnull;
   }
   if (m_mdbAllThreadsTable)
   {
@@ -4043,19 +4043,23 @@ NS_IMETHODIMP nsMsgDatabase::GetMsgHdrFo
 
   messageIdYarn.mYarn_Buf = (void *) msgID;
   messageIdYarn.mYarn_Fill = PL_strlen(msgID);
   messageIdYarn.mYarn_Form = 0;
   messageIdYarn.mYarn_Size = messageIdYarn.mYarn_Fill;
 
   nsIMdbRow *hdrRow;
   mdbOid outRowId;
-  mdb_err result = GetStore()->FindRow(GetEnv(), m_hdrRowScopeToken,
-    m_messageIdColumnToken, &messageIdYarn,  &outRowId,
-    &hdrRow);
+  mdb_err result;
+  if (m_mdbStore)
+    result = m_mdbStore->FindRow(GetEnv(), m_hdrRowScopeToken,
+      m_messageIdColumnToken, &messageIdYarn,  &outRowId,
+      &hdrRow);
+  else
+    return NS_ERROR_FAILURE;
   if (NS_SUCCEEDED(result) && hdrRow)
   {
     //Get key from row
     mdbOid outOid;
     nsMsgKey key=0;
     if (hdrRow->GetOid(GetEnv(), &outOid) == NS_OK)
       key = outOid.mOid_Id;
     rv = GetHdrFromUseCache(key, &msgHdr);
--- a/mailnews/imap/src/nsImapMailFolder.cpp
+++ b/mailnews/imap/src/nsImapMailFolder.cpp
@@ -1541,17 +1541,17 @@ NS_IMETHODIMP nsImapMailFolder::RenameLo
   nsCAutoString leafname(newName);
   nsCAutoString parentName;
   // newName always in the canonical form "greatparent/parentname/leafname"
   PRInt32 leafpos = leafname.RFindChar('/');
   if (leafpos >0)
       leafname.Cut(0, leafpos+1);
   m_msgParser = nsnull;
   PrepareToRename();
-  ForceDBClosed();
+  CloseAndBackupFolderDB(leafname);
 
   nsresult rv = NS_OK;
   nsCOMPtr<nsILocalFile> oldPathFile;
   rv = GetFilePath(getter_AddRefs(oldPathFile));
   if (NS_FAILED(rv)) return rv;
 
   nsCOMPtr<nsILocalFile> parentPathFile;
   rv = parent->GetFilePath(getter_AddRefs(parentPathFile));
@@ -2502,20 +2502,32 @@ NS_IMETHODIMP nsImapMailFolder::UpdateIm
     if (NS_FAILED(rv)) return rv;
 
     nsCOMPtr<nsIMsgDBService> msgDBService = do_GetService(NS_MSGDB_SERVICE_CONTRACTID, &rv);
     NS_ENSURE_SUCCESS(rv, rv);
 
     nsCOMPtr <nsIDBFolderInfo> transferInfo;
     if (dbFolderInfo)
       dbFolderInfo->GetTransferInfo(getter_AddRefs(transferInfo));
+
+    // A backup message database might have been created earlier, for example
+    // if the user requested a reindex. We'll use the earlier one if we can,
+    // otherwise we'll try to backup at this point.
+    nsresult rvbackup = OpenBackupMsgDatabase();
     if (mDatabase)
     {
       dbFolderInfo = nsnull;
-      mDatabase->ForceClosed();
+      if (NS_FAILED(rvbackup))
+      {
+        CloseAndBackupFolderDB(EmptyCString());
+        if (NS_FAILED(OpenBackupMsgDatabase()))
+          mBackupDatabase = nsnull;
+      }
+      else
+        mDatabase->ForceClosed();
     }
     mDatabase = nsnull;
 
     nsCOMPtr <nsILocalFile> summaryFile;
     GetSummaryFileLocation(pathFile, getter_AddRefs(summaryFile));
     // Remove summary file.
     summaryFile->Remove(PR_FALSE);
 
@@ -2736,16 +2748,18 @@ nsresult nsImapMailFolder::SetupHeaderPa
     nsresult rv;
     m_msgParser = do_CreateInstance(kParseMailMsgStateCID, &rv);
     NS_ENSURE_SUCCESS(rv, rv);
   }
   else
     m_msgParser->Clear();
 
   m_msgParser->SetMailDB(mDatabase);
+  if (mBackupDatabase)
+    m_msgParser->SetBackupMailDB(mBackupDatabase);
   return m_msgParser->SetState(nsIMsgParseMailMsgState::ParseHeadersState);
 }
 
 nsresult nsImapMailFolder::ParseAdoptedHeaderLine(const char *aMessageLine, PRUint32 aMsgKey)
 {
   // we can get blocks that contain more than one line,
   // but they never contain partial lines
   const char *str = aMessageLine;
@@ -5026,16 +5040,19 @@ nsImapMailFolder::GetMessageId(nsIImapUr
 }
 
 NS_IMETHODIMP
 nsImapMailFolder::HeaderFetchCompleted(nsIImapProtocol* aProtocol)
 {
   nsCOMPtr <nsIMsgWindow> msgWindow; // we might need this for the filter plugins.
   if (mDatabase)
     mDatabase->Commit(nsMsgDBCommitType::kLargeCommit);
+  if (mBackupDatabase)
+    RemoveBackupMsgDatabase();
+
   SetSizeOnDisk(mFolderSize);
   PRInt32 numNewBiffMsgs = 0;
   if (m_performingBiff)
     GetNumNewMessages(PR_FALSE, &numNewBiffMsgs);
 
   PRBool pendingMoves = m_moveCoalescer && m_moveCoalescer->HasPendingMoves();
   PlaybackCoalescedOperations();
   if (aProtocol)
--- a/mailnews/local/public/nsIMsgParseMailMsgState.idl
+++ b/mailnews/local/public/nsIMsgParseMailMsgState.idl
@@ -43,21 +43,28 @@ interface nsIOutputStream;
 
 %{C++
 #include "nsIMsgDatabase.h"
 #include "nsIMsgHdr.h"
 %}
 
 typedef long nsMailboxParseState;
 
-[scriptable, uuid(BE2B880F-B696-46FB-B78A-18FB3C795AEB)]
+[scriptable, uuid(9BE9B91D-5D5B-4e85-84B2-79FBFF56F5D4)]
 interface nsIMsgParseMailMsgState : nsISupports {
 
     attribute unsigned long envelopePos;
     void SetMailDB(in nsIMsgDatabase aDatabase);
+    /*
+     * Set a backup mail database, whose data will be read during parsing to
+     * attempt to recover message metadata
+     *
+     * @param aDatabase   the backup database
+     */
+    void SetBackupMailDB(in nsIMsgDatabase aDatabase);
     void setDBFolderStream(in nsIOutputStream fileStream);
     void Clear();
 
     void ParseAFolderLine(in string line, in unsigned long lineLength);
     nsIMsgDBHdr GetNewMsgHdr();
     void FinishHeader();
 
     long GetAllHeaders(out string headers);
--- a/mailnews/local/src/nsLocalMailFolder.cpp
+++ b/mailnews/local/src/nsLocalMailFolder.cpp
@@ -515,17 +515,29 @@ NS_IMETHODIMP nsMsgLocalMailFolder::GetD
         {
           if (folderOpen == NS_MSG_ERROR_FOLDER_SUMMARY_MISSING)
             dbFolderInfo->SetFlags(mFlags);
           dbFolderInfo->SetNumMessages(0);
           dbFolderInfo->SetNumUnreadMessages(0);
           dbFolderInfo->GetTransferInfo(getter_AddRefs(transferInfo));
         }
         dbFolderInfo = nsnull;
-        mDatabase->ForceClosed();
+
+        // A backup message database might have been created earlier, for example
+        // if the user requested a reindex. We'll use the earlier one if we can,
+        // otherwise we'll try to backup at this point.
+        if (NS_FAILED(OpenBackupMsgDatabase()))
+        {
+          CloseAndBackupFolderDB(EmptyCString());
+          if (NS_FAILED(OpenBackupMsgDatabase()))
+            mBackupDatabase = nsnull;
+        }
+        else
+          mDatabase->ForceClosed();
+
         mDatabase = nsnull;
       }
       nsCOMPtr <nsILocalFile> summaryFile;
       rv = GetSummaryFileLocation(pathFile, getter_AddRefs(summaryFile));
       NS_ENSURE_SUCCESS(rv, rv);
       // Remove summary file.
       summaryFile->Remove(PR_FALSE);
 
--- a/mailnews/local/src/nsParseMailbox.cpp
+++ b/mailnews/local/src/nsParseMailbox.cpp
@@ -158,16 +158,25 @@ NS_IMETHODIMP nsMsgMailboxParser::OnStar
             if (msgDBService)
             {
                 //Use OpenFolderDB to always open the db so that db's m_folder is set correctly.
                 rv = msgDBService->OpenFolderDB(folder, PR_TRUE, PR_TRUE, (nsIMsgDatabase **) getter_AddRefs(m_mailDB));
                 if (m_mailDB)
                     m_mailDB->AddListener(this);
             }
             NS_ASSERTION(m_mailDB, "failed to open mail db parsing folder");
+
+            // try to get a backup message database
+            nsresult rvignore = folder->GetBackupMsgDatabase(
+                getter_AddRefs(m_backupMailDB));
+
+            // We'll accept failures and move on, as we're dealing with some
+            // sort of unknown problem to begin with.
+            if (NS_FAILED(rvignore))
+              m_backupMailDB = nsnull;
         }
     }
 
     // need to get the mailbox name out of the url and call SetMailboxName with it.
     // then, we need to open the mail db for this parser.
     return rv;
 }
 
@@ -335,16 +344,25 @@ void nsMsgMailboxParser::DoneParsingFold
   PublishMsgHeader(nsnull);
 
   // only mark the db valid if we've succeeded.
   if (NS_SUCCEEDED(status) && m_mailDB)  // finished parsing, so flush db folder info
     UpdateDBFolderInfo();
   else if (m_mailDB)
     m_mailDB->SetSummaryValid(PR_FALSE);
 
+  // remove the backup database
+  if (m_backupMailDB)
+  {
+    nsCOMPtr<nsIMsgFolder> folder = do_QueryReferent(m_folder);
+    if (folder)
+      folder->RemoveBackupMsgDatabase();
+    m_backupMailDB = nsnull;
+  }
+
   //  if (m_folder != nsnull)
   //    m_folder->SummaryChanged();
   FreeBuffers();
 }
 
 void nsMsgMailboxParser::FreeBuffers()
 {
   /* We're done reading the folder - we don't need these things
@@ -650,16 +668,22 @@ PRInt32 nsParseMailMessageState::ParseFo
 }
 
 NS_IMETHODIMP nsParseMailMessageState::SetMailDB(nsIMsgDatabase *mailDB)
 {
   m_mailDB = mailDB;
   return NS_OK;
 }
 
+NS_IMETHODIMP nsParseMailMessageState::SetBackupMailDB(nsIMsgDatabase *aBackupMailDB)
+{
+  m_backupMailDB = aBackupMailDB;
+  return NS_OK;
+}
+
 NS_IMETHODIMP nsParseMailMessageState::SetDBFolderStream(nsIOutputStream *fileStream)
 {
   NS_ASSERTION(m_mailDB, "m_mailDB is not set");
   if (m_mailDB)
     m_mailDB->SetFolderStream(fileStream);
   return NS_OK;
 }
 
@@ -1259,17 +1283,47 @@ int nsParseMailMessageState::FinalizeHea
   {
     PRUint32 flags2 = 0;
     sscanf(mozstatus2->value, " %x ", &flags2);
     flags |= flags2;
   }
 
   if (!(flags & MSG_FLAG_EXPUNGED))  // message was deleted, don't bother creating a hdr.
   {
-    nsresult ret = m_mailDB->CreateNewHdr(m_envelope_pos, getter_AddRefs(m_newMsgHdr));
+    // We'll need the message id first to recover data from the backup database
+    nsCAutoString rawMsgId;
+    /* Take off <> around message ID. */
+    if (id)
+    {
+      if (id->value[0] == '<')
+        id->value++, id->length--;
+      if (id->value[id->length - 1] == '>')
+        /* generate a new null-terminated string without the final > */
+        rawMsgId.Assign(id->value, id->length - 1);
+      else
+        rawMsgId.Assign(id->value);
+    }
+
+    /*
+     * Try to copy the data from the backup database, referencing the MessageID
+     * If that fails, just create a new header
+     */
+    nsCOMPtr<nsIMsgDBHdr> oldHeader;
+    nsresult ret = NS_ERROR_FAILURE;
+
+    if (m_backupMailDB && !rawMsgId.IsEmpty())
+      ret = m_backupMailDB->GetMsgHdrForMessageID(
+              rawMsgId.get(), getter_AddRefs(oldHeader));
+
+    if (NS_SUCCEEDED(ret) && oldHeader)
+        ret = m_mailDB->CopyHdrFromExistingHdr(m_envelope_pos,
+                oldHeader, PR_FALSE, getter_AddRefs(m_newMsgHdr));
+    else
+      ret = m_mailDB->CreateNewHdr(m_envelope_pos, getter_AddRefs(m_newMsgHdr));
+
     if (NS_SUCCEEDED(ret) && m_newMsgHdr)
     {
       PRUint32 origFlags;
       (void)m_newMsgHdr->GetFlags(&origFlags);
       if (origFlags & MSG_FLAG_HAS_RE)
         flags |= MSG_FLAG_HAS_RE;
       else
         flags &= ~MSG_FLAG_HAS_RE;
@@ -1383,28 +1437,20 @@ int nsParseMailMessageState::FinalizeHea
               md5_b64 = hash.get();
           }
           PR_snprintf (md5_data, sizeof(md5_data), "<md5:%s>", md5_b64);
           md5_header.value = md5_data;
           md5_header.length = strlen(md5_data);
           id = &md5_header;
         }
 
-        /* Take off <> around message ID. */
-        if (id->value[0] == '<')
-          id->value++, id->length--;
-
-        if (id->value[id->length-1] == '>') {
-          /* generate a new null-terminated string without the final > */
-          nsCAutoString rawMsgId;
-          rawMsgId.Assign(id->value, id->length - 1);
+        if (!rawMsgId.IsEmpty())
           m_newMsgHdr->SetMessageId(rawMsgId.get());
-        } else {
+        else
           m_newMsgHdr->SetMessageId(id->value);
-        }
 
         if (!mozstatus && statush)
         {
           /* Parse a little bit of the Berkeley Mail status header. */
           for (s = statush->value; *s; s++) {
             PRUint32 msgFlags = 0;
             (void)m_newMsgHdr->GetFlags(&msgFlags);
             switch (*s)
@@ -1477,17 +1523,36 @@ int nsParseMailMessageState::FinalizeHea
         }
         m_newMsgHdr->SetUint32Property("dateReceived", rcvTimeSecs);
 
         if (priority)
           m_newMsgHdr->SetPriorityString(priority->value);
         else if (priorityFlags == nsMsgPriority::notSet)
           m_newMsgHdr->SetPriority(nsMsgPriority::none);
         if (keywords)
-          m_newMsgHdr->SetStringProperty("keywords", keywords->value);
+        {
+          // When there are many keywords, some may not have been written
+          // to the message file, so add extra keywords from the backup
+          nsCAutoString oldKeywords;
+          m_newMsgHdr->GetStringProperty("keywords", getter_Copies(oldKeywords));
+          nsCStringArray newKeywordArray, oldKeywordArray;
+          newKeywordArray.ParseString(keywords->value, " ");
+          oldKeywordArray.ParseString(oldKeywords.get(), " ");
+          for (PRInt32 i = 0; i < oldKeywordArray.Count(); i++)
+            if (newKeywordArray.IndexOf(*oldKeywordArray.CStringAt(i)) < 0)
+              newKeywordArray.AppendCString(*oldKeywordArray.CStringAt(i));
+          nsCAutoString newKeywords;
+          for (PRInt32 i = 0; i < newKeywordArray.Count(); i++)
+          {
+            if (i)
+              newKeywords.Append(" ");
+            newKeywords.Append(*newKeywordArray.CStringAt(i));
+          }
+          m_newMsgHdr->SetStringProperty("keywords", newKeywords.get());
+        }
         for (PRInt32 i = 0; i < m_customDBHeaders.Count(); i++)
         {
           if (m_customDBHeaderValues[i].length)
             m_newMsgHdr->SetStringProperty((m_customDBHeaders[i])->get(), m_customDBHeaderValues[i].value);
         }
         if (content_type)
         {
           char *substring = PL_strstr(content_type->value, "charset");
@@ -1608,16 +1673,18 @@ nsParseNewMailState::Init(nsIMsgFolder *
   m_disableFilters = PR_FALSE;
   return NS_OK;
 }
 
 nsParseNewMailState::~nsParseNewMailState()
 {
   if (m_mailDB)
     m_mailDB->Close(PR_TRUE);
+  if (m_backupMailDB)
+    m_backupMailDB->ForceClosed();
 #ifdef DOING_JSFILTERS
   JSFilter_cleanup();
 #endif
 }
 
 // not an IMETHOD so we don't need to do error checking or return an error.
 // We only have one caller.
 void nsParseNewMailState::GetMsgWindow(nsIMsgWindow **aMsgWindow)
--- a/mailnews/local/src/nsParseMailbox.h
+++ b/mailnews/local/src/nsParseMailbox.h
@@ -96,16 +96,17 @@ public:
 
   static PRBool  IsEnvelopeLine(const char *buf, PRInt32 buf_size);
   static int  msg_UnHex(char C);
 
   nsCOMPtr<nsIMsgHeaderParser> m_HeaderAddressParser;
 
   nsCOMPtr<nsIMsgDBHdr> m_newMsgHdr; /* current message header we're building */
   nsCOMPtr<nsIMsgDatabase>  m_mailDB;
+  nsCOMPtr<nsIMsgDatabase> m_backupMailDB;
 
   nsMailboxParseState   m_state;
   PRUint32              m_position;
   PRUint32              m_envelope_pos;
   PRUint32              m_headerstartpos;
 
   nsByteArray           m_headers;