Bug 1110446 P2 Cleanup stale caches/bodies if last session didn't shutdown cleanly. r=ehsan
authorBen Kelly <ben@wanderview.com>
Thu, 25 Jun 2015 22:22:46 -0700
changeset 281083 b89856b358ea1e5673da81e7e5c11e6b4a2dfaf3
parent 281082 f5b3cfa8649fba3373edf27bfeebb959d7d39f95
child 281084 a9f10e13ec411dc65660b04ac6772d2baefb915d
push id4932
push userjlund@mozilla.com
push dateMon, 10 Aug 2015 18:23:06 +0000
treeherdermozilla-beta@6dd5a4f5f745 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs1110446
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1110446 P2 Cleanup stale caches/bodies if last session didn't shutdown cleanly. r=ehsan
dom/cache/DBSchema.cpp
dom/cache/DBSchema.h
dom/cache/FileUtils.cpp
dom/cache/FileUtils.h
dom/cache/Manager.cpp
--- a/dom/cache/DBSchema.cpp
+++ b/dom/cache/DBSchema.cpp
@@ -533,16 +533,72 @@ IsCacheOrphaned(mozIStorageConnection* a
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   *aOrphanedOut = refCount == 0;
 
   return rv;
 }
 
 nsresult
+FindOrphanedCacheIds(mozIStorageConnection* aConn,
+                     nsTArray<CacheId>& aOrphanedListOut)
+{
+  nsCOMPtr<mozIStorageStatement> state;
+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT id FROM caches "
+    "WHERE id NOT IN (SELECT cache_id from storage);"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {
+    CacheId cacheId = INVALID_CACHE_ID;
+    rv = state->GetInt64(0, &cacheId);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    aOrphanedListOut.AppendElement(cacheId);
+  }
+
+  return rv;
+}
+
+nsresult
+GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConn);
+
+  nsCOMPtr<mozIStorageStatement> state;
+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT request_body_id, response_body_id FROM entries;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {
+    // extract 0 to 2 nsID structs per row
+    for (uint32_t i = 0; i < 2; ++i) {
+      bool isNull = false;
+
+      rv = state->GetIsNull(i, &isNull);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      if (!isNull) {
+        nsID id;
+        rv = ExtractId(state, i, &id);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+        aBodyIdListOut.AppendElement(id);
+      }
+    }
+  }
+
+  return rv;
+}
+
+nsresult
 CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId,
            const CacheRequest& aRequest,
            const CacheQueryParams& aParams,
            bool* aFoundResponseOut,
            SavedResponse* aSavedResponseOut)
 {
   MOZ_ASSERT(!NS_IsMainThread());
   MOZ_ASSERT(aConn);
--- a/dom/cache/DBSchema.h
+++ b/dom/cache/DBSchema.h
@@ -44,16 +44,23 @@ DeleteCacheId(mozIStorageConnection* aCo
               nsTArray<nsID>& aDeletedBodyIdListOut);
 
 // TODO: Consider removing unused IsCacheOrphaned after writing cleanup code. (bug 1110446)
 nsresult
 IsCacheOrphaned(mozIStorageConnection* aConn, CacheId aCacheId,
                 bool* aOrphanedOut);
 
 nsresult
+FindOrphanedCacheIds(mozIStorageConnection* aConn,
+                     nsTArray<CacheId>& aOrphanedListOut);
+
+nsresult
+GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut);
+
+nsresult
 CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId,
            const CacheRequest& aRequest, const CacheQueryParams& aParams,
            bool* aFoundResponseOut, SavedResponse* aSavedResponseOut);
 
 nsresult
 CacheMatchAll(mozIStorageConnection* aConn, CacheId aCacheId,
               const CacheRequestOrVoid& aRequestOrVoid,
               const CacheQueryParams& aParams,
--- a/dom/cache/FileUtils.cpp
+++ b/dom/cache/FileUtils.cpp
@@ -315,28 +315,142 @@ BodyIdToFile(nsIFile* aBaseDir, const ns
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   return rv;
 }
 
 } // anonymous namespace
 
 nsresult
-CreateMarkerFile(const QuotaInfo& aQuotaInfo)
+BodyDeleteOrphanedFiles(nsIFile* aBaseDir, nsTArray<nsID>& aKnownBodyIdList)
 {
+  MOZ_ASSERT(aBaseDir);
+
+  // body files are stored in a directory structure like:
+  //
+  //  /morgue/01/{01fdddb2-884d-4c3d-95ba-0c8062f6c325}.final
+  //  /morgue/02/{02fdddb2-884d-4c3d-95ba-0c8062f6c325}.tmp
+
+  nsCOMPtr<nsIFile> dir;
+  nsresult rv = aBaseDir->Clone(getter_AddRefs(dir));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Add the root morgue directory
+  rv = dir->Append(NS_LITERAL_STRING("morgue"));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  nsCOMPtr<nsISimpleEnumerator> entries;
+  rv = dir->GetDirectoryEntries(getter_AddRefs(entries));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Iterate over all the intermediate morgue subdirs
+  bool hasMore = false;
+  while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
+    nsCOMPtr<nsISupports> entry;
+    rv = entries->GetNext(getter_AddRefs(entry));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    nsCOMPtr<nsIFile> subdir = do_QueryInterface(entry);
+
+    bool isDir = false;
+    rv = subdir->IsDirectory(&isDir);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // If a file got in here somehow, try to remove it and move on
+    if (NS_WARN_IF(!isDir)) {
+      rv = subdir->Remove(false /* recursive */);
+      MOZ_ASSERT(NS_SUCCEEDED(rv));
+      continue;
+    }
+
+    nsCOMPtr<nsISimpleEnumerator> subEntries;
+    rv = subdir->GetDirectoryEntries(getter_AddRefs(subEntries));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // Now iterate over all the files in the subdir
+    bool subHasMore = false;
+    while(NS_SUCCEEDED(rv = subEntries->HasMoreElements(&subHasMore)) &&
+          subHasMore) {
+      nsCOMPtr<nsISupports> subEntry;
+      rv = subEntries->GetNext(getter_AddRefs(subEntry));
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      nsCOMPtr<nsIFile> file = do_QueryInterface(subEntry);
+
+      nsAutoCString leafName;
+      rv = file->GetNativeLeafName(leafName);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      // Delete all tmp files regardless of known bodies.  These are
+      // all considered orphans.
+      if (StringEndsWith(leafName, NS_LITERAL_CSTRING(".tmp"))) {
+        // remove recursively in case its somehow a directory
+        rv = file->Remove(true /* recursive */);
+        MOZ_ASSERT(NS_SUCCEEDED(rv));
+        continue;
+      }
+
+      nsCString suffix(NS_LITERAL_CSTRING(".final"));
+
+      // Otherwise, it must be a .final file.  If its not, then just
+      // skip it.
+      if (NS_WARN_IF(!StringEndsWith(leafName, suffix) ||
+                     leafName.Length() != NSID_LENGTH - 1 + suffix.Length())) {
+        continue;
+      }
+
+      // Finally, parse the uuid out of the name.  If its fails to parse,
+      // the ignore the file.
+      nsID id;
+      if (NS_WARN_IF(!id.Parse(leafName.BeginReading()))) {
+        continue;
+      }
+
+      if (!aKnownBodyIdList.Contains(id)) {
+        // remove recursively in case its somehow a directory
+        rv = file->Remove(true /* recursive */);
+        MOZ_ASSERT(NS_SUCCEEDED(rv));
+      }
+    }
+  }
+
+  return rv;
+}
+
+namespace {
+
+nsresult
+GetMarkerFileHandle(const QuotaInfo& aQuotaInfo, nsIFile** aFileOut)
+{
+  MOZ_ASSERT(aFileOut);
+
   nsCOMPtr<nsIFile> marker;
   nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(marker));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = marker->Append(NS_LITERAL_STRING("cache"));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = marker->Append(NS_LITERAL_STRING("context_open.marker"));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
+  marker.forget(aFileOut);
+
+  return rv;
+}
+
+} // anonymous namespace
+
+nsresult
+CreateMarkerFile(const QuotaInfo& aQuotaInfo)
+{
+  nsCOMPtr<nsIFile> marker;
+  nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
   rv = marker->Create(nsIFile::NORMAL_FILE_TYPE, 0644);
   if (rv == NS_ERROR_FILE_ALREADY_EXISTS) {
     rv = NS_OK;
   }
 
   // Note, we don't need to fsync here.  We only care about actually
   // writing the marker if later modifications to the Cache are
   // actually flushed to the disk.  If the OS crashes before the marker
@@ -345,34 +459,42 @@ CreateMarkerFile(const QuotaInfo& aQuota
 
   return rv;
 }
 
 nsresult
 DeleteMarkerFile(const QuotaInfo& aQuotaInfo)
 {
   nsCOMPtr<nsIFile> marker;
-  nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(marker));
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
-  rv = marker->Append(NS_LITERAL_STRING("cache"));
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
-  rv = marker->Append(NS_LITERAL_STRING("context_open.marker"));
+  nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = marker->Remove(/* recursive = */ false);
   if (rv == NS_ERROR_FILE_NOT_FOUND ||
       rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) {
     rv = NS_OK;
   }
 
   // Again, no fsync is necessary.  If the OS crashes before the file
   // removal is flushed, then the Cache will search for stale data on
   // startup.  This will cause the next Cache access to be a bit slow, but
   // it seems appropriate after an OS crash.
 
   return NS_OK;
 }
 
+bool
+MarkerFileExists(const QuotaInfo& aQuotaInfo)
+{
+  nsCOMPtr<nsIFile> marker;
+  nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return false; }
+
+  bool exists = false;
+  rv = marker->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return false; }
+
+  return exists;
+}
+
 } // namespace cache
 } // namespace dom
 } // namespace mozilla
--- a/dom/cache/FileUtils.h
+++ b/dom/cache/FileUtils.h
@@ -46,18 +46,24 @@ BodyFinalizeWrite(nsIFile* aBaseDir, con
 nsresult
 BodyOpen(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir, const nsID& aId,
          nsIInputStream** aStreamOut);
 
 nsresult
 BodyDeleteFiles(nsIFile* aBaseDir, const nsTArray<nsID>& aIdList);
 
 nsresult
+BodyDeleteOrphanedFiles(nsIFile* aBaseDir, nsTArray<nsID>& aKnownBodyIdList);
+
+nsresult
 CreateMarkerFile(const QuotaInfo& aQuotaInfo);
 
 nsresult
 DeleteMarkerFile(const QuotaInfo& aQuotaInfo);
 
+bool
+MarkerFileExists(const QuotaInfo& aQuotaInfo);
+
 } // namespace cache
 } // namespace dom
 } // namespace mozilla
 
 #endif // mozilla_dom_cache_FileUtils_h
--- a/dom/cache/Manager.cpp
+++ b/dom/cache/Manager.cpp
@@ -25,57 +25,87 @@
 #include "nsAutoPtr.h"
 #include "nsIInputStream.h"
 #include "nsID.h"
 #include "nsIFile.h"
 #include "nsIThread.h"
 #include "nsThreadUtils.h"
 #include "nsTObserverArray.h"
 
-namespace {
 
-using mozilla::unused;
-using mozilla::dom::cache::Action;
-using mozilla::dom::cache::BodyCreateDir;
-using mozilla::dom::cache::BodyDeleteFiles;
-using mozilla::dom::cache::QuotaInfo;
-using mozilla::dom::cache::SyncDBAction;
-using mozilla::dom::cache::db::CreateSchema;
+namespace mozilla {
+namespace dom {
+namespace cache {
+
+namespace {
 
 // An Action that is executed when a Context is first created.  It ensures that
 // the directory and database are setup properly.  This lets other actions
 // not worry about these details.
 class SetupAction final : public SyncDBAction
 {
 public:
   SetupAction()
     : SyncDBAction(DBAction::Create)
   { }
 
   virtual nsresult
   RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir,
                         mozIStorageConnection* aConn) override
   {
-    // TODO: init maintainance marker (bug 1110446)
-    // TODO: perform maintainance if necessary (bug 1110446)
-    // TODO: find orphaned caches in database (bug 1110446)
-    // TODO: have Context create/delete marker files in constructor/destructor
-    //       and only do expensive maintenance if that marker is present (bug 1110446)
-
     nsresult rv = BodyCreateDir(aDBDir);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-    mozStorageTransaction trans(aConn, false,
-                                mozIStorageConnection::TRANSACTION_IMMEDIATE);
+    {
+      mozStorageTransaction trans(aConn, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+      rv = db::CreateSchema(aConn);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      rv = trans.Commit();
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    }
 
-    rv = CreateSchema(aConn);
-    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    // If the Context marker file exists, then the last session was
+    // not cleanly shutdown.  In these cases sqlite will ensure that
+    // the database is valid, but we might still orphan data.  Both
+    // Cache objects and body files can be referenced by DOM objects
+    // after they are "removed" from their parent.  So we need to
+    // look and see if any of these late access objects have been
+    // orphaned.
+    //
+    // Note, this must be done after any schema version updates to
+    // ensure our DBSchema methods work correctly.
+    if (MarkerFileExists(aQuotaInfo)) {
+      NS_WARNING("Cache not shutdown cleanly! Cleaning up stale data...");
+      mozStorageTransaction trans(aConn, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
 
-    rv = trans.Commit();
-    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      // Clean up orphaned Cache objects
+      nsAutoTArray<CacheId, 8> orphanedCacheIdList;
+      nsresult rv = db::FindOrphanedCacheIds(aConn, orphanedCacheIdList);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      for (uint32_t i = 0; i < orphanedCacheIdList.Length(); ++i) {
+        nsAutoTArray<nsID, 16> deletedBodyIdList;
+        rv = db::DeleteCacheId(aConn, orphanedCacheIdList[i], deletedBodyIdList);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+        rv = BodyDeleteFiles(aDBDir, deletedBodyIdList);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      }
+
+      // Clean up orphaned body objects
+      nsAutoTArray<nsID, 64> knownBodyIdList;
+      rv = db::GetKnownBodyIds(aConn, knownBodyIdList);
+
+      rv = BodyDeleteOrphanedFiles(aDBDir, knownBodyIdList);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    }
 
     return rv;
   }
 };
 
 // ----------------------------------------------------------------------------
 
 // Action that is executed when we determine that content has stopped using
@@ -119,24 +149,16 @@ public:
 
     aResolver->Resolve(rv);
   }
 
 private:
   nsTArray<nsID> mDeletedBodyIdList;
 };
 
-} // anonymous namespace
-
-namespace mozilla {
-namespace dom {
-namespace cache {
-
-namespace {
-
 bool IsHeadRequest(CacheRequest aRequest, CacheQueryParams aParams)
 {
   return !aParams.ignoreMethod() && aRequest.method().LowerCaseEqualsLiteral("head");
 }
 
 bool IsHeadRequest(CacheRequestOrVoid aRequest, CacheQueryParams aParams)
 {
   if (aRequest.type() == CacheRequestOrVoid::TCacheRequest) {