Bug 1540668 - IDB: Add a timer for ShutdownWorkThreads; r=asuth, dr=chutten
authorJan Varga <jan.varga@gmail.com>
Tue, 02 Apr 2019 13:16:55 +0200
changeset 467686 74be8815ccfce81f1ca6568f8cfebfc847a83ba2
parent 467685 56c9e5b7f7ae99d43c874cc03e96e65ba1cfe6f2
child 467687 d4109c8cfb7ddc7be522d6b36e2408a0f3fd1c9f
child 467845 a574ac3c62644b9580343bfaa0abdb66b157e699
child 467901 82ba38c4aa6acc7b322bcb56c417c0f64d5d3660
push id112647
push userjvarga@mozilla.com
push dateWed, 03 Apr 2019 18:03:28 +0000
treeherdermozilla-inbound@74be8815ccfc [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersasuth
bugs1540668
milestone68.0a1
first release with
nightly linux32
74be8815ccfc / 68.0a1 / 20190403204605 / files
nightly linux64
74be8815ccfc / 68.0a1 / 20190403204605 / files
nightly mac
74be8815ccfc / 68.0a1 / 20190403204605 / files
nightly win32
74be8815ccfc / 68.0a1 / 20190403204605 / files
nightly win64
74be8815ccfc / 68.0a1 / 20190403204605 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540668 - IDB: Add a timer for ShutdownWorkThreads; r=asuth, dr=chutten Differential Revision: https://phabricator.services.mozilla.com/D25745
dom/indexedDB/ActorsParent.cpp
toolkit/crashreporter/CrashAnnotations.yaml
--- a/dom/indexedDB/ActorsParent.cpp
+++ b/dom/indexedDB/ActorsParent.cpp
@@ -69,16 +69,17 @@
 #include "mozilla/Scoped.h"
 #include "mozilla/storage/Variant.h"
 #include "nsAutoPtr.h"
 #include "nsCharSeparatedTokenizer.h"
 #include "nsClassHashtable.h"
 #include "nsCOMPtr.h"
 #include "nsDataHashtable.h"
 #include "nsEscape.h"
+#include "nsExceptionHandler.h"
 #include "nsHashKeys.h"
 #include "nsNetUtil.h"
 #include "nsIAsyncInputStream.h"
 #include "nsISimpleEnumerator.h"
 #include "nsIEventTarget.h"
 #include "nsIFile.h"
 #include "nsIFileURL.h"
 #include "nsIFileProtocolHandler.h"
@@ -268,16 +269,30 @@ const char kPrefFileHandleEnabled[] = "d
 // routine is run again to ensure that the database and all of its related files
 // are removed. The primary goal of this mechanism is to avoid situations where
 // a database has been partially deleted, leading to inconsistent state for the
 // origin.
 #define IDB_DELETION_MARKER_FILE_PREFIX "idb-deleting-"
 
 const uint32_t kDeleteTimeoutMs = 1000;
 
+/**
+ * Automatically crash the browser if IndexedDB shutdown takes this long.  We've
+ * chosen a value that is longer than the value for QuotaManager shutdown timer
+ * which is currently set to 30 seconds.  We've also chosen a value that is long
+ * long enough that it is unlikely for the problem to be falsely triggered by
+ * slow system I/O.  We've also chosen a value long enough so that automated
+ * tests should time out and fail if IndexedDB shutdown hangs.  Also, this value
+ * is long enough so that testers can notice the IndexedDB shutdown hang; we
+ * want to know about the hangs, not hide them.  On the other hand this value is
+ * less than 60 seconds which is used by nsTerminator to crash a hung main
+ * process.
+ */
+#define SHUTDOWN_TIMEOUT_MS 50000
+
 #ifdef DEBUG
 
 const int32_t kDEBUGThreadPriority = nsISupportsPriority::PRIORITY_NORMAL;
 const uint32_t kDEBUGThreadSleepMS = 0;
 
 const int32_t kDEBUGTransactionThreadPriority =
     nsISupportsPriority::PRIORITY_NORMAL;
 const uint32_t kDEBUGTransactionThreadSleepMS = 0;
@@ -7845,16 +7860,18 @@ class QuotaClient final : public mozilla
 
   void StopIdleMaintenance() override;
 
   void ShutdownWorkThreads() override;
 
  private:
   ~QuotaClient() override;
 
+  void ShutdownTimedOut();
+
   static void DeleteTimerCallback(nsITimer* aTimer, void* aClosure);
 
   nsresult GetDirectory(PersistenceType aPersistenceType,
                         const nsACString& aOrigin, nsIFile** aDirectory);
 
   // The aObsoleteFiles will collect files based on the marker files. For now,
   // InitOrigin() is the only consumer of this argument because it checks those
   // unfinished deletion and clean them up after that.
@@ -16204,23 +16221,36 @@ void QuotaClient::StopIdleMaintenance() 
 void QuotaClient::ShutdownWorkThreads() {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(!mShutdownRequested);
 
   mShutdownRequested = true;
 
   AbortOperations(VoidCString());
 
+  nsCOMPtr<nsITimer> timer = NS_NewTimer();
+
+  MOZ_ALWAYS_SUCCEEDS(timer->InitWithNamedFuncCallback(
+      [](nsITimer* aTimer, void* aClosure) {
+        auto quotaClient = static_cast<QuotaClient*>(aClosure);
+
+        quotaClient->ShutdownTimedOut();
+      },
+      this, SHUTDOWN_TIMEOUT_MS, nsITimer::TYPE_ONE_SHOT,
+      "indexeddb::QuotaClient::ShutdownWorkThreads::SpinEventLoopTimer"));
+
   // This should release any IDB related quota objects or directory locks.
   MOZ_ALWAYS_TRUE(SpinEventLoopUntil([&]() {
     return (!gFactoryOps || gFactoryOps->IsEmpty()) &&
            (!gLiveDatabaseHashtable || !gLiveDatabaseHashtable->Count()) &&
            !mCurrentMaintenance;
   }));
 
+  MOZ_ALWAYS_SUCCEEDS(timer->Cancel());
+
   // And finally, shutdown all threads.
   RefPtr<ConnectionPool> connectionPool = gConnectionPool.get();
   if (connectionPool) {
     connectionPool->Shutdown();
 
     gConnectionPool = nullptr;
   }
 
@@ -16238,16 +16268,47 @@ void QuotaClient::ShutdownWorkThreads() 
   }
 
   if (mDeleteTimer) {
     MOZ_ALWAYS_SUCCEEDS(mDeleteTimer->Cancel());
     mDeleteTimer = nullptr;
   }
 }
 
+void QuotaClient::ShutdownTimedOut() {
+  AssertIsOnBackgroundThread();
+  MOZ_DIAGNOSTIC_ASSERT(
+      (gFactoryOps && !gFactoryOps->IsEmpty()) ||
+      (gLiveDatabaseHashtable && gLiveDatabaseHashtable->Count()) ||
+      mCurrentMaintenance);
+
+  nsCString data;
+
+  if (gFactoryOps && !gFactoryOps->IsEmpty()) {
+    data.Append("gFactoryOps: ");
+    data.AppendInt(static_cast<uint32_t>(gFactoryOps->Length()));
+    data.Append("\n");
+  }
+
+  if (gLiveDatabaseHashtable && gLiveDatabaseHashtable->Count()) {
+    data.Append("gLiveDatabaseHashtable: ");
+    data.AppendInt(gLiveDatabaseHashtable->Count());
+    data.Append("\n");
+  }
+
+  if (mCurrentMaintenance) {
+    data.Append("mCurrentMaintenance\n");
+  }
+
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::IndexedDBShutdownTimeout, data);
+
+  MOZ_CRASH("IndexedDB shutdown timed out");
+}
+
 void QuotaClient::DeleteTimerCallback(nsITimer* aTimer, void* aClosure) {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aTimer);
 
   auto* self = static_cast<QuotaClient*>(aClosure);
   MOZ_ASSERT(self);
   MOZ_ASSERT(self->mDeleteTimer);
   MOZ_ASSERT(SameCOMIdentity(self->mDeleteTimer, aTimer));
--- a/toolkit/crashreporter/CrashAnnotations.yaml
+++ b/toolkit/crashreporter/CrashAnnotations.yaml
@@ -359,16 +359,25 @@ HasDeviceTouchScreen:
   type: boolean
 
 IAccessibleConfig:
   description: >
     Set when something is seriously wrong with the IAccessible configuration in
     the computer's registry. The value is always set to "NoSystemTypeLibOrPS"
   type: string
 
+IndexedDBShutdownTimeout:
+  description: >
+    This annotation is present if IndexedDB shutdown was not finished in time
+    and the browser was crashed instead of waiting for IndexedDB shutdown to
+    finish. The condition that caused the hang is contained in the annotation.
+  type: string
+  content: false
+  ping: true
+
 InstallTime:
   description: >
     The time when Firefox was installed expressed as seconds since the Epoch
   type: integer
 
 InterfaceRegistrationInfoChild:
   description: >
     Microsoft COM interface registration annotation for the child process.