Bug 1360308 - Part 2: Use callback in CrashReporterHost::GenerateMinidumpAndPair() and up the caller chain. r=gsvelto
authorCervantes Yu <cyu@mozilla.com>
Thu, 22 Jun 2017 18:52:58 +0800
changeset 365492 2b28e4ee269cb5559f0f28a1421d284cb0c98658
parent 365491 fef8b4d5bd1d3abbbd5ed278fb51a49a59575793
child 365493 4f17147356744f39fcd1756c019540a90eda9192
push id32077
push userkwierso@gmail.com
push dateThu, 22 Jun 2017 21:02:06 +0000
treeherdermozilla-central@4c6668cbaccb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgsvelto
bugs1360308
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1360308 - Part 2: Use callback in CrashReporterHost::GenerateMinidumpAndPair() and up the caller chain. r=gsvelto This changes CrashReporterHost::GenerateMinidumpAndPair() and up the caller chain to use callbacks so we may call it synchronously or asynchronously. MozReview-Commit-ID: 4PQH6cVdOk0
dom/ipc/ContentParent.cpp
dom/ipc/ContentParent.h
dom/ipc/ProcessHangMonitor.cpp
dom/plugins/ipc/PluginBridge.h
dom/plugins/ipc/PluginHangUIParent.cpp
dom/plugins/ipc/PluginModuleParent.cpp
dom/plugins/ipc/PluginModuleParent.h
ipc/glue/CrashReporterHost.cpp
ipc/glue/CrashReporterHost.h
--- a/dom/ipc/ContentParent.cpp
+++ b/dom/ipc/ContentParent.cpp
@@ -2971,27 +2971,46 @@ ContentParent::KillHard(const char* aRea
     mCrashReporter->AddNote(
       NS_LITERAL_CSTRING("additional_minidumps"),
       additionalDumps);
     nsDependentCString reason(aReason);
     mCrashReporter->AddNote(
       NS_LITERAL_CSTRING("ipc_channel_error"),
       reason);
 
+    Telemetry::Accumulate(Telemetry::SUBPROCESS_KILL_HARD, reason, 1);
+
+    RefPtr<ContentParent> self = this;
+    std::function<void(bool)> callback = [self](bool aResult) {
+      self->OnGenerateMinidumpComplete(aResult);
+    };
     // Generate the report and insert into the queue for submittal.
-    if (mCrashReporter->GenerateMinidumpAndPair(Process(),
-                                                nullptr,
-                                                NS_LITERAL_CSTRING("browser")))
-    {
-      mCreatedPairedMinidumps = mCrashReporter->FinalizeCrashReport();
-    }
-
-    Telemetry::Accumulate(Telemetry::SUBPROCESS_KILL_HARD, reason, 1);
+    mCrashReporter->GenerateMinidumpAndPair(Process(),
+                                            nullptr,
+                                            NS_LITERAL_CSTRING("browser"),
+                                            Move(callback),
+                                            true);
+    return;
   }
 #endif
+  OnGenerateMinidumpComplete(false);
+}
+
+void
+ContentParent::OnGenerateMinidumpComplete(bool aDumpResult)
+{
+#if defined(MOZ_CRASHREPORTER) && !defined(MOZ_B2G)
+  if (mCrashReporter && aDumpResult) {
+    // CrashReporterHost::GenerateMinidumpAndPair() is successful.
+    mCreatedPairedMinidumps = mCrashReporter->FinalizeCrashReport();
+  }
+#endif
+
+  Unused << aDumpResult; // Don't care about result if no minidump was requested.
+
   ProcessHandle otherProcessHandle;
   if (!base::OpenProcessHandle(OtherPid(), &otherProcessHandle)) {
     NS_ERROR("Failed to open child process when attempting kill.");
     return;
   }
 
   if (!KillProcess(otherProcessHandle, base::PROCESS_END_KILLED_BY_USER,
            false)) {
--- a/dom/ipc/ContentParent.h
+++ b/dom/ipc/ContentParent.h
@@ -6,16 +6,17 @@
 
 #ifndef mozilla_dom_ContentParent_h
 #define mozilla_dom_ContentParent_h
 
 #include "mozilla/dom/PContentParent.h"
 #include "mozilla/dom/nsIContentParent.h"
 #include "mozilla/gfx/gfxVarReceiver.h"
 #include "mozilla/gfx/GPUProcessListener.h"
+#include "mozilla/ipc/CrashReporterHost.h"
 #include "mozilla/ipc/GeckoChildProcessHost.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/FileUtils.h"
 #include "mozilla/HalTypes.h"
 #include "mozilla/LinkedList.h"
 #include "mozilla/MemoryReportingProcess.h"
 #include "mozilla/StaticPtr.h"
 #include "mozilla/TimeStamp.h"
@@ -70,17 +71,16 @@ namespace embedding {
 class PrintingParent;
 }
 
 namespace ipc {
 class OptionalURIParams;
 class PFileDescriptorSetParent;
 class URIParams;
 class TestShellParent;
-class CrashReporterHost;
 } // namespace ipc
 
 namespace jsipc {
 class PJavaScriptParent;
 } // namespace jsipc
 
 namespace layers {
 struct TextureFactoryIdentifier;
@@ -808,16 +808,18 @@ private:
 
   // Perform any steps necesssary to gracefully shtudown the message
   // manager and null out mMessageManager.
   void ShutDownMessageManager();
 
   // Start the force-kill timer on shutdown.
   void StartForceKillTimer();
 
+  void OnGenerateMinidumpComplete(bool aDumpResult);
+
   // Ensure that the permissions for the giben Permission key are set in the
   // content process.
   //
   // See nsIPermissionManager::GetPermissionsForKey for more information on
   // these keys.
   void EnsurePermissionsByKey(const nsCString& aKey);
 
   static void ForceKillTimerCallback(nsITimer* aTimer, void* aClosure);
--- a/dom/ipc/ProcessHangMonitor.cpp
+++ b/dom/ipc/ProcessHangMonitor.cpp
@@ -245,16 +245,18 @@ public:
   bool IsOnThread() { return mHangMonitor->IsOnThread(); }
 
 private:
   bool TakeBrowserMinidump(const PluginHangData& aPhd, nsString& aCrashId);
 
   void SendHangNotification(const HangData& aHangData,
                             const nsString& aBrowserDumpId,
                             bool aTakeMinidump);
+  void OnTakeFullMinidumpComplete(const HangData& aHangData,
+                                  const nsString& aDumpId);
 
   void ClearHangNotification();
 
   void ForcePaintOnThread(TabId aTabId, uint64_t aLayerObserverEpoch);
 
   void ShutdownOnThread();
 
   const RefPtr<ProcessHangMonitor> mHangMonitor;
@@ -668,34 +670,50 @@ HangMonitorParent::Bind(Endpoint<PProces
 void
 HangMonitorParent::SendHangNotification(const HangData& aHangData,
                                         const nsString& aBrowserDumpId,
                                         bool aTakeMinidump)
 {
   // chrome process, main thread
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
 
-  nsString dumpId;
   if ((aHangData.type() == HangData::TPluginHangData) && aTakeMinidump) {
     // We've been handed a partial minidump; complete it with plugin and
     // content process dumps.
     const PluginHangData& phd = aHangData.get_PluginHangData();
-    plugins::TakeFullMinidump(phd.pluginId(), phd.contentProcessId(),
-                              aBrowserDumpId, dumpId);
-    UpdateMinidump(phd.pluginId(), dumpId);
+
+    std::function<void(nsString)> callback =
+      [this, aHangData](nsString aResult) {
+        this->UpdateMinidump(aHangData.get_PluginHangData().pluginId(),
+                       aResult);
+        this->OnTakeFullMinidumpComplete(aHangData, aResult);
+      };
+
+    plugins::TakeFullMinidump(phd.pluginId(),
+                              phd.contentProcessId(),
+                              aBrowserDumpId,
+                              Move(callback),
+                              true);
   } else {
     // We already have a full minidump; go ahead and use it.
-    dumpId = aBrowserDumpId;
+    OnTakeFullMinidumpComplete(aHangData, aBrowserDumpId);
   }
+}
 
-  mProcess->SetHangData(aHangData, dumpId);
+void
+HangMonitorParent::OnTakeFullMinidumpComplete(const HangData& aHangData,
+                                              const nsString& aDumpId)
+{
+  mProcess->SetHangData(aHangData, aDumpId);
 
   nsCOMPtr<nsIObserverService> observerService =
     mozilla::services::GetObserverService();
-  observerService->NotifyObservers(mProcess, "process-hang-report", nullptr);
+  observerService->NotifyObservers(mProcess,
+                                   "process-hang-report",
+                                   nullptr);
 }
 
 void
 HangMonitorParent::ClearHangNotification()
 {
   // chrome process, main thread
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
   mProcess->ClearHang();
@@ -989,22 +1007,30 @@ HangMonitoredProcess::TerminatePlugin()
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
   if (mHangData.type() != HangData::TPluginHangData) {
     return NS_ERROR_UNEXPECTED;
   }
 
   // Use the multi-process crash report generated earlier.
   uint32_t id = mHangData.get_PluginHangData().pluginId();
   base::ProcessId contentPid = mHangData.get_PluginHangData().contentProcessId();
-  plugins::TerminatePlugin(id, contentPid, NS_LITERAL_CSTRING("HangMonitor"),
-                           mDumpId);
 
-  if (mActor) {
-    mActor->CleanupPluginHang(id, false);
-  }
+  RefPtr<HangMonitoredProcess> self{this};
+  std::function<void(bool)> callback =
+    [self, id](bool aResult) {
+      if (self->mActor) {
+        self->mActor->CleanupPluginHang(id, false);
+      }
+    };
+
+  plugins::TerminatePlugin(id,
+                           contentPid,
+                           NS_LITERAL_CSTRING("HangMonitor"),
+                           mDumpId,
+                           Move(callback));
   return NS_OK;
 }
 
 NS_IMETHODIMP
 HangMonitoredProcess::IsReportForBrowser(nsIFrameLoader* aFrameLoader, bool* aResult)
 {
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
 
--- a/dom/plugins/ipc/PluginBridge.h
+++ b/dom/plugins/ipc/PluginBridge.h
@@ -2,16 +2,18 @@
  * vim: sw=2 ts=2 et :
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef mozilla_plugins_PluginBridge_h
 #define mozilla_plugins_PluginBridge_h
 
+#include <functional>
+
 #include "base/process.h"
 
 namespace mozilla {
 
 namespace dom {
 class ContentParent;
 } // namespace dom
 
@@ -34,20 +36,21 @@ FindPluginsForContent(uint32_t aPluginEp
                       nsTArray<PluginTag>* aPlugins,
                       nsTArray<FakePluginTag>* aFakePlugins,
                       uint32_t* aNewPluginEpoch);
 
 void
 TakeFullMinidump(uint32_t aPluginId,
                  base::ProcessId aContentProcessId,
                  const nsAString& aBrowserDumpId,
-                 nsString& aDumpId);
+                 std::function<void(nsString)>&& aCallback,
+                 bool aAsync);
 
 void
 TerminatePlugin(uint32_t aPluginId,
                 base::ProcessId aContentProcessId,
                 const nsCString& aMonitorDescription,
-                const nsAString& aDumpId);
-
+                const nsAString& aDumpId,
+                std::function<void(bool)>&& aCallback);
 } // namespace plugins
 } // namespace mozilla
 
 #endif // mozilla_plugins_PluginBridge_h
--- a/dom/plugins/ipc/PluginHangUIParent.cpp
+++ b/dom/plugins/ipc/PluginHangUIParent.cpp
@@ -350,20 +350,23 @@ PluginHangUIParent::RecvUserResponse(con
   }
   mLastUserResponse = aResponse;
   mResponseTicks = ::GetTickCount();
   mIsShowing = false;
   // responseCode: 1 = Stop, 2 = Continue, 3 = Cancel
   int responseCode;
   if (aResponse & HANGUI_USER_RESPONSE_STOP) {
     // User clicked Stop
+    std::function<void(bool)> callback = [](bool aResult) { };
     mModule->TerminateChildProcess(mMainThreadMessageLoop,
                                    mozilla::ipc::kInvalidProcessId,
                                    NS_LITERAL_CSTRING("ModalHangUI"),
-                                   EmptyString());
+                                   EmptyString(),
+                                   mModule->DummyCallback<bool>(),
+                                   /* aAsync = */ false);
     responseCode = 1;
   } else if(aResponse & HANGUI_USER_RESPONSE_CONTINUE) {
     mModule->OnHangUIContinue();
     // User clicked Continue
     responseCode = 2;
   } else {
     // Dialog was cancelled
     responseCode = 3;
--- a/dom/plugins/ipc/PluginModuleParent.cpp
+++ b/dom/plugins/ipc/PluginModuleParent.cpp
@@ -370,40 +370,51 @@ PluginModuleChromeParentForId(const uint
 
   return static_cast<PluginModuleChromeParent*>(plugin->GetLibrary());
 }
 
 void
 mozilla::plugins::TakeFullMinidump(uint32_t aPluginId,
                                    base::ProcessId aContentProcessId,
                                    const nsAString& aBrowserDumpId,
-                                   nsString& aDumpId)
+                                   std::function<void(nsString)>&& aCallback,
+                                   bool aAsync)
 {
   PluginModuleChromeParent* chromeParent =
     PluginModuleChromeParentForId(aPluginId);
 
   if (chromeParent) {
-    chromeParent->TakeFullMinidump(aContentProcessId, aBrowserDumpId, aDumpId);
+    chromeParent->TakeFullMinidump(aContentProcessId,
+                                   aBrowserDumpId,
+                                   Move(aCallback),
+                                   aAsync);
+  } else {
+    aCallback(EmptyString());
   }
 }
 
 void
 mozilla::plugins::TerminatePlugin(uint32_t aPluginId,
                                   base::ProcessId aContentProcessId,
                                   const nsCString& aMonitorDescription,
-                                  const nsAString& aDumpId)
+                                  const nsAString& aDumpId,
+                                  std::function<void(bool)>&& aCallback)
 {
   PluginModuleChromeParent* chromeParent =
     PluginModuleChromeParentForId(aPluginId);
 
   if (chromeParent) {
     chromeParent->TerminateChildProcess(MessageLoop::current(),
                                         aContentProcessId,
                                         aMonitorDescription,
-                                        aDumpId);
+                                        aDumpId,
+                                        Move(aCallback),
+                                        true); // Always runs asynchronously.
+  } else {
+    aCallback(true);
   }
 }
 
 /* static */ PluginLibrary*
 PluginModuleContentParent::LoadModule(uint32_t aPluginId,
                                       nsPluginTag* aPluginTag)
 {
     PluginModuleMapping::NotifyLoadingModule loadingModule;
@@ -1151,20 +1162,25 @@ PluginModuleChromeParent::ShouldContinue
 #ifdef XP_WIN
     if (LaunchHangUI()) {
         return true;
     }
     // If LaunchHangUI returned false then we should proceed with the 
     // original plugin hang behaviour and kill the plugin container.
     FinishHangUI();
 #endif // XP_WIN
+
+    // Terminate the child process synchronously because this function can be
+    // called in sync IPC.
     TerminateChildProcess(MessageLoop::current(),
                           mozilla::ipc::kInvalidProcessId,
                           NS_LITERAL_CSTRING("ModalHangUI"),
-                          EmptyString());
+                          EmptyString(),
+                          DummyCallback<bool>(),
+                          /* aAsync = */ false);
     GetIPCChannel()->CloseWithTimeout();
     return false;
 }
 
 bool
 PluginModuleContentParent::ShouldContinueFromReplyTimeout()
 {
     RefPtr<ProcessHangMonitor> monitor = ProcessHangMonitor::Get();
@@ -1179,66 +1195,154 @@ void
 PluginModuleContentParent::OnExitedSyncSend()
 {
     ProcessHangMonitor::ClearHang();
 }
 
 void
 PluginModuleChromeParent::TakeFullMinidump(base::ProcessId aContentPid,
                                            const nsAString& aBrowserDumpId,
-                                           nsString& aDumpId)
+                                           std::function<void(nsString)>&& aCallback,
+                                           bool aAsync)
 {
 #ifdef MOZ_CRASHREPORTER
     mozilla::MutexAutoLock lock(mCrashReporterMutex);
 
-    if (!mCrashReporter) {
+    if (!mCrashReporter || !mTakeFullMinidumpCallback.IsEmpty()) {
+        aCallback(EmptyString());
         return;
     }
-
-    bool reportsReady = false;
+    mTakeFullMinidumpCallback.Init(Move(aCallback), aAsync);
+
+    nsString browserDumpId{aBrowserDumpId};
 
     // Check to see if we already have a browser dump id - with e10s plugin
     // hangs we take this earlier (see ProcessHangMonitor) from a background
     // thread. We do this before we message the main thread about the hang
     // since the posted message will trash our browser stack state.
-    nsCOMPtr<nsIFile> browserDumpFile;
     if (CrashReporter::GetMinidumpForID(aBrowserDumpId,
-                                        getter_AddRefs(browserDumpFile))) {
+                                        getter_AddRefs(mBrowserDumpFile))) {
+
+        // Hold a ref to mPlugin to keep *this* alive until the callback runs.
+        RetainPluginRef();
+        std::function<void(bool)> callback =
+            [this, aContentPid, browserDumpId, aAsync](bool aResult) {
+                if (aAsync) {
+                    this->mCrashReporterMutex.Lock();
+                }
+
+                this->TakeBrowserAndPluginMinidumps(aResult,
+                                                    aContentPid,
+                                                    browserDumpId,
+                                                    aAsync);
+                if (aAsync) {
+                    this->mCrashReporterMutex.Unlock();
+                }
+
+                this->ReleasePluginRef();
+             };
         // We have a single browser report, generate a new plugin process parent
         // report and pair it up with the browser report handed in.
-        reportsReady = mCrashReporter->GenerateMinidumpAndPair(
-          Process(),
-          browserDumpFile,
-          NS_LITERAL_CSTRING("browser"));
-
-        if (!reportsReady) {
-          browserDumpFile = nullptr;
-          CrashReporter::DeleteMinidumpFilesForID(aBrowserDumpId);
-        }
+        mCrashReporter->GenerateMinidumpAndPair(Process(), mBrowserDumpFile,
+                                                NS_LITERAL_CSTRING("browser"),
+                                                Move(callback), aAsync);
+    } else {
+        TakeBrowserAndPluginMinidumps(false, aContentPid, browserDumpId, aAsync);
+    }
+#else // MOZ_CRASHREPORTER
+    aCallback(NS_LITERAL_STRING(""));
+#endif
+}
+
+#ifdef MOZ_CRASHREPORTER
+void
+PluginModuleChromeParent::RetainPluginRef()
+{
+    if (!mPlugin) {
+        return;
     }
 
+    if (NS_IsMainThread()) {
+        mPlugin->AddRef();
+    } else {
+        // XXX We can't sync-dispatch to the main thread because doing that
+        // deadlocks when we are called from
+        // PluginHangUIParent::RecvUserResponse().
+        Unused << NS_DispatchToMainThread(
+            NewNonOwningRunnableMethod(mPlugin, &nsNPAPIPlugin::AddRef));
+    }
+}
+
+void
+PluginModuleChromeParent::ReleasePluginRef()
+{
+    if (!mPlugin) {
+        return;
+    }
+
+    if (NS_IsMainThread()) {
+        mPlugin->Release();
+    } else {
+        // Async release the reference to mPlugin.
+        Unused << NS_DispatchToMainThread(
+            NewNonOwningRunnableMethod(mPlugin, &nsNPAPIPlugin::Release));
+    }
+}
+
+void
+PluginModuleChromeParent::TakeBrowserAndPluginMinidumps(bool aReportsReady,
+                                                        base::ProcessId aContentPid,
+                                                        const nsAString& aBrowserDumpId,
+                                                        bool aAsync)
+{
+    mCrashReporterMutex.AssertCurrentThreadOwns();
+
     // Generate crash report including plugin and browser process minidumps.
     // The plugin process is the parent report with additional dumps including
     // the browser process, content process when running under e10s, and
     // various flash subprocesses if we're the flash module.
-    if (!reportsReady) {
-        reportsReady = mCrashReporter->GenerateMinidumpAndPair(
-          Process(),
-          nullptr, // Pair with a dump of this process and thread.
-          NS_LITERAL_CSTRING("browser"));
+    if (!aReportsReady) {
+        mBrowserDumpFile = nullptr;
+        CrashReporter::DeleteMinidumpFilesForID(aBrowserDumpId);
+
+        nsString browserDumpId{aBrowserDumpId};
+
+        RetainPluginRef();
+        std::function<void(bool)> callback =
+            [this, aContentPid, browserDumpId](bool aResult) {
+                this->OnTakeFullMinidumpComplete(aResult,
+                                                 aContentPid,
+                                                 browserDumpId);
+                this->ReleasePluginRef();
+            };
+        mCrashReporter->GenerateMinidumpAndPair(Process(),
+                                                nullptr, // Pair with a dump of this process and thread.
+                                                NS_LITERAL_CSTRING("browser"),
+                                                Move(callback),
+                                                aAsync);
+    } else {
+        OnTakeFullMinidumpComplete(aReportsReady, aContentPid, aBrowserDumpId);
     }
-
-    if (reportsReady) {
-        aDumpId = mCrashReporter->MinidumpID();
+}
+
+void
+PluginModuleChromeParent::OnTakeFullMinidumpComplete(bool aReportsReady,
+                                                     base::ProcessId aContentPid,
+                                                     const nsAString& aBrowserDumpId)
+{
+    mCrashReporterMutex.AssertCurrentThreadOwns();
+
+    if (aReportsReady) {
+        nsString dumpId = mCrashReporter->MinidumpID();
         PLUGIN_LOG_DEBUG(
-                ("generated paired browser/plugin minidumps: %s)",
-                 NS_ConvertUTF16toUTF8(aDumpId).get()));
+                         ("generated paired browser/plugin minidumps: %s)",
+                          NS_ConvertUTF16toUTF8(dumpId).get()));
         nsAutoCString additionalDumps("browser");
         nsCOMPtr<nsIFile> pluginDumpFile;
-        if (GetMinidumpForID(aDumpId, getter_AddRefs(pluginDumpFile))) {
+        if (GetMinidumpForID(dumpId, getter_AddRefs(pluginDumpFile))) {
 #ifdef MOZ_CRASHREPORTER_INJECTOR
             // If we have handles to the flash sandbox processes on Windows,
             // include those minidumps as well.
             if (CreatePluginMinidump(mFlashProcess1, 0, pluginDumpFile,
                                      NS_LITERAL_CSTRING("flash1"))) {
                 additionalDumps.AppendLiteral(",flash1");
             }
             if (CreatePluginMinidump(mFlashProcess2, 0, pluginDumpFile,
@@ -1250,44 +1354,84 @@ PluginModuleChromeParent::TakeFullMinidu
                 // Include the content process minidump
                 if (CreatePluginMinidump(aContentPid, 0,
                                          pluginDumpFile,
                                          NS_LITERAL_CSTRING("content"))) {
                     additionalDumps.AppendLiteral(",content");
                 }
             }
         }
-        mCrashReporter->AddNote(
-            NS_LITERAL_CSTRING("additional_minidumps"),
-            additionalDumps);
+        mCrashReporter->AddNote(NS_LITERAL_CSTRING("additional_minidumps"),
+                                additionalDumps);
+
+        mTakeFullMinidumpCallback.Invoke(mCrashReporter->MinidumpID());
     } else {
+        mTakeFullMinidumpCallback.Invoke(EmptyString());
         NS_WARNING("failed to capture paired minidumps from hang");
     }
+}
+
 #endif // MOZ_CRASHREPORTER
-}
 
 void
 PluginModuleChromeParent::TerminateChildProcess(MessageLoop* aMsgLoop,
                                                 base::ProcessId aContentPid,
                                                 const nsCString& aMonitorDescription,
-                                                const nsAString& aDumpId)
+                                                const nsAString& aDumpId,
+                                                std::function<void(bool)>&& aCallback,
+                                                bool aAsync)
 {
+    if (!mTerminateChildProcessCallback.IsEmpty()) {
+        aCallback(false);
+        return;
+    }
+    mTerminateChildProcessCallback.Init(Move(aCallback), aAsync);
+
 #ifdef MOZ_CRASHREPORTER
     // Start by taking a full minidump if necessary, this is done early
     // because it also needs to lock the mCrashReporterMutex and Mutex doesn't
     // support recursive locking.
-    nsAutoString dumpId;
     if (aDumpId.IsEmpty()) {
-        TakeFullMinidump(aContentPid, EmptyString(), dumpId);
+
+      RetainPluginRef();
+      std::function<void(nsString)> callback =
+            [this, aMsgLoop, aMonitorDescription, aAsync](nsString aResult) {
+                if (aAsync) {
+                    this->mCrashReporterMutex.Lock();
+                }
+                this->TerminateChildProcessOnDumpComplete(aMsgLoop,
+                                                          aMonitorDescription);
+                if (aAsync) {
+                    this->mCrashReporterMutex.Unlock();
+                }
+
+                this->ReleasePluginRef();
+            };
+
+        TakeFullMinidump(aContentPid, EmptyString(), Move(callback), aAsync);
+    } else {
+        TerminateChildProcessOnDumpComplete(aMsgLoop, aMonitorDescription);
     }
 
-    mozilla::MutexAutoLock lock(mCrashReporterMutex);
+#else
+    TerminateChildProcessOnDumpComplete(aMsgLoop, aMonitorDescription);
+#endif
+}
+
+void
+PluginModuleChromeParent::TerminateChildProcessOnDumpComplete(MessageLoop* aMsgLoop,
+                                                              const nsCString& aMonitorDescription)
+{
+#ifdef MOZ_CRASHREPORTER
+    mCrashReporterMutex.AssertCurrentThreadOwns();
+
     if (!mCrashReporter) {
         // If mCrashReporter is null then the hang has ended, the plugin module
         // is shutting down. There's nothing to do here.
+        mTerminateChildProcessCallback.Invoke(true);
         return;
     }
     mCrashReporter->AddNote(NS_LITERAL_CSTRING("PluginHang"),
                             NS_LITERAL_CSTRING("1"));
     mCrashReporter->AddNote(NS_LITERAL_CSTRING("HangMonitorDescription"),
                             aMonitorDescription);
 #ifdef XP_WIN
     if (mHangUIParent) {
@@ -1326,28 +1470,30 @@ PluginModuleChromeParent::TerminateChild
         base::OpenProcessHandle(mFlashProcess2, &flashSandboxProcess.rwget())) {
         processHandles.AppendElement(flashSandboxProcess);
     }
 #endif
 
     if (!GetProcessCpuUsage(processHandles, mPluginCpuUsageOnHang)) {
       mPluginCpuUsageOnHang.Clear();
     }
-#endif
+#endif // MOZ_CRASHREPORTER
 
     // this must run before the error notification from the channel,
     // or not at all
     bool isFromHangUI = aMsgLoop != MessageLoop::current();
     aMsgLoop->PostTask(
         mChromeTaskFactory.NewRunnableMethod(
             &PluginModuleChromeParent::CleanupFromTimeout, isFromHangUI));
 
     if (!childOpened || !KillProcess(geckoChildProcess, 1, false)) {
         NS_WARNING("failed to kill subprocess!");
     }
+
+    mTerminateChildProcessCallback.Invoke(true);
 }
 
 bool
 PluginModuleParent::GetPluginDetails()
 {
     RefPtr<nsPluginHost> host = nsPluginHost::GetInst();
     if (!host) {
         return false;
--- a/dom/plugins/ipc/PluginModuleParent.h
+++ b/dom/plugins/ipc/PluginModuleParent.h
@@ -6,16 +6,17 @@
 
 #ifndef mozilla_plugins_PluginModuleParent_h
 #define mozilla_plugins_PluginModuleParent_h
 
 #include "base/process.h"
 #include "mozilla/FileUtils.h"
 #include "mozilla/HangAnnotations.h"
 #include "mozilla/PluginLibrary.h"
+#include "mozilla/ipc/CrashReporterHost.h"
 #include "mozilla/plugins/PluginProcessParent.h"
 #include "mozilla/plugins/PPluginModuleParent.h"
 #include "mozilla/plugins/PluginMessageUtils.h"
 #include "mozilla/plugins/PluginTypes.h"
 #include "mozilla/ipc/TaskFactory.h"
 #include "mozilla/TimeStamp.h"
 #include "mozilla/Unused.h"
 #include "npapi.h"
@@ -33,19 +34,16 @@
 #ifdef MOZ_CRASHREPORTER
 #include "nsExceptionHandler.h"
 #endif
 
 class nsPluginTag;
 
 namespace mozilla {
 
-namespace ipc {
-class CrashReporterHost;
-} // namespace ipc
 namespace layers {
 class TextureClientRecycleAllocator;
 } // namespace layers
 
 namespace plugins {
 //-----------------------------------------------------------------------------
 
 class BrowserStreamParent;
@@ -415,16 +413,20 @@ class PluginModuleContentParent : public
     uint32_t mPluginId;
 };
 
 class PluginModuleChromeParent
     : public PluginModuleParent
     , public mozilla::HangMonitor::Annotator
 {
     friend class mozilla::ipc::CrashReporterHost;
+    using TerminateChildProcessCallback =
+        mozilla::ipc::CrashReporterHost::CallbackWrapper<bool>;
+    using TakeFullMinidumpCallback =
+        mozilla::ipc::CrashReporterHost::CallbackWrapper<nsString>;
   public:
     /**
      * LoadModule
      *
      * This may or may not launch a plugin child process,
      * and may or may not be very expensive.
      */
     static PluginLibrary* LoadModule(const char* aFilePath, uint32_t aPluginId,
@@ -447,22 +449,26 @@ class PluginModuleChromeParent
      * Takes ownership of the file associated with aBrowserDumpId.
      *
      * @param aContentPid PID of the e10s content process from which a hang was
      *   reported. May be kInvalidProcessId if not applicable.
      * @param aBrowserDumpId (optional) previously taken browser dump id. If
      *   provided TakeFullMinidump will use this dump file instead of
      *   generating a new one. If not provided a browser dump will be taken at
      *   the time of this call.
-     * @param aDumpId Returns the ID of the newly generated crash dump. Left
-     *   untouched upon failure.
+     * @param aCallback a callback invoked when the operation completes. The ID
+     *   of the newly generated crash dump is provided in the callback argument.
+     *   An empty string will be provided upon failure.
+     * @param aAsync whether to perform the dump asynchronously.
      */
-    void TakeFullMinidump(base::ProcessId aContentPid,
-                          const nsAString& aBrowserDumpId,
-                          nsString& aDumpId);
+    void
+    TakeFullMinidump(base::ProcessId aContentPid,
+                     const nsAString& aBrowserDumpId,
+                     std::function<void(nsString)>&& aCallback,
+                     bool aAsync);
 
     /*
      * Terminates the plugin process associated with this plugin module. Also
      * generates appropriate crash reports unless an existing one is provided.
      * Takes ownership of the file associated with aDumpId on success.
      *
      * @param aMsgLoop the main message pump associated with the module
      *   protocol.
@@ -470,21 +476,57 @@ class PluginModuleChromeParent
      *   reported. May be kInvalidProcessId if not applicable.
      * @param aMonitorDescription a string describing the hang monitor that
      *   is making this call. This string is added to the crash reporter
      *   annotations for the plugin process.
      * @param aDumpId (optional) previously taken dump id. If provided
      *   TerminateChildProcess will use this dump file instead of generating a
      *   multi-process crash report. If not provided a multi-process dump will
      *   be taken at the time of this call.
+     * @param aCallback a callback invoked when the operation completes. The
+     *   argument denotes whether the operation succeeded.
+     * @param aAsync whether to perform the operation asynchronously.
      */
-    void TerminateChildProcess(MessageLoop* aMsgLoop,
-                               base::ProcessId aContentPid,
-                               const nsCString& aMonitorDescription,
-                               const nsAString& aDumpId);
+    void
+    TerminateChildProcess(MessageLoop* aMsgLoop,
+                          base::ProcessId aContentPid,
+                          const nsCString& aMonitorDescription,
+                          const nsAString& aDumpId,
+                          std::function<void(bool)>&& aCallback,
+                          bool aAsync);
+
+    /**
+     * Helper for passing a dummy callback in calling the above function if it
+     * is called synchronously and the caller doesn't care about the callback
+     * result.
+     */
+    template<typename T>
+    static std::function<void(T)> DummyCallback()
+    {
+        return std::function<void(T)>([](T aResult) { });
+    }
+
+  private:
+#ifdef MOZ_CRASHREPORTER
+    // The following methods are callbacks invoked after calling
+    // TakeFullMinidump(). The methods are invoked in the following order:
+    void TakeBrowserAndPluginMinidumps(bool aReportsReady,
+                                       base::ProcessId aContentPid,
+                                       const nsAString& aBrowserDumpId,
+                                       bool aAsync);
+    void OnTakeFullMinidumpComplete(bool aReportsReady,
+                                    base::ProcessId aContentPid,
+                                    const nsAString& aBrowserDumpId);
+
+#endif
+    // The following method is the callback invoked after calling
+    // TerminateChidlProcess().
+    void TerminateChildProcessOnDumpComplete(MessageLoop* aMsgLoop,
+                                             const nsCString& aMonitorDescription);
+  public:
 
 #ifdef XP_WIN
     /**
      * Called by Plugin Hang UI to notify that the user has clicked continue.
      * Used for chrome hang annotations.
      */
     void
     OnHangUIContinue();
@@ -532,16 +574,18 @@ private:
     virtual void
     AnnotateHang(mozilla::HangMonitor::HangAnnotations& aAnnotations) override;
 
     virtual bool ShouldContinueFromReplyTimeout() override;
 
 #ifdef MOZ_CRASHREPORTER
     void ProcessFirstMinidump();
     void WriteExtraDataForMinidump();
+    void RetainPluginRef();
+    void ReleasePluginRef();
 #endif
 
     PluginProcessParent* Process() const { return mSubprocess; }
     base::ProcessHandle ChildProcessHandle() { return mSubprocess->GetChildProcessHandle(); }
 
 #if defined(XP_UNIX) && !defined(XP_MACOSX) && !defined(MOZ_WIDGET_GONK)
     virtual nsresult NP_Initialize(NPNetscapeFuncs* bFuncs, NPPluginFuncs* pFuncs, NPError* error) override;
 #else
@@ -661,14 +705,20 @@ private:
     // processes in existence!
     dom::ContentParent* mContentParent;
     nsCOMPtr<nsIObserver> mPluginOfflineObserver;
     bool mIsBlocklisted;
     static bool sInstantiated;
 #if defined(XP_WIN) && defined(MOZ_SANDBOX)
     mozilla::SandboxPermissions mSandboxPermissions;
 #endif
+
+#ifdef MOZ_CRASHREPORTER
+    nsCOMPtr<nsIFile> mBrowserDumpFile;
+    TakeFullMinidumpCallback mTakeFullMinidumpCallback;
+#endif
+    TerminateChildProcessCallback mTerminateChildProcessCallback;
 };
 
 } // namespace plugins
 } // namespace mozilla
 
 #endif // mozilla_plugins_PluginModuleParent_h
--- a/ipc/glue/CrashReporterHost.cpp
+++ b/ipc/glue/CrashReporterHost.cpp
@@ -100,42 +100,58 @@ CrashReporterHost::FinalizeCrashReport()
   // Use mExtraNotes, since NotifyCrashService looks for "PluginHang" which is
   // set in the parent process.
   NotifyCrashService(mProcessType, mDumpID, &mExtraNotes);
 
   mFinalized = true;
   return true;
 }
 
-bool
+void
 CrashReporterHost::GenerateMinidumpAndPair(GeckoChildProcessHost* aChildProcess,
                                            nsIFile* aMinidumpToPair,
-                                           const nsACString& aPairName)
+                                           const nsACString& aPairName,
+                                           std::function<void(bool)>&& aCallback,
+                                           bool aAsync)
 {
   base::ProcessHandle childHandle;
 #ifdef XP_MACOSX
   childHandle = aChildProcess->GetChildTask();
 #else
   childHandle = aChildProcess->GetChildProcessHandle();
+#endif
+
+  if (!mCreateMinidumpCallback.IsEmpty()) {
+    aCallback(false);
+    return;
+  }
+  mCreateMinidumpCallback.Init(Move(aCallback), aAsync);
+
   if (!childHandle) {
     NS_WARNING("Failed to get child process handle.");
-    return false;
+    mCreateMinidumpCallback.Invoke(false);
+    return;
   }
-#endif
 
   nsCOMPtr<nsIFile> targetDump;
   if (!CrashReporter::CreateMinidumpsAndPair(childHandle,
                                              mThreadId,
                                              aPairName,
                                              aMinidumpToPair,
                                              getter_AddRefs(targetDump))) {
-    return false;
+    mCreateMinidumpCallback.Invoke(false);
+    return;
   }
 
-  return CrashReporter::GetIDFromMinidump(targetDump, mDumpID);
+  if (!CrashReporter::GetIDFromMinidump(targetDump, mDumpID)) {
+    mCreateMinidumpCallback.Invoke(false);
+    return;
+  }
+
+  mCreateMinidumpCallback.Invoke(true);
 }
 
 /* static */ void
 CrashReporterHost::NotifyCrashService(GeckoProcessType aProcessType,
                                       const nsString& aChildDumpID,
                                       const AnnotationTable* aNotes)
 {
   if (!NS_IsMainThread()) {
--- a/ipc/glue/CrashReporterHost.h
+++ b/ipc/glue/CrashReporterHost.h
@@ -2,16 +2,18 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef mozilla_ipc_CrashReporterHost_h
 #define mozilla_ipc_CrashReporterHost_h
 
+#include <functional>
+
 #include "mozilla/UniquePtr.h"
 #include "mozilla/ipc/Shmem.h"
 #include "base/process.h"
 #ifdef MOZ_CRASHREPORTER
 #include "nsExceptionHandler.h"
 #endif
 
 namespace mozilla {
@@ -31,16 +33,71 @@ class CrashReporterHost
   typedef CrashReporter::AnnotationTable AnnotationTable;
   typedef CrashReporter::ThreadId ThreadId;
 #else
   // unused in this case
   typedef int32_t ThreadId;
 #endif
 
 public:
+
+  template <typename T>
+  class CallbackWrapper {
+  public:
+    void Init(std::function<void(T)>&& aCallback, bool aAsync)
+    {
+      mCallback = Move(aCallback);
+      mAsync = aAsync;
+      if (IsAsync()) {
+        // Don't call do_GetCurrentThread() is this is called synchronously
+        // because 1. it's unnecessary, and 2. more importantly, it might create
+        // one if called from a native thread, and the thread will be leaked.
+        mTargetThread = do_GetCurrentThread();
+      }
+    }
+
+    bool IsEmpty()
+    {
+      return !mCallback;
+    }
+
+    bool IsAsync()
+    {
+      return mAsync;
+    }
+
+    void Invoke(T aResult)
+    {
+      if (IsAsync()) {
+        decltype(mCallback) callback = Move(mCallback);
+        mTargetThread->
+          Dispatch(NS_NewRunnableFunction([callback, aResult](){
+                     callback(aResult);
+                   }), NS_DISPATCH_NORMAL);
+      } else {
+        MOZ_ASSERT(!mTargetThread);
+        mCallback(aResult);
+      }
+
+      Clear();
+    }
+
+  private:
+    void Clear()
+    {
+      mCallback = nullptr;
+      mTargetThread = nullptr;
+      mAsync = false;
+    }
+
+    bool mAsync;
+    std::function<void(T)> mCallback;
+    nsCOMPtr<nsIThread> mTargetThread;
+  };
+
   CrashReporterHost(GeckoProcessType aProcessType,
                     const Shmem& aShmem,
                     ThreadId aThreadId);
 
 #ifdef MOZ_CRASHREPORTER
   // Helper function for generating a crash report for a process that probably
   // crashed (i.e., had an AbnormalShutdown in ActorDestroy). Returns true if
   // the process has a minidump attached and we were able to generate a report.
@@ -56,22 +113,25 @@ public:
 
   // If a minidump was already captured (e.g. via the hang reporter), this
   // finalizes the existing report by attaching metadata and notifying the
   // crash service.
   bool FinalizeCrashReport();
 
   // Generate a paired minidump. This does not take the crash report, as
   // GenerateCrashReport does. After this, FinalizeCrashReport may be called.
-  //
-  // This calls TakeCrashedChildMinidump and FinalizeCrashReport.
-  bool
+  // Minidump(s) can be generated synchronously or asynchronously, specified in
+  // argument aAsync. When the operation completes, aCallback is invoked, where
+  // the callback argument denotes whether the operation succeeded.
+  void
   GenerateMinidumpAndPair(GeckoChildProcessHost* aChildProcess,
                           nsIFile* aMinidumpToPair,
-                          const nsACString& aPairName);
+                          const nsACString& aPairName,
+                          std::function<void(bool)>&& aCallback,
+                          bool aAsync);
 
   // This is a static helper function to notify the crash service that a
   // crash has occurred. When PCrashReporter is removed, we can make this
   // a member function. This can be called from any thread, and if not
   // called from the main thread, will post a synchronous message to the
   // main thread.
   static void NotifyCrashService(
     GeckoProcessType aProcessType,
@@ -89,16 +149,17 @@ public:
   }
 #endif
 
 private:
   static void AsyncAddCrash(int32_t aProcessType, int32_t aCrashType,
                             const nsString& aChildDumpID);
 
 private:
+  CallbackWrapper<bool> mCreateMinidumpCallback;
   GeckoProcessType mProcessType;
   Shmem mShmem;
   ThreadId mThreadId;
   time_t mStartTime;
 #ifdef MOZ_CRASHREPORTER
   AnnotationTable mExtraNotes;
 #endif
   nsString mDumpID;