Bug 1038342 - Add a Shutdown watchdog. r=froydnj, r=ted, sr=bsmedberg
authorDavid Rajchenbach-Teller <dteller@mozilla.com>
Tue, 26 Aug 2014 14:54:43 +0200
changeset 223598 b1223e628e8f6e64f3b045e14dd28602db7a301b
parent 223597 d9c0f0fa42dceef2c1135fa47e9fe44af51197a1
child 223599 cb96975c76817d0a2867a90a4076e2174d5aa934
push id3979
push userraliiev@mozilla.com
push dateMon, 13 Oct 2014 16:35:44 +0000
treeherdermozilla-beta@30f2cc610691 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj, ted, bsmedberg
bugs1038342
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1038342 - Add a Shutdown watchdog. r=froydnj, r=ted, sr=bsmedberg
toolkit/components/build/nsToolkitCompsModule.cpp
toolkit/components/moz.build
toolkit/components/terminator/moz.build
toolkit/components/terminator/nsTerminator.cpp
toolkit/components/terminator/nsTerminator.h
toolkit/components/terminator/terminator.manifest
toolkit/crashreporter/test/unit/test_crash_terminator.js
toolkit/crashreporter/test/unit/xpcshell.ini
--- a/toolkit/components/build/nsToolkitCompsModule.cpp
+++ b/toolkit/components/build/nsToolkitCompsModule.cpp
@@ -1,14 +1,15 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "mozilla/ModuleUtils.h"
 #include "nsAppStartup.h"
+#include "nsTerminator.h"
 #include "nsUserInfo.h"
 #include "nsToolkitCompsCID.h"
 #include "nsFindService.h"
 #if defined(USE_MOZ_UPDATER)
 #include "nsUpdateDriver.h"
 #endif
 
 #if !defined(MOZ_DISABLE_PARENTAL_CONTROLS)
@@ -37,16 +38,18 @@
 #include "mozilla/NativeOSFileInternals.h"
 #include "mozilla/AddonPathService.h"
 
 using namespace mozilla;
 
 /////////////////////////////////////////////////////////////////////////////
 
 NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsAppStartup, Init)
+NS_GENERIC_FACTORY_CONSTRUCTOR(nsTerminator)
+
 NS_GENERIC_FACTORY_CONSTRUCTOR(nsUserInfo)
 NS_GENERIC_FACTORY_CONSTRUCTOR(nsFindService)
 
 #if !defined(MOZ_DISABLE_PARENTAL_CONTROLS)
 NS_GENERIC_FACTORY_CONSTRUCTOR(nsParentalControlsService)
 #endif
 
 NS_GENERIC_FACTORY_CONSTRUCTOR(nsAlertsService)
@@ -90,16 +93,17 @@ NS_GENERIC_FACTORY_CONSTRUCTOR(nsBrowser
 NS_GENERIC_FACTORY_CONSTRUCTOR(nsUpdateProcessor)
 #endif
 NS_GENERIC_FACTORY_CONSTRUCTOR(FinalizationWitnessService)
 NS_GENERIC_FACTORY_CONSTRUCTOR(NativeOSFileInternalsService)
 
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(AddonPathService, AddonPathService::GetInstance)
 
 NS_DEFINE_NAMED_CID(NS_TOOLKIT_APPSTARTUP_CID);
+NS_DEFINE_NAMED_CID(NS_TOOLKIT_TERMINATOR_CID);
 NS_DEFINE_NAMED_CID(NS_USERINFO_CID);
 NS_DEFINE_NAMED_CID(NS_ALERTSSERVICE_CID);
 #if !defined(MOZ_DISABLE_PARENTAL_CONTROLS)
 NS_DEFINE_NAMED_CID(NS_PARENTALCONTROLSSERVICE_CID);
 #endif
 NS_DEFINE_NAMED_CID(NS_DOWNLOADMANAGER_CID);
 NS_DEFINE_NAMED_CID(NS_DOWNLOADPLATFORM_CID);
 NS_DEFINE_NAMED_CID(NS_DOWNLOAD_CID);
@@ -117,16 +121,17 @@ NS_DEFINE_NAMED_CID(NS_BROWSERSTATUSFILT
 NS_DEFINE_NAMED_CID(NS_UPDATEPROCESSOR_CID);
 #endif
 NS_DEFINE_NAMED_CID(FINALIZATIONWITNESSSERVICE_CID);
 NS_DEFINE_NAMED_CID(NATIVE_OSFILE_INTERNALS_SERVICE_CID);
 NS_DEFINE_NAMED_CID(NS_ADDON_PATH_SERVICE_CID);
 
 static const Module::CIDEntry kToolkitCIDs[] = {
   { &kNS_TOOLKIT_APPSTARTUP_CID, false, nullptr, nsAppStartupConstructor },
+  { &kNS_TOOLKIT_TERMINATOR_CID, false, nullptr, nsTerminatorConstructor },
   { &kNS_USERINFO_CID, false, nullptr, nsUserInfoConstructor },
   { &kNS_ALERTSSERVICE_CID, false, nullptr, nsAlertsServiceConstructor },
 #if !defined(MOZ_DISABLE_PARENTAL_CONTROLS)
   { &kNS_PARENTALCONTROLSSERVICE_CID, false, nullptr, nsParentalControlsServiceConstructor },
 #endif
   { &kNS_DOWNLOADMANAGER_CID, false, nullptr, nsDownloadManagerConstructor },
   { &kNS_DOWNLOADPLATFORM_CID, false, nullptr, DownloadPlatformConstructor },
   { &kNS_DOWNLOAD_CID, false, nullptr, nsDownloadProxyConstructor },
@@ -146,16 +151,17 @@ static const Module::CIDEntry kToolkitCI
   { &kFINALIZATIONWITNESSSERVICE_CID, false, nullptr, FinalizationWitnessServiceConstructor },
   { &kNATIVE_OSFILE_INTERNALS_SERVICE_CID, false, nullptr, NativeOSFileInternalsServiceConstructor },
   { &kNS_ADDON_PATH_SERVICE_CID, false, nullptr, AddonPathServiceConstructor },
   { nullptr }
 };
 
 static const Module::ContractIDEntry kToolkitContracts[] = {
   { NS_APPSTARTUP_CONTRACTID, &kNS_TOOLKIT_APPSTARTUP_CID },
+  { NS_TOOLKIT_TERMINATOR_CONTRACTID, &kNS_TOOLKIT_TERMINATOR_CID },
   { NS_USERINFO_CONTRACTID, &kNS_USERINFO_CID },
   { NS_ALERTSERVICE_CONTRACTID, &kNS_ALERTSSERVICE_CID },
 #if !defined(MOZ_DISABLE_PARENTAL_CONTROLS)
   { NS_PARENTALCONTROLSSERVICE_CONTRACTID, &kNS_PARENTALCONTROLSSERVICE_CID },
 #endif
   { NS_DOWNLOADMANAGER_CONTRACTID, &kNS_DOWNLOADMANAGER_CID },
   { NS_DOWNLOADPLATFORM_CONTRACTID, &kNS_DOWNLOADPLATFORM_CID },
   { NS_TRANSFER_CONTRACTID, &kNS_DOWNLOAD_CID },
--- a/toolkit/components/moz.build
+++ b/toolkit/components/moz.build
@@ -37,16 +37,17 @@ DIRS += [
     'promiseworker',
     'prompts',
     'protobuf',
     'reflect',
     'sqlite',
     'startup',
     'statusfilter',
     'telemetry',
+    'terminator',
     'thumbnails',
     'typeaheadfind',
     'urlformatter',
     'viewconfig',
     'viewsource',
     'workerloader',
     'workerlz4',
     'xulstore'
new file mode 100644
--- /dev/null
+++ b/toolkit/components/terminator/moz.build
@@ -0,0 +1,19 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+SOURCES += [
+    'nsTerminator.cpp',
+]
+
+EXPORTS += [
+    'nsTerminator.h',
+]
+
+EXTRA_COMPONENTS += [
+    'terminator.manifest',
+]
+
+FINAL_LIBRARY = 'xul'
new file mode 100644
--- /dev/null
+++ b/toolkit/components/terminator/nsTerminator.cpp
@@ -0,0 +1,205 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * A watchdog designed to terminate shutdown if it lasts too long.
+ *
+ * This watchdog is designed as a worst-case problem container for the
+ * common case in which Firefox just won't shutdown.
+ *
+ * We spawn a thread during quit-application. If any of the shutdown
+ * steps takes more than n milliseconds (63000 by default), kill the
+ * process as fast as possible, without any cleanup.
+ */
+
+#include "nsTerminator.h"
+
+#include "prthread.h"
+#include "nsString.h"
+#include "nsServiceManagerUtils.h"
+
+#include "nsIObserverService.h"
+#include "nsIPrefService.h"
+#if defined(MOZ_CRASHREPORTER)
+#include "nsExceptionHandler.h"
+#endif
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/Services.h"
+#include "mozilla/UniquePtr.h"
+#include "mozilla/unused.h"
+
+// Normally, the number of milliseconds that AsyncShutdown waits until
+// it decides to crash is specified as a preference. We use the
+// following value as a fallback if for some reason the preference is
+// absent.
+#define FALLBACK_ASYNCSHUTDOWN_CRASH_AFTER_MS 60000
+
+// Additional number of milliseconds to wait until we decide to exit
+// forcefully.
+#define ADDITIONAL_WAIT_BEFORE_CRASH_MS 3000
+
+// One second, in ticks.
+#define TICK_DURATION 1000
+
+namespace mozilla {
+
+namespace {
+
+/**
+ * Set to `true` by the main thread whenever we pass a shutdown phase,
+ * which means that the shutdown is still ongoing. Reset to `false` by
+ * the Terminator thread, once it has acknowledged the progress.
+ */
+Atomic<bool> gProgress(false);
+
+struct Options {
+  int32_t crashAfterMS;
+};
+
+void
+Run(void* arg)
+{
+  PR_SetCurrentThreadName("Shutdown Hang Terminator");
+
+  // Let's copy and deallocate options, that's one less leak to worry
+  // about.
+  UniquePtr<Options> options((Options*)arg);
+  int32_t crashAfterMS = options->crashAfterMS;
+  options = nullptr;
+
+  int32_t timeToLive = crashAfterMS;
+  while (true) {
+    //
+    // We do not want to sleep for the entire duration,
+    // as putting the computer to sleep would suddenly
+    // cause us to timeout on wakeup.
+    //
+    // Rather, we prefer sleeping for at most 1 second
+    // at a time. If the computer sleeps then wakes up,
+    // we have lost at most one second, which is much
+    // more reasonable.
+    //
+    PR_Sleep(TICK_DURATION);
+    if (gProgress.exchange(false)) {
+      // We have passed at least one shutdown phase while waiting.
+      // Shutdown is still alive, reset the countdown.
+      timeToLive = crashAfterMS;
+      continue;
+    }
+    timeToLive -= TICK_DURATION;
+    if (timeToLive >= 0) {
+      continue;
+    }
+
+    // Shutdown is apparently dead. Crash the process.
+    MOZ_CRASH("Shutdown too long, probably frozen, causing a crash.");
+  }
+}
+
+} // anonymous namespace
+
+
+static char const *const sObserverTopics[] = {
+  "quit-application",
+  "profile-change-teardown",
+  "profile-before-change",
+  "xpcom-will-shutdown",
+  "xpcom-shutdown",
+};
+
+NS_IMPL_ISUPPORTS(nsTerminator, nsIObserver)
+
+nsTerminator::nsTerminator()
+  : mInitialized(false)
+{
+}
+
+// During startup, register as an observer for all interesting topics.
+nsresult
+nsTerminator::SelfInit()
+{
+  nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
+  if (!os) {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  for (size_t i = 0; i < ArrayLength(sObserverTopics); ++i) {
+    DebugOnly<nsresult> rv = os->AddObserver(this, sObserverTopics[i], false);
+#if defined(DEBUG)
+    NS_WARN_IF(NS_FAILED(rv));
+#endif // defined(DEBUG)
+  }
+  return NS_OK;
+}
+
+// Actually launch the thread. This takes place at the first sign of shutdown.
+void
+nsTerminator::Start() {
+  // Determine how long we need to wait
+
+  int32_t crashAfterMS =
+    Preferences::GetInt("toolkit.asyncshutdown.crash_timeout",
+                        FALLBACK_ASYNCSHUTDOWN_CRASH_AFTER_MS);
+
+  // Add a little padding, to ensure that we do not crash before
+  // AsyncShutdown.
+  crashAfterMS += ADDITIONAL_WAIT_BEFORE_CRASH_MS;
+
+  UniquePtr<Options> options(new Options());
+  options->crashAfterMS = crashAfterMS;
+
+  // Allocate and start the thread.
+  // By design, it will never finish, nor be deallocated.
+  PRThread* thread = PR_CreateThread(
+    PR_SYSTEM_THREAD, /* This thread will not prevent the process from terminating */
+    Run,
+    options.release(),
+    PR_PRIORITY_LOW,
+    PR_GLOBAL_THREAD /* Make sure that the thread is never cooperatively scheduled */,
+    PR_UNJOINABLE_THREAD,
+    0 /* Use default stack size */
+  );
+
+  MOZ_ASSERT(thread);
+  mInitialized = true;
+}
+
+
+NS_IMETHODIMP
+nsTerminator::Observe(nsISupports *, const char *aTopic, const char16_t *)
+{
+  if (strcmp(aTopic, "profile-after-change") == 0) {
+    return SelfInit();
+  }
+
+  // Other notifications are shutdown-related.
+
+  // As we have seen examples in the wild of shutdown notifications
+  // not being sent (or not being sent in the expected order), we do
+  // not assume a specific order.
+  if (!mInitialized) {
+    Start();
+  }
+
+  // Inform the thread that we have advanced by one phase.
+  gProgress.exchange(true);
+
+#if defined(MOZ_CRASHREPORTER)
+  // In case of crash, we wish to know where in shutdown we are
+  unused << CrashReporter::AnnotateCrashReport(NS_LITERAL_CSTRING("ShutdownProgress"),
+                                               nsAutoCString(aTopic));
+#endif // defined(MOZ_CRASH_REPORTER)
+
+  nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
+  MOZ_RELEASE_ASSERT(os);
+  (void)os->RemoveObserver(this, aTopic);
+  return NS_OK;
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/toolkit/components/terminator/nsTerminator.h
@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsTerminator_h__
+#define nsTerminator_h__
+
+#include "nsISupports.h"
+#include "nsIObserver.h"
+
+namespace mozilla {
+
+class nsTerminator MOZ_FINAL: public nsIObserver {
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIOBSERVER
+
+  nsTerminator();
+
+private:
+  nsresult SelfInit();
+  void Start();
+
+  ~nsTerminator() {}
+
+  bool mInitialized;
+};
+
+}
+
+#define NS_TOOLKIT_TERMINATOR_CID { 0x2e59cc70, 0xf83a, 0x412f, \
+  { 0x89, 0xd4, 0x45, 0x38, 0x85, 0x83, 0x72, 0x17 } }
+#define NS_TOOLKIT_TERMINATOR_CONTRACTID "@mozilla.org/toolkit/shutdown-terminator;1"
+
+#endif // nsTerminator_h__
new file mode 100644
--- /dev/null
+++ b/toolkit/components/terminator/terminator.manifest
@@ -0,0 +1,2 @@
+category profile-after-change nsTerminator @mozilla.org/toolkit/shutdown-terminator;1
+
new file mode 100644
--- /dev/null
+++ b/toolkit/crashreporter/test/unit/test_crash_terminator.js
@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the Shutdown Terminator report errors correctly
+
+function setup_crash() {
+  Components.utils.import("resource://gre/modules/Services.jsm");
+
+  Services.prefs.setBoolPref("toolkit.terminator.testing", true);
+  Services.prefs.setIntPref("toolkit.asyncshutdown.crash_timeout", 10);
+
+  // Initialize the terminator
+  // (normally, this is done through the manifest file, but xpcshell
+  // doesn't take them into account).
+  let terminator = Components.classes["@mozilla.org/toolkit/shutdown-terminator;1"].
+    createInstance(Components.interfaces.nsIObserver);
+  terminator.observe(null, "profile-after-change", null);
+
+  // Inform the terminator that shutdown has started
+  // Pick an arbitrary notification
+  terminator.observe(null, "xpcom-will-shutdown", null);
+
+  dump("Waiting (actively) for the crash\n");
+  while(true) {
+    Services.tm.currentThread.processNextEvent(true);
+  }
+};
+
+
+function after_crash(mdump, extra) {
+  Assert.equal(extra.ShutdownProgress, "xpcom-will-shutdown");
+}
+
+function run_test() {
+  do_crash(setup_crash, after_crash);
+}
--- a/toolkit/crashreporter/test/unit/xpcshell.ini
+++ b/toolkit/crashreporter/test/unit/xpcshell.ini
@@ -27,8 +27,9 @@ run-if = os == 'win'
 
 [test_crashreporter_appmem.js]
 run-if = os == 'win' || os == 'linux'
 # we need to skip this due to bug 838613
 skip-if = os=='linux' && bits==32
 
 [test_crash_AsyncShutdown.js]
 [test_event_files.js]
+[test_crash_terminator.js]