Bug 1263774 - Include memory reports in content process crash reports. r=mccr8, r=aklotz
authorKirk Steuber <ksteuber@mozilla.com>
Tue, 21 Jun 2016 11:45:25 -0700
changeset 342421 982855e49ded007bc554c3ec5d3652ed5b1789d5
parent 342420 6e96476b8d490a6fd5331cdc820230c0a7c81ff0
child 342422 c407d7f3ba963138172ac2061910b08e21a389f9
push id6389
push userraliiev@mozilla.com
push dateMon, 19 Sep 2016 13:38:22 +0000
treeherdermozilla-beta@01d67bfe6c81 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmccr8, aklotz
bugs1263774
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1263774 - Include memory reports in content process crash reports. r=mccr8, r=aklotz MozReview-Commit-ID: 7y3GFBZxjsS
dom/ipc/ContentParent.cpp
dom/ipc/ContentParent.h
dom/ipc/PContent.ipdl
toolkit/crashreporter/docs/index.rst
toolkit/crashreporter/nsExceptionHandler.cpp
toolkit/crashreporter/nsExceptionHandler.h
toolkit/xre/nsAppRunner.cpp
xpcom/threads/nsThread.cpp
xpcom/threads/nsThread.h
--- a/dom/ipc/ContentParent.cpp
+++ b/dom/ipc/ContentParent.cpp
@@ -268,16 +268,20 @@ using namespace mozilla::system;
 #ifndef MOZ_SIMPLEPUSH
 #include "mozilla/dom/PushNotifier.h"
 #endif
 
 #ifdef XP_WIN
 #include "mozilla/widget/AudioSession.h"
 #endif
 
+#ifdef MOZ_CRASHREPORTER
+#include "nsThread.h"
+#endif
+
 #include "VRManagerParent.h"            // for VRManagerParent
 
 // For VP9Benchmark::sBenchmarkFpsPref
 #include "Benchmark.h"
 
 static NS_DEFINE_CID(kCClipboardCID, NS_CLIPBOARD_CID);
 
 #if defined(XP_WIN)
@@ -5833,16 +5837,25 @@ ContentParent::RecvNotifyPushSubscriptio
 {
 #ifndef MOZ_SIMPLEPUSH
   PushSubscriptionModifiedDispatcher dispatcher(aScope, aPrincipal);
   Unused << NS_WARN_IF(NS_FAILED(dispatcher.NotifyObservers()));
 #endif
   return true;
 }
 
+bool
+ContentParent::RecvNotifyLowMemory()
+{
+#ifdef MOZ_CRASHREPORTER
+  nsThread::SaveMemoryReportNearOOM(nsThread::ShouldSaveMemoryReport::kForceReport);
+#endif
+  return true;
+}
+
 } // namespace dom
 } // namespace mozilla
 
 NS_IMPL_ISUPPORTS(ParentIdleListener, nsIObserver)
 
 NS_IMETHODIMP
 ParentIdleListener::Observe(nsISupports*, const char* aTopic, const char16_t* aData)
 {
--- a/dom/ipc/ContentParent.h
+++ b/dom/ipc/ContentParent.h
@@ -1175,16 +1175,18 @@ private:
                                                InfallibleTArray<uint8_t>&& aData) override;
 
   virtual bool RecvNotifyPushSubscriptionChangeObservers(const nsCString& aScope,
                                                          const IPC::Principal& aPrincipal) override;
 
   virtual bool RecvNotifyPushSubscriptionModifiedObservers(const nsCString& aScope,
                                                            const IPC::Principal& aPrincipal) override;
 
+  virtual bool RecvNotifyLowMemory() override;
+
   // If you add strong pointers to cycle collected objects here, be sure to
   // release these objects in ShutDownProcess.  See the comment there for more
   // details.
 
   GeckoChildProcessHost* mSubprocess;
   ContentParent* mOpener;
 
   ContentParentId mChildID;
--- a/dom/ipc/PContent.ipdl
+++ b/dom/ipc/PContent.ipdl
@@ -1202,16 +1202,23 @@ parent:
                                       nsString messageId, uint8_t[] data);
 
     /**
      * Notify `push-subscription-change` observers in the parent.
      */
     async NotifyPushSubscriptionChangeObservers(nsCString scope,
                                                 Principal principal);
 
+    /**
+     * Tell the parent process that the child process is low on memory. This
+     * allows the parent process to save a memory report that can potentially be
+     * sent with a crash report from the content process.
+     */
+     async NotifyLowMemory();
+
 both:
      async AsyncMessage(nsString aMessage, CpowEntry[] aCpows,
                         Principal aPrincipal, ClonedMessageData aData);
 
     /**
      * Notify `push-subscription-modified` observers in the parent and child.
      */
     async NotifyPushSubscriptionModifiedObservers(nsCString scope,
--- a/toolkit/crashreporter/docs/index.rst
+++ b/toolkit/crashreporter/docs/index.rst
@@ -120,16 +120,33 @@ is responsible for calling
 appropriate crash annotations specific to the crash. All child-process
 crashes are annotated with a ``ProcessType`` annotation, such as "content" or
 "plugin".
 
 Submission of child process crashes is handled by application code. This
 code prompts the user to submit crashes in context-appropriate UI and then
 submits the crashes using ``CrashSubmit.jsm``.
 
+Memory Reports
+==============
+
+When a process detects that it is running low on memory, a memory report is
+saved. If the process crashes, the memory report will be included with the crash
+report. ``nsThread::SaveMemoryReportNearOOM()`` checks to see if the process is
+low on memory every 30 seconds at most and saves a report every 3 minutes at
+most. Since a child process cannot actually save to the hard drive, it instead
+notifies its parent process, which saves the report for it. If a crash does
+occur, the memory report is moved to the *pending* directory with the other dump
+data and an annotation is added to indicate the presence of the report. This
+happens in ``nsExceptionHandler.cpp``, but occurs in different functions
+depending on what process crashed. When the main process crashes, this happens
+in ``MinidumpCallback()``. When a child process crashes, it happens in
+``OnChildProcessDumpRequested()``, with the annotation being added in
+``WriteExtraData()``.
+
 Flash Process Crashes
 =====================
 
 On Windows Vista+, the Adobe Flash plugin creates two extra processes in its
 Firefox plugin to implement OS-level sandboxing. In order to catch crashes in
 these processes, Firefox injects a crash report handler into the process using the code at ``InjectCrashReporter.cpp``. When these crashes occur, the
 ProcessType=plugin annotation is present, and an additional annotation
 FlashProcessDump has the value "Sandbox" or "Broker".
--- a/toolkit/crashreporter/nsExceptionHandler.cpp
+++ b/toolkit/crashreporter/nsExceptionHandler.cpp
@@ -169,16 +169,17 @@ typedef std::string xpstring;
 
 #ifndef XP_LINUX
 static const XP_CHAR dumpFileExtension[] = XP_TEXT(".dmp");
 #endif
 
 static const XP_CHAR childCrashAnnotationBaseName[] = XP_TEXT("GeckoChildCrash");
 static const XP_CHAR extraFileExtension[] = XP_TEXT(".extra");
 static const XP_CHAR memoryReportExtension[] = XP_TEXT(".memory.json.gz");
+static xpstring *defaultMemoryReportPath = nullptr;
 
 // A whitelist of crash annotations which do not contain sensitive data
 // and are saved in the crash record and sent with Firefox Health Report.
 static char const * const kCrashEventAnnotations[] = {
   "AsyncShutdownTimeout",
   "BuildID",
   "TelemetryEnvironment",
   "ProductID",
@@ -2746,16 +2747,41 @@ SetMemoryReportFile(nsIFile* aFile)
   memoryReportPath = reinterpret_cast<wchar_t*>(ToNewUnicode(path));
 #else
   nsCString path;
   aFile->GetNativePath(path);
   memoryReportPath = ToNewCString(path);
 #endif
 }
 
+nsresult
+GetDefaultMemoryReportFile(nsIFile** aFile)
+{
+  nsCOMPtr<nsIFile> defaultMemoryReportFile;
+  if (!defaultMemoryReportPath) {
+    nsresult rv = NS_GetSpecialDirectory(NS_APP_PROFILE_DIR_STARTUP,
+                                         getter_AddRefs(defaultMemoryReportFile));
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+    defaultMemoryReportFile->AppendNative(NS_LITERAL_CSTRING("memory-report.json.gz"));
+    defaultMemoryReportPath = CreatePathFromFile(defaultMemoryReportFile);
+    if (!defaultMemoryReportPath) {
+      return NS_ERROR_FAILURE;
+    }
+  } else {
+    CreateFileFromPath(*defaultMemoryReportPath,
+                       getter_AddRefs(defaultMemoryReportFile));
+    if (!defaultMemoryReportFile) {
+      return NS_ERROR_FAILURE;
+    }
+  }
+  defaultMemoryReportFile.forget(aFile);
+  return NS_OK;
+}
 
 void
 SetTelemetrySessionId(const nsACString& id)
 {
   if (!gExceptionHandler) {
     return;
   }
   if (currentSessionId) {
@@ -2938,16 +2964,23 @@ static void
 WriteAnnotation(PRFileDesc* fd, const nsACString& key, const nsACString& value)
 {
   PR_Write(fd, key.BeginReading(), key.Length());
   PR_Write(fd, "=", 1);
   PR_Write(fd, value.BeginReading(), value.Length());
   PR_Write(fd, "\n", 1);
 }
 
+template<int N>
+void
+WriteLiteral(PRFileDesc* fd, const char (&str)[N])
+{
+  PR_Write(fd, str, N - 1);
+}
+
 static bool
 WriteExtraData(nsIFile* extraFile,
                const AnnotationTable& data,
                const Blacklist& blacklist,
                bool writeCrashTime=false,
                bool truncate=false)
 {
   PRFileDesc* fd;
@@ -2982,16 +3015,20 @@ WriteExtraData(nsIFile* extraFile,
     char uptimeTSString[64];
     SimpleNoCLibDtoA(uptimeTS, uptimeTSString, sizeof(uptimeTSString));
 
     WriteAnnotation(fd,
                     nsDependentCString("UptimeTS"),
                     nsDependentCString(uptimeTSString));
   }
 
+  if (memoryReportPath) {
+    WriteLiteral(fd, "ContainsMemoryReport=1\n");
+  }
+
   PR_Close(fd);
   return true;
 }
 
 bool
 AppendExtraData(nsIFile* extraFile, const AnnotationTable& data)
 {
   return WriteExtraData(extraFile, data, Blacklist());
@@ -3132,31 +3169,47 @@ WriteExtraForMinidump(nsIFile* minidump,
 
   extra.forget(extraFile);
 
   return true;
 }
 
 // It really only makes sense to call this function when
 // ShouldReport() is true.
+// Uses dumpFile's filename to generate memoryReport's filename (same name with
+// a different extension)
 static bool
-MoveToPending(nsIFile* dumpFile, nsIFile* extraFile)
+MoveToPending(nsIFile* dumpFile, nsIFile* extraFile, nsIFile* memoryReport)
 {
   nsCOMPtr<nsIFile> pendingDir;
   if (!GetPendingDir(getter_AddRefs(pendingDir)))
     return false;
 
   if (NS_FAILED(dumpFile->MoveTo(pendingDir, EmptyString()))) {
     return false;
   }
 
   if (extraFile && NS_FAILED(extraFile->MoveTo(pendingDir, EmptyString()))) {
     return false;
   }
 
+  if (memoryReport) {
+    nsAutoString leafName;
+    nsresult rv = dumpFile->GetLeafName(leafName);
+    if (NS_FAILED(rv)) {
+      return false;
+    }
+    // Generate the correct memory report filename from the dumpFile's name
+    leafName.Replace(leafName.Length() - 4, 4,
+                     static_cast<nsString>(CONVERT_XP_CHAR_TO_UTF16(memoryReportExtension)));
+    if (NS_FAILED(memoryReport->MoveTo(pendingDir, leafName))) {
+      return false;
+    }
+  }
+
   return true;
 }
 
 static void
 OnChildProcessDumpRequested(void* aContext,
 #ifdef XP_MACOSX
                             const ClientInfo& aClientInfo,
                             const xpstring& aFilePath
@@ -3192,18 +3245,24 @@ OnChildProcessDumpRequested(void* aConte
 #endif
 
   if (!WriteExtraForMinidump(minidump, pid,
                              Blacklist(kSubprocessBlacklist,
                                        ArrayLength(kSubprocessBlacklist)),
                              getter_AddRefs(extraFile)))
     return;
 
-  if (ShouldReport())
-    MoveToPending(minidump, extraFile);
+  if (ShouldReport()) {
+    nsCOMPtr<nsIFile> memoryReport;
+    if (memoryReportPath) {
+      CreateFileFromPath(memoryReportPath, getter_AddRefs(memoryReport));
+      MOZ_ASSERT(memoryReport);
+    }
+    MoveToPending(minidump, extraFile, memoryReport);
+  }
 
   {
 
 #ifdef MOZ_CRASHREPORTER_INJECTOR
     bool runCallback;
 #endif
     {
       MutexAutoLock lock(*dumpMapLock);
@@ -3475,20 +3534,26 @@ CheckForLastRunCrash()
   }
 
   nsCOMPtr<nsIFile> lastExtraFile;
   if (!GetExtraFileForMinidump(lastMinidumpFile,
                                getter_AddRefs(lastExtraFile))) {
     return false;
   }
 
+  nsCOMPtr<nsIFile> memoryReportFile;
+  nsresult rv = GetDefaultMemoryReportFile(getter_AddRefs(memoryReportFile));
+  if (NS_FAILED(rv) || NS_FAILED(memoryReportFile->Exists(&exists)) || !exists) {
+    memoryReportFile = nullptr;
+  }
+
   FindPendingDir();
 
-  // Move {dump,extra} to pending folder
-  if (!MoveToPending(lastMinidumpFile, lastExtraFile)) {
+  // Move {dump,extra,memory} to pending folder
+  if (!MoveToPending(lastMinidumpFile, lastExtraFile, memoryReportFile)) {
     return false;
   }
 
   lastRunCrashID = new nsString();
   return GetIDFromMinidump(lastMinidumpFile, *lastRunCrashID);
 }
 
 bool
@@ -3800,17 +3865,17 @@ bool TakeMinidump(nsIFile** aResult, boo
          true,
 #endif
          PairedDumpCallback,
          static_cast<void*>(aResult))) {
     return false;
   }
 
   if (aMoveToPending) {
-    MoveToPending(*aResult, nullptr);
+    MoveToPending(*aResult, nullptr, nullptr);
   }
   return true;
 }
 
 bool
 CreateMinidumpsAndPair(ProcessHandle aTargetPid,
                        ThreadId aTargetBlamedThread,
                        const nsACString& aIncomingPairName,
@@ -3864,18 +3929,18 @@ CreateMinidumpsAndPair(ProcessHandle aTa
     }
   } else {
     incomingDump = aIncomingDumpToPair;
   }
 
   RenameAdditionalHangMinidump(incomingDump, targetMinidump, aIncomingPairName);
 
   if (ShouldReport()) {
-    MoveToPending(targetMinidump, targetExtra);
-    MoveToPending(incomingDump, nullptr);
+    MoveToPending(targetMinidump, targetExtra, nullptr);
+    MoveToPending(incomingDump, nullptr, nullptr);
   }
 
   targetMinidump.forget(aMainDumpOut);
 
   return true;
 }
 
 bool
--- a/toolkit/crashreporter/nsExceptionHandler.h
+++ b/toolkit/crashreporter/nsExceptionHandler.h
@@ -46,16 +46,17 @@ nsresult UnsetExceptionHandler();
  *    the environment variable except for tests and other atypical setups.
  * 2. <profile>/crashes/events
  * 3. <UAppData>/Crash Reports/events
  */
 void SetUserAppDataDirectory(nsIFile* aDir);
 void SetProfileDirectory(nsIFile* aDir);
 void UpdateCrashEventsDir();
 void SetMemoryReportFile(nsIFile* aFile);
+nsresult GetDefaultMemoryReportFile(nsIFile** aFile);
 void SetTelemetrySessionId(const nsACString& id);
 
 /**
  * Get the path where crash event files should be written.
  */
 bool     GetCrashEventsDir(nsAString& aPath);
 
 bool     GetEnabled();
--- a/toolkit/xre/nsAppRunner.cpp
+++ b/toolkit/xre/nsAppRunner.cpp
@@ -1374,22 +1374,20 @@ nsXULAppInfo::UpdateCrashEventsDir()
 
 NS_IMETHODIMP
 nsXULAppInfo::SaveMemoryReport()
 {
   if (!CrashReporter::GetEnabled()) {
     return NS_ERROR_NOT_INITIALIZED;
   }
   nsCOMPtr<nsIFile> file;
-  nsresult rv = NS_GetSpecialDirectory(NS_APP_PROFILE_DIR_STARTUP,
-                                       getter_AddRefs(file));
+  nsresult rv = CrashReporter::GetDefaultMemoryReportFile(getter_AddRefs(file));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
-  file->AppendNative(NS_LITERAL_CSTRING("memory-report.json.gz"));
 
   nsString path;
   file->GetPath(path);
 
   nsCOMPtr<nsIMemoryInfoDumper> dumper =
     do_GetService("@mozilla.org/memory-info-dumper;1");
   if (NS_WARN_IF(!dumper)) {
     return NS_ERROR_UNEXPECTED;
--- a/xpcom/threads/nsThread.cpp
+++ b/xpcom/threads/nsThread.cpp
@@ -34,16 +34,17 @@
 #include "mozilla/TimeStamp.h"
 #include "mozilla/unused.h"
 #include "nsThreadSyncDispatch.h"
 #include "LeakRefPtr.h"
 
 #ifdef MOZ_CRASHREPORTER
 #include "nsServiceManagerUtils.h"
 #include "nsICrashReporter.h"
+#include "mozilla/dom/ContentChild.h"
 #endif
 
 #ifdef MOZ_NUWA_PROCESS
 #include "private/pprthred.h"
 #endif
 
 #ifdef XP_LINUX
 #include <sys/time.h>
@@ -517,40 +518,77 @@ nsThread::ThreadFunc(void* aArg)
   NS_RELEASE(self);
 }
 
 //-----------------------------------------------------------------------------
 
 #ifdef MOZ_CRASHREPORTER
 // Tell the crash reporter to save a memory report if our heuristics determine
 // that an OOM failure is likely to occur soon.
-static bool SaveMemoryReportNearOOM()
+// Memory usage will not be checked more than every 30 seconds or saved more
+// than every 3 minutes
+// If |aShouldSave == kForceReport|, a report will be saved regardless of
+// whether the process is low on memory or not. However, it will still not be
+// saved if a report was saved less than 3 minutes ago.
+bool
+nsThread::SaveMemoryReportNearOOM(ShouldSaveMemoryReport aShouldSave)
 {
-  bool needMemoryReport = false;
+  // Keep an eye on memory usage (cheap, ~7ms) somewhat frequently,
+  // but save memory reports (expensive, ~75ms) less frequently.
+  const size_t kLowMemoryCheckSeconds = 30;
+  const size_t kLowMemorySaveSeconds = 3 * 60;
 
+  static TimeStamp nextCheck = TimeStamp::NowLoRes()
+    + TimeDuration::FromSeconds(kLowMemoryCheckSeconds);
+  static bool recentlySavedReport = false; // Keeps track of whether a report
+                                           // was saved last time we checked
+
+  // Are we checking again too soon?
+  TimeStamp now = TimeStamp::NowLoRes();
+  if ((aShouldSave == ShouldSaveMemoryReport::kMaybeReport ||
+      recentlySavedReport) && now < nextCheck) {
+    return false;
+  }
+
+  bool needMemoryReport = (aShouldSave == ShouldSaveMemoryReport::kForceReport);
 #ifdef XP_WIN // XXX implement on other platforms as needed
-  const size_t LOWMEM_THRESHOLD_VIRTUAL = 200 * 1024 * 1024;
-  MEMORYSTATUSEX statex;
-  statex.dwLength = sizeof(statex);
-  if (GlobalMemoryStatusEx(&statex)) {
-    if (statex.ullAvailVirtual < LOWMEM_THRESHOLD_VIRTUAL) {
-      needMemoryReport = true;
+  // If the report is forced there is no need to check whether it is necessary
+  if (aShouldSave != ShouldSaveMemoryReport::kForceReport) {
+    const size_t LOWMEM_THRESHOLD_VIRTUAL = 200 * 1024 * 1024;
+    MEMORYSTATUSEX statex;
+    statex.dwLength = sizeof(statex);
+    if (GlobalMemoryStatusEx(&statex)) {
+      if (statex.ullAvailVirtual < LOWMEM_THRESHOLD_VIRTUAL) {
+        needMemoryReport = true;
+      }
     }
   }
 #endif
 
   if (needMemoryReport) {
-    nsCOMPtr<nsICrashReporter> cr =
-      do_GetService("@mozilla.org/toolkit/crash-reporter;1");
-    if (cr) {
-      cr->SaveMemoryReport();
+    if (XRE_IsContentProcess()) {
+      dom::ContentChild* cc = dom::ContentChild::GetSingleton();
+      if (cc) {
+        cc->SendNotifyLowMemory();
+      }
+    } else {
+      nsCOMPtr<nsICrashReporter> cr =
+        do_GetService("@mozilla.org/toolkit/crash-reporter;1");
+      if (cr) {
+        cr->SaveMemoryReport();
+      }
     }
+    recentlySavedReport = true;
+    nextCheck = now + TimeDuration::FromSeconds(kLowMemorySaveSeconds);
+  } else {
+    recentlySavedReport = false;
+    nextCheck = now + TimeDuration::FromSeconds(kLowMemoryCheckSeconds);
   }
 
-  return needMemoryReport;
+  return recentlySavedReport;
 }
 #endif
 
 #ifdef MOZ_CANARY
 int sCanaryOutputFD = -1;
 #endif
 
 nsThread::nsThread(MainThreadFlag aMainThread, uint32_t aStackSize)
@@ -1267,32 +1305,17 @@ nsThread::DoMainThreadSpecificProcessing
       } else {
         NS_WARNING("Can't get observer service!");
       }
     }
   }
 
 #ifdef MOZ_CRASHREPORTER
   if (!ShuttingDown()) {
-    // Keep an eye on memory usage (cheap, ~7ms) somewhat frequently,
-    // but save memory reports (expensive, ~75ms) less frequently.
-    const size_t LOW_MEMORY_CHECK_SECONDS = 30;
-    const size_t LOW_MEMORY_SAVE_SECONDS = 3 * 60;
-
-    static TimeStamp nextCheck = TimeStamp::NowLoRes()
-      + TimeDuration::FromSeconds(LOW_MEMORY_CHECK_SECONDS);
-
-    TimeStamp now = TimeStamp::NowLoRes();
-    if (now >= nextCheck) {
-      if (SaveMemoryReportNearOOM()) {
-        nextCheck = now + TimeDuration::FromSeconds(LOW_MEMORY_SAVE_SECONDS);
-      } else {
-        nextCheck = now + TimeDuration::FromSeconds(LOW_MEMORY_CHECK_SECONDS);
-      }
-    }
+    SaveMemoryReportNearOOM(ShouldSaveMemoryReport::kMaybeReport);
   }
 #endif
 }
 
 //-----------------------------------------------------------------------------
 
 NS_IMPL_ISUPPORTS(nsThread::nsNestedEventTarget, nsIEventTarget)
 
--- a/xpcom/threads/nsThread.h
+++ b/xpcom/threads/nsThread.h
@@ -76,16 +76,26 @@ public:
 
   uint32_t
   RecursionDepth() const;
 
   void ShutdownComplete(NotNull<struct nsThreadShutdownContext*> aContext);
 
   void WaitForAllAsynchronousShutdowns();
 
+#ifdef MOZ_CRASHREPORTER
+  enum class ShouldSaveMemoryReport
+  {
+    kMaybeReport,
+    kForceReport
+  };
+
+  static bool SaveMemoryReportNearOOM(ShouldSaveMemoryReport aShouldSave);
+#endif
+
 private:
   void DoMainThreadSpecificProcessing(bool aReallyWait);
 
 protected:
   class nsChainedEventQueue;
 
   class nsNestedEventTarget;
   friend class nsNestedEventTarget;