Bug 1263774 - Include memory reports in content process crash reports. r=mccr8, r=aklotz, a=lizzard
authorKirk Steuber <ksteuber@mozilla.com>
Tue, 21 Jun 2016 11:45:25 -0700
changeset 341765 df92cb65023ba9c46beae00e98d6da882471b231
parent 341764 fdd969907fe7f521be4a410a7739caca9554790e
child 341766 c83de296c78d277a35e67cdc950162ee8e75bf39
push id1183
push userraliiev@mozilla.com
push dateMon, 05 Sep 2016 20:01:49 +0000
treeherdermozilla-release@3148731bed45 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmccr8, aklotz, lizzard
bugs1263774
milestone49.0a2
Bug 1263774 - Include memory reports in content process crash reports. r=mccr8, r=aklotz, a=lizzard MozReview-Commit-ID: 7y3GFBZxjsS
dom/ipc/ContentParent.cpp
dom/ipc/ContentParent.h
dom/ipc/PContent.ipdl
toolkit/crashreporter/docs/index.rst
toolkit/crashreporter/nsExceptionHandler.cpp
toolkit/crashreporter/nsExceptionHandler.h
toolkit/xre/nsAppRunner.cpp
xpcom/threads/nsThread.cpp
xpcom/threads/nsThread.h
--- a/dom/ipc/ContentParent.cpp
+++ b/dom/ipc/ContentParent.cpp
@@ -270,16 +270,20 @@ using namespace mozilla::system;
 #ifndef MOZ_SIMPLEPUSH
 #include "mozilla/dom/PushNotifier.h"
 #endif
 
 #ifdef XP_WIN
 #include "mozilla/widget/AudioSession.h"
 #endif
 
+#ifdef MOZ_CRASHREPORTER
+#include "nsThread.h"
+#endif
+
 #include "VRManagerParent.h"            // for VRManagerParent
 
 // For VP9Benchmark::sBenchmarkFpsPref
 #include "Benchmark.h"
 
 static NS_DEFINE_CID(kCClipboardCID, NS_CLIPBOARD_CID);
 
 #if defined(XP_WIN)
@@ -5882,16 +5886,25 @@ ContentParent::RecvNotifyPushSubscriptio
 {
 #ifndef MOZ_SIMPLEPUSH
   PushSubscriptionModifiedDispatcher dispatcher(aScope, aPrincipal);
   Unused << NS_WARN_IF(NS_FAILED(dispatcher.NotifyObservers()));
 #endif
   return true;
 }
 
+bool
+ContentParent::RecvNotifyLowMemory()
+{
+#ifdef MOZ_CRASHREPORTER
+  nsThread::SaveMemoryReportNearOOM(nsThread::ShouldSaveMemoryReport::kForceReport);
+#endif
+  return true;
+}
+
 } // namespace dom
 } // namespace mozilla
 
 NS_IMPL_ISUPPORTS(ParentIdleListener, nsIObserver)
 
 NS_IMETHODIMP
 ParentIdleListener::Observe(nsISupports*, const char* aTopic, const char16_t* aData)
 {
--- a/dom/ipc/ContentParent.h
+++ b/dom/ipc/ContentParent.h
@@ -1117,16 +1117,18 @@ private:
                                                InfallibleTArray<uint8_t>&& aData) override;
 
   virtual bool RecvNotifyPushSubscriptionChangeObservers(const nsCString& aScope,
                                                          const IPC::Principal& aPrincipal) override;
 
   virtual bool RecvNotifyPushSubscriptionModifiedObservers(const nsCString& aScope,
                                                            const IPC::Principal& aPrincipal) override;
 
+  virtual bool RecvNotifyLowMemory() override;
+
   // If you add strong pointers to cycle collected objects here, be sure to
   // release these objects in ShutDownProcess.  See the comment there for more
   // details.
 
   GeckoChildProcessHost* mSubprocess;
   ContentParent* mOpener;
 
   ContentParentId mChildID;
--- a/dom/ipc/PContent.ipdl
+++ b/dom/ipc/PContent.ipdl
@@ -1199,16 +1199,23 @@ parent:
                                       nsString messageId, uint8_t[] data);
 
     /**
      * Notify `push-subscription-change` observers in the parent.
      */
     async NotifyPushSubscriptionChangeObservers(nsCString scope,
                                                 Principal principal);
 
+    /**
+     * Tell the parent process that the child process is low on memory. This
+     * allows the parent process to save a memory report that can potentially be
+     * sent with a crash report from the content process.
+     */
+     async NotifyLowMemory();
+
 both:
      async AsyncMessage(nsString aMessage, CpowEntry[] aCpows,
                         Principal aPrincipal, ClonedMessageData aData);
 
     /**
      * Notify `push-subscription-modified` observers in the parent and child.
      */
     async NotifyPushSubscriptionModifiedObservers(nsCString scope,
--- a/toolkit/crashreporter/docs/index.rst
+++ b/toolkit/crashreporter/docs/index.rst
@@ -120,16 +120,33 @@ is responsible for calling
 appropriate crash annotations specific to the crash. All child-process
 crashes are annotated with a ``ProcessType`` annotation, such as "content" or
 "plugin".
 
 Submission of child process crashes is handled by application code. This
 code prompts the user to submit crashes in context-appropriate UI and then
 submits the crashes using ``CrashSubmit.jsm``.
 
+Memory Reports
+==============
+
+When a process detects that it is running low on memory, a memory report is
+saved. If the process crashes, the memory report will be included with the crash
+report. ``nsThread::SaveMemoryReportNearOOM()`` checks to see if the process is
+low on memory every 30 seconds at most and saves a report every 3 minutes at
+most. Since a child process cannot actually save to the hard drive, it instead
+notifies its parent process, which saves the report for it. If a crash does
+occur, the memory report is moved to the *pending* directory with the other dump
+data and an annotation is added to indicate the presence of the report. This
+happens in ``nsExceptionHandler.cpp``, but occurs in different functions
+depending on what process crashed. When the main process crashes, this happens
+in ``MinidumpCallback()``. When a child process crashes, it happens in
+``OnChildProcessDumpRequested()``, with the annotation being added in
+``WriteExtraData()``.
+
 Flash Process Crashes
 =====================
 
 On Windows Vista+, the Adobe Flash plugin creates two extra processes in its
 Firefox plugin to implement OS-level sandboxing. In order to catch crashes in
 these processes, Firefox injects a crash report handler into the process using the code at ``InjectCrashReporter.cpp``. When these crashes occur, the
 ProcessType=plugin annotation is present, and an additional annotation
 FlashProcessDump has the value "Sandbox" or "Broker".
--- a/toolkit/crashreporter/nsExceptionHandler.cpp
+++ b/toolkit/crashreporter/nsExceptionHandler.cpp
@@ -169,16 +169,17 @@ typedef std::string xpstring;
 
 #ifndef XP_LINUX
 static const XP_CHAR dumpFileExtension[] = XP_TEXT(".dmp");
 #endif
 
 static const XP_CHAR childCrashAnnotationBaseName[] = XP_TEXT("GeckoChildCrash");
 static const XP_CHAR extraFileExtension[] = XP_TEXT(".extra");
 static const XP_CHAR memoryReportExtension[] = XP_TEXT(".memory.json.gz");
+static xpstring *defaultMemoryReportPath = nullptr;
 
 // A whitelist of crash annotations which do not contain sensitive data
 // and are saved in the crash record and sent with Firefox Health Report.
 static char const * const kCrashEventAnnotations[] = {
   "AsyncShutdownTimeout",
   "BuildID",
   "TelemetryEnvironment",
   "ProductID",
@@ -2746,16 +2747,41 @@ SetMemoryReportFile(nsIFile* aFile)
   memoryReportPath = reinterpret_cast<wchar_t*>(ToNewUnicode(path));
 #else
   nsCString path;
   aFile->GetNativePath(path);
   memoryReportPath = ToNewCString(path);
 #endif
 }
 
+nsresult
+GetDefaultMemoryReportFile(nsIFile** aFile)
+{
+  nsCOMPtr<nsIFile> defaultMemoryReportFile;
+  if (!defaultMemoryReportPath) {
+    nsresult rv = NS_GetSpecialDirectory(NS_APP_PROFILE_DIR_STARTUP,
+                                         getter_AddRefs(defaultMemoryReportFile));
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+    defaultMemoryReportFile->AppendNative(NS_LITERAL_CSTRING("memory-report.json.gz"));
+    defaultMemoryReportPath = CreatePathFromFile(defaultMemoryReportFile);
+    if (!defaultMemoryReportPath) {
+      return NS_ERROR_FAILURE;
+    }
+  } else {
+    CreateFileFromPath(*defaultMemoryReportPath,
+                       getter_AddRefs(defaultMemoryReportFile));
+    if (!defaultMemoryReportFile) {
+      return NS_ERROR_FAILURE;
+    }
+  }
+  defaultMemoryReportFile.forget(aFile);
+  return NS_OK;
+}
 
 void
 SetTelemetrySessionId(const nsACString& id)
 {
   if (!gExceptionHandler) {
     return;
   }
   if (currentSessionId) {
@@ -2938,16 +2964,23 @@ static void
 WriteAnnotation(PRFileDesc* fd, const nsACString& key, const nsACString& value)
 {
   PR_Write(fd, key.BeginReading(), key.Length());
   PR_Write(fd, "=", 1);
   PR_Write(fd, value.BeginReading(), value.Length());
   PR_Write(fd, "\n", 1);
 }
 
+template<int N>
+void
+WriteLiteral(PRFileDesc* fd, const char (&str)[N])
+{
+  PR_Write(fd, str, N - 1);
+}
+
 static bool
 WriteExtraData(nsIFile* extraFile,
                const AnnotationTable& data,
                const Blacklist& blacklist,
                bool writeCrashTime=false,
                bool truncate=false)
 {
   PRFileDesc* fd;
@@ -2982,16 +3015,20 @@ WriteExtraData(nsIFile* extraFile,
     char uptimeTSString[64];
     SimpleNoCLibDtoA(uptimeTS, uptimeTSString, sizeof(uptimeTSString));
 
     WriteAnnotation(fd,
                     nsDependentCString("UptimeTS"),
                     nsDependentCString(uptimeTSString));
   }
 
+  if (memoryReportPath) {
+    WriteLiteral(fd, "ContainsMemoryReport=1\n");
+  }
+
   PR_Close(fd);
   return true;
 }
 
 bool
 AppendExtraData(nsIFile* extraFile, const AnnotationTable& data)
 {
   return WriteExtraData(extraFile, data, Blacklist());
@@ -3132,31 +3169,47 @@ WriteExtraForMinidump(nsIFile* minidump,
 
   extra.forget(extraFile);
 
   return true;
 }
 
 // It really only makes sense to call this function when
 // ShouldReport() is true.
+// Uses dumpFile's filename to generate memoryReport's filename (same name with
+// a different extension)
 static bool
-MoveToPending(nsIFile* dumpFile, nsIFile* extraFile)
+MoveToPending(nsIFile* dumpFile, nsIFile* extraFile, nsIFile* memoryReport)
 {
   nsCOMPtr<nsIFile> pendingDir;
   if (!GetPendingDir(getter_AddRefs(pendingDir)))
     return false;
 
   if (NS_FAILED(dumpFile->MoveTo(pendingDir, EmptyString()))) {
     return false;
   }
 
   if (extraFile && NS_FAILED(extraFile->MoveTo(pendingDir, EmptyString()))) {
     return false;
   }
 
+  if (memoryReport) {
+    nsAutoString leafName;
+    nsresult rv = dumpFile->GetLeafName(leafName);
+    if (NS_FAILED(rv)) {
+      return false;
+    }
+    // Generate the correct memory report filename from the dumpFile's name
+    leafName.Replace(leafName.Length() - 4, 4,
+                     static_cast<nsString>(CONVERT_XP_CHAR_TO_UTF16(memoryReportExtension)));
+    if (NS_FAILED(memoryReport->MoveTo(pendingDir, leafName))) {
+      return false;
+    }
+  }
+
   return true;
 }
 
 static void
 OnChildProcessDumpRequested(void* aContext,
 #ifdef XP_MACOSX
                             const ClientInfo& aClientInfo,
                             const xpstring& aFilePath
@@ -3192,18 +3245,24 @@ OnChildProcessDumpRequested(void* aConte
 #endif
 
   if (!WriteExtraForMinidump(minidump, pid,
                              Blacklist(kSubprocessBlacklist,
                                        ArrayLength(kSubprocessBlacklist)),
                              getter_AddRefs(extraFile)))
     return;
 
-  if (ShouldReport())
-    MoveToPending(minidump, extraFile);
+  if (ShouldReport()) {
+    nsCOMPtr<nsIFile> memoryReport;
+    if (memoryReportPath) {
+      CreateFileFromPath(memoryReportPath, getter_AddRefs(memoryReport));
+      MOZ_ASSERT(memoryReport);
+    }
+    MoveToPending(minidump, extraFile, memoryReport);
+  }
 
   {
 
 #ifdef MOZ_CRASHREPORTER_INJECTOR
     bool runCallback;
 #endif
     {
       MutexAutoLock lock(*dumpMapLock);
@@ -3475,20 +3534,26 @@ CheckForLastRunCrash()
   }
 
   nsCOMPtr<nsIFile> lastExtraFile;
   if (!GetExtraFileForMinidump(lastMinidumpFile,
                                getter_AddRefs(lastExtraFile))) {
     return false;
   }
 
+  nsCOMPtr<nsIFile> memoryReportFile;
+  nsresult rv = GetDefaultMemoryReportFile(getter_AddRefs(memoryReportFile));
+  if (NS_FAILED(rv) || NS_FAILED(memoryReportFile->Exists(&exists)) || !exists) {
+    memoryReportFile = nullptr;
+  }
+
   FindPendingDir();
 
-  // Move {dump,extra} to pending folder
-  if (!MoveToPending(lastMinidumpFile, lastExtraFile)) {
+  // Move {dump,extra,memory} to pending folder
+  if (!MoveToPending(lastMinidumpFile, lastExtraFile, memoryReportFile)) {
     return false;
   }
 
   lastRunCrashID = new nsString();
   return GetIDFromMinidump(lastMinidumpFile, *lastRunCrashID);
 }
 
 bool
@@ -3800,17 +3865,17 @@ bool TakeMinidump(nsIFile** aResult, boo
          true,
 #endif
          PairedDumpCallback,
          static_cast<void*>(aResult))) {
     return false;
   }
 
   if (aMoveToPending) {
-    MoveToPending(*aResult, nullptr);
+    MoveToPending(*aResult, nullptr, nullptr);
   }
   return true;
 }
 
 bool
 CreateMinidumpsAndPair(ProcessHandle aTargetPid,
                        ThreadId aTargetBlamedThread,
                        const nsACString& aIncomingPairName,
@@ -3864,18 +3929,18 @@ CreateMinidumpsAndPair(ProcessHandle aTa
     }
   } else {
     incomingDump = aIncomingDumpToPair;
   }
 
   RenameAdditionalHangMinidump(incomingDump, targetMinidump, aIncomingPairName);
 
   if (ShouldReport()) {
-    MoveToPending(targetMinidump, targetExtra);
-    MoveToPending(incomingDump, nullptr);
+    MoveToPending(targetMinidump, targetExtra, nullptr);
+    MoveToPending(incomingDump, nullptr, nullptr);
   }
 
   targetMinidump.forget(aMainDumpOut);
 
   return true;
 }
 
 bool
--- a/toolkit/crashreporter/nsExceptionHandler.h
+++ b/toolkit/crashreporter/nsExceptionHandler.h
@@ -46,16 +46,17 @@ nsresult UnsetExceptionHandler();
  *    the environment variable except for tests and other atypical setups.
  * 2. <profile>/crashes/events
  * 3. <UAppData>/Crash Reports/events
  */
 void SetUserAppDataDirectory(nsIFile* aDir);
 void SetProfileDirectory(nsIFile* aDir);
 void UpdateCrashEventsDir();
 void SetMemoryReportFile(nsIFile* aFile);
+nsresult GetDefaultMemoryReportFile(nsIFile** aFile);
 void SetTelemetrySessionId(const nsACString& id);
 
 /**
  * Get the path where crash event files should be written.
  */
 bool     GetCrashEventsDir(nsAString& aPath);
 
 bool     GetEnabled();
--- a/toolkit/xre/nsAppRunner.cpp
+++ b/toolkit/xre/nsAppRunner.cpp
@@ -1356,22 +1356,20 @@ nsXULAppInfo::UpdateCrashEventsDir()
 
 NS_IMETHODIMP
 nsXULAppInfo::SaveMemoryReport()
 {
   if (!CrashReporter::GetEnabled()) {
     return NS_ERROR_NOT_INITIALIZED;
   }
   nsCOMPtr<nsIFile> file;
-  nsresult rv = NS_GetSpecialDirectory(NS_APP_PROFILE_DIR_STARTUP,
-                                       getter_AddRefs(file));
+  nsresult rv = CrashReporter::GetDefaultMemoryReportFile(getter_AddRefs(file));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
-  file->AppendNative(NS_LITERAL_CSTRING("memory-report.json.gz"));
 
   nsString path;
   file->GetPath(path);
 
   nsCOMPtr<nsIMemoryInfoDumper> dumper =
     do_GetService("@mozilla.org/memory-info-dumper;1");
   if (NS_WARN_IF(!dumper)) {
     return NS_ERROR_UNEXPECTED;
--- a/xpcom/threads/nsThread.cpp
+++ b/xpcom/threads/nsThread.cpp
@@ -34,16 +34,17 @@
 #include "mozilla/TimeStamp.h"
 #include "mozilla/unused.h"
 #include "nsThreadSyncDispatch.h"
 #include "LeakRefPtr.h"
 
 #ifdef MOZ_CRASHREPORTER
 #include "nsServiceManagerUtils.h"
 #include "nsICrashReporter.h"
+#include "mozilla/dom/ContentChild.h"
 #endif
 
 #ifdef MOZ_NUWA_PROCESS
 #include "private/pprthred.h"
 #endif
 
 #ifdef XP_LINUX
 #include <sys/time.h>
@@ -517,40 +518,77 @@ nsThread::ThreadFunc(void* aArg)
   NS_RELEASE(self);
 }
 
 //-----------------------------------------------------------------------------
 
 #ifdef MOZ_CRASHREPORTER
 // Tell the crash reporter to save a memory report if our heuristics determine
 // that an OOM failure is likely to occur soon.
-static bool SaveMemoryReportNearOOM()
+// Memory usage will not be checked more than every 30 seconds or saved more
+// than every 3 minutes
+// If |aShouldSave == kForceReport|, a report will be saved regardless of
+// whether the process is low on memory or not. However, it will still not be
+// saved if a report was saved less than 3 minutes ago.
+bool
+nsThread::SaveMemoryReportNearOOM(ShouldSaveMemoryReport aShouldSave)
 {
-  bool needMemoryReport = false;
+  // Keep an eye on memory usage (cheap, ~7ms) somewhat frequently,
+  // but save memory reports (expensive, ~75ms) less frequently.
+  const size_t kLowMemoryCheckSeconds = 30;
+  const size_t kLowMemorySaveSeconds = 3 * 60;
 
+  static TimeStamp nextCheck = TimeStamp::NowLoRes()
+    + TimeDuration::FromSeconds(kLowMemoryCheckSeconds);
+  static bool recentlySavedReport = false; // Keeps track of whether a report
+                                           // was saved last time we checked
+
+  // Are we checking again too soon?
+  TimeStamp now = TimeStamp::NowLoRes();
+  if ((aShouldSave == ShouldSaveMemoryReport::kMaybeReport ||
+      recentlySavedReport) && now < nextCheck) {
+    return false;
+  }
+
+  bool needMemoryReport = (aShouldSave == ShouldSaveMemoryReport::kForceReport);
 #ifdef XP_WIN // XXX implement on other platforms as needed
-  const size_t LOWMEM_THRESHOLD_VIRTUAL = 200 * 1024 * 1024;
-  MEMORYSTATUSEX statex;
-  statex.dwLength = sizeof(statex);
-  if (GlobalMemoryStatusEx(&statex)) {
-    if (statex.ullAvailVirtual < LOWMEM_THRESHOLD_VIRTUAL) {
-      needMemoryReport = true;
+  // If the report is forced there is no need to check whether it is necessary
+  if (aShouldSave != ShouldSaveMemoryReport::kForceReport) {
+    const size_t LOWMEM_THRESHOLD_VIRTUAL = 200 * 1024 * 1024;
+    MEMORYSTATUSEX statex;
+    statex.dwLength = sizeof(statex);
+    if (GlobalMemoryStatusEx(&statex)) {
+      if (statex.ullAvailVirtual < LOWMEM_THRESHOLD_VIRTUAL) {
+        needMemoryReport = true;
+      }
     }
   }
 #endif
 
   if (needMemoryReport) {
-    nsCOMPtr<nsICrashReporter> cr =
-      do_GetService("@mozilla.org/toolkit/crash-reporter;1");
-    if (cr) {
-      cr->SaveMemoryReport();
+    if (XRE_IsContentProcess()) {
+      dom::ContentChild* cc = dom::ContentChild::GetSingleton();
+      if (cc) {
+        cc->SendNotifyLowMemory();
+      }
+    } else {
+      nsCOMPtr<nsICrashReporter> cr =
+        do_GetService("@mozilla.org/toolkit/crash-reporter;1");
+      if (cr) {
+        cr->SaveMemoryReport();
+      }
     }
+    recentlySavedReport = true;
+    nextCheck = now + TimeDuration::FromSeconds(kLowMemorySaveSeconds);
+  } else {
+    recentlySavedReport = false;
+    nextCheck = now + TimeDuration::FromSeconds(kLowMemoryCheckSeconds);
   }
 
-  return needMemoryReport;
+  return recentlySavedReport;
 }
 #endif
 
 #ifdef MOZ_CANARY
 int sCanaryOutputFD = -1;
 #endif
 
 nsThread::nsThread(MainThreadFlag aMainThread, uint32_t aStackSize)
@@ -1267,32 +1305,17 @@ nsThread::DoMainThreadSpecificProcessing
       } else {
         NS_WARNING("Can't get observer service!");
       }
     }
   }
 
 #ifdef MOZ_CRASHREPORTER
   if (!ShuttingDown()) {
-    // Keep an eye on memory usage (cheap, ~7ms) somewhat frequently,
-    // but save memory reports (expensive, ~75ms) less frequently.
-    const size_t LOW_MEMORY_CHECK_SECONDS = 30;
-    const size_t LOW_MEMORY_SAVE_SECONDS = 3 * 60;
-
-    static TimeStamp nextCheck = TimeStamp::NowLoRes()
-      + TimeDuration::FromSeconds(LOW_MEMORY_CHECK_SECONDS);
-
-    TimeStamp now = TimeStamp::NowLoRes();
-    if (now >= nextCheck) {
-      if (SaveMemoryReportNearOOM()) {
-        nextCheck = now + TimeDuration::FromSeconds(LOW_MEMORY_SAVE_SECONDS);
-      } else {
-        nextCheck = now + TimeDuration::FromSeconds(LOW_MEMORY_CHECK_SECONDS);
-      }
-    }
+    SaveMemoryReportNearOOM(ShouldSaveMemoryReport::kMaybeReport);
   }
 #endif
 }
 
 //-----------------------------------------------------------------------------
 
 NS_IMPL_ISUPPORTS(nsThread::nsNestedEventTarget, nsIEventTarget)
 
--- a/xpcom/threads/nsThread.h
+++ b/xpcom/threads/nsThread.h
@@ -76,16 +76,26 @@ public:
 
   uint32_t
   RecursionDepth() const;
 
   void ShutdownComplete(NotNull<struct nsThreadShutdownContext*> aContext);
 
   void WaitForAllAsynchronousShutdowns();
 
+#ifdef MOZ_CRASHREPORTER
+  enum class ShouldSaveMemoryReport
+  {
+    kMaybeReport,
+    kForceReport
+  };
+
+  static bool SaveMemoryReportNearOOM(ShouldSaveMemoryReport aShouldSave);
+#endif
+
 private:
   void DoMainThreadSpecificProcessing(bool aReallyWait);
 
 protected:
   class nsChainedEventQueue;
 
   class nsNestedEventTarget;
   friend class nsNestedEventTarget;