Bug 1481009 Part 5 - Generate a minidump when reporting a fatal record/replay error, r=froydnj.
authorBrian Hackett <bhackett1024@gmail.com>
Mon, 13 Aug 2018 20:47:35 +0000
changeset 431343 305a6de8a4579c3a2546d693a927f143e4c4e906
parent 431342 d605332894bff2a8eee8667463e35b76e0f2f791
child 431344 41d3f63a86da3ac2a57c46afc40a99ab411539df
push id34437
push userebalazs@mozilla.com
push dateTue, 14 Aug 2018 09:31:09 +0000
treeherdermozilla-central@914b3b370ad0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1481009
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1481009 Part 5 - Generate a minidump when reporting a fatal record/replay error, r=froydnj.
toolkit/recordreplay/Callback.cpp
toolkit/recordreplay/DirtyMemoryHandler.cpp
toolkit/recordreplay/File.cpp
toolkit/recordreplay/MemorySnapshot.cpp
toolkit/recordreplay/ProcessRecordReplay.cpp
toolkit/recordreplay/Thread.cpp
toolkit/recordreplay/Trigger.cpp
toolkit/recordreplay/ipc/ChildIPC.cpp
toolkit/recordreplay/ipc/ChildIPC.h
toolkit/recordreplay/ipc/ChildInternal.h
toolkit/recordreplay/ipc/DisabledIPC.cpp
toolkit/recordreplay/moz.build
--- a/toolkit/recordreplay/Callback.cpp
+++ b/toolkit/recordreplay/Callback.cpp
@@ -122,19 +122,17 @@ PassThroughThreadEventsAllowCallbacks(co
       child::EndIdleTime();
     }
     thread->SetPassThrough(false);
     thread->Events().RecordOrReplayThreadEvent(ThreadEvent::CallbacksFinished);
   } else {
     while (true) {
       ThreadEvent ev = (ThreadEvent) thread->Events().ReadScalar();
       if (ev != ThreadEvent::ExecuteCallback) {
-        if (ev != ThreadEvent::CallbacksFinished) {
-          child::ReportFatalError("Unexpected event while replaying callback events");
-        }
+        MOZ_RELEASE_ASSERT(ev == ThreadEvent::CallbacksFinished);
         break;
       }
       size_t id = thread->Events().ReadScalar();
       ReplayInvokeCallback(id);
     }
   }
 }
 
--- a/toolkit/recordreplay/DirtyMemoryHandler.cpp
+++ b/toolkit/recordreplay/DirtyMemoryHandler.cpp
@@ -1,17 +1,17 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "DirtyMemoryHandler.h"
 
-#include "ipc/ChildIPC.h"
+#include "ipc/ChildInternal.h"
 #include "mozilla/Sprintf.h"
 #include "MemorySnapshot.h"
 #include "Thread.h"
 
 #include <mach/exc.h>
 #include <mach/mach.h>
 #include <mach/mach_vm.h>
 #include <sys/time.h>
@@ -60,21 +60,25 @@ DirtyMemoryExceptionHandlerThread(void*)
         request.body.Head.msgh_id == sExceptionId &&
         request.body.exception == EXC_BAD_ACCESS &&
         request.body.codeCnt == 2)
     {
       uint8_t* faultingAddress = (uint8_t*) request.body.code[1];
       if (HandleDirtyMemoryFault(faultingAddress)) {
         replyCode = KERN_SUCCESS;
       } else {
-        child::ReportFatalError("HandleDirtyMemoryFault failed %p %s", faultingAddress,
-                                gMozCrashReason ? gMozCrashReason : "");
+        child::MinidumpInfo info(request.body.exception,
+                                 request.body.code[0], request.body.code[1],
+                                 request.body.thread.name);
+        child::ReportFatalError(Some(info), "HandleDirtyMemoryFault failed %p %s",
+                                faultingAddress, gMozCrashReason ? gMozCrashReason : "");
       }
     } else {
-      child::ReportFatalError("DirtyMemoryExceptionHandlerThread mach_msg returned unexpected data");
+      child::ReportFatalError(Nothing(),
+                              "DirtyMemoryExceptionHandlerThread mach_msg returned unexpected data");
     }
 
     __Reply__exception_raise_t reply;
     reply.Head.msgh_bits = MACH_MSGH_BITS(MACH_MSGH_BITS_REMOTE(request.body.Head.msgh_bits), 0);
     reply.Head.msgh_size = sizeof(reply);
     reply.Head.msgh_remote_port = request.body.Head.msgh_remote_port;
     reply.Head.msgh_local_port = MACH_PORT_NULL;
     reply.Head.msgh_id = request.body.Head.msgh_id + 100;
--- a/toolkit/recordreplay/File.cpp
+++ b/toolkit/recordreplay/File.cpp
@@ -1,17 +1,17 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "File.h"
 
-#include "ipc/ChildIPC.h"
+#include "ipc/ChildInternal.h"
 #include "mozilla/Compression.h"
 #include "mozilla/Sprintf.h"
 #include "ProcessRewind.h"
 #include "SpinLock.h"
 
 #include <algorithm>
 
 namespace mozilla {
@@ -156,17 +156,18 @@ Stream::WriteScalar(size_t aValue)
 }
 
 void
 Stream::CheckInput(size_t aValue)
 {
   size_t oldValue = aValue;
   RecordOrReplayScalar(&oldValue);
   if (oldValue != aValue) {
-    child::ReportFatalError("Input Mismatch: Recorded: %zu Replayed %zu\n", oldValue, aValue);
+    child::ReportFatalError(Nothing(), "Input Mismatch: Recorded: %zu Replayed %zu\n",
+                            oldValue, aValue);
     Unreachable();
   }
 }
 
 void
 Stream::EnsureMemory(UniquePtr<char[]>* aBuf, size_t* aSize,
                      size_t aNeededSize, size_t aMaxSize, ShouldCopy aCopy)
 {
--- a/toolkit/recordreplay/MemorySnapshot.cpp
+++ b/toolkit/recordreplay/MemorySnapshot.cpp
@@ -405,17 +405,17 @@ AutoCountdown::~AutoCountdown()
 
 static void
 CountdownThreadMain(void*)
 {
   while (true) {
     if (gMemoryInfo->mCountdown && --gMemoryInfo->mCountdown == 0) {
       // When debugging hangs in the child process, we can break here in lldb
       // to inspect what the process is doing.
-      child::ReportFatalError("CountdownThread activated");
+      child::ReportFatalError(Nothing(), "CountdownThread activated");
     }
     ThreadYield();
   }
 }
 
 #endif // WANT_COUNTDOWN_THREAD
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -660,21 +660,23 @@ HandleDirtyMemoryFault(uint8_t* aAddress
   gMemoryInfo->mActiveDirty.insert(aAddress, DirtyPage(aAddress, original, executable));
   DirectUnprotectMemory(aAddress, PageSize, executable);
   return true;
 }
 
 void
 UnrecoverableSnapshotFailure()
 {
-  AutoSpinLock lock(gMemoryInfo->mTrackedRegionsLock);
-  DirectUnprotectMemory(PageBase(&errno), PageSize, false);
-  for (auto region : gMemoryInfo->mTrackedRegionsByAllocationOrder) {
-    DirectUnprotectMemory(region.mBase, region.mSize, region.mExecutable,
-                          /* aIgnoreFailures = */ true);
+  if (gMemoryInfo) {
+    AutoSpinLock lock(gMemoryInfo->mTrackedRegionsLock);
+    DirectUnprotectMemory(PageBase(&errno), PageSize, false);
+    for (auto region : gMemoryInfo->mTrackedRegionsByAllocationOrder) {
+      DirectUnprotectMemory(region.mBase, region.mSize, region.mExecutable,
+                            /* aIgnoreFailures = */ true);
+    }
   }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 // Initial Memory Region Processing
 ///////////////////////////////////////////////////////////////////////////////
 
 void
@@ -1059,24 +1061,24 @@ CheckFixedMemory(void* aAddress, size_t 
     // of which entirely contains this memory.
     AutoSpinLock lock(gMemoryInfo->mTrackedRegionsLock);
     for (size_t offset = 0; offset < aSize; offset += PageSize) {
       uint8_t* page = (uint8_t*)aAddress + offset;
       Maybe<AllocatedMemoryRegion> region =
         gMemoryInfo->mTrackedRegions.lookupClosestLessOrEqual(page);
       if (!region.isSome() ||
           !MemoryContains(region.ref().mBase, region.ref().mSize, page, PageSize)) {
-        child::ReportFatalError("Fixed memory is not tracked!");
+        MOZ_CRASH("Fixed memory is not tracked!");
       }
     }
   }
 
   // The memory should not be free.
   if (gFreeRegions.Intersects(aAddress, aSize)) {
-    child::ReportFatalError("Fixed memory is currently free!");
+    MOZ_CRASH("Fixed memory is currently free!");
   }
 }
 
 void
 RestoreWritableFixedMemory(void* aAddress, size_t aSize)
 {
   MOZ_RELEASE_ASSERT(aAddress == PageBase(aAddress));
   MOZ_RELEASE_ASSERT(aSize == RoundupSizeToPageBoundary(aSize));
--- a/toolkit/recordreplay/ProcessRecordReplay.cpp
+++ b/toolkit/recordreplay/ProcessRecordReplay.cpp
@@ -200,19 +200,19 @@ RecordReplayInterface_InternalRecordRepl
   thread->Events().CheckInput(aSize);
   thread->Events().RecordOrReplayBytes(aData, aSize);
 }
 
 MOZ_EXPORT void
 RecordReplayInterface_InternalInvalidateRecording(const char* aWhy)
 {
   if (IsRecording()) {
-    child::ReportFatalError("Recording invalidated: %s", aWhy);
+    child::ReportFatalError(Nothing(), "Recording invalidated: %s", aWhy);
   } else {
-    child::ReportFatalError("Recording invalidated while replaying: %s", aWhy);
+    child::ReportFatalError(Nothing(), "Recording invalidated while replaying: %s", aWhy);
   }
   Unreachable();
 }
 
 } // extern "C"
 
 // How many recording endpoints have been flushed to the recording.
 static size_t gNumEndpoints;
@@ -466,17 +466,18 @@ RecordReplayInterface_InternalRecordRepl
         }
       }
 
       {
         AutoPassThroughThreadEvents pt;
         SetCurrentStackString(text, text + strlen(text), sizeof(text) - strlen(text));
       }
 
-      child::ReportFatalError("Assertion Mismatch: Thread %d\n"
+      child::ReportFatalError(Nothing(),
+                              "Assertion Mismatch: Thread %d\n"
                               "Recorded: %s [%d,%d]\n"
                               "Replayed: %s [%d,%d]\n",
                               (int) thread->Id(), buffer, (int) streamPos, (int) progress, text,
                               (int) thread->Events().StreamPosition(),
                               (int) (thread->IsMainThread() ? *ExecutionProgressCounter() : 0));
       Unreachable();
     }
 
@@ -542,17 +543,18 @@ RecordReplayInterface_InternalRecordRepl
             }
           }
         }
         if (mismatches == MAX_MISMATCHES) {
           Print("Position ...\n");
         }
       }
 
-      child::ReportFatalError("Byte Comparison Check Failed: Position %d %d Length %d %d\n",
+      child::ReportFatalError(Nothing(),
+                              "Byte Comparison Check Failed: Position %d %d Length %d %d\n",
                               (int) streamPos, (int) thread->Events().StreamPosition(),
                               (int) oldSize, (int) aSize);
       Unreachable();
     }
 
     thread->RestoreBuffer(buffer);
   }
 #endif // INCLUDE_RECORD_REPLAY_ASSERTIONS
--- a/toolkit/recordreplay/Thread.cpp
+++ b/toolkit/recordreplay/Thread.cpp
@@ -251,17 +251,17 @@ Thread::StartThread(Callback aStart, voi
     // Look for an idle thread.
     for (id = MainThreadId + 1; id <= MaxRecordedThreadId; id++) {
       Thread* targetThread = Thread::GetById(id);
       if (!targetThread->mStart && !targetThread->mNeedsJoin) {
         break;
       }
     }
     if (id >= MaxRecordedThreadId) {
-      child::ReportFatalError("Too many threads");
+      child::ReportFatalError(Nothing(), "Too many threads");
     }
     MOZ_RELEASE_ASSERT(id <= MaxRecordedThreadId);
   }
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::CreateThread);
   thread->Events().RecordOrReplayScalar(&id);
 
   Thread* targetThread = GetById(id);
 
--- a/toolkit/recordreplay/Trigger.cpp
+++ b/toolkit/recordreplay/Trigger.cpp
@@ -183,17 +183,17 @@ RecordReplayInterface_ExecuteTriggers()
     thread->Events().RecordOrReplayThreadEvent(ThreadEvent::ExecuteTriggersFinished);
   } else {
     // Execute the same callbacks which were executed at this point while
     // recording.
     while (true) {
       ThreadEvent ev = (ThreadEvent) thread->Events().ReadScalar();
       if (ev != ThreadEvent::ExecuteTrigger) {
         if (ev != ThreadEvent::ExecuteTriggersFinished) {
-          child::ReportFatalError("ExecuteTrigger Mismatch");
+          child::ReportFatalError(Nothing(), "ExecuteTrigger Mismatch");
           Unreachable();
         }
         break;
       }
       size_t id = thread->Events().ReadScalar();
       InvokeTriggerCallback(id);
     }
   }
--- a/toolkit/recordreplay/ipc/ChildIPC.cpp
+++ b/toolkit/recordreplay/ipc/ChildIPC.cpp
@@ -9,16 +9,17 @@
 
 #include "ChildInternal.h"
 
 #include "base/message_loop.h"
 #include "base/task.h"
 #include "chrome/common/child_thread.h"
 #include "chrome/common/mach_ipc_mac.h"
 #include "ipc/Channel.h"
+#include "mac/handler/exception_handler.h"
 #include "mozilla/dom/ContentChild.h"
 #include "mozilla/layers/ImageDataSerializer.h"
 #include "mozilla/Sprintf.h"
 #include "mozilla/VsyncDispatcher.h"
 
 #include "InfallibleVector.h"
 #include "MemorySnapshot.h"
 #include "ParentInternal.h"
@@ -277,17 +278,17 @@ InitRecordingOrReplayingProcess(int* aAr
   MOZ_RELEASE_ASSERT(*aArgc >= 1);
   MOZ_RELEASE_ASSERT(gParentArgv.back() == nullptr);
 
   *aArgc = gParentArgv.length() - 1; // For the trailing null.
   *aArgv = gParentArgv.begin();
 
   // If we failed to initialize then report it to the user.
   if (gInitializationFailureMessage) {
-    ReportFatalError("%s", gInitializationFailureMessage);
+    ReportFatalError(Nothing(), "%s", gInitializationFailureMessage);
     Unreachable();
   }
 }
 
 base::ProcessId
 MiddlemanProcessId()
 {
   return gMiddlemanPid;
@@ -295,18 +296,31 @@ MiddlemanProcessId()
 
 base::ProcessId
 ParentProcessId()
 {
   return gParentPid;
 }
 
 void
-ReportFatalError(const char* aFormat, ...)
+ReportFatalError(const Maybe<MinidumpInfo>& aMinidump, const char* aFormat, ...)
 {
+  // Unprotect any memory which might be written while producing the minidump.
+  UnrecoverableSnapshotFailure();
+
+  AutoEnsurePassThroughThreadEvents pt;
+
+#ifdef MOZ_CRASHREPORTER
+  MinidumpInfo info = aMinidump.isSome()
+                      ? aMinidump.ref()
+                      : MinidumpInfo(EXC_CRASH, 1, 0, mach_thread_self());
+  google_breakpad::ExceptionHandler::WriteForwardedExceptionMinidump
+    (info.mExceptionType, info.mCode, info.mSubcode, info.mThread);
+#endif
+
   va_list ap;
   va_start(ap, aFormat);
   char buf[2048];
   VsprintfLiteral(buf, aFormat, ap);
   va_end(ap);
 
   // Construct a FatalErrorMessage on the stack, to avoid touching the heap.
   char msgBuf[4096];
@@ -318,18 +332,16 @@ ReportFatalError(const char* aFormat, ..
 
   // Don't take the message lock when sending this, to avoid touching the heap.
   gChannel->SendMessage(*msg);
 
   DirectPrint("***** Fatal Record/Replay Error *****\n");
   DirectPrint(buf);
   DirectPrint("\n");
 
-  UnrecoverableSnapshotFailure();
-
   // Block until we get a terminate message and die.
   Thread::WaitForeverNoIdle();
 }
 
 void
 NotifyFlushedRecording()
 {
   gChannel->SendMessage(RecordingFlushedMessage());
--- a/toolkit/recordreplay/ipc/ChildIPC.h
+++ b/toolkit/recordreplay/ipc/ChildIPC.h
@@ -55,19 +55,16 @@ void WaitForPaintToComplete();
 already_AddRefed<gfx::DrawTarget> DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize);
 
 // Notify the middleman that the recording was flushed.
 void NotifyFlushedRecording();
 
 // Notify the middleman about an AlwaysMarkMajorCheckpoints directive.
 void NotifyAlwaysMarkMajorCheckpoints();
 
-// Report a fatal error to the middleman process.
-void ReportFatalError(const char* aFormat, ...);
-
 // Mark a time span when the main thread is idle.
 void BeginIdleTime();
 void EndIdleTime();
 
 } // namespace child
 } // namespace recordreplay
 } // namespace mozilla
 
--- a/toolkit/recordreplay/ipc/ChildInternal.h
+++ b/toolkit/recordreplay/ipc/ChildInternal.h
@@ -73,16 +73,34 @@ void AfterCheckpoint(const CheckpointId&
 namespace child {
 
 void RespondToRequest(const js::CharBuffer& aBuffer);
 
 void HitCheckpoint(size_t aId, bool aRecordingEndpoint);
 
 void HitBreakpoint(bool aRecordingEndpoint, const uint32_t* aBreakpoints, size_t aNumBreakpoints);
 
+// Optional information about a crash that occurred. If not provided to
+// ReportFatalError, the current thread will be treated as crashed.
+struct MinidumpInfo
+{
+  int mExceptionType;
+  int mCode;
+  int mSubcode;
+  mach_port_t mThread;
+
+  MinidumpInfo(int aExceptionType, int aCode, int aSubcode, mach_port_t aThread)
+    : mExceptionType(aExceptionType), mCode(aCode), mSubcode(aSubcode), mThread(aThread)
+  {}
+};
+
+// Generate a minidump and report a fatal error to the middleman process.
+void ReportFatalError(const Maybe<MinidumpInfo>& aMinidumpInfo,
+                      const char* aFormat, ...);
+
 // Monitor used for various synchronization tasks.
 extern Monitor* gMonitor;
 
 } // namespace child
 
 } // namespace recordreplay
 } // namespace mozilla
 
--- a/toolkit/recordreplay/ipc/DisabledIPC.cpp
+++ b/toolkit/recordreplay/ipc/DisabledIPC.cpp
@@ -88,22 +88,16 @@ NotifyFlushedRecording()
 
 void
 NotifyAlwaysMarkMajorCheckpoints()
 {
   MOZ_CRASH();
 }
 
 void
-ReportFatalError(const char* aFormat, ...)
-{
-  MOZ_CRASH();
-}
-
-void
 BeginIdleTime()
 {
   MOZ_CRASH();
 }
 
 void
 EndIdleTime()
 {
--- a/toolkit/recordreplay/moz.build
+++ b/toolkit/recordreplay/moz.build
@@ -44,14 +44,16 @@ else:
     UNIFIED_SOURCES += [
         'ipc/DisabledIPC.cpp',
     ]
 
 LOCAL_INCLUDES += [
     '!/ipc/ipdl/_ipdlheaders',
     '/ipc/chromium/src',
     '/js/xpconnect/src',
+    '/toolkit/crashreporter/breakpad-client',
+    '/toolkit/crashreporter/google-breakpad/src',
 ]
 
 FINAL_LIBRARY = 'xul'
 
 with Files('**'):
     BUG_COMPONENT = ('Core', 'Web Replay')