Bug 874437 - Only enable ipc sync wait deferred Windows message handling for plugin protocols, everything else should use standard blocking waits. r=bsmedberg
authorJim Mathies <jmathies@mozilla.com>
Fri, 15 Aug 2014 12:12:37 -0500
changeset 200019 ce3f106abcd27d6b992627a2a5b3b8c1059d43c3
parent 200018 e5a893c6e64c40e5b7648cc80cfb2dce5b27594a
child 200020 356e4cb982faaf4b34d0e82497df1eb35b4606c5
push idunknown
push userunknown
push dateunknown
reviewersbsmedberg
bugs874437
milestone34.0a1
Bug 874437 - Only enable ipc sync wait deferred Windows message handling for plugin protocols, everything else should use standard blocking waits. r=bsmedberg
dom/plugins/ipc/PluginModuleChild.cpp
dom/plugins/ipc/PluginModuleParent.cpp
ipc/glue/MessageChannel.cpp
ipc/glue/MessageChannel.h
ipc/glue/WindowsMessageLoop.cpp
--- a/dom/plugins/ipc/PluginModuleChild.cpp
+++ b/dom/plugins/ipc/PluginModuleChild.cpp
@@ -138,16 +138,21 @@ PluginModuleChild::Init(const std::strin
                         base::ProcessHandle aParentProcessHandle,
                         MessageLoop* aIOLoop,
                         IPC::Channel* aChannel)
 {
     PLUGIN_LOG_DEBUG_METHOD;
 
     GetIPCChannel()->SetAbortOnError(true);
 
+    // Request Windows message deferral behavior on our channel. This
+    // applies to the top level and all sub plugin protocols since they
+    // all share the same channel.
+    GetIPCChannel()->SetChannelFlags(MessageChannel::REQUIRE_DEFERRED_MESSAGE_PROTECTION);
+
 #ifdef XP_WIN
     COMMessageFilter::Initialize(this);
 #endif
 
     NS_ASSERTION(aChannel, "need a channel");
 
     if (!InitGraphics())
         return false;
--- a/dom/plugins/ipc/PluginModuleParent.cpp
+++ b/dom/plugins/ipc/PluginModuleParent.cpp
@@ -98,16 +98,21 @@ PluginModuleParent::LoadModule(const cha
     if (!launched) {
         // We never reached open
         parent->mShutdown = true;
         return nullptr;
     }
     parent->Open(parent->mSubprocess->GetChannel(),
                  parent->mSubprocess->GetChildProcessHandle());
 
+    // Request Windows message deferral behavior on our channel. This
+    // applies to the top level and all sub plugin protocols since they
+    // all share the same channel.
+    parent->GetIPCChannel()->SetChannelFlags(MessageChannel::REQUIRE_DEFERRED_MESSAGE_PROTECTION);
+
     TimeoutChanged(CHILD_TIMEOUT_PREF, parent);
 
 #ifdef MOZ_CRASHREPORTER
     // If this fails, we're having IPC troubles, and we're doomed anyways.
     if (!CrashReporterParent::CreateCrashReporter(parent.get())) {
         parent->Close();
         return nullptr;
     }
--- a/ipc/glue/MessageChannel.cpp
+++ b/ipc/glue/MessageChannel.cpp
@@ -229,17 +229,18 @@ MessageChannel::MessageChannel(MessageLi
     mPendingUrgentReplies(0),
     mPendingRPCReplies(0),
     mCurrentRPCTransaction(0),
     mDispatchingSyncMessage(false),
     mDispatchingUrgentMessageCount(0),
     mRemoteStackDepthGuess(false),
     mSawInterruptOutMsg(false),
     mAbortOnError(false),
-    mBlockScripts(false)
+    mBlockScripts(false),
+    mFlags(REQUIRE_DEFAULT)
 {
     MOZ_COUNT_CTOR(ipc::MessageChannel);
 
 #ifdef OS_WIN
     mTopFrame = nullptr;
     mIsSyncWaitingOnNonMainThread = false;
 #endif
 
--- a/ipc/glue/MessageChannel.h
+++ b/ipc/glue/MessageChannel.h
@@ -58,23 +58,23 @@ class MessageChannel : HasResultCodes
     typedef mozilla::ipc::Transport Transport;
 
     explicit MessageChannel(MessageListener *aListener);
     ~MessageChannel();
 
     // "Open" from the perspective of the transport layer; the underlying
     // socketpair/pipe should already be created.
     //
-    // Returns true iff the transport layer was successfully connected,
+    // Returns true if the transport layer was successfully connected,
     // i.e., mChannelState == ChannelConnected.
     bool Open(Transport* aTransport, MessageLoop* aIOLoop=0, Side aSide=UnknownSide);
 
     // "Open" a connection to another thread in the same process.
     //
-    // Returns true iff the transport layer was successfully connected,
+    // Returns true if the transport layer was successfully connected,
     // i.e., mChannelState == ChannelConnected.
     //
     // For more details on the process of opening a channel between
     // threads, see the extended comment on this function
     // in MessageChannel.cpp.
     bool Open(MessageChannel *aTargetChan, MessageLoop *aTargetLoop, Side aSide);
 
     // Close the underlying transport channel.
@@ -84,16 +84,29 @@ class MessageChannel : HasResultCodes
     // for process links only, not thread links.
     void CloseWithError();
 
     void SetAbortOnError(bool abort)
     {
         mAbortOnError = true;
     }
 
+    // Misc. behavioral traits consumers can request for this channel
+    enum ChannelFlags {
+      REQUIRE_DEFAULT                         = 0,
+      // Windows: if this channel operates on the UI thread, indicates
+      // WindowsMessageLoop code should enable deferred native message
+      // handling to prevent deadlocks. Should only be used for protocols
+      // that manage child processes which might create native UI, like
+      // plugins.
+      REQUIRE_DEFERRED_MESSAGE_PROTECTION     = 1 << 0
+    };
+    void SetChannelFlags(ChannelFlags aFlags) { mFlags = aFlags; }
+    ChannelFlags GetChannelFlags() { return mFlags; }
+
     void BlockScripts();
 
     bool ShouldBlockScripts() const
     {
         return mBlockScripts;
     }
 
     // Asynchronously send a message to the other side of the channel
@@ -644,16 +657,19 @@ class MessageChannel : HasResultCodes
 #endif
 
     // Should the channel abort the process from the I/O thread when
     // a channel error occurs?
     bool mAbortOnError;
 
     // Should we prevent scripts from running while dispatching urgent messages?
     bool mBlockScripts;
+
+    // See SetChannelFlags
+    ChannelFlags mFlags;
 };
 
 bool
 ProcessingUrgentMessages();
 
 } // namespace ipc
 } // namespace mozilla
 
--- a/ipc/glue/WindowsMessageLoop.cpp
+++ b/ipc/glue/WindowsMessageLoop.cpp
@@ -652,42 +652,44 @@ InitUIThread()
   MOZ_ASSERT(gUIThreadId == GetCurrentThreadId(),
              "Called InitUIThread multiple times on different threads!");
 }
 
 } // namespace windows
 } // namespace ipc
 } // namespace mozilla
 
+// See SpinInternalEventLoop below
 MessageChannel::SyncStackFrame::SyncStackFrame(MessageChannel* channel, bool interrupt)
   : mInterrupt(interrupt)
   , mSpinNestedEvents(false)
   , mListenerNotified(false)
   , mChannel(channel)
   , mPrev(mChannel->mTopFrame)
   , mStaticPrev(sStaticTopFrame)
 {
-  // Only track stack frames when we are on the gui thread.
-  if (GetCurrentThreadId() != gUIThreadId) {
+  // Only track stack frames when Windows message deferral behavior
+  // is request for the channel.
+  if (!(mChannel->GetChannelFlags() & REQUIRE_DEFERRED_MESSAGE_PROTECTION)) {
     return;
   }
 
   mChannel->mTopFrame = this;
   sStaticTopFrame = this;
 
   if (!mStaticPrev) {
     NS_ASSERTION(!gNeuteredWindows, "Should only set this once!");
     gNeuteredWindows = new nsAutoTArray<HWND, 20>();
     NS_ASSERTION(gNeuteredWindows, "Out of memory!");
   }
 }
 
 MessageChannel::SyncStackFrame::~SyncStackFrame()
 {
-  if (GetCurrentThreadId() != gUIThreadId) {
+  if (!(mChannel->GetChannelFlags() & REQUIRE_DEFERRED_MESSAGE_PROTECTION)) {
     return;
   }
 
   NS_ASSERTION(this == mChannel->mTopFrame,
                "Mismatched interrupt stack frames");
   NS_ASSERTION(this == sStaticTopFrame,
                "Mismatched static Interrupt stack frames");
 
@@ -804,21 +806,19 @@ IsTimeoutExpired(PRIntervalTime aStart, 
 
 bool
 MessageChannel::WaitForSyncNotify()
 {
   mMonitor->AssertCurrentThreadOwns();
 
   MOZ_ASSERT(gUIThreadId, "InitUIThread was not called!");
 
-  // We jump through a lot of unique hoops in dealing with Windows message
-  // trapping to prevent re-entrancy when we are blocked waiting on a sync
-  // reply or new rpc in-call. However none of this overhead is needed when
-  // we aren't on the main (gui) thread. 
-  if (GetCurrentThreadId() != gUIThreadId) {
+  // Use a blocking wait if this channel does not require
+  // Windows message deferral behavior.
+  if (!(mFlags & REQUIRE_DEFERRED_MESSAGE_PROTECTION)) {
     PRIntervalTime timeout = (kNoTimeout == mTimeoutMs) ?
                              PR_INTERVAL_NO_TIMEOUT :
                              PR_MillisecondsToInterval(mTimeoutMs);
     PRIntervalTime waitStart = 0;
 
     if (timeout != PR_INTERVAL_NO_TIMEOUT) {
       waitStart = PR_IntervalNow();
     }
@@ -832,19 +832,18 @@ MessageChannel::WaitForSyncNotify()
     mIsSyncWaitingOnNonMainThread = false;
 
     // If the timeout didn't expire, we know we received an event. The
     // converse is not true.
     return WaitResponse(timeout == PR_INTERVAL_NO_TIMEOUT ?
                         false : IsTimeoutExpired(waitStart, timeout));
   }
 
-  NS_ASSERTION(GetCurrentThreadId() == gUIThreadId,
-               "Shouldn't be on a non-main thread in here!");
-
+  NS_ASSERTION(mFlags & REQUIRE_DEFERRED_MESSAGE_PROTECTION,
+               "Shouldn't be here for channels that don't use message deferral!");
   NS_ASSERTION(mTopFrame && !mTopFrame->mInterrupt,
                "Top frame is not a sync frame!");
 
   MonitorAutoUnlock unlock(*mMonitor);
 
   bool timedout = false;
 
   UINT_PTR timerId = 0;
@@ -956,31 +955,29 @@ MessageChannel::WaitForSyncNotify()
 
 bool
 MessageChannel::WaitForInterruptNotify()
 {
   mMonitor->AssertCurrentThreadOwns();
 
   MOZ_ASSERT(gUIThreadId, "InitUIThread was not called!");
 
-  // Re-use sync notification wait code when we aren't on the
-  // gui thread, which bypasses the gui thread hoops we jump
-  // through in dealing with Windows messaging.
-  if (GetCurrentThreadId() != gUIThreadId) {
+  // Re-use sync notification wait code if this channel does not require
+  // Windows message deferral behavior. 
+  if (!(mFlags & REQUIRE_DEFERRED_MESSAGE_PROTECTION)) {
     return WaitForSyncNotify();
   }
 
-  NS_ASSERTION(GetCurrentThreadId() == gUIThreadId,
-               "Shouldn't be on a non-main thread in here!");
-
   if (!InterruptStackDepth()) {
     // There is currently no way to recover from this condition.
     NS_RUNTIMEABORT("StackDepth() is 0 in call to MessageChannel::WaitForNotify!");
   }
 
+  NS_ASSERTION(mFlags & REQUIRE_DEFERRED_MESSAGE_PROTECTION,
+               "Shouldn't be here for channels that don't use message deferral!");
   NS_ASSERTION(mTopFrame && mTopFrame->mInterrupt,
                "Top frame is not a sync frame!");
 
   MonitorAutoUnlock unlock(*mMonitor);
 
   bool timedout = false;
 
   UINT_PTR timerId = 0;