Bug 1151318 - Add quirks flag to help Unity plugin release mouse capture. r=jimm, a=lizzard
authorAaron Klotz <aklotz@mozilla.com>
Wed, 20 May 2015 13:13:16 -0600
changeset 266186 a75365b95a17
parent 266185 516ef88d8790
child 266187 f16daa2effd1
push id4780
push userryanvm@gmail.com
push date2015-06-04 17:55 +0000
treeherdermozilla-beta@62bb5056f458 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjimm, lizzard
bugs1151318
milestone39.0
Bug 1151318 - Add quirks flag to help Unity plugin release mouse capture. r=jimm, a=lizzard
dom/plugins/base/nsPluginHost.cpp
dom/plugins/base/nsPluginHost.h
dom/plugins/ipc/PluginInstanceChild.cpp
dom/plugins/ipc/PluginInstanceChild.h
dom/plugins/ipc/PluginModuleChild.cpp
dom/plugins/ipc/PluginModuleChild.h
--- a/dom/plugins/base/nsPluginHost.cpp
+++ b/dom/plugins/base/nsPluginHost.cpp
@@ -1648,16 +1648,20 @@ nsPluginHost::GetSpecialType(const nsACS
     NS_WARNING("You are loading RealPlayer");
     return eSpecialType_RealPlayer;
   }
 
   if (aMIMEType.LowerCaseEqualsASCII("application/pdf")) {
     return eSpecialType_PDF;
   }
 
+  if (aMIMEType.LowerCaseEqualsASCII("application/vnd.unity")) {
+    return eSpecialType_Unity;
+  }
+
   // Java registers variants of its MIME with parameters, e.g.
   // application/x-java-vm;version=1.3
   const nsACString &noParam = Substring(aMIMEType, 0, aMIMEType.FindChar(';'));
 
   // The java mime pref may well not be one of these,
   // e.g. application/x-java-test used in the test suite
   nsAdoptingCString javaMIME = Preferences::GetCString(kPrefJavaMIME);
   if ((!javaMIME.IsEmpty() && noParam.LowerCaseEqualsASCII(javaMIME)) ||
--- a/dom/plugins/base/nsPluginHost.h
+++ b/dom/plugins/base/nsPluginHost.h
@@ -194,17 +194,19 @@ public:
                      // Binds to the <applet> tag, has various special
                      // rules around opening channels, codebase, ...
                      eSpecialType_Java,
                      // Some IPC quirks
                      eSpecialType_Silverlight,
                      // Native widget quirks
                      eSpecialType_PDF,
                      // Native widget quirks
-                     eSpecialType_RealPlayer };
+                     eSpecialType_RealPlayer,
+                     // Native widget quirks
+                     eSpecialType_Unity };
   static SpecialType GetSpecialType(const nsACString & aMIMEType);
 
   static nsresult PostPluginUnloadEvent(PRLibrary* aLibrary);
 
   void PluginCrashed(nsNPAPIPlugin* plugin,
                      const nsAString& pluginDumpID,
                      const nsAString& browserDumpID);
 
--- a/dom/plugins/ipc/PluginInstanceChild.cpp
+++ b/dom/plugins/ipc/PluginInstanceChild.cpp
@@ -27,16 +27,17 @@ using mozilla::gfx::SharedDIBSurface;
 #endif
 #include "gfxSharedImageSurface.h"
 #include "gfxUtils.h"
 #include "gfxAlphaRecovery.h"
 
 #include "mozilla/ArrayUtils.h"
 #include "mozilla/ipc/MessageChannel.h"
 #include "mozilla/AutoRestore.h"
+#include "mozilla/StaticPtr.h"
 #include "ImageContainer.h"
 
 using namespace mozilla;
 using mozilla::ipc::ProcessChild;
 using namespace mozilla::plugins;
 using namespace mozilla::layers;
 using namespace mozilla::gfx;
 using namespace std;
@@ -143,16 +144,18 @@ PluginInstanceChild::PluginInstanceChild
 #if defined(OS_WIN)
     , mPluginWindowHWND(0)
     , mPluginWndProc(0)
     , mPluginParentHWND(0)
     , mCachedWinlessPluginHWND(0)
     , mWinlessPopupSurrogateHWND(0)
     , mWinlessThrottleOldWndProc(0)
     , mWinlessHiddenMsgHWND(0)
+    , mUnityGetMessageHook(NULL)
+    , mUnitySendMessageHook(NULL)
 #endif // OS_WIN
     , mAsyncCallMutex("PluginInstanceChild::mAsyncCallMutex")
 #if defined(MOZ_WIDGET_COCOA)
 #if defined(__i386__)
     , mEventModel(NPEventModelCarbon)
 #endif
     , mShColorSpace(nullptr)
     , mShContext(nullptr)
@@ -189,23 +192,29 @@ PluginInstanceChild::PluginInstanceChild
     mWsInfo.display = DefaultXDisplay();
 #endif
 #endif // MOZ_X11 && XP_UNIX && !XP_MACOSX
 #if defined(OS_WIN)
     memset(&mAlphaExtract, 0, sizeof(mAlphaExtract));
 #endif // OS_WIN
 #if defined(OS_WIN)
     InitPopupMenuHook();
+    if (GetQuirks() & PluginModuleChild::QUIRK_UNITY_FIXUP_MOUSE_CAPTURE) {
+        SetUnityHooks();
+    }
 #endif // OS_WIN
 }
 
 PluginInstanceChild::~PluginInstanceChild()
 {
 #if defined(OS_WIN)
     NS_ASSERTION(!mPluginWindowHWND, "Destroying PluginInstanceChild without NPP_Destroy?");
+    if (GetQuirks() & PluginModuleChild::QUIRK_UNITY_FIXUP_MOUSE_CAPTURE) {
+        ClearUnityHooks();
+    }
 #endif
 #if defined(MOZ_WIDGET_COCOA)
     if (mShColorSpace) {
         ::CGColorSpaceRelease(mShColorSpace);
     }
     if (mShContext) {
         ::CGContextRelease(mShContext);
     }
@@ -1697,16 +1706,149 @@ PluginInstanceChild::HookSetWindowLongPt
         sUser32Intercept.AddHook("SetWindowLongA", reinterpret_cast<intptr_t>(SetWindowLongAHook),
                                  (void**) &sUser32SetWindowLongAHookStub);
     if (!sUser32SetWindowLongWHookStub)
         sUser32Intercept.AddHook("SetWindowLongW", reinterpret_cast<intptr_t>(SetWindowLongWHook),
                                  (void**) &sUser32SetWindowLongWHookStub);
 #endif
 }
 
+class SetCaptureHookData
+{
+public:
+    explicit SetCaptureHookData(HWND aHwnd)
+        : mHwnd(aHwnd)
+        , mHaveRect(false)
+    {
+        MOZ_ASSERT(aHwnd);
+        mHaveRect = !!GetClientRect(aHwnd, &mCaptureRect);
+    }
+
+    /**
+     * @return true if capture was released
+     */
+    bool HandleMouseMsg(const MSG& aMsg)
+    {
+        // If the window belongs to Unity, the mouse button is up, and the mouse
+        // has moved outside the client rect of the Unity window, release capture.
+        if (aMsg.hwnd != mHwnd || !mHaveRect) {
+            return false;
+        }
+        if (aMsg.message != WM_MOUSEMOVE && aMsg.message != WM_LBUTTONUP) {
+            return false;
+        }
+        if ((aMsg.message == WM_MOUSEMOVE && (aMsg.wParam & MK_LBUTTON))) {
+            return false;
+        }
+        POINT pt = { GET_X_LPARAM(aMsg.lParam), GET_Y_LPARAM(aMsg.lParam) };
+        if (PtInRect(&mCaptureRect, pt)) {
+            return false;
+        }
+        return !!ReleaseCapture();
+    }
+
+    bool IsUnityLosingCapture(const CWPSTRUCT& aInfo) const
+    {
+        return aInfo.message == WM_CAPTURECHANGED &&
+               aInfo.hwnd == mHwnd;
+    }
+
+private:
+    HWND mHwnd;
+    bool mHaveRect;
+    RECT mCaptureRect;
+};
+
+static StaticAutoPtr<SetCaptureHookData> sSetCaptureHookData;
+typedef HWND (WINAPI* User32SetCapture)(HWND);
+static User32SetCapture sUser32SetCaptureHookStub = nullptr;
+
+HWND WINAPI
+PluginInstanceChild::SetCaptureHook(HWND aHwnd)
+{
+    // Don't do anything unless aHwnd belongs to Unity
+    wchar_t className[256] = {0};
+    int numChars = GetClassNameW(aHwnd, className, ArrayLength(className));
+    NS_NAMED_LITERAL_STRING(unityClassName, "Unity.WebPlayer");
+    if (numChars == unityClassName.Length() && unityClassName == className) {
+        sSetCaptureHookData = new SetCaptureHookData(aHwnd);
+    }
+    return sUser32SetCaptureHookStub(aHwnd);
+}
+
+void
+PluginInstanceChild::SetUnityHooks()
+{
+    if (!(GetQuirks() & PluginModuleChild::QUIRK_UNITY_FIXUP_MOUSE_CAPTURE)) {
+        return;
+    }
+
+    sUser32Intercept.Init("user32.dll");
+    if (!sUser32SetCaptureHookStub) {
+        sUser32Intercept.AddHook("SetCapture",
+                                 reinterpret_cast<intptr_t>(SetCaptureHook),
+                                 (void**) &sUser32SetCaptureHookStub);
+    }
+    if (!mUnityGetMessageHook) {
+        mUnityGetMessageHook = SetWindowsHookEx(WH_GETMESSAGE,
+                                                &UnityGetMessageHookProc, NULL,
+                                                GetCurrentThreadId());
+    }
+    if (!mUnitySendMessageHook) {
+        mUnitySendMessageHook = SetWindowsHookEx(WH_CALLWNDPROC,
+                                                 &UnitySendMessageHookProc,
+                                                 NULL, GetCurrentThreadId());
+    }
+}
+
+void
+PluginInstanceChild::ClearUnityHooks()
+{
+    if (mUnityGetMessageHook) {
+        UnhookWindowsHookEx(mUnityGetMessageHook);
+        mUnityGetMessageHook = NULL;
+    }
+    if (mUnitySendMessageHook) {
+        UnhookWindowsHookEx(mUnitySendMessageHook);
+        mUnitySendMessageHook = NULL;
+    }
+    sSetCaptureHookData = nullptr;
+}
+
+LRESULT CALLBACK
+PluginInstanceChild::UnityGetMessageHookProc(int aCode, WPARAM aWparam,
+                                             LPARAM aLParam)
+{
+    if (aCode >= 0) {
+        MSG* info = reinterpret_cast<MSG*>(aLParam);
+        MOZ_ASSERT(info);
+        if (sSetCaptureHookData && sSetCaptureHookData->HandleMouseMsg(*info)) {
+            sSetCaptureHookData = nullptr;
+        }
+    }
+
+    return CallNextHookEx(0, aCode, aWparam, aLParam);
+}
+
+LRESULT CALLBACK
+PluginInstanceChild::UnitySendMessageHookProc(int aCode, WPARAM aWparam,
+                                              LPARAM aLParam)
+{
+    if (aCode >= 0) {
+        CWPSTRUCT* info = reinterpret_cast<CWPSTRUCT*>(aLParam);
+        MOZ_ASSERT(info);
+        if (sSetCaptureHookData &&
+            sSetCaptureHookData->IsUnityLosingCapture(*info)) {
+            sSetCaptureHookData = nullptr;
+        }
+    }
+
+    return CallNextHookEx(0, aCode, aWparam, aLParam);
+}
+
 /* windowless track popup menu helpers */
 
 BOOL
 WINAPI
 PluginInstanceChild::TrackPopupHookProc(HMENU hMenu,
                                         UINT uFlags,
                                         int x,
                                         int y,
--- a/dom/plugins/ipc/PluginInstanceChild.h
+++ b/dom/plugins/ipc/PluginInstanceChild.h
@@ -287,16 +287,18 @@ private:
     void SizePluginWindow(int width, int height);
     int16_t WinlessHandleEvent(NPEvent& event);
     void CreateWinlessPopupSurrogate();
     void DestroyWinlessPopupSurrogate();
     void InitPopupMenuHook();
     void SetupFlashMsgThrottle();
     void UnhookWinlessFlashThrottle();
     void HookSetWindowLongPtr();
+    void SetUnityHooks();
+    void ClearUnityHooks();
     static inline bool SetWindowLongHookCheck(HWND hWnd,
                                                 int nIndex,
                                                 LONG_PTR newLong);
     void FlashThrottleMessage(HWND, UINT, WPARAM, LPARAM, bool);
     static LRESULT CALLBACK DummyWindowProc(HWND hWnd,
                                             UINT message,
                                             WPARAM wParam,
                                             LPARAM lParam);
@@ -306,16 +308,19 @@ private:
                                              LPARAM lParam);
     static BOOL WINAPI TrackPopupHookProc(HMENU hMenu,
                                           UINT uFlags,
                                           int x,
                                           int y,
                                           int nReserved,
                                           HWND hWnd,
                                           CONST RECT *prcRect);
+    static HWND WINAPI SetCaptureHook(HWND aHwnd);
+    static LRESULT CALLBACK UnityGetMessageHookProc(int aCode, WPARAM aWparam, LPARAM aLParam);
+    static LRESULT CALLBACK UnitySendMessageHookProc(int aCode, WPARAM aWparam, LPARAM aLParam);
     static BOOL CALLBACK EnumThreadWindowsCallback(HWND hWnd,
                                                    LPARAM aParam);
     static LRESULT CALLBACK WinlessHiddenFlashWndProc(HWND hWnd,
                                                       UINT message,
                                                       WPARAM wParam,
                                                       LPARAM lParam);
 #ifdef _WIN64
     static LONG_PTR WINAPI SetWindowLongPtrAHook(HWND hWnd,
@@ -397,16 +402,18 @@ private:
     WNDPROC mPluginWndProc;
     HWND mPluginParentHWND;
     int mNestedEventLevelDepth;
     HWND mCachedWinlessPluginHWND;
     HWND mWinlessPopupSurrogateHWND;
     nsIntPoint mPluginSize;
     WNDPROC mWinlessThrottleOldWndProc;
     HWND mWinlessHiddenMsgHWND;
+    HHOOK mUnityGetMessageHook;
+    HHOOK mUnitySendMessageHook;
 #endif
 
     friend class ChildAsyncCall;
 
     Mutex mAsyncCallMutex;
     nsTArray<ChildAsyncCall*> mPendingAsyncCalls;
     nsTArray<nsAutoPtr<ChildTimer> > mTimers;
 
--- a/dom/plugins/ipc/PluginModuleChild.cpp
+++ b/dom/plugins/ipc/PluginModuleChild.cpp
@@ -2152,16 +2152,22 @@ PluginModuleChild::InitQuirksModes(const
     NS_NAMED_LITERAL_CSTRING(quicktime, "QuickTime Plugin.plugin");
     if (specialType == nsPluginHost::eSpecialType_Flash) {
         mQuirks |= QUIRK_FLASH_AVOID_CGMODE_CRASHES;
         mQuirks |= QUIRK_ALLOW_OFFLINE_RENDERER;
     } else if (FindInReadable(quicktime, mPluginFilename)) {
         mQuirks |= QUIRK_ALLOW_OFFLINE_RENDERER;
     }
 #endif
+
+#ifdef OS_WIN
+    if (specialType == nsPluginHost::eSpecialType_Unity) {
+        mQuirks |= QUIRK_UNITY_FIXUP_MOUSE_CAPTURE;
+    }
+#endif
 }
 
 bool
 PluginModuleChild::RecvPPluginInstanceConstructor(PPluginInstanceChild* aActor,
                                                   const nsCString& aMimeType,
                                                   const uint16_t& aMode,
                                                   InfallibleTArray<nsCString>&& aNames,
                                                   InfallibleTArray<nsCString>&& aValues)
--- a/dom/plugins/ipc/PluginModuleChild.h
+++ b/dom/plugins/ipc/PluginModuleChild.h
@@ -282,16 +282,18 @@ public:
         // in CoreGraphics mode:  The Flash plugin sometimes accesses the
         // CGContextRef we pass to it in NPP_HandleEvent(NPCocoaEventDrawRect)
         // outside of that call.  See bug 804606.
         QUIRK_FLASH_AVOID_CGMODE_CRASHES                = 1 << 10,
         // Work around a Flash bug where it fails to check the error code of a
         // NPN_GetValue(NPNVdocumentOrigin) call before trying to dereference
         // its char* output.
         QUIRK_FLASH_RETURN_EMPTY_DOCUMENT_ORIGIN        = 1 << 11,
+        // Win: Addresses a Unity bug with mouse capture.
+        QUIRK_UNITY_FIXUP_MOUSE_CAPTURE                 = 1 << 12,
     };
 
     int GetQuirks() { return mQuirks; }
 
     const PluginSettings& Settings() const { return mCachedSettings; }
 
 private:
     NPError DoNP_Initialize(const PluginSettings& aSettings);