Bug 1460022: Part 7 - Update plugin code to work with revised DLL interceptor interface; r=handyman
authorAaron Klotz <aklotz@mozilla.com>
Wed, 27 Jun 2018 11:50:50 -0600
changeset 424937 88494d07fbcd14ced0c83f00c12b4bae5d566b48
parent 424936 137f27a03ba3b56b87a698adc34cbcf4b2def52f
child 424938 528b5685d597db2680e356bbd67fc52e575e3853
push id104944
push useraklotz@mozilla.com
push dateWed, 04 Jul 2018 02:23:52 +0000
treeherdermozilla-inbound@ff25e66da36e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewershandyman
bugs1460022
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1460022: Part 7 - Update plugin code to work with revised DLL interceptor interface; r=handyman
dom/plugins/base/nsPluginNativeWindowWin.cpp
dom/plugins/ipc/FunctionHook.cpp
dom/plugins/ipc/FunctionHook.h
dom/plugins/ipc/PluginInstanceChild.cpp
--- a/dom/plugins/base/nsPluginNativeWindowWin.cpp
+++ b/dom/plugins/base/nsPluginNativeWindowWin.cpp
@@ -352,29 +352,33 @@ static WindowsDllInterceptor sUser32Inte
 typedef LONG_PTR
   (WINAPI *User32SetWindowLongPtrA)(HWND hWnd,
                                     int nIndex,
                                     LONG_PTR dwNewLong);
 typedef LONG_PTR
   (WINAPI *User32SetWindowLongPtrW)(HWND hWnd,
                                     int nIndex,
                                     LONG_PTR dwNewLong);
-static User32SetWindowLongPtrA sUser32SetWindowLongAHookStub = nullptr;
-static User32SetWindowLongPtrW sUser32SetWindowLongWHookStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongPtrA>
+  sUser32SetWindowLongAHookStub;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongPtrW>
+  sUser32SetWindowLongWHookStub;
 #else
 typedef LONG
 (WINAPI *User32SetWindowLongA)(HWND hWnd,
                                int nIndex,
                                LONG dwNewLong);
 typedef LONG
 (WINAPI *User32SetWindowLongW)(HWND hWnd,
                                int nIndex,
                                LONG dwNewLong);
-static User32SetWindowLongA sUser32SetWindowLongAHookStub = nullptr;
-static User32SetWindowLongW sUser32SetWindowLongWHookStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongA>
+  sUser32SetWindowLongAHookStub;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongW>
+  sUser32SetWindowLongWHookStub;
 #endif
 static inline bool
 SetWindowLongHookCheck(HWND hWnd,
                        int nIndex,
                        LONG_PTR newLong)
 {
   nsPluginNativeWindowWin * win =
     (nsPluginNativeWindowWin *)GetProp(hWnd, NS_PLUGIN_WINDOW_PROPERTY_ASSOCIATION);
@@ -443,33 +447,25 @@ SetWindowLongWHook(HWND hWnd,
   return proc;
 }
 
 static void
 HookSetWindowLongPtr()
 {
   sUser32Intercept.Init("user32.dll");
 #ifdef _WIN64
-  if (!sUser32SetWindowLongAHookStub)
-    sUser32Intercept.AddHook("SetWindowLongPtrA",
-                             reinterpret_cast<intptr_t>(SetWindowLongPtrAHook),
-                             (void**) &sUser32SetWindowLongAHookStub);
-  if (!sUser32SetWindowLongWHookStub)
-    sUser32Intercept.AddHook("SetWindowLongPtrW",
-                             reinterpret_cast<intptr_t>(SetWindowLongPtrWHook),
-                             (void**) &sUser32SetWindowLongWHookStub);
+  sUser32SetWindowLongAHookStub.Set(sUser32Intercept, "SetWindowLongPtrA",
+                                    &SetWindowLongPtrAHook);
+  sUser32SetWindowLongWHookStub.Set(sUser32Intercept, "SetWindowLongPtrW",
+                                    &SetWindowLongPtrWHook);
 #else
-  if (!sUser32SetWindowLongAHookStub)
-    sUser32Intercept.AddHook("SetWindowLongA",
-                             reinterpret_cast<intptr_t>(SetWindowLongAHook),
-                             (void**) &sUser32SetWindowLongAHookStub);
-  if (!sUser32SetWindowLongWHookStub)
-    sUser32Intercept.AddHook("SetWindowLongW",
-                             reinterpret_cast<intptr_t>(SetWindowLongWHook),
-                             (void**) &sUser32SetWindowLongWHookStub);
+  sUser32SetWindowLongAHookStub.Set(sUser32Intercept, "SetWindowLongA",
+                                    &SetWindowLongAHook);
+  sUser32SetWindowLongWHookStub.Set(sUser32Intercept, "SetWindowLongW",
+                                    &SetWindowLongWHook);
 #endif
 }
 
 /**
  *   nsPluginNativeWindowWin implementation
  */
 nsPluginNativeWindowWin::nsPluginNativeWindowWin() : nsPluginNativeWindow()
 {
--- a/dom/plugins/ipc/FunctionHook.cpp
+++ b/dom/plugins/ipc/FunctionHook.cpp
@@ -163,23 +163,23 @@ BOOL WINAPI PrintDlgWHook(LPPRINTDLGW aD
 
 // Hooking CreateFileW for protected-mode magic
 static WindowsDllInterceptor sKernel32Intercept;
 typedef HANDLE (WINAPI *CreateFileWPtr)(LPCWSTR aFname, DWORD aAccess,
                                         DWORD aShare,
                                         LPSECURITY_ATTRIBUTES aSecurity,
                                         DWORD aCreation, DWORD aFlags,
                                         HANDLE aFTemplate);
-static CreateFileWPtr sCreateFileWStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<CreateFileWPtr> sCreateFileWStub;
 typedef HANDLE (WINAPI *CreateFileAPtr)(LPCSTR aFname, DWORD aAccess,
                                         DWORD aShare,
                                         LPSECURITY_ATTRIBUTES aSecurity,
                                         DWORD aCreation, DWORD aFlags,
                                         HANDLE aFTemplate);
-static CreateFileAPtr sCreateFileAStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<CreateFileAPtr> sCreateFileAStub;
 
 // Windows 8 RTM (kernelbase's version is 6.2.9200.16384) doesn't call
 // CreateFileW from CreateFileA.
 // So we hook CreateFileA too to use CreateFileW hook.
 static HANDLE WINAPI
 CreateFileAHookFn(LPCSTR aFname, DWORD aAccess, DWORD aShare,
                   LPSECURITY_ATTRIBUTES aSecurity, DWORD aCreation, DWORD aFlags,
                   HANDLE aFTemplate)
@@ -258,17 +258,17 @@ CreateFileWHookFn(LPCWSTR aFname, DWORD 
     if (GetTempFileNameW(tempPath, L"fx", 0, tempFile) == 0) {
       break;
     }
     HANDLE replacement =
       sCreateFileWStub(tempFile, GENERIC_READ | GENERIC_WRITE, aShare,
                        aSecurity, TRUNCATE_EXISTING,
                        FILE_ATTRIBUTE_TEMPORARY |
                          FILE_FLAG_DELETE_ON_CLOSE,
-                       NULL);
+                       nullptr);
     if (replacement == INVALID_HANDLE_VALUE) {
       break;
     }
 
     HANDLE original = sCreateFileWStub(aFname, aAccess, aShare, aSecurity,
                                        aCreation, aFlags, aFTemplate);
     if (original != INVALID_HANDLE_VALUE) {
       // copy original to replacement
@@ -295,33 +295,22 @@ CreateFileWHookFn(LPCWSTR aFname, DWORD 
     return replacement;
   }
   return sCreateFileWStub(aFname, aAccess, aShare, aSecurity, aCreation, aFlags,
                           aFTemplate);
 }
 
 void FunctionHook::HookProtectedMode()
 {
-  // Make sure we only do this once.
-  static bool sRunOnce = false;
-  if (sRunOnce) {
-    return;
-  }
-  sRunOnce = true;
-
   // Legacy code.  Uses the nsWindowsDLLInterceptor directly instead of
   // using the FunctionHook
   sKernel32Intercept.Init("kernel32.dll");
   MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Plugin);
-  sKernel32Intercept.AddHook("CreateFileW",
-                             reinterpret_cast<intptr_t>(CreateFileWHookFn),
-                             (void**) &sCreateFileWStub);
-  sKernel32Intercept.AddHook("CreateFileA",
-                             reinterpret_cast<intptr_t>(CreateFileAHookFn),
-                             (void**) &sCreateFileAStub);
+  sCreateFileWStub.Set(sKernel32Intercept, "CreateFileW", &CreateFileWHookFn);
+  sCreateFileAStub.Set(sKernel32Intercept, "CreateFileA", &CreateFileAHookFn);
 }
 
 #endif // defined(XP_WIN)
 
 #define FUN_HOOK(x) static_cast<FunctionHook*>(x)
 
 void
 FunctionHook::AddFunctionHooks(FunctionHookArray& aHooks)
--- a/dom/plugins/ipc/FunctionHook.h
+++ b/dom/plugins/ipc/FunctionHook.h
@@ -4,16 +4,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef dom_plugins_ipc_functionhook_h
 #define dom_plugins_ipc_functionhook_h 1
 
 #include "IpdlTuple.h"
 #include "base/process.h"
+#include "mozilla/Atomics.h"
 
 #if defined(XP_WIN)
 #include "nsWindowsDllInterceptor.h"
 #endif
 
 namespace mozilla {
 namespace plugins {
 
@@ -91,22 +92,29 @@ public:
 };
 
 // Type of function that returns true if a function should be hooked according to quirks.
 typedef bool(ShouldHookFunc)(int aQuirks);
 
 template<FunctionHookId functionId, typename FunctionType>
 class BasicFunctionHook : public FunctionHook
 {
+#if defined(XP_WIN)
+  using FuncHookType = WindowsDllInterceptor::FuncHookType<FunctionType*>;
+#endif // defined(XP_WIN)
+
 public:
   BasicFunctionHook(const char* aModuleName,
                     const char* aFunctionName, FunctionType* aOldFunction,
-                    FunctionType* aNewFunction) :
-    mOldFunction(aOldFunction), mRegistration(UNREGISTERED), mModuleName(aModuleName),
-    mFunctionName(aFunctionName), mNewFunction(aNewFunction)
+                    FunctionType* aNewFunction)
+    : mOldFunction(aOldFunction)
+    , mRegistration(UNREGISTERED)
+    , mModuleName(aModuleName)
+    , mFunctionName(aFunctionName)
+    , mNewFunction(aNewFunction)
   {
     MOZ_ASSERT(mOldFunction);
     MOZ_ASSERT(mNewFunction);
   }
 
   /**
    * Hooks the function if we haven't already and if ShouldHook() says to.
    */
@@ -123,17 +131,20 @@ public:
   FunctionHookId FunctionId() const override { return functionId; }
 
   FunctionType* OriginalFunction() const { return mOldFunction; }
 
 protected:
   // Once the function is hooked, this field will take the value of a pointer to
   // a function that performs the old behavior.  Before that, it is a pointer to
   // the original function.
-  FunctionType* mOldFunction;
+  Atomic<FunctionType*> mOldFunction;
+#if defined(XP_WIN)
+  FuncHookType mStub;
+#endif // defined(XP_WIN)
 
   enum RegistrationStatus { UNREGISTERED, FAILED, SUCCEEDED };
   RegistrationStatus mRegistration;
 
   // The name of the module containing the function to hook.  E.g. "user32.dll".
   const nsCString mModuleName;
   // The name of the function in the module.
   const nsCString mFunctionName;
@@ -165,24 +176,26 @@ BasicFunctionHook<functionId, FunctionTy
 
 #if defined(XP_WIN)
   WindowsDllInterceptor* dllInterceptor =
     FunctionHook::GetDllInterceptorFor(mModuleName.Data());
   if (!dllInterceptor) {
     return false;
   }
 
-  isHooked =
-    dllInterceptor->AddHook(mFunctionName.Data(), reinterpret_cast<intptr_t>(mNewFunction),
-                            reinterpret_cast<void**>(&mOldFunction));
+  isHooked = mStub.Set(*dllInterceptor, mFunctionName.Data(), mNewFunction);
 #endif
 
   if (isHooked) {
+#if defined(XP_WIN)
+    mOldFunction = mStub.GetStub();
+#endif
     mRegistration = SUCCEEDED;
   }
+
   HOOK_LOG(LogLevel::Debug,
            ("Registering to intercept function '%s' : '%s'", mFunctionName.Data(),
             SuccessMsg(isHooked)));
 
   return isHooked;
 }
 
 }
--- a/dom/plugins/ipc/PluginInstanceChild.cpp
+++ b/dom/plugins/ipc/PluginInstanceChild.cpp
@@ -68,31 +68,27 @@ typedef BOOL (WINAPI *User32TrackPopupMe
                                             UINT uFlags,
                                             int x,
                                             int y,
                                             int nReserved,
                                             HWND hWnd,
                                             CONST RECT *prcRect);
 static WindowsDllInterceptor sUser32Intercept;
 static HWND sWinlessPopupSurrogateHWND = nullptr;
-static User32TrackPopupMenu sUser32TrackPopupMenuStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<User32TrackPopupMenu> sUser32TrackPopupMenuStub;
 
 static WindowsDllInterceptor sImm32Intercept;
-static decltype(ImmGetContext)* sImm32ImmGetContextStub = nullptr;
-static decltype(ImmGetCompositionStringW)* sImm32ImmGetCompositionStringStub =
-                                             nullptr;
-static decltype(ImmSetCandidateWindow)* sImm32ImmSetCandidateWindowStub =
-                                          nullptr;
-static decltype(ImmNotifyIME)* sImm32ImmNotifyIME = nullptr;
-static decltype(ImmAssociateContextEx)* sImm32ImmAssociateContextExStub =
-                                          nullptr;
+static WindowsDllInterceptor::FuncHookType<decltype(&ImmGetContext)> sImm32ImmGetContextStub;
+static WindowsDllInterceptor::FuncHookType<decltype(&ImmGetCompositionStringW)> sImm32ImmGetCompositionStringStub;
+static WindowsDllInterceptor::FuncHookType<decltype(&ImmSetCandidateWindow)> sImm32ImmSetCandidateWindowStub;
+static WindowsDllInterceptor::FuncHookType<decltype(&ImmNotifyIME)> sImm32ImmNotifyIME;
+static WindowsDllInterceptor::FuncHookType<decltype(&ImmAssociateContextEx)> sImm32ImmAssociateContextExStub;
+
 static PluginInstanceChild* sCurrentPluginInstance = nullptr;
 static const HIMC sHookIMC = (const HIMC)0xefefefef;
-static bool sPopupMenuHookSet;
-static bool sSetWindowLongHookSet;
 
 using mozilla::gfx::SharedDIB;
 
 // Flash WM_USER message delay time for PostDelayedTask. Borrowed
 // from Chromium's web plugin delegate src. See 'flash msg throttling
 // helpers' section for details.
 const int kFlashWMUSERMessageThrottleDelayMs = 5;
 
@@ -1788,29 +1784,29 @@ PluginInstanceChild::MaybePostKeyMessage
 typedef LONG_PTR
   (WINAPI *User32SetWindowLongPtrA)(HWND hWnd,
                                     int nIndex,
                                     LONG_PTR dwNewLong);
 typedef LONG_PTR
   (WINAPI *User32SetWindowLongPtrW)(HWND hWnd,
                                     int nIndex,
                                     LONG_PTR dwNewLong);
-static User32SetWindowLongPtrA sUser32SetWindowLongAHookStub = nullptr;
-static User32SetWindowLongPtrW sUser32SetWindowLongWHookStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongPtrA> sUser32SetWindowLongAHookStub;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongPtrW> sUser32SetWindowLongWHookStub;
 #else
 typedef LONG
 (WINAPI *User32SetWindowLongA)(HWND hWnd,
                                int nIndex,
                                LONG dwNewLong);
 typedef LONG
 (WINAPI *User32SetWindowLongW)(HWND hWnd,
                                int nIndex,
                                LONG dwNewLong);
-static User32SetWindowLongA sUser32SetWindowLongAHookStub = nullptr;
-static User32SetWindowLongW sUser32SetWindowLongWHookStub = nullptr;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongA> sUser32SetWindowLongAHookStub;
+static WindowsDllInterceptor::FuncHookType<User32SetWindowLongW> sUser32SetWindowLongWHookStub;
 #endif
 
 extern LRESULT CALLBACK
 NeuteredWindowProc(HWND hwnd,
                    UINT uMsg,
                    WPARAM wParam,
                    LPARAM lParam);
 
@@ -1910,37 +1906,27 @@ PluginInstanceChild::SetWindowLongWHook(
 
 void
 PluginInstanceChild::HookSetWindowLongPtr()
 {
     if (!(GetQuirks() & QUIRK_FLASH_HOOK_SETLONGPTR)) {
         return;
     }
 
-    // Only pass through here once
-    if (sSetWindowLongHookSet) {
-        return;
-    }
-    sSetWindowLongHookSet = true;
-
     sUser32Intercept.Init("user32.dll");
 #ifdef _WIN64
-    if (!sUser32SetWindowLongAHookStub)
-        sUser32Intercept.AddHook("SetWindowLongPtrA", reinterpret_cast<intptr_t>(SetWindowLongPtrAHook),
-                                 (void**) &sUser32SetWindowLongAHookStub);
-    if (!sUser32SetWindowLongWHookStub)
-        sUser32Intercept.AddHook("SetWindowLongPtrW", reinterpret_cast<intptr_t>(SetWindowLongPtrWHook),
-                                 (void**) &sUser32SetWindowLongWHookStub);
+    sUser32SetWindowLongAHookStub.Set(sUser32Intercept, "SetWindowLongPtrA",
+                                      &SetWindowLongPtrAHook);
+    sUser32SetWindowLongWHookStub.Set(sUser32Intercept, "SetWindowLongPtrW",
+                                      &SetWindowLongPtrWHook);
 #else
-    if (!sUser32SetWindowLongAHookStub)
-        sUser32Intercept.AddHook("SetWindowLongA", reinterpret_cast<intptr_t>(SetWindowLongAHook),
-                                 (void**) &sUser32SetWindowLongAHookStub);
-    if (!sUser32SetWindowLongWHookStub)
-        sUser32Intercept.AddHook("SetWindowLongW", reinterpret_cast<intptr_t>(SetWindowLongWHook),
-                                 (void**) &sUser32SetWindowLongWHookStub);
+    sUser32SetWindowLongAHookStub.Set(sUser32Intercept, "SetWindowLongA",
+                                      &SetWindowLongAHook);
+    sUser32SetWindowLongWHookStub.Set(sUser32Intercept, "SetWindowLongW",
+                                      &SetWindowLongWHook);
 #endif
 }
 
 /* windowless track popup menu helpers */
 
 BOOL
 WINAPI
 PluginInstanceChild::TrackPopupHookProc(HMENU hMenu,
@@ -1998,29 +1984,23 @@ PluginInstanceChild::TrackPopupHookProc(
 
 void
 PluginInstanceChild::InitPopupMenuHook()
 {
     if (!(GetQuirks() & QUIRK_WINLESS_TRACKPOPUP_HOOK)) {
         return;
     }
 
-    // Only pass through here once
-    if (sPopupMenuHookSet) {
-        return;
-    }
-    sPopupMenuHookSet = true;
-
     // Note, once WindowsDllInterceptor is initialized for a module,
     // it remains initialized for that particular module for it's
     // lifetime. Additional instances are needed if other modules need
     // to be hooked.
     sUser32Intercept.Init("user32.dll");
-    sUser32Intercept.AddHook("TrackPopupMenu", reinterpret_cast<intptr_t>(TrackPopupHookProc),
-                             (void**) &sUser32TrackPopupMenuStub);
+    sUser32TrackPopupMenuStub.Set(sUser32Intercept, "TrackPopupMenu",
+                                  &TrackPopupHookProc);
 }
 
 void
 PluginInstanceChild::CreateWinlessPopupSurrogate()
 {
     // already initialized
     if (mWinlessPopupSurrogateHWND)
         return;
@@ -2152,46 +2132,33 @@ PluginInstanceChild::ImmAssociateContext
 
 void
 PluginInstanceChild::InitImm32Hook()
 {
     if (!(GetQuirks() & QUIRK_WINLESS_HOOK_IME)) {
         return;
     }
 
-    if (sImm32ImmGetContextStub) {
-        return;
-    }
-
     // When using windowless plugin, IMM API won't work due ot OOP.
     //
     // ImmReleaseContext on Windows 7+ just returns TRUE only, so we don't
     // need to hook this.
 
     sImm32Intercept.Init("imm32.dll");
-    sImm32Intercept.AddHook(
-        "ImmGetContext",
-        reinterpret_cast<intptr_t>(ImmGetContextProc),
-        (void**)&sImm32ImmGetContextStub);
-    sImm32Intercept.AddHook(
-        "ImmGetCompositionStringW",
-        reinterpret_cast<intptr_t>(ImmGetCompositionStringProc),
-        (void**)&sImm32ImmGetCompositionStringStub);
-    sImm32Intercept.AddHook(
-        "ImmSetCandidateWindow",
-        reinterpret_cast<intptr_t>(ImmSetCandidateWindowProc),
-        (void**)&sImm32ImmSetCandidateWindowStub);
-    sImm32Intercept.AddHook(
-        "ImmNotifyIME",
-        reinterpret_cast<intptr_t>(ImmNotifyIME),
-        (void**)&sImm32ImmNotifyIME);
-    sImm32Intercept.AddHook(
-        "ImmAssociateContextEx",
-        reinterpret_cast<intptr_t>(ImmAssociateContextExProc),
-        (void**)&sImm32ImmAssociateContextExStub);
+    sImm32ImmGetContextStub.Set(sImm32Intercept, "ImmGetContext",
+                                &ImmGetContextProc);
+    sImm32ImmGetCompositionStringStub.Set(sImm32Intercept,
+                                          "ImmGetCompositionStringW",
+                                          &ImmGetCompositionStringProc);
+    sImm32ImmSetCandidateWindowStub.Set(sImm32Intercept,
+                                        "ImmSetCandidateWindow",
+                                        &ImmSetCandidateWindowProc);
+    sImm32ImmNotifyIME.Set(sImm32Intercept, "ImmNotifyIME", &ImmNotifyIME);
+    sImm32ImmAssociateContextExStub.Set(sImm32Intercept, "ImmAssociateContextEx",
+                                        &ImmAssociateContextExProc);
 }
 
 void
 PluginInstanceChild::DestroyWinlessPopupSurrogate()
 {
     if (mWinlessPopupSurrogateHWND)
         DestroyWindow(mWinlessPopupSurrogateHWND);
     mWinlessPopupSurrogateHWND = nullptr;