Bug 1382251: Part 3 - Add mechanism for automatically hooking DLL functions r=jimm
☠☠ backed out by 6159c8eb5442 ☠ ☠
authorDavid Parks <dparks@mozilla.com>
Mon, 06 Nov 2017 10:04:19 -0800
changeset 453105 5dc95b140fd75283c58a86793791759b98550937
parent 453104 a1ca7804d84876782d6d14c448ade93049e77d49
child 453106 e8262c37dfaca6409a2df0bc5146efbb331c73a2
push id1648
push usermtabara@mozilla.com
push dateThu, 01 Mar 2018 12:45:47 +0000
treeherdermozilla-release@cbb9688c2eeb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjimm
bugs1382251
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1382251: Part 3 - Add mechanism for automatically hooking DLL functions r=jimm FunctionHook uses the DLL interceptor to redirect Win32 calls to a user-supplied function.
dom/plugins/ipc/FunctionHook.cpp
dom/plugins/ipc/FunctionHook.h
new file mode 100644
--- /dev/null
+++ b/dom/plugins/ipc/FunctionHook.cpp
@@ -0,0 +1,247 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "FunctionHook.h"
+#include "FunctionBroker.h"
+#include "nsClassHashtable.h"
+#include "mozilla/ClearOnShutdown.h"
+
+#if defined(XP_WIN)
+#include <shlobj.h>
+#endif
+
+namespace mozilla {
+namespace plugins {
+
+StaticAutoPtr<FunctionHookArray> FunctionHook::sFunctionHooks;
+
+bool AlwaysHook(int) { return true; }
+
+FunctionHookArray*
+FunctionHook::GetHooks()
+{
+  if (sFunctionHooks) {
+    return sFunctionHooks;
+  }
+
+  // sFunctionHooks is the StaticAutoPtr to the singleton array of FunctionHook
+  // objects.  We free it by clearing the StaticAutoPtr on shutdown.
+  sFunctionHooks = new FunctionHookArray();
+  ClearOnShutdown(&sFunctionHooks);
+  sFunctionHooks->SetLength(ID_FunctionHookCount);
+
+  AddFunctionHooks(*sFunctionHooks);
+  AddBrokeredFunctionHooks(*sFunctionHooks);
+  return sFunctionHooks;
+}
+
+void
+FunctionHook::HookFunctions(int aQuirks)
+{
+  MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Plugin);
+  FunctionHookArray* hooks = FunctionHook::GetHooks();
+  MOZ_ASSERT(hooks);
+  for(size_t i=0; i < hooks->Length(); ++i) {
+    FunctionHook* mhb = hooks->ElementAt(i);
+    // Check that the FunctionHook array is in the same order as the
+    // FunctionHookId enum.
+    MOZ_ASSERT((size_t)mhb->FunctionId() == i);
+    mhb->Register(aQuirks);
+  }
+}
+
+#if defined(XP_WIN)
+
+// This cache is created when a DLL is registered with a FunctionHook.
+// It is cleared on a call to ClearDllInterceptorCache().  It
+// must be freed before exit to avoid leaks.
+typedef nsClassHashtable<nsCStringHashKey, WindowsDllInterceptor> DllInterceptors;
+DllInterceptors* sDllInterceptorCache = nullptr;
+
+WindowsDllInterceptor*
+FunctionHook::GetDllInterceptorFor(const char* aModuleName)
+{
+  if (!sDllInterceptorCache) {
+    sDllInterceptorCache = new DllInterceptors();
+  }
+
+  WindowsDllInterceptor* ret =
+    sDllInterceptorCache->LookupOrAdd(nsCString(aModuleName), aModuleName);
+  MOZ_ASSERT(ret);
+  return ret;
+}
+
+void
+FunctionHook::ClearDllInterceptorCache()
+{
+  delete sDllInterceptorCache;
+  sDllInterceptorCache = nullptr;
+}
+
+// Hooking CreateFileW for protected-mode magic
+static WindowsDllInterceptor sKernel32Intercept;
+typedef HANDLE (WINAPI *CreateFileWPtr)(LPCWSTR aFname, DWORD aAccess,
+                                        DWORD aShare,
+                                        LPSECURITY_ATTRIBUTES aSecurity,
+                                        DWORD aCreation, DWORD aFlags,
+                                        HANDLE aFTemplate);
+static CreateFileWPtr sCreateFileWStub = nullptr;
+typedef HANDLE (WINAPI *CreateFileAPtr)(LPCSTR aFname, DWORD aAccess,
+                                        DWORD aShare,
+                                        LPSECURITY_ATTRIBUTES aSecurity,
+                                        DWORD aCreation, DWORD aFlags,
+                                        HANDLE aFTemplate);
+static CreateFileAPtr sCreateFileAStub = nullptr;
+
+// Windows 8 RTM (kernelbase's version is 6.2.9200.16384) doesn't call
+// CreateFileW from CreateFileA.
+// So we hook CreateFileA too to use CreateFileW hook.
+static HANDLE WINAPI
+CreateFileAHookFn(LPCSTR aFname, DWORD aAccess, DWORD aShare,
+                  LPSECURITY_ATTRIBUTES aSecurity, DWORD aCreation, DWORD aFlags,
+                  HANDLE aFTemplate)
+{
+  while (true) { // goto out
+    // Our hook is for mms.cfg into \Windows\System32\Macromed\Flash
+    // We don't require supporting too long path.
+    WCHAR unicodeName[MAX_PATH];
+    size_t len = strlen(aFname);
+
+    if (len >= MAX_PATH) {
+      break;
+    }
+
+    // We call to CreateFileW for workaround of Windows 8 RTM
+    int newLen = MultiByteToWideChar(CP_ACP, MB_ERR_INVALID_CHARS, aFname,
+                                     len, unicodeName, MAX_PATH);
+    if (newLen == 0 || newLen >= MAX_PATH) {
+      break;
+    }
+    unicodeName[newLen] = '\0';
+
+    return CreateFileW(unicodeName, aAccess, aShare, aSecurity, aCreation, aFlags, aFTemplate);
+  }
+
+  return sCreateFileAStub(aFname, aAccess, aShare, aSecurity, aCreation, aFlags,
+                          aFTemplate);
+}
+
+static bool
+GetLocalLowTempPath(size_t aLen, LPWSTR aPath)
+{
+  NS_NAMED_LITERAL_STRING(tempname, "\\Temp");
+  LPWSTR path;
+  if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppDataLow, 0,
+                                     nullptr, &path))) {
+    if (wcslen(path) + tempname.Length() < aLen) {
+        wcscpy(aPath, path);
+        wcscat(aPath, tempname.get());
+        CoTaskMemFree(path);
+        return true;
+    }
+    CoTaskMemFree(path);
+  }
+
+  // XP doesn't support SHGetKnownFolderPath and LocalLow
+  if (!GetTempPathW(aLen, aPath)) {
+    return false;
+  }
+  return true;
+}
+
+HANDLE WINAPI
+CreateFileWHookFn(LPCWSTR aFname, DWORD aAccess, DWORD aShare,
+                  LPSECURITY_ATTRIBUTES aSecurity, DWORD aCreation, DWORD aFlags,
+                  HANDLE aFTemplate)
+{
+  static const WCHAR kConfigFile[] = L"mms.cfg";
+  static const size_t kConfigLength = ArrayLength(kConfigFile) - 1;
+
+  while (true) { // goto out, in sheep's clothing
+    size_t len = wcslen(aFname);
+    if (len < kConfigLength) {
+      break;
+    }
+    if (wcscmp(aFname + len - kConfigLength, kConfigFile) != 0) {
+      break;
+    }
+
+    // This is the config file we want to rewrite
+    WCHAR tempPath[MAX_PATH+1];
+    if (GetLocalLowTempPath(MAX_PATH, tempPath) == 0) {
+      break;
+    }
+    WCHAR tempFile[MAX_PATH+1];
+    if (GetTempFileNameW(tempPath, L"fx", 0, tempFile) == 0) {
+      break;
+    }
+    HANDLE replacement =
+      sCreateFileWStub(tempFile, GENERIC_READ | GENERIC_WRITE, aShare,
+                       aSecurity, TRUNCATE_EXISTING,
+                       FILE_ATTRIBUTE_TEMPORARY |
+                         FILE_FLAG_DELETE_ON_CLOSE,
+                       NULL);
+    if (replacement == INVALID_HANDLE_VALUE) {
+      break;
+    }
+
+    HANDLE original = sCreateFileWStub(aFname, aAccess, aShare, aSecurity,
+                                       aCreation, aFlags, aFTemplate);
+    if (original != INVALID_HANDLE_VALUE) {
+      // copy original to replacement
+      static const size_t kBufferSize = 1024;
+      char buffer[kBufferSize];
+      DWORD bytes;
+      while (ReadFile(original, buffer, kBufferSize, &bytes, NULL)) {
+        if (bytes == 0) {
+          break;
+        }
+        DWORD wbytes;
+        WriteFile(replacement, buffer, bytes, &wbytes, NULL);
+        if (bytes < kBufferSize) {
+          break;
+        }
+      }
+      CloseHandle(original);
+    }
+    static const char kSettingString[] = "\nProtectedMode=0\n";
+    DWORD wbytes;
+    WriteFile(replacement, static_cast<const void*>(kSettingString),
+              sizeof(kSettingString) - 1, &wbytes, NULL);
+    SetFilePointer(replacement, 0, NULL, FILE_BEGIN);
+    return replacement;
+  }
+  return sCreateFileWStub(aFname, aAccess, aShare, aSecurity, aCreation, aFlags,
+                          aFTemplate);
+}
+
+void FunctionHook::HookProtectedMode()
+{
+  // Legacy code.  Uses the nsWindowsDLLInterceptor directly instead of
+  // using the FunctionHook
+  MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Plugin);
+  WindowsDllInterceptor k32Intercept("kernel32.dll");
+  k32Intercept.AddHook("CreateFileW",
+                       reinterpret_cast<intptr_t>(CreateFileWHookFn),
+                       (void**) &sCreateFileWStub);
+  k32Intercept.AddHook("CreateFileA",
+                       reinterpret_cast<intptr_t>(CreateFileAHookFn),
+                       (void**) &sCreateFileAStub);
+}
+
+#endif // defined(XP_WIN)
+
+#define FUN_HOOK(x) static_cast<FunctionHook*>(x)
+
+void
+FunctionHook::AddFunctionHooks(FunctionHookArray& aHooks)
+{
+}
+
+#undef FUN_HOOK
+
+} // namespace plugins
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/plugins/ipc/FunctionHook.h
@@ -0,0 +1,183 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef dom_plugins_ipc_functionhook_h
+#define dom_plugins_ipc_functionhook_h 1
+
+#include "IpdlTuple.h"
+#include "base/process.h"
+
+#if defined(XP_WIN)
+#include "nsWindowsDllInterceptor.h"
+#endif
+
+namespace mozilla {
+namespace plugins {
+
+// "PluginHooks" logging helpers
+extern mozilla::LazyLogModule sPluginHooksLog;
+#define HOOK_LOG(lvl, msg) MOZ_LOG(mozilla::plugins::sPluginHooksLog, lvl, msg);
+inline const char *SuccessMsg(bool aVal) { return aVal ? "succeeded" : "failed";  }
+
+class FunctionHook;
+class FunctionHookArray;
+
+class FunctionHook
+{
+public:
+  virtual ~FunctionHook() {}
+
+  virtual FunctionHookId FunctionId() const = 0;
+
+  /**
+   * Register to hook the function represented by this class.
+   * Returns false if we should have hooked but didn't.
+   */
+  virtual bool Register(int aQuirks) = 0;
+
+  /**
+   * Run the original function with parameters stored in a tuple.
+   * This is only supported on server-side and for auto-brokered methods.
+   */
+  virtual bool RunOriginalFunction(base::ProcessId aClientId,
+                                   const IPC::IpdlTuple &aInTuple,
+                                   IPC::IpdlTuple *aOutTuple) const = 0;
+
+  /**
+   * Hook the Win32 methods needed by the plugin process.
+   */
+  static void HookFunctions(int aQuirks);
+
+  static FunctionHookArray* GetHooks();
+
+#if defined(XP_WIN)
+  /**
+   * Special handler for hooking some kernel32.dll methods that we use to
+   * disable Flash protected mode.
+   */
+  static void HookProtectedMode();
+
+  /**
+   * Get the WindowsDllInterceptor for the given module.  Creates a cache of
+   * WindowsDllInterceptors by name.
+   */
+  static WindowsDllInterceptor* GetDllInterceptorFor(const char* aModuleName);
+
+  /**
+   * Must be called to clear the cache created by calls to GetDllInterceptorFor.
+   */
+  static void ClearDllInterceptorCache();
+#endif // defined(XP_WIN)
+
+private:
+  static StaticAutoPtr<FunctionHookArray> sFunctionHooks;
+  static void AddFunctionHooks(FunctionHookArray& aHooks);
+};
+
+// The FunctionHookArray deletes its FunctionHook objects when freed.
+class FunctionHookArray : public nsTArray<FunctionHook*> {
+public:
+  ~FunctionHookArray()
+  {
+    for (uint32_t idx = 0; idx < Length(); ++idx) {
+      FunctionHook* elt = ElementAt(idx);
+      MOZ_ASSERT(elt);
+      delete elt;
+    }
+  }
+};
+
+// Type of function that returns true if a function should be hooked according to quirks.
+typedef bool(ShouldHookFunc)(int aQuirks);
+
+template<FunctionHookId functionId, typename FunctionType>
+class BasicFunctionHook : public FunctionHook
+{
+public:
+  BasicFunctionHook(const char* aModuleName,
+                    const char* aFunctionName, FunctionType* aOldFunction,
+                    FunctionType* aNewFunction) :
+    mOldFunction(aOldFunction), mIsHooked(false), mModuleName(aModuleName),
+    mFunctionName(aFunctionName), mNewFunction(aNewFunction)
+  {
+    MOZ_ASSERT(mOldFunction);
+    MOZ_ASSERT(mNewFunction);
+  }
+
+  /**
+   * Hooks the function if we haven't already and if ShouldHook() says to.
+   */
+  bool Register(int aQuirks) override;
+
+  /**
+   * Can be specialized to perform "extra" operations when running the
+   * function on the server side.
+   */
+  bool RunOriginalFunction(base::ProcessId aClientId,
+                           const IPC::IpdlTuple &aInTuple,
+                           IPC::IpdlTuple *aOutTuple) const override { return false; }
+
+  FunctionHookId FunctionId() const override { return functionId; }
+
+  FunctionType* OriginalFunction() const { return mOldFunction; }
+
+protected:
+  // Once the function is hooked, this field will take the value of a pointer to
+  // a function that performs the old behavior.  Before that, it is a pointer to
+  // the original function.
+  FunctionType* mOldFunction;
+  // True if we have already hooked the function.
+  bool mIsHooked;
+
+  // The name of the module containing the function to hook.  E.g. "user32.dll".
+  const nsCString mModuleName;
+  // The name of the function in the module.
+  const nsCString mFunctionName;
+  // The function that we should replace functionName with.  The signature of
+  // newFunction must match that of functionName.
+  FunctionType* const mNewFunction;
+  static ShouldHookFunc* const mShouldHook;
+};
+
+// Default behavior is to hook every registered function.
+extern bool AlwaysHook(int);
+template<FunctionHookId functionId, typename FunctionType>
+ShouldHookFunc* const BasicFunctionHook<functionId, FunctionType>::mShouldHook = AlwaysHook;
+
+template <FunctionHookId functionId, typename FunctionType>
+bool
+BasicFunctionHook<functionId, FunctionType>::Register(int aQuirks)
+{
+  MOZ_RELEASE_ASSERT(XRE_IsPluginProcess());
+
+  // If we have already hooked or if quirks tell us not to then don't hook.
+  if (mIsHooked || !mShouldHook(aQuirks)) {
+    return true;
+  }
+
+#if defined(XP_WIN)
+  WindowsDllInterceptor* dllInterceptor =
+    FunctionHook::GetDllInterceptorFor(mModuleName.Data());
+  if (!dllInterceptor) {
+    return false;
+  }
+
+  mIsHooked =
+    dllInterceptor->AddHook(mFunctionName.Data(), reinterpret_cast<intptr_t>(mNewFunction),
+                            reinterpret_cast<void**>(&mOldFunction));
+#endif
+
+  HOOK_LOG(LogLevel::Debug,
+           ("Registering to intercept function '%s' : '%s'", mFunctionName.Data(),
+            SuccessMsg(mIsHooked)));
+
+  return mIsHooked;
+}
+
+}
+}
+
+#endif // dom_plugins_ipc_functionhook_h