Bug 1218473: Add check for presence of NVIDIA Optimus drivers to WindowsNopSpacePatcher; r=ehsan
authorAaron Klotz <aklotz@mozilla.com>
Thu, 13 Oct 2016 17:10:52 -0600
changeset 317915 fcbf4915f370
parent 317914 609a153be0a5
child 317916 711480e99c8c
push id30819
push usercbook@mozilla.com
push date2016-10-14 09:59 +0000
treeherdermozilla-central@1391a2889aeb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs1218473
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1218473: Add check for presence of NVIDIA Optimus drivers to WindowsNopSpacePatcher; r=ehsan MozReview-Commit-ID: 7WhWWbRHOw7
toolkit/xre/test/win/moz.build
xpcom/build/nsWindowsDllInterceptor.h
--- a/toolkit/xre/test/win/moz.build
+++ b/toolkit/xre/test/win/moz.build
@@ -14,15 +14,16 @@ CppUnitTests([
 
 DEFINES['NS_NO_XPCOM'] = True
 
 LOCAL_INCLUDES += [
     '/config',
     '/toolkit/xre',
 ]
 
+DISABLE_STL_WRAPPING = True
 USE_STATIC_LIBS = True
 
 OS_LIBS += [
     'comctl32',
     'ws2_32',
     'shell32',
 ]
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -3,17 +3,21 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef NS_WINDOWS_DLL_INTERCEPTOR_H_
 #define NS_WINDOWS_DLL_INTERCEPTOR_H_
 
 #include "mozilla/Assertions.h"
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/UniquePtr.h"
+#include "nsWindowsHelpers.h"
 
+#include <wchar.h>
 #include <windows.h>
 #include <winternl.h>
 
 /*
  * Simple function interception.
  *
  * We have two separate mechanisms for intercepting a function: We can use the
  * built-in nop space, if it exists, or we can create a detour.
@@ -112,16 +116,17 @@ class WindowsDllNopSpacePatcher
   int mPatchedFnsLen;
 
 public:
   WindowsDllNopSpacePatcher()
     : mModule(0)
     , mPatchedFnsLen(0)
   {}
 
+#if defined(_M_IX86)
   ~WindowsDllNopSpacePatcher()
   {
     // Restore the mov edi, edi to the beginning of each function we patched.
 
     for (int i = 0; i < mPatchedFnsLen; i++) {
       byteptr_t fn = mPatchedFns[i];
 
       // Ensure we can write to the code.
@@ -138,30 +143,117 @@ public:
       FlushInstructionCache(GetCurrentProcess(),
                             /* ignored */ nullptr,
                             /* ignored */ 0);
     }
   }
 
   void Init(const char* aModuleName)
   {
+    if (!IsCompatible()) {
+#if defined(MOZILLA_INTERNAL_API)
+      NS_WARNING("NOP space patching is unavailable for compatibility reasons");
+#endif
+      return;
+    }
+
     mModule = LoadLibraryExA(aModuleName, nullptr, 0);
     if (!mModule) {
       //printf("LoadLibraryEx for '%s' failed\n", aModuleName);
       return;
     }
   }
 
-#if defined(_M_IX86)
+  /**
+   * NVIDIA Optimus drivers utilize Microsoft Detours 2.x to patch functions
+   * in our address space. There is a bug in Detours 2.x that causes it to
+   * patch at the wrong address when attempting to detour code that is already
+   * NOP space patched. This function is an effort to detect the presence of
+   * this NVIDIA code in our address space and disable NOP space patching if it
+   * is. We also check AppInit_DLLs since this is the mechanism that the Optimus
+   * drivers use to inject into our process.
+   */
+  static bool IsCompatible()
+  {
+    // These DLLs are known to have bad interactions with this style of patching
+    const wchar_t* kIncompatibleDLLs[] = {
+      L"detoured.dll",
+      L"_etoured.dll",
+      L"nvd3d9wrap.dll",
+      L"nvdxgiwrap.dll"
+    };
+    // See if the infringing DLLs are already loaded
+    for (unsigned int i = 0; i < mozilla::ArrayLength(kIncompatibleDLLs); ++i) {
+      if (GetModuleHandleW(kIncompatibleDLLs[i])) {
+        return false;
+      }
+    }
+    if (GetModuleHandleW(L"user32.dll")) {
+      // user32 is loaded but the infringing DLLs are not, assume we're safe to
+      // proceed.
+      return true;
+    }
+    // If user32 has not loaded yet, check AppInit_DLLs to ensure that Optimus
+    // won't be loaded once user32 is initialized.
+    HKEY hkey = NULL;
+    if (!RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+          L"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows",
+          0, KEY_QUERY_VALUE, &hkey)) {
+      nsAutoRegKey key(hkey);
+      DWORD numBytes = 0;
+      const wchar_t kAppInitDLLs[] = L"AppInit_DLLs";
+      // Query for required buffer size
+      LONG status = RegQueryValueExW(hkey, kAppInitDLLs, nullptr,
+                                     nullptr, nullptr, &numBytes);
+      mozilla::UniquePtr<wchar_t[]> data;
+      if (!status) {
+        // Allocate the buffer and query for the actual data
+        data = mozilla::MakeUnique<wchar_t[]>(numBytes / sizeof(wchar_t));
+        status = RegQueryValueExW(hkey, kAppInitDLLs, nullptr,
+                                  nullptr, (LPBYTE)data.get(), &numBytes);
+      }
+      if (!status) {
+        // For each token, split up the filename components and then check the
+        // name of the file.
+        const wchar_t kDelimiters[] = L", ";
+        wchar_t* tokenContext = nullptr;
+        wchar_t* token = wcstok_s(data.get(), kDelimiters, &tokenContext);
+        while (token) {
+          wchar_t fname[_MAX_FNAME] = {0};
+          if (!_wsplitpath_s(token, nullptr, 0, nullptr, 0,
+                             fname, mozilla::ArrayLength(fname),
+                             nullptr, 0)) {
+            // nvinit.dll is responsible for bootstrapping the DLL injection, so
+            // that is the library that we check for here
+            const wchar_t kNvInitName[] = L"nvinit";
+            if (!_wcsnicmp(fname, kNvInitName,
+                           mozilla::ArrayLength(kNvInitName))) {
+              return false;
+            }
+          }
+          token = wcstok_s(nullptr, kDelimiters, &tokenContext);
+        }
+      }
+    }
+    return true;
+  }
+
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     if (!mModule) {
       return false;
     }
 
+    if (!IsCompatible()) {
+#if defined(MOZILLA_INTERNAL_API)
+      NS_WARNING("NOP space patching is unavailable for compatibility reasons");
+#endif
+      return false;
+    }
+
     if (mPatchedFnsLen == maxPatchedFns) {
       // printf ("No space for hook in mPatchedFns.\n");
       return false;
     }
 
     byteptr_t fn = reinterpret_cast<byteptr_t>(GetProcAddress(mModule, aName));
     if (!fn) {
       //printf ("GetProcAddress failed\n");
@@ -241,16 +333,21 @@ private:
     // we resolve redirected address from import table.
     if (aOriginalFunction[0] == 0xff && aOriginalFunction[1] == 0x25) {
       return (byteptr_t)(**((uint32_t**) (aOriginalFunction + 2)));
     }
 
     return aOriginalFunction;
   }
 #else
+  void Init(const char* aModuleName)
+  {
+    // Not implemented except on x86-32.
+  }
+
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     // Not implemented except on x86-32.
     return false;
   }
 #endif
 };