Bug 1218473: Add check for presence of NVIDIA Optimus drivers to WindowsNopSpacePatcher; r=ehsan
authorAaron Klotz <aklotz@mozilla.com>
Thu, 13 Oct 2016 17:10:52 -0600
changeset 317915 fcbf4915f370
parent 317914 609a153be0a5
child 317916 711480e99c8c
push id30819
push usercbook@mozilla.com
push date2016-10-14 09:59 +0000
treeherdermozilla-central@1391a2889aeb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs1218473
milestone52.0a1
Bug 1218473: Add check for presence of NVIDIA Optimus drivers to WindowsNopSpacePatcher; r=ehsan

MozReview-Commit-ID: 7WhWWbRHOw7
toolkit/xre/test/win/moz.build
xpcom/build/nsWindowsDllInterceptor.h
--- a/toolkit/xre/test/win/moz.build
+++ b/toolkit/xre/test/win/moz.build
@@ -14,15 +14,16 @@ CppUnitTests([
 
 DEFINES['NS_NO_XPCOM'] = True
 
 LOCAL_INCLUDES += [
     '/config',
     '/toolkit/xre',
 ]
 
+DISABLE_STL_WRAPPING = True
 USE_STATIC_LIBS = True
 
 OS_LIBS += [
     'comctl32',
     'ws2_32',
     'shell32',
 ]
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -3,17 +3,21 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef NS_WINDOWS_DLL_INTERCEPTOR_H_
 #define NS_WINDOWS_DLL_INTERCEPTOR_H_
 
 #include "mozilla/Assertions.h"
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/UniquePtr.h"
+#include "nsWindowsHelpers.h"
 
+#include <wchar.h>
 #include <windows.h>
 #include <winternl.h>
 
 /*
  * Simple function interception.
  *
  * We have two separate mechanisms for intercepting a function: We can use the
  * built-in nop space, if it exists, or we can create a detour.
@@ -112,16 +116,17 @@ class WindowsDllNopSpacePatcher
   int mPatchedFnsLen;
 
 public:
   WindowsDllNopSpacePatcher()
     : mModule(0)
     , mPatchedFnsLen(0)
   {}
 
+#if defined(_M_IX86)
   ~WindowsDllNopSpacePatcher()
   {
     // Restore the mov edi, edi to the beginning of each function we patched.
 
     for (int i = 0; i < mPatchedFnsLen; i++) {
       byteptr_t fn = mPatchedFns[i];
 
       // Ensure we can write to the code.
@@ -138,30 +143,117 @@ public:
       FlushInstructionCache(GetCurrentProcess(),
                             /* ignored */ nullptr,
                             /* ignored */ 0);
     }
   }
 
   void Init(const char* aModuleName)
   {
+    if (!IsCompatible()) {
+#if defined(MOZILLA_INTERNAL_API)
+      NS_WARNING("NOP space patching is unavailable for compatibility reasons");
+#endif
+      return;
+    }
+
     mModule = LoadLibraryExA(aModuleName, nullptr, 0);
     if (!mModule) {
       //printf("LoadLibraryEx for '%s' failed\n", aModuleName);
       return;
     }
   }
 
-#if defined(_M_IX86)
+  /**
+   * NVIDIA Optimus drivers utilize Microsoft Detours 2.x to patch functions
+   * in our address space. There is a bug in Detours 2.x that causes it to
+   * patch at the wrong address when attempting to detour code that is already
+   * NOP space patched. This function is an effort to detect the presence of
+   * this NVIDIA code in our address space and disable NOP space patching if it
+   * is. We also check AppInit_DLLs since this is the mechanism that the Optimus
+   * drivers use to inject into our process.
+   */
+  static bool IsCompatible()
+  {
+    // These DLLs are known to have bad interactions with this style of patching
+    const wchar_t* kIncompatibleDLLs[] = {
+      L"detoured.dll",
+      L"_etoured.dll",
+      L"nvd3d9wrap.dll",
+      L"nvdxgiwrap.dll"
+    };
+    // See if the infringing DLLs are already loaded
+    for (unsigned int i = 0; i < mozilla::ArrayLength(kIncompatibleDLLs); ++i) {
+      if (GetModuleHandleW(kIncompatibleDLLs[i])) {
+        return false;
+      }
+    }
+    if (GetModuleHandleW(L"user32.dll")) {
+      // user32 is loaded but the infringing DLLs are not, assume we're safe to
+      // proceed.
+      return true;
+    }
+    // If user32 has not loaded yet, check AppInit_DLLs to ensure that Optimus
+    // won't be loaded once user32 is initialized.
+    HKEY hkey = NULL;
+    if (!RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+          L"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows",
+          0, KEY_QUERY_VALUE, &hkey)) {
+      nsAutoRegKey key(hkey);
+      DWORD numBytes = 0;
+      const wchar_t kAppInitDLLs[] = L"AppInit_DLLs";
+      // Query for required buffer size
+      LONG status = RegQueryValueExW(hkey, kAppInitDLLs, nullptr,
+                                     nullptr, nullptr, &numBytes);
+      mozilla::UniquePtr<wchar_t[]> data;
+      if (!status) {
+        // Allocate the buffer and query for the actual data
+        data = mozilla::MakeUnique<wchar_t[]>(numBytes / sizeof(wchar_t));
+        status = RegQueryValueExW(hkey, kAppInitDLLs, nullptr,
+                                  nullptr, (LPBYTE)data.get(), &numBytes);
+      }
+      if (!status) {
+        // For each token, split up the filename components and then check the
+        // name of the file.
+        const wchar_t kDelimiters[] = L", ";
+        wchar_t* tokenContext = nullptr;
+        wchar_t* token = wcstok_s(data.get(), kDelimiters, &tokenContext);
+        while (token) {
+          wchar_t fname[_MAX_FNAME] = {0};
+          if (!_wsplitpath_s(token, nullptr, 0, nullptr, 0,
+                             fname, mozilla::ArrayLength(fname),
+                             nullptr, 0)) {
+            // nvinit.dll is responsible for bootstrapping the DLL injection, so
+            // that is the library that we check for here
+            const wchar_t kNvInitName[] = L"nvinit";
+            if (!_wcsnicmp(fname, kNvInitName,
+                           mozilla::ArrayLength(kNvInitName))) {
+              return false;
+            }
+          }
+          token = wcstok_s(nullptr, kDelimiters, &tokenContext);
+        }
+      }
+    }
+    return true;
+  }
+
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     if (!mModule) {
       return false;
     }
 
+    if (!IsCompatible()) {
+#if defined(MOZILLA_INTERNAL_API)
+      NS_WARNING("NOP space patching is unavailable for compatibility reasons");
+#endif
+      return false;
+    }
+
     if (mPatchedFnsLen == maxPatchedFns) {
       // printf ("No space for hook in mPatchedFns.\n");
       return false;
     }
 
     byteptr_t fn = reinterpret_cast<byteptr_t>(GetProcAddress(mModule, aName));
     if (!fn) {
       //printf ("GetProcAddress failed\n");
@@ -241,16 +333,21 @@ private:
     // we resolve redirected address from import table.
     if (aOriginalFunction[0] == 0xff && aOriginalFunction[1] == 0x25) {
       return (byteptr_t)(**((uint32_t**) (aOriginalFunction + 2)));
     }
 
     return aOriginalFunction;
   }
 #else
+  void Init(const char* aModuleName)
+  {
+    // Not implemented except on x86-32.
+  }
+
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     // Not implemented except on x86-32.
     return false;
   }
 #endif
 };