Bug 1218473: Add check for presence of NVIDIA Optimus drivers to WindowsNopSpacePatcher; r=ehsan
💩💩 backed out by 4d7efb9a8026 💩 💩
authorAaron Klotz <aklotz@mozilla.com>
Fri, 27 Nov 2015 13:01:23 -0700
changeset 274477 a25efba616ac
parent 274476 a16c2903506a
child 274478 4c2fc03f6c17
push id68601
push useraklotz@mozilla.com
push date2015-11-28 00:14 +0000
treeherdermozilla-inbound@a25efba616ac [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs1218473
milestone45.0a1
Bug 1218473: Add check for presence of NVIDIA Optimus drivers to WindowsNopSpacePatcher; r=ehsan
toolkit/xre/test/win/moz.build
xpcom/build/nsWindowsDllInterceptor.h
--- a/toolkit/xre/test/win/moz.build
+++ b/toolkit/xre/test/win/moz.build
@@ -14,15 +14,16 @@ CppUnitTests([
 
 DEFINES['NS_NO_XPCOM'] = True
 
 LOCAL_INCLUDES += [
     '/config',
     '/toolkit/xre',
 ]
 
+DISABLE_STL_WRAPPING = True
 USE_STATIC_LIBS = True
 
 OS_LIBS += [
     'comctl32',
     'ws2_32',
     'shell32',
 ]
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -1,19 +1,25 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef NS_WINDOWS_DLL_INTERCEPTOR_H_
 #define NS_WINDOWS_DLL_INTERCEPTOR_H_
+
+#include <wchar.h>
 #include <windows.h>
 #include <winternl.h>
 
+#include "mozilla/ArrayUtils.h"
+#include "nsAutoPtr.h"
+#include "nsWindowsHelpers.h"
+
 /*
  * Simple function interception.
  *
  * We have two separate mechanisms for intercepting a function: We can use the
  * built-in nop space, if it exists, or we can create a detour.
  *
  * Using the built-in nop space works as follows: On x86-32, DLL functions
  * begin with a two-byte nop (mov edi, edi) and are preceeded by five bytes of
@@ -135,29 +141,115 @@ public:
       FlushInstructionCache(GetCurrentProcess(),
                             /* ignored */ nullptr,
                             /* ignored */ 0);
     }
   }
 
   void Init(const char* aModuleName)
   {
+    if (!IsCompatible()) {
+#if defined(MOZILLA_INTERNAL_API)
+      NS_WARNING("NOP space patching is unavailable for compatibility reasons");
+#endif
+      return;
+    }
     mModule = LoadLibraryExA(aModuleName, nullptr, 0);
     if (!mModule) {
       //printf("LoadLibraryEx for '%s' failed\n", aModuleName);
       return;
     }
   }
 
+  /**
+   * NVIDIA Optimus drivers utilize Microsoft Detours 2.x to patch functions
+   * in our address space. There is a bug in Detours 2.x that causes it to
+   * patch at the wrong address when attempting to detour code that is already
+   * NOP space patched. This function is an effort to detect the presence of
+   * this NVIDIA code in our address space and disable NOP space patching if it
+   * is. We also check AppInit_DLLs since this is the mechanism that the Optimus
+   * drivers use to inject into our process.
+   */
+  static bool IsCompatible()
+  {
+    // These DLLs are known to have bad interactions with this style of patching
+    const wchar_t* kIncompatibleDLLs[] = {
+      MOZ_UTF16("detoured.dll"),
+      MOZ_UTF16("_etoured.dll"),
+      MOZ_UTF16("nvd3d9wrap.dll"),
+      MOZ_UTF16("nvdxgiwrap.dll")
+    };
+    // See if the infringing DLLs are already loaded
+    for (unsigned int i = 0; i < mozilla::ArrayLength(kIncompatibleDLLs); ++i) {
+      if (GetModuleHandleW(kIncompatibleDLLs[i])) {
+        return false;
+      }
+    }
+    if (GetModuleHandleW(MOZ_UTF16("user32.dll"))) {
+      // user32 is loaded but the infringing DLLs are not, assume we're safe to
+      // proceed.
+      return true;
+    }
+    // If user32 has not loaded yet, check AppInit_DLLs to ensure that Optimus
+    // won't be loaded once user32 is initialized.
+    HKEY hkey = NULL;
+    if (!RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+          MOZ_UTF16("SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows"),
+          0, KEY_QUERY_VALUE, &hkey)) {
+      nsAutoRegKey key(hkey);
+      DWORD numBytes = 0;
+      const wchar_t kAppInitDLLs[] = MOZ_UTF16("AppInit_DLLs");
+      // Query for required buffer size
+      LONG status = RegQueryValueExW(hkey, kAppInitDLLs, nullptr,
+                                     nullptr, nullptr, &numBytes);
+      nsAutoArrayPtr<wchar_t> data;
+      if (!status) {
+        // Allocate the buffer and query for the actual data
+        data = new wchar_t[numBytes / sizeof(wchar_t)];
+        status = RegQueryValueExW(hkey, kAppInitDLLs, nullptr,
+                                  nullptr, (LPBYTE)data.get(), &numBytes);
+      }
+      if (!status) {
+        // For each token, split up the filename components and then check the
+        // name of the file.
+        const wchar_t kDelimiters[] = MOZ_UTF16(", ");
+        wchar_t* tokenContext = nullptr;
+        wchar_t* token = wcstok_s(data, kDelimiters, &tokenContext);
+        while (token) {
+          wchar_t fname[_MAX_FNAME] = {0};
+          if (!_wsplitpath_s(token, nullptr, 0, nullptr, 0,
+                             fname, mozilla::ArrayLength(fname),
+                             nullptr, 0)) {
+            // nvinit.dll is responsible for bootstrapping the DLL injection, so
+            // that is the library that we check for here
+            const wchar_t kNvInitName[] = MOZ_UTF16("nvinit");
+            if (!_wcsnicmp(fname, kNvInitName,
+                           mozilla::ArrayLength(kNvInitName))) {
+              return false;
+            }
+          }
+          token = wcstok_s(nullptr, kDelimiters, &tokenContext);
+        }
+      }
+    }
+    return true;
+  }
+
 #if defined(_M_IX86)
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     if (!mModule) {
       return false;
     }
+    if (!IsCompatible()) {
+#if defined(MOZILLA_INTERNAL_API)
+      NS_WARNING("NOP space patching is unavailable for compatibility reasons");
+#endif
+      return false;
+    }
 
     if (mPatchedFnsLen == maxPatchedFns) {
       // printf ("No space for hook in mPatchedFns.\n");
       return false;
     }
 
     byteptr_t fn = reinterpret_cast<byteptr_t>(GetProcAddress(mModule, aName));
     if (!fn) {