Bug 604302 - Windows DLL blocklist support for Windows x64. r=vlad
authorMakoto Kato <m_kato@ga2.so-net.ne.jp>
Mon, 25 Apr 2011 10:02:07 +0900
changeset 68501 c062731105cf04ef462a78debeaf00a4e492167d
parent 68500 0eaee388eeadf2da8850c88f4bf3d31673fa255b
child 68502 246b6dbc417a28dbbd3b29c691200af91e94f06e
child 68647 c9d5e9a67657a0a18d313d0ffba87cc1d3f21f1e
child 69184 a06f8f79f7f4f901c8649351c207819b8bc5af7a
push id19663
push userm_kato@ga2.so-net.ne.jp
push dateMon, 25 Apr 2011 04:49:30 +0000
treeherdermozilla-central@c062731105cf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersvlad
bugs604302
milestone6.0a1
first release with
nightly linux32
c062731105cf / 6.0a1 / 20110425030532 / files
nightly linux64
c062731105cf / 6.0a1 / 20110425030532 / files
nightly mac
c062731105cf / 6.0a1 / 20110425030532 / files
nightly win32
c062731105cf / 6.0a1 / 20110425030532 / files
nightly win64
c062731105cf / 6.0a1 / 20110425030208 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 604302 - Windows DLL blocklist support for Windows x64. r=vlad
toolkit/xre/nsWindowsDllBlocklist.cpp
toolkit/xre/nsWindowsDllInterceptor.h
toolkit/xre/nsWindowsWMain.cpp
--- a/toolkit/xre/nsWindowsDllBlocklist.cpp
+++ b/toolkit/xre/nsWindowsDllBlocklist.cpp
@@ -51,22 +51,16 @@
 
 #ifndef STATUS_DLL_NOT_FOUND
 #define STATUS_DLL_NOT_FOUND ((DWORD)0xC0000135L)
 #endif
 
 // define this for very verbose dll load debug spew
 #undef DEBUG_very_verbose
 
-// The signature for LdrLoadDll changed at some point, with the second arg
-// becoming a PULONG instead of a ULONG.  This should only matter on 64-bit
-// systems, for which there was no support earlier -- on 32-bit systems,
-// they should be the same size.
-PR_STATIC_ASSERT(sizeof(PULONG) == sizeof(ULONG));
-
 typedef NTSTATUS (NTAPI *LdrLoadDll_func) (PWCHAR filePath, PULONG flags, PUNICODE_STRING moduleFileName, PHANDLE handle);
 
 static LdrLoadDll_func stub_LdrLoadDll = 0;
 
 static NTSTATUS NTAPI
 patched_LdrLoadDll (PWCHAR filePath, PULONG flags, PUNICODE_STRING moduleFileName, PHANDLE handle)
 {
   // We have UCS2 (UTF16?), we want ASCII, but we also just want the filename portion
--- a/toolkit/xre/nsWindowsDllInterceptor.h
+++ b/toolkit/xre/nsWindowsDllInterceptor.h
@@ -146,16 +146,17 @@ protected:
   {
     byteptr_t tramp = FindTrampolineSpace();
     if (!tramp)
       return 0;
 
     byteptr_t origBytes = (byteptr_t) origFunction;
 
     int nBytes = 0;
+#if defined(_M_IX86)
     while (nBytes < 5) {
       // Understand some simple instructions that might be found in a
       // prologue; we might need to extend this as necessary.
       //
       // Note!  If we ever need to understand jump instructions, we'll
       // need to rewrite the displacement argument.
       if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
         // various MOVs; but only handle the case where it truly is a 2-byte instruction
@@ -178,40 +179,191 @@ protected:
       } else if (origBytes[nBytes] == 0x6A) {
         // PUSH imm8
         nBytes += 2;
       } else {
         //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nBytes]);
         return 0;
       }
     }
+#elif defined(_M_X64)
+    int pJmp32 = 0;
+
+    while (nBytes < 13) {
+
+      // if found JMP 32bit offset, next bytes must be NOP 
+      if (pJmp32) {
+        if (origBytes[nBytes++] != 0x90)
+          return 0;
+
+        continue;
+      } 
+        
+      if (origBytes[nBytes] == 0x41) {
+        // REX.B
+        nBytes++;
+
+        if ((origBytes[nBytes] & 0xf0) == 0x50) {
+          // push/pop with Rx register
+          nBytes++;
+        } else if (origBytes[nBytes] >= 0xb8 && origBytes[nBytes] <= 0xbf) {
+          // mov r32, imm32
+          nBytes += 5;
+        } else {
+          return 0;
+        }
+      } else if (origBytes[nBytes] == 0x45) {
+        // REX.R & REX.B
+        nBytes++;
+
+        if (origBytes[nBytes] == 0x33) {
+          // xor r32, r32
+          nBytes += 2;
+        } else {
+          return 0;
+        }
+      } else if (origBytes[nBytes] == 0x48) {
+        // REX.W
+        nBytes++;
+
+        if (origBytes[nBytes] == 0x81 && (origBytes[nBytes+1] & 0xf8) == 0xe8) {
+          // sub r, dword
+          nBytes += 6;
+        } else if (origBytes[nBytes] == 0x83 &&
+                  (origBytes[nBytes+1] & 0xf8) == 0xe8) {
+          // sub r, byte
+          nBytes += 3;
+        } else if (origBytes[nBytes] == 0x83 &&
+                  (origBytes[nBytes+1] & 0xf8) == 0x60) {
+          // and [r+d], imm8
+          nBytes += 5;
+        } else if (origBytes[nBytes] == 0x89) {
+          // MOV r/m64, r64
+          if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
+            if ((origBytes[nBytes+1] & 0x7) == 0x04) {
+              // mov [SIB+disp8], r64
+              nBytes += 4;
+            } else {
+              // mov [r64+disp8], r64
+              nBytes += 3;
+            }
+          } else {
+            // complex mov
+            return 0;
+          }
+        } else if (origBytes[nBytes] == 0x8b) {
+          // mov r64, r/m64
+          if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
+            if ((origBytes[nBytes+1] & 0x7) == 0x04) {
+              // mov r64, [SIB+disp8]
+              nBytes += 4;
+            } else {
+              // mov r64, [r64+disp8]
+              nBytes += 3;
+            }
+          } else if ((origBytes[nBytes+1] & 0xc0) == 0xc0) {
+            // MOV r64, r64
+            nBytes += 2;
+          } else {
+            // complex MOV
+            return 0;
+          }
+        } else {
+          // not support yet!
+          return 0;
+        }
+      } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
+        // 1-byte push/pop
+        nBytes++;
+      } else if (origBytes[nBytes] == 0x90) {
+        // nop
+        nBytes++;
+      } else if (origBytes[nBytes] == 0xe9) {
+        pJmp32 = nBytes;
+        // jmp 32bit offset
+        nBytes += 5;
+      } else if (origBytes[nBytes] == 0xff) {
+        nBytes++;
+        if ((origBytes[nBytes] & 0xf8) == 0xf0) {
+          // push r64
+          nBytes++;
+        } else {
+          return 0;
+        }
+      } else {
+        return 0;
+      }
+    }
+#else
+#error "Unknown processor type"
+#endif
 
     if (nBytes > 100) {
       //printf ("Too big!");
       return 0;
     }
 
     memcpy(tramp, origFunction, nBytes);
 
     // OrigFunction+N, the target of the trampoline
     byteptr_t trampDest = origBytes + nBytes;
 
+#if defined(_M_IX86)
     tramp[nBytes] = 0xE9; // jmp
     *((intptr_t*)(tramp+nBytes+1)) = (intptr_t)trampDest - (intptr_t)(tramp+nBytes+5); // target displacement
+#elif defined(_M_X64)
+    // If JMP32 opcode found, we don't insert to trampoline jump 
+    if (pJmp32) {
+      // convert JMP 32bit offset to JMP 64bit direct
+      byteptr_t directJmpAddr = origBytes + pJmp32 + 5 + (*((LONG*)(origBytes+pJmp32+1)));
+      // mov r11, address
+      tramp[pJmp32]   = 0x49;
+      tramp[pJmp32+1] = 0xbb;
+      *((intptr_t*)(tramp+pJmp32+2)) = (intptr_t)directJmpAddr;
+
+      // jmp r11
+      tramp[pJmp32+10] = 0x41;
+      tramp[pJmp32+11] = 0xff;
+      tramp[pJmp32+12] = 0xe3;
+    } else {
+      // mov r11, address
+      tramp[nBytes] = 0x49;
+      tramp[nBytes+1] = 0xbb;
+      *((intptr_t*)(tramp+nBytes+2)) = (intptr_t)trampDest;
+
+      // jmp r11
+      tramp[nBytes+10] = 0x41;
+      tramp[nBytes+11] = 0xff;
+      tramp[nBytes+12] = 0xe3;
+    }
+#endif
 
     // ensure we can modify the original code
     DWORD op;
     if (!VirtualProtectEx(GetCurrentProcess(), origFunction, nBytes, PAGE_EXECUTE_READWRITE, &op)) {
       //printf ("VirtualProtectEx failed! %d\n", GetLastError());
       return 0;
     }
 
+#if defined(_M_IX86)
     // now modify the original bytes
     origBytes[0] = 0xE9; // jmp
     *((intptr_t*)(origBytes+1)) = (intptr_t)dest - (intptr_t)(origBytes+5); // target displacement
+#elif defined(_M_X64)
+    // mov r11, address
+    origBytes[0] = 0x49;
+    origBytes[1] = 0xbb;
+
+    *((intptr_t*)(origBytes+2)) = (intptr_t)dest;
+
+    // jmp r11
+    origBytes[10] = 0x41;
+    origBytes[11] = 0xff;
+    origBytes[12] = 0xe3;
+#endif
 
     // restore protection; if this fails we can't really do anything about it
     VirtualProtectEx(GetCurrentProcess(), origFunction, nBytes, op, &op);
 
     return tramp;
   }
 
   byteptr_t FindTrampolineSpace() {
--- a/toolkit/xre/nsWindowsWMain.cpp
+++ b/toolkit/xre/nsWindowsWMain.cpp
@@ -7,17 +7,17 @@
 #endif
 
 #include "nsUTF8Utils.h"
 
 #ifndef XRE_DONT_PROTECT_DLL_LOAD
 #include "nsSetDllDirectory.h"
 #endif
 
-#if defined(_MSC_VER) && defined(_M_IX86) && defined(XRE_WANT_DLL_BLOCKLIST)
+#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) && defined(XRE_WANT_DLL_BLOCKLIST)
 #include "nsWindowsDllBlocklist.cpp"
 #else
 #undef XRE_WANT_DLL_BLOCKLIST
 #endif
 
 
 #ifdef __MINGW32__