Bug 1240848: Adds additional instructions to x64 detour patcher; r=ehsan
authorAaron Klotz <aklotz@mozilla.com>
Tue, 19 Jan 2016 15:14:24 -0700
changeset 317629 a786af9186eba708a37ac59df94fa5e148dbe4a3
parent 317628 90e42416b6c9faa0f88cca915a26a1c569c1738b
child 317630 31f08cfe084d7e57a28866ac0a9fa36a4ea50040
push id1079
push userjlund@mozilla.com
push dateFri, 15 Apr 2016 21:02:33 +0000
treeherdermozilla-release@575fbf6786d5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs1240848
milestone46.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1240848: Adds additional instructions to x64 detour patcher; r=ehsan
toolkit/xre/test/win/TestDllInterceptor.cpp
xpcom/build/nsWindowsDllInterceptor.h
--- a/toolkit/xre/test/win/TestDllInterceptor.cpp
+++ b/toolkit/xre/test/win/TestDllInterceptor.cpp
@@ -157,15 +157,18 @@ int main()
       TestHook("kernel32.dll", "VirtualAlloc") &&
       TestHook("kernel32.dll", "MapViewOfFile") &&
       TestHook("gdi32.dll", "CreateDIBSection") &&
       TestHook("kernel32.dll", "CreateFileW") &&
 #endif
       TestHook("imm32.dll", "ImmGetContext") &&
       TestHook("imm32.dll", "ImmGetCompositionStringW") &&
       TestHook("imm32.dll", "ImmSetCandidateWindow") &&
+#ifdef _M_X64
+      TestHook("user32.dll", "CreateWindowExW") &&
+#endif
       TestDetour("ntdll.dll", "LdrLoadDll")) {
     printf("TEST-PASS | WindowsDllInterceptor | all checks passed\n");
     return 0;
   }
 
   return 1;
 }
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -733,25 +733,53 @@ protected:
           } else {
             // not support yet!
             return;
           }
         } else {
           // not support yet!
           return;
         }
+      } else if (origBytes[nBytes] == 0x66) {
+        // operand override prefix
+        nBytes += 1;
+        // This is the same as the x86 version
+        if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
+          // various MOVs
+          unsigned char b = origBytes[nBytes + 1];
+          if (((b & 0xc0) == 0xc0) ||
+              (((b & 0xc0) == 0x00) &&
+               ((b & 0x07) != 0x04) && ((b & 0x07) != 0x05))) {
+            // REG=r, R/M=r or REG=r, R/M=[r]
+            nBytes += 2;
+          } else if ((b & 0xc0) == 0x40) {
+            if ((b & 0x07) == 0x04) {
+              // REG=r, R/M=[SIB + disp8]
+              nBytes += 4;
+            } else {
+              // REG=r, R/M=[r + disp8]
+              nBytes += 3;
+            }
+          } else {
+            // complex MOV, bail
+            return;
+          }
+        }
       } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
         // 1-byte push/pop
         nBytes++;
       } else if (origBytes[nBytes] == 0x90) {
         // nop
         nBytes++;
       } else if (origBytes[nBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         nBytes += 5;
+      } else if (origBytes[nBytes] == 0x33) {
+        // xor r32, r/m32
+        nBytes += 2;
       } else if (origBytes[nBytes] == 0xc3) {
         // ret
         nBytes++;
       } else if (origBytes[nBytes] == 0xcc) {
         // int 3
         nBytes++;
       } else if (origBytes[nBytes] == 0xe9) {
         // jmp 32bit offset