Bug 899026 - some API hooks don't work on Windows 8 x64. r=ehsan
authorMakoto Kato <m_kato@ga2.so-net.ne.jp>
Mon, 05 Aug 2013 11:13:53 +0900
changeset 153627 4951cadc9980588b1f40495e95c68dd872dae5be
parent 153626 2e3cdffc3e6505c860cadda56f354d0e8c3b4438
child 153628 26b10ebb6140e41572d780afdf647f631238db2b
push id2859
push userakeybl@mozilla.com
push dateMon, 16 Sep 2013 19:14:59 +0000
treeherdermozilla-beta@87d3c51cd2bf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs899026
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 899026 - some API hooks don't work on Windows 8 x64. r=ehsan
toolkit/xre/nsWindowsDllInterceptor.h
--- a/toolkit/xre/nsWindowsDllInterceptor.h
+++ b/toolkit/xre/nsWindowsDllInterceptor.h
@@ -388,16 +388,18 @@ protected:
         // jmp 32bit offset
         nBytes += 5;
       } else {
         //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nBytes]);
         return;
       }
     }
 #elif defined(_M_X64)
+    byteptr_t directJmpAddr;
+
     while (nBytes < 13) {
 
       // if found JMP 32bit offset, next bytes must be NOP 
       if (pJmp32 >= 0) {
         if (origBytes[nBytes++] != 0x90)
           return;
 
         continue;
@@ -473,21 +475,34 @@ protected:
             // REG=r64, R/M=r64 or REG=r64, R/M=[r64]
             nBytes += 2;
           } else {
             // complex MOV
             return;
           }
         } else if (origBytes[nBytes] == 0xc7) {
           // MOV r/m64, imm32
-          if (origBytes[nBytes + 1] & 0xf8 == 0x40) {
+          if ((origBytes[nBytes + 1] & 0xf8) == 0x40) {
             nBytes += 6;
           } else {
             return;
           }
+        } else if (origBytes[nBytes] == 0xff) {
+          pJmp32 = nBytes - 1;
+          // JMP /4
+          if ((origBytes[nBytes+1] & 0xc0) == 0x0 &&
+              (origBytes[nBytes+1] & 0x07) == 0x5) {
+            // [rip+disp32]
+            // convert JMP 32bit offset to JMP 64bit direct
+            directJmpAddr = (byteptr_t)*((uint64_t*)(origBytes + nBytes + 6 + (*((uint32_t*)(origBytes + nBytes + 2)))));
+            nBytes += 6;
+          } else {
+            // not support yet!
+            return;
+          }
         } else {
           // not support yet!
           return;
         }
       } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
         // 1-byte push/pop
         nBytes++;
       } else if (origBytes[nBytes] == 0x90) {
@@ -496,16 +511,18 @@ protected:
       } else if (origBytes[nBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         nBytes += 5;
       } else if (origBytes[nBytes] == 0xc3) {
         // ret
         nBytes++;
       } else if (origBytes[nBytes] == 0xe9) {
         pJmp32 = nBytes;
+        // convert JMP 32bit offset to JMP 64bit direct
+        directJmpAddr = origBytes + pJmp32 + 5 + (*((uint32_t*)(origBytes + pJmp32 + 1)));
         // jmp 32bit offset
         nBytes += 5;
       } else if (origBytes[nBytes] == 0xff) {
         nBytes++;
         if ((origBytes[nBytes] & 0xf8) == 0xf0) {
           // push r64
           nBytes++;
         } else {
@@ -542,18 +559,16 @@ protected:
       *((intptr_t*)(tramp+pJmp32+1)) += origBytes + pJmp32 - tramp;
     } else {
       tramp[nBytes] = 0xE9; // jmp
       *((intptr_t*)(tramp+nBytes+1)) = (intptr_t)trampDest - (intptr_t)(tramp+nBytes+5); // target displacement
     }
 #elif defined(_M_X64)
     // If JMP32 opcode found, we don't insert to trampoline jump 
     if (pJmp32 >= 0) {
-      // convert JMP 32bit offset to JMP 64bit direct
-      byteptr_t directJmpAddr = origBytes + pJmp32 + 5 + (*((LONG*)(origBytes+pJmp32+1)));
       // mov r11, address
       tramp[pJmp32]   = 0x49;
       tramp[pJmp32+1] = 0xbb;
       *((intptr_t*)(tramp+pJmp32+2)) = (intptr_t)directJmpAddr;
 
       // jmp r11
       tramp[pJmp32+10] = 0x41;
       tramp[pJmp32+11] = 0xff;