Bug 681238 part 0 - Support a few more opcodes and fix x86 mov r, [r] test. r=vlad
☠☠ backed out by 9f7ccdc68e3f ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Mon, 29 Aug 2011 13:23:21 +0200
changeset 76177 76e73aad0fabaeb8e26a3de4a5d6df4220de3050
parent 76176 153a1de6cd67e8f803f75288f0105e8c2e3b4e0b
child 76178 be03169f563597cc243a9d8459353598c56acae5
push id3
push userfelipc@gmail.com
push dateFri, 30 Sep 2011 20:09:13 +0000
reviewersvlad
bugs681238
milestone9.0a1
Bug 681238 part 0 - Support a few more opcodes and fix x86 mov r, [r] test. r=vlad
toolkit/xre/nsWindowsDllInterceptor.h
--- a/toolkit/xre/nsWindowsDllInterceptor.h
+++ b/toolkit/xre/nsWindowsDllInterceptor.h
@@ -191,27 +191,41 @@ protected:
 #if defined(_M_IX86)
     while (nBytes < 5) {
       // Understand some simple instructions that might be found in a
       // prologue; we might need to extend this as necessary.
       //
       // Note!  If we ever need to understand jump instructions, we'll
       // need to rewrite the displacement argument.
       if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
-        // various MOVs; but only handle the case where it truly is a 2-byte instruction
+        // various MOVs
         unsigned char b = origBytes[nBytes+1];
         if (((b & 0xc0) == 0xc0) ||
             (((b & 0xc0) == 0x00) &&
-             ((b & 0x38) != 0x20) && ((b & 0x38) != 0x28)))
+             ((b & 0x07) != 0x04) && ((b & 0x07) != 0x05)))
         {
+          // REG=r, R/M=r or REG=r, R/M=[r]
           nBytes += 2;
+        } else if (((b & 0xc0) == 0x40) && ((b & 0x38) != 0x20)) {
+          // REG=r, R/M=[r + disp8]
+          nBytes += 3;
         } else {
           // complex MOV, bail
           return 0;
         }
+      } else if (origBytes[nBytes] == 0x83) {
+        // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r/m, imm8
+        unsigned char b = origBytes[nBytes+1];
+        if ((b & 0xc0) == 0xc0) {
+          // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r, imm8
+          nBytes += 3;
+        } else {
+          // bail
+          return 0;
+        }
       } else if (origBytes[nBytes] == 0x68) {
         // PUSH with 4-byte operand
         nBytes += 5;
       } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
         // 1-byte PUSH/POP
         nBytes++;
       } else if (origBytes[nBytes] == 0x6A) {
         // PUSH imm8
@@ -252,57 +266,45 @@ protected:
         nBytes++;
 
         if (origBytes[nBytes] == 0x33) {
           // xor r32, r32
           nBytes += 2;
         } else {
           return 0;
         }
-      } else if (origBytes[nBytes] == 0x48) {
-        // REX.W
+      } else if ((origBytes[nBytes] & 0xfb) == 0x48) {
+        // REX.W | REX.WR
         nBytes++;
 
         if (origBytes[nBytes] == 0x81 && (origBytes[nBytes+1] & 0xf8) == 0xe8) {
           // sub r, dword
           nBytes += 6;
         } else if (origBytes[nBytes] == 0x83 &&
                   (origBytes[nBytes+1] & 0xf8) == 0xe8) {
           // sub r, byte
           nBytes += 3;
         } else if (origBytes[nBytes] == 0x83 &&
                   (origBytes[nBytes+1] & 0xf8) == 0x60) {
           // and [r+d], imm8
           nBytes += 5;
-        } else if (origBytes[nBytes] == 0x89) {
-          // MOV r/m64, r64
+        } else if ((origBytes[nBytes] & 0xfd) == 0x89) {
+          // MOV r/m64, r64 | MOV r64, r/m64
           if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
             if ((origBytes[nBytes+1] & 0x7) == 0x04) {
-              // mov [SIB+disp8], r64
+              // R/M=[SIB+disp8], REG=r64
               nBytes += 4;
             } else {
-              // mov [r64+disp8], r64
+              // R/M=[r64+disp8], REG=r64
               nBytes += 3;
             }
-          } else {
-            // complex mov
-            return 0;
-          }
-        } else if (origBytes[nBytes] == 0x8b) {
-          // mov r64, r/m64
-          if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
-            if ((origBytes[nBytes+1] & 0x7) == 0x04) {
-              // mov r64, [SIB+disp8]
-              nBytes += 4;
-            } else {
-              // mov r64, [r64+disp8]
-              nBytes += 3;
-            }
-          } else if ((origBytes[nBytes+1] & 0xc0) == 0xc0) {
-            // MOV r64, r64
+          } else if (((origBytes[nBytes+1] & 0xc0) == 0xc0) ||
+                     (((origBytes[nBytes+1] & 0xc0) == 0x00) &&
+                      ((origBytes[nBytes+1] & 0x07) != 0x04) && ((origBytes[nBytes+1] & 0x07) != 0x05))) {
+            // REG=r64, R/M=r64 or REG=r64, R/M=[r64]
             nBytes += 2;
           } else {
             // complex MOV
             return 0;
           }
         } else {
           // not support yet!
           return 0;