Bug 1284897 - Add missing hooked methods to TestDllInterceptor. r=aklotz
☠☠ backed out by 1112eb3a5f8c ☠ ☠
authorDavid Parks <dparks@mozilla.com>
Tue, 07 Feb 2017 12:00:45 -0800
changeset 373200 4e81ec8850dcaf402ce8b23f146859cfea377f41
parent 373199 a176abd99d2b2a12288694f0a8378266372d1f84
child 373201 71b9ac06a60a570b6a7ce4560a685f642122a9d4
push id10863
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 23:02:23 +0000
treeherdermozilla-aurora@0931190cd725 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaklotz
bugs1284897
milestone54.0a1
Bug 1284897 - Add missing hooked methods to TestDllInterceptor. r=aklotz Added ASSERTions to nsWindowsDllInterceptor in case of a failed detour hook, with an exception for the RET opcode that appears in ImmReleaseContext. Added documentation about TestDllInterceptor.
toolkit/xre/test/win/TestDllInterceptor.cpp
xpcom/build/nsWindowsDllInterceptor.h
--- a/toolkit/xre/test/win/TestDllInterceptor.cpp
+++ b/toolkit/xre/test/win/TestDllInterceptor.cpp
@@ -204,25 +204,30 @@ int main()
       TestHook("ntdll.dll", "NtQueryFullAttributesFile") &&
       // Bug 733892: toolkit/crashreporter/nsExceptionHandler.cpp
       TestHook("kernel32.dll", "SetUnhandledExceptionFilter") &&
 #ifdef _M_IX86
       // Bug 670967: xpcom/base/AvailableMemoryTracker.cpp
       TestHook("kernel32.dll", "VirtualAlloc") &&
       TestHook("kernel32.dll", "MapViewOfFile") &&
       TestHook("gdi32.dll", "CreateDIBSection") &&
-      TestHook("kernel32.dll", "CreateFileW") &&
+      TestHook("kernel32.dll", "CreateFileW") &&    // see Bug 1316415
 #endif
+      TestHook("kernel32.dll", "CreateFileA") &&
       TestDetour("user32.dll", "CreateWindowExW") &&
       TestHook("user32.dll", "InSendMessageEx") &&
       TestHook("imm32.dll", "ImmGetContext") &&
+      // TestHook("imm32.dll", "ImmReleaseContext") &&    // see Bug 1316415
       TestHook("imm32.dll", "ImmGetCompositionStringW") &&
       TestHook("imm32.dll", "ImmSetCandidateWindow") &&
+      TestHook("imm32.dll", "ImmNotifyIME") &&
+      TestHook("comdlg32.dll", "GetSaveFileNameW") &&
+      TestHook("comdlg32.dll", "GetOpenFileNameW") &&
 #ifdef _M_X64
-      TestHook("user32.dll", "GetKeyState") &&
+      TestHook("user32.dll", "GetKeyState") &&    // see Bug 1316415
 #endif
       MaybeTestHook(ShouldTestTipTsf(), "tiptsf.dll", "ProcessCaretEvents") &&
 #ifdef _M_IX86
       TestHook("user32.dll", "SendMessageTimeoutW") &&
 #endif
       TestDetour("ntdll.dll", "LdrLoadDll")) {
     printf("TEST-PASS | WindowsDllInterceptor | all checks passed\n");
     return 0;
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -695,25 +695,27 @@ protected:
       //
       // Note!  If we ever need to understand jump instructions, we'll
       // need to rewrite the displacement argument.
       unsigned char prefixGroups;
       int numPrefixBytes = CountPrefixBytes(origBytes, nOrigBytes, &prefixGroups);
       if (numPrefixBytes < 0 || (prefixGroups & (ePrefixGroup3 | ePrefixGroup4))) {
         // Either the prefix sequence was bad, or there are prefixes that
         // we don't currently support (groups 3 and 4)
+        MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
         return;
       }
       nOrigBytes += numPrefixBytes;
       if (origBytes[nOrigBytes] >= 0x88 &&
           origBytes[nOrigBytes] <= 0x8B) {
         // various MOVs
         ++nOrigBytes;
         int len = CountModRmSib(origBytes + nOrigBytes);
         if (len < 0) {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized MOV opcode sequence");
           return;
         }
         nOrigBytes += len;
       } else if (origBytes[nOrigBytes] == 0xA1) {
         // MOV eax, [seg:offset]
         nOrigBytes += 5;
       } else if (origBytes[nOrigBytes] == 0xB8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
@@ -728,16 +730,17 @@ protected:
       } else if (origBytes[nOrigBytes] == 0x83) {
         // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r/m, imm8
         unsigned char b = origBytes[nOrigBytes + 1];
         if ((b & 0xc0) == 0xc0) {
           // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r, imm8
           nOrigBytes += 3;
         } else {
           // bail
+          MOZ_ASSERT_UNREACHABLE("Unrecognized bit opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x68) {
         // PUSH with 4-byte operand
         nOrigBytes += 5;
       } else if ((origBytes[nOrigBytes] & 0xf0) == 0x50) {
         // 1-byte PUSH/POP
         nOrigBytes++;
@@ -754,17 +757,18 @@ protected:
         nOrigBytes += 6;
       } else if (origBytes[nOrigBytes] == 0xc2) {
         // ret imm16.  We can't handle this but it happens.  We don't ASSERT but we do fail to hook.
 #if defined(MOZILLA_INTERNAL_API)
         NS_WARNING("Cannot hook method -- RET opcode found");
 #endif
         return;
       } else {
-        //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nBytes]);
+        //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nOrigBytes]);
+        MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
         return;
       }
     }
 
     // The trampoline is a copy of the instructions that we just traced,
     // followed by a jump that we add below.
     memcpy(tramp, aOrigFunction, nOrigBytes);
 #elif defined(_M_X64)
@@ -781,64 +785,69 @@ protected:
       // then its _probably_ the target of a JMP somewhere else and we
       // will be overwriting it, which would be tragic.  This seems
       // highly unlikely.
       if (foundJmp) {
         if (origBytes[nOrigBytes] == 0x90 || origBytes[nOrigBytes] == 0xcc) {
           nOrigBytes++;
           continue;
         }
+        MOZ_ASSERT_UNREACHABLE("Opcode sequence includes commands after JMP");
         return;
       }
       if (origBytes[nOrigBytes] == 0x0f) {
         COPY_CODES(1);
         if (origBytes[nOrigBytes] == 0x1f) {
           // nop (multibyte)
           COPY_CODES(1);
           if ((origBytes[nOrigBytes] & 0xc0) == 0x40 &&
               (origBytes[nOrigBytes] & 0x7) == 0x04) {
             COPY_CODES(3);
           } else {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else if (origBytes[nOrigBytes] == 0x05) {
           // syscall
           COPY_CODES(1);
         } else if (origBytes[nOrigBytes] == 0x84) {
           // je rel32
           JumpPatch jump(nTrampBytes - 1,  // overwrite the 0x0f we copied above
                           (intptr_t)(origBytes + nOrigBytes + 5 +
                                      *(reinterpret_cast<int32_t*>(origBytes + nOrigBytes + 1))),
                           JumpType::Je);
           nTrampBytes = jump.GenerateJump(tramp);
           nOrigBytes += 5;
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x40 ||
                  origBytes[nOrigBytes] == 0x41) {
         // Plain REX or REX.B
         COPY_CODES(1);
         if ((origBytes[nOrigBytes] & 0xf0) == 0x50) {
           // push/pop with Rx register
           COPY_CODES(1);
         } else if (origBytes[nOrigBytes] >= 0xb8 && origBytes[nOrigBytes] <= 0xbf) {
           // mov r32, imm32
           COPY_CODES(5);
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x45) {
         // REX.R & REX.B
         COPY_CODES(1);
 
         if (origBytes[nOrigBytes] == 0x33) {
           // xor r32, r32
           COPY_CODES(2);
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if ((origBytes[nOrigBytes] & 0xfb) == 0x48) {
         // REX.W | REX.WR
         COPY_CODES(1);
 
         if (origBytes[nOrigBytes] == 0x81 &&
             (origBytes[nOrigBytes + 1] & 0xf8) == 0xe8) {
@@ -860,33 +869,36 @@ protected:
                    (origBytes[nOrigBytes + 1] & kMaskMod) == kModReg) {
           // sub r64, r64
           COPY_CODES(2);
         } else if (origBytes[nOrigBytes] == 0x85) {
           // 85 /r => TEST r/m32, r32
           if ((origBytes[nOrigBytes + 1] & 0xc0) == 0xc0) {
             COPY_CODES(2);
           } else {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else if ((origBytes[nOrigBytes] & 0xfd) == 0x89) {
           COPY_CODES(1);
           // MOV r/m64, r64 | MOV r64, r/m64
           int len = CountModRmSib(origBytes + nOrigBytes);
           if (len < 0) {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
           COPY_CODES(len);
         } else if (origBytes[nOrigBytes] == 0xc7) {
           // MOV r/m64, imm32
           if (origBytes[nOrigBytes + 1] == 0x44) {
             // MOV [r64+disp8], imm32
             // ModR/W + SIB + disp8 + imm32
             COPY_CODES(8);
           } else {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else if (origBytes[nOrigBytes] == 0xff) {
           // JMP /4
           if ((origBytes[nOrigBytes + 1] & 0xc0) == 0x0 &&
               (origBytes[nOrigBytes + 1] & 0x07) == 0x5) {
             // [rip+disp32]
             // convert JMP 32bit offset to JMP 64bit direct
@@ -894,20 +906,22 @@ protected:
                            *reinterpret_cast<intptr_t*>(origBytes + nOrigBytes + 6 +
                                                         *reinterpret_cast<int32_t*>(origBytes + nOrigBytes + 2)),
                            JumpType::Jmp);
             nTrampBytes = jump.GenerateJump(tramp);
             nOrigBytes += 6;
             foundJmp = true;
           } else {
             // not support yet!
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else {
           // not support yet!
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x66) {
         // operand override prefix
         COPY_CODES(1);
         // This is the same as the x86 version
         if (origBytes[nOrigBytes] >= 0x88 && origBytes[nOrigBytes] <= 0x8B) {
           // various MOVs
@@ -922,16 +936,17 @@ protected:
               // REG=r, R/M=[SIB + disp8]
               COPY_CODES(4);
             } else {
               // REG=r, R/M=[r + disp8]
               COPY_CODES(3);
             }
           } else {
             // complex MOV, bail
+            MOZ_ASSERT_UNREACHABLE("Unrecognized MOV opcode sequence");
             return;
           }
         }
       } else if ((origBytes[nOrigBytes] & 0xf0) == 0x50) {
         // 1-byte push/pop
         COPY_CODES(1);
       } else if (origBytes[nOrigBytes] == 0x65) {
         // GS prefix
@@ -940,20 +955,22 @@ protected:
         // 65 48 8b 04 25 30 00 00 00    mov   rax,qword ptr gs:[30h]
         // (GS prefix + REX + MOV (0x8b) ...)
         if (origBytes[nOrigBytes + 1] == 0x48 &&
             (origBytes[nOrigBytes + 2] >= 0x88 && origBytes[nOrigBytes + 2] <= 0x8b)) {
           COPY_CODES(3);
           int len = CountModRmSib(origBytes + nOrigBytes);
           if (len < 0) {
             // no way to support this yet.
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
           COPY_CODES(len);
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x90) {
         // nop
         COPY_CODES(1);
       } else if (origBytes[nOrigBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         COPY_CODES(5);
@@ -963,16 +980,17 @@ protected:
       } else if (origBytes[nOrigBytes] == 0xf6) {
         // test r/m8, imm8 (used by ntdll on Windows 10 x64)
         // (no flags are affected by near jmp since there is no task switch,
         // so it is ok for a jmp to be written immediately after a test)
         BYTE subOpcode = 0;
         int nModRmSibBytes = CountModRmSib(&origBytes[nOrigBytes + 1], &subOpcode);
         if (nModRmSibBytes < 0 || subOpcode != 0) {
           // Unsupported
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
         COPY_CODES(2 + nModRmSibBytes);
       } else if (origBytes[nOrigBytes] == 0xd1 &&
                   (origBytes[nOrigBytes+1] & kMaskMod) == kModReg) {
         // bit shifts/rotates : (SA|SH|RO|RC)(R|L) r32
         // (e.g. 0xd1 0xe0 is SAL, 0xd1 0xc8 is ROR)
         COPY_CODES(2);
@@ -1002,19 +1020,21 @@ protected:
           foundJmp = true;
           int32_t offset = *(reinterpret_cast<int32_t*>(origBytes + nOrigBytes + 1));
           int64_t* ptrToJmpDest = reinterpret_cast<int64_t*>(origBytes + nOrigBytes + 5 + offset);
           intptr_t jmpDest = static_cast<intptr_t>(*ptrToJmpDest);
           JumpPatch jump(nTrampBytes, jmpDest, JumpType::Jmp);
           nTrampBytes = jump.GenerateJump(tramp);
           nOrigBytes += 5;
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else {
+        MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
         return;
       }
     }
 #else
 #error "Unknown processor type"
 #endif
 
     if (nOrigBytes > 100) {
@@ -1140,32 +1160,52 @@ public:
 
   void LockHooks()
   {
     if (mDetourPatcher.Initialized()) {
       mDetourPatcher.LockHooks();
     }
   }
 
+  /**
+   * Hook/detour the method aName from the DLL we set in Init so that it calls
+   * aHookDest instead.  Returns the original method pointer in aOrigFunc
+   * and returns true if successful.
+   *
+   * IMPORTANT: If you use this method, please add your case to the
+   * TestDllInterceptor in order to detect future failures.  Even if this
+   * succeeds now, updates to the hooked DLL could cause it to fail in
+   * the future.
+   */
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     // Use a nop space patch if possible, otherwise fall back to a detour.
     // This should be the preferred method for adding hooks.
 
     if (!mModuleName) {
       return false;
     }
 
     if (mNopSpacePatcher.AddHook(aName, aHookDest, aOrigFunc)) {
       return true;
     }
 
     return AddDetour(aName, aHookDest, aOrigFunc);
   }
 
+  /**
+   * Detour the method aName from the DLL we set in Init so that it calls
+   * aHookDest instead.  Returns the original method pointer in aOrigFunc
+   * and returns true if successful.
+   *
+   * IMPORTANT: If you use this method, please add your case to the
+   * TestDllInterceptor in order to detect future failures.  Even if this
+   * succeeds now, updates to the detoured DLL could cause it to fail in
+   * the future.
+   */
   bool AddDetour(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     // Generally, code should not call this method directly. Use AddHook unless
     // there is a specific need to avoid nop space patches.
 
     if (!mModuleName) {
       return false;
     }