Bug 1240977: DLL interceptor improvements for Windows 10 x64 ntdll APIs; r=ehsan
authorAaron Klotz <aklotz@mozilla.com>
Thu, 13 Oct 2016 15:15:22 -0600
changeset 317927 78344bbde6134a8be1c22d159730189c876205af
parent 317926 332c2d9bc2151c885afea77b0f2a2b5db6b7498b
child 317928 609a153be0a59c7cdb79d06c0254dfdeb33dbaa4
push id33170
push usercbook@mozilla.com
push dateFri, 14 Oct 2016 10:37:07 +0000
treeherderautoland@0d101ebfd95c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersehsan
bugs1240977
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1240977: DLL interceptor improvements for Windows 10 x64 ntdll APIs; r=ehsan MozReview-Commit-ID: rM8XWK5y2n
xpcom/build/nsWindowsDllInterceptor.h
--- a/xpcom/build/nsWindowsDllInterceptor.h
+++ b/xpcom/build/nsWindowsDllInterceptor.h
@@ -2,16 +2,18 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef NS_WINDOWS_DLL_INTERCEPTOR_H_
 #define NS_WINDOWS_DLL_INTERCEPTOR_H_
 
+#include "mozilla/Assertions.h"
+
 #include <windows.h>
 #include <winternl.h>
 
 /*
  * Simple function interception.
  *
  * We have two separate mechanisms for intercepting a function: We can use the
  * built-in nop space, if it exists, or we can create a detour.
@@ -62,67 +64,16 @@
  *
  */
 
 #include <stdint.h>
 
 namespace mozilla {
 namespace internal {
 
-// Get op length of '/r'
-static size_t
-GetOpLengthByModRM(const uint8_t* aOp)
-{
-  uint8_t mod = *aOp >> 6;
-  uint8_t rm = *aOp & 0x7;
-
-  switch (mod) {
-  case 0:
-    if (rm == 4) {
-      // SIB
-      if ((*(aOp + 1) & 0x7) == 5) {
-        // disp32
-        return 6;
-      }
-      return 2;
-    } else if (rm == 5) {
-      // [RIP/EIP + disp32]
-      // Since we don't modify relative offset, we should mark as impossible
-      // code.
-      return 0;
-    }
-    // [r/m]
-    return 1;
-
-  case 1:
-    if (rm == 4) {
-      // [SIB + imm8]
-      return 3;
-    }
-    // [r/m + imm8]
-    return 2;
-
-  case 2:
-    if (rm == 4) {
-      // [SIB + imm32]
-      return 6;
-    }
-    // [r/m + imm32]
-    return 5;
-
-  case 3:
-    // r/w
-    return 1;
-
-  default:
-     break;
-  }
-  return 0;
-}
-
 class AutoVirtualProtect
 {
 public:
   AutoVirtualProtect(void* aFunc, size_t aSize, DWORD aProtect)
     : mFunc(aFunc), mSize(aSize), mNewProtect(aProtect), mOldProtect(0),
       mSuccess(false)
   {}
 
@@ -418,16 +369,79 @@ protected:
   const static int kPageSize = 4096;
   const static int kHookSize = 128;
 
   HMODULE mModule;
   byteptr_t mHookPage;
   int mMaxHooks;
   int mCurHooks;
 
+  // rex bits
+  static const BYTE kMaskHighNibble = 0xF0;
+  static const BYTE kRexOpcode = 0x40;
+  static const BYTE kMaskRexW = 0x08;
+  static const BYTE kMaskRexR = 0x04;
+  static const BYTE kMaskRexX = 0x02;
+  static const BYTE kMaskRexB = 0x01;
+
+  // mod r/m bits
+  static const BYTE kRegFieldShift = 3;
+  static const BYTE kMaskMod = 0xC0;
+  static const BYTE kMaskReg = 0x38;
+  static const BYTE kMaskRm = 0x07;
+  static const BYTE kRmNeedSib = 0x04;
+  static const BYTE kModReg = 0xC0;
+  static const BYTE kModDisp32 = 0x80;
+  static const BYTE kModDisp8 = 0x40;
+  static const BYTE kModNoRegDisp = 0x00;
+  static const BYTE kRmNoRegDispDisp32 = 0x05;
+
+  // sib bits
+  static const BYTE kMaskSibScale = 0xC0;
+  static const BYTE kMaskSibIndex = 0x38;
+  static const BYTE kMaskSibBase = 0x07;
+  static const BYTE kSibBaseEbp = 0x05;
+
+  int CountModRmSib(const BYTE *aModRm, BYTE* aSubOpcode = nullptr)
+  {
+    if (!aModRm) {
+      return -1;
+    }
+    int numBytes = 1; // Start with 1 for mod r/m byte itself
+    switch (*aModRm & kMaskMod) {
+      case kModReg:
+        return numBytes;
+      case kModDisp8:
+        numBytes += 1;
+        break;
+      case kModDisp32:
+        numBytes += 4;
+        break;
+      case kModNoRegDisp:
+        if ((*aModRm & kMaskRm) == kRmNoRegDispDisp32 ||
+            ((*aModRm & kMaskRm) == kRmNeedSib &&
+             (*(aModRm + 1) & kMaskSibBase) == kSibBaseEbp)) {
+          numBytes += 4;
+        }
+        break;
+      default:
+        // This should not be reachable
+        MOZ_ASSERT_UNREACHABLE("Impossible value for modr/m byte mod bits");
+        return -1;
+    }
+    if ((*aModRm & kMaskRm) == kRmNeedSib) {
+      // SIB byte
+      numBytes += 1;
+    }
+    if (aSubOpcode) {
+      *aSubOpcode = (*aModRm & kMaskReg) >> kRegFieldShift;
+    }
+    return numBytes;
+  }
+
 #if defined(_M_X64)
   // To patch for JMP and JE
 
   enum JumpType {
    Je,
    Jmp
   };
 
@@ -700,31 +714,42 @@ protected:
         // GS prefix
         //
         // The entry of GetKeyState on Windows 10 has the following code.
         // 65 48 8b 04 25 30 00 00 00    mov   rax,qword ptr gs:[30h]
         // (GS prefix + REX + MOV (0x8b) ...)
         if (origBytes[nBytes + 1] == 0x48 &&
             (origBytes[nBytes + 2] >= 0x88 && origBytes[nBytes + 2] <= 0x8b)) {
           nBytes += 3;
-          size_t len = GetOpLengthByModRM(origBytes + nBytes);
-          if (!len) {
+          int len = CountModRmSib(origBytes + nBytes);
+          if (len < 0) {
             // no way to support this yet.
             return;
           }
           nBytes += len;
         } else {
           return;
         }
       } else if (origBytes[nBytes] == 0x90) {
         // nop
         nBytes++;
       } else if (origBytes[nBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         nBytes += 5;
+      } else if (origBytes[nBytes] == 0xf6) {
+        // test r/m8, imm8 (used by ntdll on Windows 10 x64)
+        // (no flags are affected by near jmp since there is no task switch,
+        // so it is ok for a jmp to be written immediately after a test)
+        BYTE subOpcode = 0;
+        int nModRmSibBytes = CountModRmSib(&origBytes[nBytes + 1], &subOpcode);
+        if (nModRmSibBytes < 0 || subOpcode != 0) {
+          // Unsupported
+          return;
+        }
+        nBytes += 2 + nModRmSibBytes;
       } else if (origBytes[nBytes] == 0xc3) {
         // ret
         nBytes++;
       } else if (origBytes[nBytes] == 0xcc) {
         // int 3
         nBytes++;
       } else if (origBytes[nBytes] == 0xe9) {
         // jmp 32bit offset