Bug 1436953 - ARM64 assembler fixes. r=sstangl
authorLars T Hansen <lhansen@mozilla.com>
Fri, 15 Dec 2017 11:19:00 -0600
changeset 405327 0d3462c103e91b79c79a58564582a866d4f5cbd6
parent 405326 d2ffffb30b31aaaaf4d1d5623eb77894b0455806
child 405328 a37627e154f3c3d599ce439b18ff2a7a5d4c6f94
push id33518
push usertoros@mozilla.com
push dateMon, 26 Feb 2018 22:20:13 +0000
treeherdermozilla-central@580d833df9c4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssstangl
bugs1436953
milestone60.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1436953 - ARM64 assembler fixes. r=sstangl - Implement `InvertCondition(DoubleCondition)` - Implement wasm buffer management - Implement `bindLater()` - Implement a better definition of `Unreachable()` that does not change the PC or the registers - Add `IsMovz()` and `IsMovk()` predicates, we'll need them - Bugfix: Patching functions must flush the icache for the updated locs - Bugfix: `AbiArgIter()` must handle 64-bit ints - Bugfix: The wasm TLS register must be a non-volatile register - Bugfix: HINT + NOP is not that hard, so clean it up
js/src/jit/arm64/Assembler-arm64.cpp
js/src/jit/arm64/Assembler-arm64.h
js/src/jit/arm64/vixl/Assembler-vixl.h
js/src/jit/arm64/vixl/Instructions-vixl.h
js/src/jit/arm64/vixl/MacroAssembler-vixl.h
js/src/jit/arm64/vixl/MozAssembler-vixl.cpp
js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h
js/src/jit/arm64/vixl/MozInstructions-vixl.cpp
--- a/js/src/jit/arm64/Assembler-arm64.cpp
+++ b/js/src/jit/arm64/Assembler-arm64.cpp
@@ -30,16 +30,17 @@ using mozilla::DebugOnly;
 // Note this is used for inter-wasm calls and may pass arguments and results
 // in floating point registers even if the system ABI does not.
 
 ABIArg
 ABIArgGenerator::next(MIRType type)
 {
     switch (type) {
       case MIRType::Int32:
+      case MIRType::Int64:
       case MIRType::Pointer:
         if (intRegIndex_ == NumIntArgRegs) {
             current_ = ABIArg(stackOffset_);
             stackOffset_ += sizeof(uintptr_t);
             break;
         }
         current_ = ABIArg(Register::FromCode(intRegIndex_));
         intRegIndex_++;
@@ -82,16 +83,44 @@ Assembler::finish()
     // before writing the first entry.
     // Don't touch memory if we saw an OOM error.
     if (jumpRelocations_.length() && !oom()) {
         MOZ_ASSERT(jumpRelocations_.length() >= sizeof(uint32_t));
         *(uint32_t*)jumpRelocations_.buffer() = ExtendedJumpTable_.getOffset();
     }
 }
 
+bool
+Assembler::appendRawCode(const uint8_t* code, size_t numBytes)
+{
+    flush();
+    return armbuffer_.appendRawCode(code, numBytes);
+}
+
+bool
+Assembler::reserve(size_t size)
+{
+    // This buffer uses fixed-size chunks so there's no point in reserving
+    // now vs. on-demand.
+    return !oom();
+}
+
+bool
+Assembler::swapBuffer(wasm::Bytes& bytes)
+{
+    // For now, specialize to the one use case. As long as wasm::Bytes is a
+    // Vector, not a linked-list of chunks, there's not much we can do other
+    // than copy.
+    MOZ_ASSERT(bytes.empty());
+    if (!bytes.resize(bytesNeeded()))
+        return false;
+    armbuffer_.executableCopy(bytes.begin());
+    return true;
+}
+
 BufferOffset
 Assembler::emitExtendedJumpTable()
 {
     if (!pendingJumps_.length() || oom())
         return BufferOffset();
 
     armbuffer_.flushPool();
     armbuffer_.align(SizeOfJumpTableEntry);
@@ -281,16 +310,29 @@ Assembler::bind(RepatchLabel* label)
         return;
     }
     int branchOffset = label->offset();
     Instruction* inst = getInstructionAt(BufferOffset(branchOffset));
     inst->SetImmPCOffsetTarget(inst + nextOffset().getOffset() - branchOffset);
 }
 
 void
+Assembler::bindLater(Label* label, wasm::OldTrapDesc target)
+{
+    if (label->used()) {
+        BufferOffset b(label);
+        do {
+            append(wasm::OldTrapSite(target, b.getOffset()));
+            b = NextLink(b);
+        } while (b.assigned());
+    }
+    label->reset();
+}
+
+void
 Assembler::trace(JSTracer* trc)
 {
     for (size_t i = 0; i < pendingJumps_.length(); i++) {
         RelativePatch& rp = pendingJumps_[i];
         if (rp.kind == Relocation::JITCODE) {
             JitCode* code = JitCode::FromExecutable((uint8_t*)rp.target);
             TraceManuallyBarrieredEdge(trc, &code, "masmrel32");
             MOZ_ASSERT(code == JitCode::FromExecutable((uint8_t*)rp.target));
@@ -376,16 +418,18 @@ Assembler::ToggleToJmp(CodeLocationLabel
     Instruction* i = (Instruction*)inst_.raw();
     MOZ_ASSERT(i->IsAddSubImmediate());
 
     // Refer to instruction layout in ToggleToCmp().
     int imm19 = (int)i->Bits(23, 5);
     MOZ_ASSERT(vixl::is_int19(imm19));
 
     b(i, imm19, Always);
+
+    AutoFlushICache::flush(uintptr_t(i), 4);
 }
 
 void
 Assembler::ToggleToCmp(CodeLocationLabel inst_)
 {
     Instruction* i = (Instruction*)inst_.raw();
     MOZ_ASSERT(i->IsCondB());
 
@@ -400,16 +444,18 @@ Assembler::ToggleToCmp(CodeLocationLabel
     // 22:23 - ShiftAddSub. (OK!)
     // 10:21 - ImmAddSub. (OK!)
     // 5:9 - First source register (Rn). (OK!)
     // 0:4 - Destination Register. Must be xzr.
 
     // From the above, there is a safe 19-bit contiguous region from 5:23.
     Emit(i, vixl::ThirtyTwoBits | vixl::AddSubImmediateFixed | vixl::SUB | Flags(vixl::SetFlags) |
             Rd(vixl::xzr) | (imm19 << vixl::Rn_offset));
+
+    AutoFlushICache::flush(uintptr_t(i), 4);
 }
 
 void
 Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled)
 {
     const Instruction* first = reinterpret_cast<Instruction*>(inst_.raw());
     Instruction* load;
     Instruction* call;
@@ -426,17 +472,17 @@ Assembler::ToggleCall(CodeLocationLabel 
     // The call instruction follows the load, but there may be an injected
     // constant pool.
     call = const_cast<Instruction*>(load->InstructionAtOffset(vixl::kInstructionSize)->skipPool());
 
     if (call->IsBLR() == enabled)
         return;
 
     if (call->IsBLR()) {
-        // If the second instruction is blr(), then wehave:
+        // If the second instruction is blr(), then we have:
         //   ldr x17, [pc, offset]
         //   blr x17
         MOZ_ASSERT(load->IsLDR());
         // We want to transform this to:
         //   adr xzr, [pc, offset]
         //   nop
         int32_t offset = load->ImmLLiteral();
         adr(load, xzr, int32_t(offset));
@@ -450,16 +496,19 @@ Assembler::ToggleCall(CodeLocationLabel 
         // Transform this to:
         //   ldr x17, [pc, offset]
         //   blr x17
         int32_t offset = (int)load->ImmPCRawOffset();
         MOZ_ASSERT(vixl::is_int19(offset));
         ldr(load, ScratchReg2_64, int32_t(offset));
         blr(call, ScratchReg2_64);
     }
+
+    AutoFlushICache::flush(uintptr_t(first), 4);
+    AutoFlushICache::flush(uintptr_t(call), 8);
 }
 
 class RelocationIterator
 {
     CompactBufferReader reader_;
     uint32_t tableStart_;
     uint32_t offset_;
     uint32_t extOffset_;
--- a/js/src/jit/arm64/Assembler-arm64.h
+++ b/js/src/jit/arm64/Assembler-arm64.h
@@ -187,25 +187,19 @@ class Assembler : public vixl::Assembler
   public:
     Assembler()
       : vixl::Assembler()
     { }
 
     typedef vixl::Condition Condition;
 
     void finish();
-    bool appendRawCode(const uint8_t* code, size_t numBytes) {
-        MOZ_CRASH("NYI");
-    }
-    bool reserve(size_t size) {
-        MOZ_CRASH("NYI");
-    }
-    bool swapBuffer(wasm::Bytes& bytes) {
-        MOZ_CRASH("NYI");
-    }
+    bool appendRawCode(const uint8_t* code, size_t numBytes);
+    bool reserve(size_t size);
+    bool swapBuffer(wasm::Bytes& bytes);
     void trace(JSTracer* trc);
 
     // Emit the jump table, returning the BufferOffset to the first entry in the table.
     BufferOffset emitExtendedJumpTable();
     BufferOffset ExtendedJumpTable_;
     void executableCopy(uint8_t* buffer, bool flushICache = true);
 
     BufferOffset immPool(ARMRegister dest, uint8_t* value, vixl::LoadLiteralOp op,
@@ -215,19 +209,17 @@ class Assembler : public vixl::Assembler
     BufferOffset fImmPool(ARMFPRegister dest, uint8_t* value, vixl::LoadLiteralOp op,
                           const LiteralDoc& doc);
     BufferOffset fImmPool64(ARMFPRegister dest, double value);
     BufferOffset fImmPool32(ARMFPRegister dest, float value);
 
     void bind(Label* label) { bind(label, nextOffset()); }
     void bind(Label* label, BufferOffset boff);
     void bind(RepatchLabel* label);
-    void bindLater(Label* label, wasm::OldTrapDesc target) {
-        MOZ_CRASH("NYI");
-    }
+    void bindLater(Label* label, wasm::OldTrapDesc target);
 
     bool oom() const {
         return AssemblerShared::oom() ||
             armbuffer_.oom() ||
             jumpRelocations_.oom() ||
             dataRelocations_.oom();
     }
 
@@ -468,18 +460,18 @@ static constexpr Register ABINonArgRetur
 
 // This register is guaranteed to be clobberable during the prologue and
 // epilogue of an ABI call which must preserve both ABI argument, return
 // and non-volatile registers.
 static constexpr Register ABINonArgReturnVolatileReg = lr;
 
 // TLS pointer argument register for WebAssembly functions. This must not alias
 // any other register used for passing function arguments or return values.
-// Preserved by WebAssembly functions.
-static constexpr Register WasmTlsReg { Registers::x17 };
+// Preserved by WebAssembly functions.  Must be nonvolatile.
+static constexpr Register WasmTlsReg { Registers::x23 };
 
 // Registers used for wasm table calls. These registers must be disjoint
 // from the ABI argument registers, WasmTlsReg and each other.
 static constexpr Register WasmTableCallScratchReg = ABINonArgReg0;
 static constexpr Register WasmTableCallSigReg = ABINonArgReg1;
 static constexpr Register WasmTableCallIndexReg = ABINonArgReg2;
 
 static inline bool
--- a/js/src/jit/arm64/vixl/Assembler-vixl.h
+++ b/js/src/jit/arm64/vixl/Assembler-vixl.h
@@ -946,17 +946,48 @@ class Assembler : public MozBaseAssemble
       return mi;
     default:
       MOZ_CRASH("TODO: figure this case out.");
     }
     return static_cast<Condition>(cond ^ 1);
   }
 
   static inline DoubleCondition InvertCondition(DoubleCondition cond) {
-    MOZ_CRASH("Not yet implemented: InvertCondition(DoubleCondition)");
+      switch (cond) {
+	case DoubleOrdered:
+	  return DoubleUnordered;
+	case DoubleEqual:
+	  return DoubleNotEqualOrUnordered;
+	case DoubleNotEqual:
+	  return DoubleEqualOrUnordered;
+	case DoubleGreaterThan:
+	  return DoubleLessThanOrEqualOrUnordered;
+	case DoubleGreaterThanOrEqual:
+	  return DoubleLessThanOrUnordered;
+	case DoubleLessThan:
+	  return DoubleGreaterThanOrEqualOrUnordered;
+	case DoubleLessThanOrEqual:
+	  return DoubleGreaterThanOrUnordered;
+	case DoubleUnordered:
+	  return DoubleOrdered;
+	case DoubleEqualOrUnordered:
+	  return DoubleNotEqual;
+	case DoubleNotEqualOrUnordered:
+	  return DoubleEqual;
+	case DoubleGreaterThanOrUnordered:
+	  return DoubleLessThanOrEqual;
+	case DoubleGreaterThanOrEqualOrUnordered:
+	  return DoubleLessThan;
+	case DoubleLessThanOrUnordered:
+	  return DoubleGreaterThanOrEqual;
+	case DoubleLessThanOrEqualOrUnordered:
+	  return DoubleGreaterThan;
+	default:
+	  MOZ_CRASH("Bad condition");
+    }
   }
 
   static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
     VIXL_ASSERT(!(cond & DoubleConditionBitSpecial));
     return static_cast<Condition>(cond);
   }
 
   // Instruction set functions.
--- a/js/src/jit/arm64/vixl/Instructions-vixl.h
+++ b/js/src/jit/arm64/vixl/Instructions-vixl.h
@@ -306,16 +306,18 @@ class Instruction {
   bool IsTBZ() const;
   bool IsTBNZ() const;
   bool IsCBZ() const;
   bool IsCBNZ() const;
   bool IsLDR() const;
   bool IsNOP() const;
   bool IsADR() const;
   bool IsADRP() const;
+  bool IsMovz() const;
+  bool IsMovk() const;
   bool IsBranchLinkImm() const;
   bool IsTargetReachable(Instruction* target) const;
   ptrdiff_t ImmPCRawOffset() const;
   void SetImmPCRawOffset(ptrdiff_t offset);
   void SetBits32(int msb, int lsb, unsigned value);
 
   // Is this a stack pointer synchronization instruction as inserted by
   // MacroAssembler::syncStackPtr()?
--- a/js/src/jit/arm64/vixl/MacroAssembler-vixl.h
+++ b/js/src/jit/arm64/vixl/MacroAssembler-vixl.h
@@ -1475,19 +1475,35 @@ class MacroAssembler : public js::jit::A
     SingleEmissionCheckScope guard(this);
     umsubl(rd, rn, rm, ra);
   }
   void Unreachable() {
     SingleEmissionCheckScope guard(this);
 #ifdef JS_SIMULATOR_ARM64
     hlt(kUnreachableOpcode);
 #else
-    // Branch to 0 to generate a segfault.
-    // lr - kInstructionSize is the address of the offending instruction.
-    blr(xzr);
+    // A couple of strategies we can use here.  There are no unencoded
+    // instructions in the instruction set that are guaranteed to remain that
+    // way.  However there are some currently (as of 2018) unencoded
+    // instructions that are good candidates.
+    //
+    // Ideally, unencoded instructions should be non-destructive to the register
+    // state, and should be unencoded at all exception levels.
+    //
+    // At the trap the pc will hold the address of the offending instruction.
+
+    // Some candidates for unencoded instructions:
+    //
+    // 0xd4a00000 (essentially dcps0, a good one since it is nonsensical and may
+    //             remain unencoded in the future for that reason)
+    // 0x33000000 (bfm variant)
+    // 0xd67f0000 (br variant)
+    // 0x5ac00c00 (rbit variant)
+
+    Emit(0xd4a00000);		// "dcps0", also has 16-bit payload if needed
 #endif
   }
   void Uxtb(const Register& rd, const Register& rn) {
     VIXL_ASSERT(!rd.IsZero());
     VIXL_ASSERT(!rn.IsZero());
     SingleEmissionCheckScope guard(this);
     uxtb(rd, rn);
   }
--- a/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp
+++ b/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp
@@ -384,22 +384,22 @@ BufferOffset Assembler::tst(const Regist
 
 void Assembler::ldr(Instruction* at, const CPURegister& rt, int imm19) {
   LoadLiteralOp op = LoadLiteralOpFor(rt);
   Emit(at, op | ImmLLiteral(imm19) | Rt(rt));
 }
 
 
 BufferOffset Assembler::hint(SystemHint code) {
-  return Emit(HINT | ImmHint(code) | Rt(xzr));
+  return Emit(HINT | ImmHint(code));
 }
 
 
 void Assembler::hint(Instruction* at, SystemHint code) {
-  Emit(at, HINT | ImmHint(code) | Rt(xzr));
+  Emit(at, HINT | ImmHint(code));
 }
 
 
 void Assembler::svc(Instruction* at, int code) {
   VIXL_ASSERT(is_uint16(code));
   Emit(at, SVC | ImmException(code));
 }
 
--- a/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h
+++ b/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h
@@ -54,18 +54,18 @@ typedef js::jit::AssemblerBufferWithCons
 // Base class for vixl::Assembler, for isolating Moz-specific changes to VIXL.
 class MozBaseAssembler : public js::jit::AssemblerShared {
   // Buffer initialization constants.
   static const unsigned BufferGuardSize = 1;
   static const unsigned BufferHeaderSize = 1;
   static const size_t   BufferCodeAlignment = 8;
   static const size_t   BufferMaxPoolOffset = 1024;
   static const unsigned BufferPCBias = 0;
-  static const uint32_t BufferAlignmentFillInstruction = BRK | (0xdead << ImmException_offset);
-  static const uint32_t BufferNopFillInstruction = HINT | (31 << Rt_offset);
+  static const uint32_t BufferAlignmentFillInstruction = HINT | (NOP << ImmHint_offset);
+  static const uint32_t BufferNopFillInstruction = HINT | (NOP << ImmHint_offset);
   static const unsigned BufferNumDebugNopsToInsert = 0;
 
 #ifdef JS_DISASM_ARM64
   static constexpr const char* const InstrIndent = "        ";
   static constexpr const char* const LabelIndent = "          ";
   static constexpr const char* const TargetIndent = "                    ";
 #endif
 
--- a/js/src/jit/arm64/vixl/MozInstructions-vixl.cpp
+++ b/js/src/jit/arm64/vixl/MozInstructions-vixl.cpp
@@ -90,16 +90,27 @@ bool Instruction::IsADR() const {
 }
 
 
 bool Instruction::IsADRP() const {
   return Mask(PCRelAddressingMask) == ADRP;
 }
 
 
+bool Instruction::IsMovz() const {
+  return (Mask(MoveWideImmediateMask) == MOVZ_x) ||
+         (Mask(MoveWideImmediateMask) == MOVZ_w);
+}
+
+
+bool Instruction::IsMovk() const {
+  return (Mask(MoveWideImmediateMask) == MOVK_x) ||
+         (Mask(MoveWideImmediateMask) == MOVK_w);
+}
+
 bool Instruction::IsBranchLinkImm() const {
   return Mask(UnconditionalBranchFMask) == (UnconditionalBranchFixed | BL);
 }
 
 
 bool Instruction::IsTargetReachable(Instruction* target) const {
     VIXL_ASSERT(((target - this) & 3) == 0);
     int offset = (target - this) >> kInstructionSizeLog2;