Bug 1283177: wasm: Implement int64 load/stores on x64; r=sunfish
authorBenjamin Bouvier <benj@benj.me>
Fri, 08 Jul 2016 09:34:00 +0200
changeset 304296 39bfb8b9c58f9e2493a75d3ad385af3936a49370
parent 304295 2ea9c31e5f0871d9390a95af813fc4f330072096
child 304297 693a7e678b235e426fe040294e374c282f8f031a
push id30420
push userphilringnalda@gmail.com
push dateSat, 09 Jul 2016 15:52:07 +0000
treeherdermozilla-central@d55b1b1d51cf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssunfish
bugs1283177
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1283177: wasm: Implement int64 load/stores on x64; r=sunfish MozReview-Commit-ID: DJ1bEsHL3Jq
js/src/asmjs/WasmCompile.cpp
js/src/asmjs/WasmIonCompile.cpp
js/src/asmjs/WasmSignalHandlers.cpp
js/src/jit/Disassembler.h
js/src/jit/MIR.h
js/src/jit/shared/CodeGenerator-shared-inl.h
js/src/jit/shared/CodeGenerator-shared.h
js/src/jit/shared/LIR-shared.h
js/src/jit/shared/LOpcodes-shared.h
js/src/jit/x64/Assembler-x64.h
js/src/jit/x64/BaseAssembler-x64.h
js/src/jit/x64/CodeGenerator-x64.cpp
js/src/jit/x64/CodeGenerator-x64.h
js/src/jit/x64/Lowering-x64.cpp
js/src/jit/x86-shared/Disassembler-x86-shared.cpp
js/src/jit/x86-shared/Lowering-x86-shared.cpp
--- a/js/src/asmjs/WasmCompile.cpp
+++ b/js/src/asmjs/WasmCompile.cpp
@@ -401,49 +401,51 @@ DecodeExpr(FunctionDecoder& f)
         return f.iter().readLoad(ValType::I32, 1, nullptr);
       case Expr::I32Load16S:
       case Expr::I32Load16U:
         return f.iter().readLoad(ValType::I32, 2, nullptr);
       case Expr::I32Load:
         return f.iter().readLoad(ValType::I32, 4, nullptr);
       case Expr::I64Load8S:
       case Expr::I64Load8U:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readLoad(ValType::I64, 1, nullptr);
       case Expr::I64Load16S:
       case Expr::I64Load16U:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readLoad(ValType::I64, 2, nullptr);
       case Expr::I64Load32S:
       case Expr::I64Load32U:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readLoad(ValType::I64, 4, nullptr);
       case Expr::I64Load:
-        return f.iter().notYetImplemented("i64");
+        return f.checkI64Support() &&
+               f.iter().readLoad(ValType::I64, 8, nullptr);
       case Expr::F32Load:
         return f.iter().readLoad(ValType::F32, 4, nullptr);
       case Expr::F64Load:
         return f.iter().readLoad(ValType::F64, 8, nullptr);
       case Expr::I32Store8:
         return f.iter().readStore(ValType::I32, 1, nullptr, nullptr);
       case Expr::I32Store16:
         return f.iter().readStore(ValType::I32, 2, nullptr, nullptr);
       case Expr::I32Store:
         return f.iter().readStore(ValType::I32, 4, nullptr, nullptr);
       case Expr::I64Store8:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readStore(ValType::I64, 1, nullptr, nullptr);
       case Expr::I64Store16:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readStore(ValType::I64, 2, nullptr, nullptr);
       case Expr::I64Store32:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readStore(ValType::I64, 4, nullptr, nullptr);
       case Expr::I64Store:
-        return f.iter().notYetImplemented("i64");
+        return f.checkI64Support() &&
+               f.iter().readStore(ValType::I64, 8, nullptr, nullptr);
       case Expr::F32Store:
         return f.iter().readStore(ValType::F32, 4, nullptr, nullptr);
       case Expr::F64Store:
         return f.iter().readStore(ValType::F64, 8, nullptr, nullptr);
       case Expr::Br:
         return f.iter().readBr(nullptr, nullptr, nullptr);
       case Expr::BrIf:
         return f.iter().readBrIf(nullptr, nullptr, nullptr, nullptr);
--- a/js/src/asmjs/WasmIonCompile.cpp
+++ b/js/src/asmjs/WasmIonCompile.cpp
@@ -585,26 +585,27 @@ class FunctionCompiler
     void assign(unsigned slot, MDefinition* def)
     {
         if (inDeadCode())
             return;
         curBlock_->setSlot(info().localSlot(slot), def);
     }
 
   private:
-    MDefinition* loadHeapPrivate(MDefinition* base, const MWasmMemoryAccess& access)
+    MDefinition* loadHeapPrivate(MDefinition* base, const MWasmMemoryAccess& access,
+                                 bool isInt64 = false)
     {
         if (inDeadCode())
             return nullptr;
 
         MInstruction* load = nullptr;
         if (mg().kind == ModuleKind::Wasm) {
             if (!mg().usesSignal.forOOB)
                 curBlock_->add(MWasmBoundsCheck::New(alloc(), base, access));
-            load = MWasmLoad::New(alloc(), base, access);
+            load = MWasmLoad::New(alloc(), base, access, isInt64);
         } else {
             load = MAsmJSLoadHeap::New(alloc(), base, access);
         }
 
         curBlock_->add(load);
         return load;
     }
 
@@ -621,20 +622,20 @@ class FunctionCompiler
         } else {
             store = MAsmJSStoreHeap::New(alloc(), base, access, v);
         }
 
         curBlock_->add(store);
     }
 
   public:
-    MDefinition* loadHeap(MDefinition* base, const MWasmMemoryAccess& access)
+    MDefinition* loadHeap(MDefinition* base, const MWasmMemoryAccess& access, bool isInt64)
     {
         MOZ_ASSERT(!Scalar::isSimdType(access.accessType()), "SIMD loads should use loadSimdHeap");
-        return loadHeapPrivate(base, access);
+        return loadHeapPrivate(base, access, isInt64);
     }
     MDefinition* loadSimdHeap(MDefinition* base, const MWasmMemoryAccess& access)
     {
         MOZ_ASSERT(Scalar::isSimdType(access.accessType()), "non-SIMD loads should use loadHeap");
         return loadHeapPrivate(base, access);
     }
     MDefinition* loadAtomicHeap(MDefinition* base, const MWasmMemoryAccess& access)
     {
@@ -2076,17 +2077,17 @@ EmitSelect(FunctionCompiler& f)
 static bool
 EmitLoad(FunctionCompiler& f, ValType type, Scalar::Type viewType)
 {
     LinearMemoryAddress<MDefinition*> addr;
     if (!f.iter().readLoad(type, Scalar::byteSize(viewType), &addr))
         return false;
 
     MWasmMemoryAccess access(viewType, addr.align, addr.offset);
-    f.iter().setResult(f.loadHeap(addr.base, access));
+    f.iter().setResult(f.loadHeap(addr.base, access, type == ValType::I64));
     return true;
 }
 
 static bool
 EmitStore(FunctionCompiler& f, ValType resultType, Scalar::Type viewType)
 {
     LinearMemoryAddress<MDefinition*> addr;
     MDefinition* value;
@@ -2928,16 +2929,38 @@ EmitExpr(FunctionCompiler& f)
       case Expr::I64Eqz:
         return EmitConversion<MNot>(f, ValType::I64, ValType::I32);
       case Expr::I64Clz:
         return EmitUnary<MClz>(f, ValType::I64);
       case Expr::I64Ctz:
         return EmitUnary<MCtz>(f, ValType::I64);
       case Expr::I64Popcnt:
         return EmitUnary<MPopcnt>(f, ValType::I64);
+      case Expr::I64Load8S:
+        return EmitLoad(f, ValType::I64, Scalar::Int8);
+      case Expr::I64Load8U:
+        return EmitLoad(f, ValType::I64, Scalar::Uint8);
+      case Expr::I64Load16S:
+        return EmitLoad(f, ValType::I64, Scalar::Int16);
+      case Expr::I64Load16U:
+        return EmitLoad(f, ValType::I64, Scalar::Uint16);
+      case Expr::I64Load32S:
+        return EmitLoad(f, ValType::I64, Scalar::Int32);
+      case Expr::I64Load32U:
+        return EmitLoad(f, ValType::I64, Scalar::Uint32);
+      case Expr::I64Load:
+        return EmitLoad(f, ValType::I64, Scalar::Int64);
+      case Expr::I64Store8:
+        return EmitStore(f, ValType::I64, Scalar::Int8);
+      case Expr::I64Store16:
+        return EmitStore(f, ValType::I64, Scalar::Int16);
+      case Expr::I64Store32:
+        return EmitStore(f, ValType::I64, Scalar::Int32);
+      case Expr::I64Store:
+        return EmitStore(f, ValType::I64, Scalar::Int64);
 
       // F32
       case Expr::F32Const: {
         float f32;
         if (!f.iter().readF32Const(&f32))
             return false;
 
         f.iter().setResult(f.constant(Float32Value(f32), MIRType::Float32));
@@ -3281,27 +3304,16 @@ EmitExpr(FunctionCompiler& f)
       case Expr::I32AtomicsBinOp:
         return EmitAtomicsBinOp(f);
       case Expr::I32AtomicsCompareExchange:
         return EmitAtomicsCompareExchange(f);
       case Expr::I32AtomicsExchange:
         return EmitAtomicsExchange(f);
 
       // Future opcodes
-      case Expr::I64Load8S:
-      case Expr::I64Load16S:
-      case Expr::I64Load32S:
-      case Expr::I64Load8U:
-      case Expr::I64Load16U:
-      case Expr::I64Load32U:
-      case Expr::I64Load:
-      case Expr::I64Store8:
-      case Expr::I64Store16:
-      case Expr::I64Store32:
-      case Expr::I64Store:
       case Expr::CurrentMemory:
       case Expr::GrowMemory:
         MOZ_CRASH("NYI");
       case Expr::Limit:;
     }
 
     MOZ_CRASH("unexpected wasm opcode");
 }
--- a/js/src/asmjs/WasmSignalHandlers.cpp
+++ b/js/src/asmjs/WasmSignalHandlers.cpp
@@ -697,16 +697,18 @@ EmulateHeapAccess(EMULATOR_CONTEXT* cont
             SetRegisterToLoadedValue(context, wrappedAddress.cast<void*>(), size, access.otherOperand());
             break;
           case Disassembler::HeapAccess::LoadSext32:
             SetRegisterToLoadedValueSext32(context, wrappedAddress.cast<void*>(), size, access.otherOperand());
             break;
           case Disassembler::HeapAccess::Store:
             StoreValueFromRegister(context, wrappedAddress.cast<void*>(), size, access.otherOperand());
             break;
+          case Disassembler::HeapAccess::LoadSext64:
+            MOZ_CRASH("no int64 accesses in asm.js");
           case Disassembler::HeapAccess::Unknown:
             MOZ_CRASH("Failed to disassemble instruction");
         }
     } else {
         // We now know that this is an out-of-bounds access made by an asm.js
         // load/store that we should handle.
 
         if (memoryAccess->throwOnOOB())
@@ -721,16 +723,18 @@ EmulateHeapAccess(EMULATOR_CONTEXT* cont
             // infer the type from the register class, since all SIMD accesses
             // throw on out of bounds (see above), so the only types using FP
             // registers are float32 and double.
             SetRegisterToCoercedUndefined(context, access.size(), access.otherOperand());
             break;
           case Disassembler::HeapAccess::Store:
             // Do nothing.
             break;
+          case Disassembler::HeapAccess::LoadSext64:
+            MOZ_CRASH("no int64 accesses in asm.js");
           case Disassembler::HeapAccess::Unknown:
             MOZ_CRASH("Failed to disassemble instruction");
         }
     }
 
     return end;
 }
 
--- a/js/src/jit/Disassembler.h
+++ b/js/src/jit/Disassembler.h
@@ -197,16 +197,17 @@ class OtherOperand {
 };
 
 class HeapAccess {
   public:
     enum Kind {
         Unknown,
         Load,       // any bits not covered by the load are zeroed
         LoadSext32, // like Load, but sign-extend to 32 bits
+        LoadSext64, // like Load, but sign-extend to 64 bits
         Store
     };
 
   private:
     Kind kind_;
     size_t size_; // The number of bytes of memory accessed
     ComplexAddress address_;
     OtherOperand otherOperand_;
--- a/js/src/jit/MIR.h
+++ b/js/src/jit/MIR.h
@@ -12983,17 +12983,17 @@ class MWasmMemoryAccess
         return Scalar::isSimdType(accessType())
                ? Scalar::scalarByteSize(accessType()) * numSimdElems()
                : TypedArrayElemSize(accessType());
     }
     bool needsBoundsCheck() const { return needsBoundsCheck_; }
     unsigned numSimdElems() const { MOZ_ASSERT(Scalar::isSimdType(accessType_)); return numSimdElems_; }
     MemoryBarrierBits barrierBefore() const { return barrierBefore_; }
     MemoryBarrierBits barrierAfter() const { return barrierAfter_; }
-    bool isAtomicAccess() const { return (barrierBefore_|barrierAfter_) != MembarNobits; }
+    bool isAtomicAccess() const { return (barrierBefore_ | barrierAfter_) != MembarNobits; }
 
     void removeBoundsCheck() { needsBoundsCheck_ = false; }
     void setOffset(uint32_t o) { offset_ = o; }
 };
 
 class MWasmBoundsCheck
   : public MUnaryInstruction,
     public MWasmMemoryAccess,
@@ -13025,23 +13025,26 @@ class MWasmBoundsCheck
     }
 };
 
 class MWasmLoad
   : public MUnaryInstruction,
     public MWasmMemoryAccess,
     public NoTypePolicy::Data
 {
-    MWasmLoad(MDefinition* base, const MWasmMemoryAccess& access)
+    MWasmLoad(MDefinition* base, const MWasmMemoryAccess& access, bool isInt64)
       : MUnaryInstruction(base),
         MWasmMemoryAccess(access)
     {
         setGuard();
         MOZ_ASSERT(access.accessType() != Scalar::Uint8Clamped, "unexpected load heap in wasm");
-        setResultType(ScalarTypeToMIRType(access.accessType()));
+        if (isInt64)
+            setResultType(MIRType::Int64);
+        else
+            setResultType(ScalarTypeToMIRType(access.accessType()));
     }
 
   public:
     INSTRUCTION_HEADER(WasmLoad)
     TRIVIAL_NEW_WRAPPERS
     NAMED_OPERANDS((0, base))
 
     AliasSet getAliasSet() const override {
--- a/js/src/jit/shared/CodeGenerator-shared-inl.h
+++ b/js/src/jit/shared/CodeGenerator-shared-inl.h
@@ -298,48 +298,64 @@ CodeGeneratorShared::restoreLiveVolatile
     LSafepoint* safepoint = ins->safepoint();
     LiveRegisterSet regs;
     regs.set() = RegisterSet::Intersect(safepoint->liveRegs().set(), RegisterSet::Volatile());
     masm.PopRegsInMask(regs);
 }
 
 void
 CodeGeneratorShared::verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, bool isLoad,
-                                                 Scalar::Type type, unsigned numElems,
+                                                 bool isInt64, Scalar::Type type, unsigned numElems,
                                                  const Operand& mem, LAllocation alloc)
 {
 #ifdef DEBUG
     using namespace Disassembler;
 
-    OtherOperand op;
     Disassembler::HeapAccess::Kind kind = isLoad ? HeapAccess::Load : HeapAccess::Store;
     switch (type) {
       case Scalar::Int8:
       case Scalar::Int16:
         if (kind == HeapAccess::Load)
-            kind = HeapAccess::LoadSext32;
-        MOZ_FALLTHROUGH;
+            kind = isInt64 ? HeapAccess::LoadSext64 : HeapAccess::LoadSext32;
+        break;
+      case Scalar::Int32:
+        if (isInt64 && kind == HeapAccess::Load)
+            kind = HeapAccess::LoadSext64;
+        break;
+      default:
+        break;
+    }
+
+    OtherOperand op;
+    switch (type) {
+      case Scalar::Int8:
       case Scalar::Uint8:
+      case Scalar::Int16:
       case Scalar::Uint16:
       case Scalar::Int32:
       case Scalar::Uint32:
         if (!alloc.isConstant()) {
             op = OtherOperand(ToRegister(alloc).encoding());
         } else {
-            int32_t i = ToInt32(&alloc);
+            // x86 doesn't allow encoding an imm64 to memory move; the value
+            // is wrapped anyways.
+            int32_t i = isInt64 ? int32_t(ToInt64(&alloc)) : ToInt32(&alloc);
 
             // Sign-extend the immediate value out to 32 bits. We do this even
             // for unsigned element types so that we match what the disassembly
             // code does, as it doesn't know about signedness of stores.
             unsigned shift = 32 - TypedArrayElemSize(type) * 8;
             i = i << shift >> shift;
-
             op = OtherOperand(i);
         }
         break;
+      case Scalar::Int64:
+        // Can't encode an imm64-to-memory move.
+        op = OtherOperand(ToRegister(alloc).encoding());
+        break;
       case Scalar::Float32:
       case Scalar::Float64:
       case Scalar::Float32x4:
       case Scalar::Int8x16:
       case Scalar::Int16x8:
       case Scalar::Int32x4:
         op = OtherOperand(ToFloatRegister(alloc).encoding());
         break;
@@ -351,16 +367,32 @@ CodeGeneratorShared::verifyHeapAccessDis
     size_t size = Scalar::isSimdType(type)
                   ? Scalar::scalarByteSize(type) * numElems
                   : TypedArrayElemSize(type);
     masm.verifyHeapAccessDisassembly(begin, end,
                                      HeapAccess(kind, size, ComplexAddress(mem), op));
 #endif
 }
 
+void
+CodeGeneratorShared::verifyLoadDisassembly(uint32_t begin, uint32_t end, bool isInt64,
+                                           Scalar::Type type, unsigned numElems, const Operand& mem,
+                                           LAllocation alloc)
+{
+    verifyHeapAccessDisassembly(begin, end, true, isInt64, type, numElems, mem, alloc);
+}
+
+void
+CodeGeneratorShared::verifyStoreDisassembly(uint32_t begin, uint32_t end, bool isInt64,
+                                            Scalar::Type type, unsigned numElems,
+                                            const Operand& mem, LAllocation alloc)
+{
+    verifyHeapAccessDisassembly(begin, end, false, isInt64, type, numElems, mem, alloc);
+}
+
 inline bool
 CodeGeneratorShared::isGlobalObject(JSObject* object)
 {
     // Calling object->is<GlobalObject>() is racy because this relies on
     // checking the group and this can be changed while we are compiling off the
     // main thread.
     return object == gen->compartment->maybeGlobal();
 }
--- a/js/src/jit/shared/CodeGenerator-shared.h
+++ b/js/src/jit/shared/CodeGenerator-shared.h
@@ -536,20 +536,27 @@ class CodeGeneratorShared : public LElem
     }
     void emitTracelogIonStop() {
 #ifdef JS_TRACE_LOGGING
         emitTracelogStopEvent(TraceLogger_IonMonkey);
         emitTracelogScriptStop();
 #endif
     }
 
-    inline void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, bool isLoad,
+  protected:
+    inline void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, bool isLoad, bool isInt64,
                                             Scalar::Type type, unsigned numElems,
                                             const Operand& mem, LAllocation alloc);
 
+  public:
+    inline void verifyLoadDisassembly(uint32_t begin, uint32_t end, bool isInt64, Scalar::Type type,
+                                      unsigned numElems, const Operand& mem, LAllocation alloc);
+    inline void verifyStoreDisassembly(uint32_t begin, uint32_t end, bool isInt64, Scalar::Type type,
+                                       unsigned numElems, const Operand& mem, LAllocation alloc);
+
     bool isGlobalObject(JSObject* object);
 };
 
 // An out-of-line path is generated at the end of the function.
 class OutOfLineCode : public TempObject
 {
     Label entry_;
     Label rejoin_;
--- a/js/src/jit/shared/LIR-shared.h
+++ b/js/src/jit/shared/LIR-shared.h
@@ -7673,33 +7673,60 @@ class LWasmBoundsCheck : public LInstruc
     MWasmBoundsCheck* mir() const {
         return mir_->toWasmBoundsCheck();
     }
     const LAllocation* ptr() {
         return getOperand(0);
     }
 };
 
-class LWasmLoad : public LInstructionHelper<1, 1, 1>
-{
-  public:
-    LIR_HEADER(WasmLoad);
-    explicit LWasmLoad(const LAllocation& ptr) {
-        setOperand(0, ptr);
-        setTemp(0, LDefinition::BogusTemp());
+namespace details {
+
+// This is a base class for LWasmLoad/LWasmLoadI64.
+template<size_t Defs, size_t Temp>
+class LWasmLoadBase : public LInstructionHelper<Defs, 1, Temp>
+{
+  public:
+    typedef LInstructionHelper<Defs, 1, Temp> Base;
+    explicit LWasmLoadBase(const LAllocation& ptr) {
+        Base::setOperand(0, ptr);
     }
     MWasmLoad* mir() const {
-        return mir_->toWasmLoad();
+        return Base::mir_->toWasmLoad();
     }
     const LAllocation* ptr() {
-        return getOperand(0);
-    }
+        return Base::getOperand(0);
+    }
+};
+
+} // namespace details
+
+class LWasmLoad : public details::LWasmLoadBase<1, 1>
+{
+  public:
+    explicit LWasmLoad(const LAllocation& ptr)
+      : LWasmLoadBase(ptr)
+    {
+        setTemp(0, LDefinition::BogusTemp());
+    }
+
     const LDefinition* ptrCopy() {
-        return getTemp(0);
-    }
+        return Base::getTemp(0);
+    }
+
+    LIR_HEADER(WasmLoad);
+};
+
+class LWasmLoadI64 : public details::LWasmLoadBase<INT64_PIECES, 0>
+{
+  public:
+    explicit LWasmLoadI64(const LAllocation& ptr)
+      : LWasmLoadBase(ptr)
+    {}
+    LIR_HEADER(WasmLoadI64);
 };
 
 class LWasmStore : public LInstructionHelper<0, 2, 1>
 {
   public:
     LIR_HEADER(WasmStore);
     LWasmStore(const LAllocation& ptr, const LAllocation& value) {
         setOperand(0, ptr);
--- a/js/src/jit/shared/LOpcodes-shared.h
+++ b/js/src/jit/shared/LOpcodes-shared.h
@@ -380,16 +380,17 @@
     _(IsCallable)                   \
     _(IsConstructor)                \
     _(IsObject)                     \
     _(IsObjectAndBranch)            \
     _(HasClass)                     \
     _(AsmSelect)                    \
     _(AsmSelectI64)                 \
     _(WasmLoad)                     \
+    _(WasmLoadI64)                  \
     _(WasmStore)                    \
     _(WasmBoundsCheck)              \
     _(AsmJSLoadHeap)                \
     _(AsmJSStoreHeap)               \
     _(AsmJSLoadFuncPtr)             \
     _(AsmJSLoadGlobalVar)           \
     _(AsmJSStoreGlobalVar)          \
     _(AsmJSLoadFFIFunc)             \
--- a/js/src/jit/x64/Assembler-x64.h
+++ b/js/src/jit/x64/Assembler-x64.h
@@ -458,16 +458,54 @@ class Assembler : public AssemblerX86Sha
             MOZ_CRASH("unexpected operand kind");
         }
     }
 
     void xchgq(Register src, Register dest) {
         masm.xchgq_rr(src.encoding(), dest.encoding());
     }
 
+    void movsbq(const Operand& src, Register dest) {
+        switch (src.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.movsbq_mr(src.disp(), src.base(), dest.encoding());
+            break;
+          case Operand::MEM_SCALE:
+            masm.movsbq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+        }
+    }
+
+    void movzbq(const Operand& src, Register dest) {
+        // movzbl zero-extends to 64 bits and is one byte smaller, so use that
+        // instead.
+        movzbl(src, dest);
+    }
+
+    void movswq(const Operand& src, Register dest) {
+        switch (src.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.movswq_mr(src.disp(), src.base(), dest.encoding());
+            break;
+          case Operand::MEM_SCALE:
+            masm.movswq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+        }
+    }
+
+    void movzwq(const Operand& src, Register dest) {
+        // movzwl zero-extends to 64 bits and is one byte smaller, so use that
+        // instead.
+        movzwl(src, dest);
+    }
+
     void movslq(Register src, Register dest) {
         masm.movslq_rr(src.encoding(), dest.encoding());
     }
     void movslq(const Operand& src, Register dest) {
         switch (src.kind()) {
           case Operand::REG:
             masm.movslq_rr(src.reg(), dest.encoding());
             break;
--- a/js/src/jit/x64/BaseAssembler-x64.h
+++ b/js/src/jit/x64/BaseAssembler-x64.h
@@ -615,17 +615,16 @@ class BaseAssemblerX64 : public BaseAsse
     }
 
     void movq_i32m(int32_t imm, int32_t offset, RegisterID base)
     {
         spew("movq       $%d, " MEM_ob, imm, ADDR_ob(offset, base));
         m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, GROUP11_MOV);
         m_formatter.immediate32(imm);
     }
-
     void movq_i32m(int32_t imm, int32_t offset, RegisterID base, RegisterID index, int scale)
     {
         spew("movq       $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
         m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, index, scale, GROUP11_MOV);
         m_formatter.immediate32(imm);
     }
     void movq_i32m(int32_t imm, const void* addr)
     {
@@ -649,16 +648,38 @@ class BaseAssemblerX64 : public BaseAsse
 
     void movq_i64r(int64_t imm, RegisterID dst)
     {
         spew("movabsq    $0x%" PRIx64 ", %s", imm, GPReg64Name(dst));
         m_formatter.oneByteOp64(OP_MOV_EAXIv, dst);
         m_formatter.immediate64(imm);
     }
 
+    void movsbq_mr(int32_t offset, RegisterID base, RegisterID dst)
+    {
+        spew("movsbq     " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+        m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, dst);
+    }
+    void movsbq_mr(int32_t offset, RegisterID base, RegisterID index, int scale, RegisterID dst)
+    {
+        spew("movsbq     " MEM_obs ", %s", ADDR_obs(offset, base, index, scale), GPReg64Name(dst));
+        m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, index, scale, dst);
+    }
+
+    void movswq_mr(int32_t offset, RegisterID base, RegisterID dst)
+    {
+        spew("movswq     " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+        m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, dst);
+    }
+    void movswq_mr(int32_t offset, RegisterID base, RegisterID index, int scale, RegisterID dst)
+    {
+        spew("movswq     " MEM_obs ", %s", ADDR_obs(offset, base, index, scale), GPReg64Name(dst));
+        m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, index, scale, dst);
+    }
+
     void movslq_rr(RegisterID src, RegisterID dst)
     {
         spew("movslq     %s, %s", GPReg32Name(src), GPReg64Name(dst));
         m_formatter.oneByteOp64(OP_MOVSXD_GvEv, src, dst);
     }
     void movslq_mr(int32_t offset, RegisterID base, RegisterID dst)
     {
         spew("movslq     " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
--- a/js/src/jit/x64/CodeGenerator-x64.cpp
+++ b/js/src/jit/x64/CodeGenerator-x64.cpp
@@ -607,16 +607,17 @@ CodeGeneratorX64::loadSimd(Scalar::Type 
         masm.loadUnalignedSimd128Int(srcAddr, out);
         break;
       case Scalar::Int8:
       case Scalar::Uint8:
       case Scalar::Int16:
       case Scalar::Uint16:
       case Scalar::Int32:
       case Scalar::Uint32:
+      case Scalar::Int64:
       case Scalar::Float32:
       case Scalar::Float64:
       case Scalar::Uint8Clamped:
       case Scalar::MaxTypedArrayViewType:
         MOZ_CRASH("should only handle SIMD types");
     }
 }
 
@@ -632,42 +633,110 @@ static wasm::MemoryAccess
 WasmMemoryAccess(uint32_t before)
 {
     return wasm::MemoryAccess(before,
                               wasm::MemoryAccess::Throw,
                               wasm::MemoryAccess::DontWrapOffset);
 }
 
 void
-CodeGeneratorX64::visitWasmLoad(LWasmLoad* ins)
+CodeGeneratorX64::load(Scalar::Type type, const Operand& srcAddr, AnyRegister out)
+{
+    switch (type) {
+      case Scalar::Int8:      masm.movsbl(srcAddr, out.gpr()); break;
+      case Scalar::Uint8:     masm.movzbl(srcAddr, out.gpr()); break;
+      case Scalar::Int16:     masm.movswl(srcAddr, out.gpr()); break;
+      case Scalar::Uint16:    masm.movzwl(srcAddr, out.gpr()); break;
+      case Scalar::Int32:
+      case Scalar::Uint32:    masm.movl(srcAddr, out.gpr()); break;
+      case Scalar::Float32:   masm.loadFloat32(srcAddr, out.fpu()); break;
+      case Scalar::Float64:   masm.loadDouble(srcAddr, out.fpu()); break;
+      case Scalar::Float32x4:
+      case Scalar::Int8x16:
+      case Scalar::Int16x8:
+      case Scalar::Int32x4:
+        MOZ_CRASH("SIMD loads should be handled in emitSimdLoad");
+      case Scalar::Int64:
+        MOZ_CRASH("int64 loads must use load64");
+      case Scalar::Uint8Clamped:
+      case Scalar::MaxTypedArrayViewType:
+        MOZ_CRASH("unexpected array type");
+    }
+}
+
+void
+CodeGeneratorX64::loadI64(Scalar::Type type, const Operand& srcAddr, AnyRegister out)
 {
-    const MWasmLoad* mir = ins->mir();
+    switch (type) {
+      case Scalar::Int8:      masm.movsbq(srcAddr, out.gpr()); break;
+      case Scalar::Uint8:     masm.movzbq(srcAddr, out.gpr()); break;
+      case Scalar::Int16:     masm.movswq(srcAddr, out.gpr()); break;
+      case Scalar::Uint16:    masm.movzwq(srcAddr, out.gpr()); break;
+      case Scalar::Int32:     masm.movslq(srcAddr, out.gpr()); break;
+      // Int32 to int64 moves zero-extend by default.
+      case Scalar::Uint32:    masm.movl(srcAddr, out.gpr());   break;
+      case Scalar::Int64:     masm.movq(srcAddr, out.gpr());   break;
+      case Scalar::Float32:
+      case Scalar::Float64:
+      case Scalar::Float32x4:
+      case Scalar::Int8x16:
+      case Scalar::Int16x8:
+      case Scalar::Int32x4:
+        MOZ_CRASH("non-int64 loads should use load()");
+      case Scalar::Uint8Clamped:
+      case Scalar::MaxTypedArrayViewType:
+        MOZ_CRASH("unexpected array type");
+    }
+}
 
+void
+CodeGeneratorX64::visitWasmLoadBase(const MWasmLoad* mir, const LAllocation* ptr,
+                                    const LDefinition* output, bool isInt64)
+{
     Scalar::Type accessType = mir->accessType();
     MOZ_ASSERT(!Scalar::isSimdType(accessType), "SIMD NYI");
     MOZ_ASSERT(!mir->barrierBefore() && !mir->barrierAfter(), "atomics NYI");
 
     if (mir->offset() > INT32_MAX) {
         masm.jump(wasm::JumpTarget::OutOfBounds);
         return;
     }
 
-    const LAllocation* ptr = ins->ptr();
     Operand srcAddr = ptr->isBogus()
                       ? Operand(HeapReg, mir->offset())
                       : Operand(HeapReg, ToRegister(ptr), TimesOne, mir->offset());
 
-    AnyRegister out = ToAnyRegister(ins->output());
+    AnyRegister out = ToAnyRegister(output);
 
     uint32_t before = masm.size();
-    load(accessType, srcAddr, out);
+    if (!isInt64)
+        load(accessType, srcAddr, out);
+    else
+        loadI64(accessType, srcAddr, out);
+    uint32_t after = masm.size();
+
+    verifyLoadDisassembly(before, after, isInt64, accessType, /* numElems */ 0, srcAddr,
+                          *output->output());
+
     masm.append(WasmMemoryAccess(before));
 }
 
 void
+CodeGeneratorX64::visitWasmLoad(LWasmLoad* ins)
+{
+    visitWasmLoadBase(ins->mir(), ins->ptr(), ins->output(), /* isInt64 */ false);
+}
+
+void
+CodeGeneratorX64::visitWasmLoadI64(LWasmLoadI64* ins)
+{
+    visitWasmLoadBase(ins->mir(), ins->ptr(), ins->output(), /* isInt64 */ true);
+}
+
+void
 CodeGeneratorX64::visitWasmStore(LWasmStore* ins)
 {
     const MWasmStore* mir = ins->mir();
 
     Scalar::Type accessType = mir->accessType();
     MOZ_ASSERT(!Scalar::isSimdType(accessType), "SIMD NYI");
     MOZ_ASSERT(!mir->barrierBefore() && !mir->barrierAfter(), "atomics NYI");
 
@@ -679,16 +748,20 @@ CodeGeneratorX64::visitWasmStore(LWasmSt
     const LAllocation* value = ins->value();
     const LAllocation* ptr = ins->ptr();
     Operand dstAddr = ptr->isBogus()
                       ? Operand(HeapReg, mir->offset())
                       : Operand(HeapReg, ToRegister(ptr), TimesOne, mir->offset());
 
     uint32_t before = masm.size();
     store(accessType, value, dstAddr);
+    uint32_t after = masm.size();
+
+    verifyStoreDisassembly(before, after, mir->value()->type() == MIRType::Int64,
+                           accessType, /* numElems */ 0, dstAddr, *value);
 
     masm.append(WasmMemoryAccess(before));
 }
 
 void
 CodeGeneratorX64::emitSimdLoad(LAsmJSLoadHeap* ins)
 {
     const MAsmJSLoadHeap* mir = ins->mir();
@@ -709,69 +782,46 @@ CodeGeneratorX64::emitSimdLoad(LAsmJSLoa
             ptr->isBogus()
             ? Operand(HeapReg, 2 * sizeof(float) + mir->offset())
             : Operand(HeapReg, ToRegister(ptr), TimesOne, 2 * sizeof(float) + mir->offset());
 
         // Load XY
         uint32_t before = masm.size();
         loadSimd(type, 2, srcAddr, out);
         uint32_t after = masm.size();
-        verifyHeapAccessDisassembly(before, after, /*isLoad=*/true, type, 2, srcAddr,
-                                    *ins->output()->output());
+        verifyLoadDisassembly(before, after, /* isInt64 */ false, type, 2, srcAddr,
+                              *ins->output()->output());
         masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::Throw));
 
         // Load Z (W is zeroed)
         // This is still in bounds, as we've checked with a manual bounds check
         // or we had enough space for sure when removing the bounds check.
         before = after;
         loadSimd(type, 1, srcAddrZ, ScratchSimd128Reg);
         after = masm.size();
-        verifyHeapAccessDisassembly(before, after, /*isLoad=*/true, type, 1, srcAddrZ,
-                                    LFloatReg(ScratchSimd128Reg));
+        verifyLoadDisassembly(before, after, /* isInt64 */ false, type, 1, srcAddrZ,
+                              LFloatReg(ScratchSimd128Reg));
         masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::Throw, 8));
 
         // Move ZW atop XY
         masm.vmovlhps(ScratchSimd128Reg, out, out);
     } else {
         uint32_t before = masm.size();
         loadSimd(type, numElems, srcAddr, out);
         uint32_t after = masm.size();
-        verifyHeapAccessDisassembly(before, after, /*isLoad=*/true, type, numElems, srcAddr,
-                                    *ins->output()->output());
+        verifyLoadDisassembly(before, after, /* isInt64 */ true, type, numElems, srcAddr,
+                              *ins->output()->output());
         masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::Throw));
     }
 
     if (hasBoundsCheck)
         cleanupAfterAsmJSBoundsCheckBranch(mir, ToRegister(ptr));
 }
 
 void
-CodeGeneratorX64::load(Scalar::Type type, const Operand& srcAddr, AnyRegister out)
-{
-    switch (type) {
-      case Scalar::Int8:      masm.movsbl(srcAddr, out.gpr()); break;
-      case Scalar::Uint8:     masm.movzbl(srcAddr, out.gpr()); break;
-      case Scalar::Int16:     masm.movswl(srcAddr, out.gpr()); break;
-      case Scalar::Uint16:    masm.movzwl(srcAddr, out.gpr()); break;
-      case Scalar::Int32:
-      case Scalar::Uint32:    masm.movl(srcAddr, out.gpr()); break;
-      case Scalar::Float32:   masm.loadFloat32(srcAddr, out.fpu()); break;
-      case Scalar::Float64:   masm.loadDouble(srcAddr, out.fpu()); break;
-      case Scalar::Float32x4:
-      case Scalar::Int8x16:
-      case Scalar::Int16x8:
-      case Scalar::Int32x4:
-        MOZ_CRASH("SIMD loads should be handled in emitSimdLoad");
-      case Scalar::Uint8Clamped:
-      case Scalar::MaxTypedArrayViewType:
-        MOZ_CRASH("unexpected array type");
-    }
-}
-
-void
 CodeGeneratorX64::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins)
 {
     const MAsmJSLoadHeap* mir = ins->mir();
     Scalar::Type accessType = mir->accessType();
 
     if (Scalar::isSimdType(accessType))
         return emitSimdLoad(ins);
 
@@ -785,41 +835,44 @@ CodeGeneratorX64::visitAsmJSLoadHeap(LAs
 
     OutOfLineLoadTypedArrayOutOfBounds* ool;
     DebugOnly<bool> hasBoundsCheck = maybeEmitAsmJSLoadBoundsCheck(mir, ins, &ool);
 
     uint32_t before = masm.size();
     load(accessType, srcAddr, ToAnyRegister(out));
     uint32_t after = masm.size();
 
-    verifyHeapAccessDisassembly(before, after, /*isLoad=*/true, accessType, 0, srcAddr, *out->output());
+    verifyLoadDisassembly(before, after, /* isInt64 */ false, accessType, 0, srcAddr,
+                          *out->output());
 
     if (ool) {
         MOZ_ASSERT(hasBoundsCheck);
         cleanupAfterAsmJSBoundsCheckBranch(mir, ToRegister(ptr));
         masm.bind(ool->rejoin());
     }
 
     memoryBarrier(mir->barrierAfter());
 
     masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::CarryOn));
 }
 
 void
 CodeGeneratorX64::store(Scalar::Type type, const LAllocation* value, const Operand& dstAddr)
 {
     if (value->isConstant()) {
-        Imm32 cst(ToInt32(value));
+        const MConstant* mir = value->toConstant();
+        Imm32 cst = Imm32(mir->type() == MIRType::Int32 ? mir->toInt32() : mir->toInt64());
         switch (type) {
           case Scalar::Int8:
           case Scalar::Uint8:        masm.movb(cst, dstAddr); break;
           case Scalar::Int16:
           case Scalar::Uint16:       masm.movw(cst, dstAddr); break;
           case Scalar::Int32:
           case Scalar::Uint32:       masm.movl(cst, dstAddr); break;
+          case Scalar::Int64:
           case Scalar::Float32:
           case Scalar::Float64:
           case Scalar::Float32x4:
           case Scalar::Int8x16:
           case Scalar::Int16x8:
           case Scalar::Int32x4:
           case Scalar::Uint8Clamped:
           case Scalar::MaxTypedArrayViewType:
@@ -834,16 +887,19 @@ CodeGeneratorX64::store(Scalar::Type typ
           case Scalar::Int16:
           case Scalar::Uint16:
             masm.movw(ToRegister(value), dstAddr);
             break;
           case Scalar::Int32:
           case Scalar::Uint32:
             masm.movl(ToRegister(value), dstAddr);
             break;
+          case Scalar::Int64:
+            masm.movq(ToRegister(value), dstAddr);
+            break;
           case Scalar::Float32:
             masm.storeUncanonicalizedFloat32(ToFloatRegister(value), dstAddr);
             break;
           case Scalar::Float64:
             masm.storeUncanonicalizedDouble(ToFloatRegister(value), dstAddr);
             break;
           case Scalar::Float32x4:
           case Scalar::Int8x16:
@@ -893,16 +949,17 @@ CodeGeneratorX64::storeSimd(Scalar::Type
         masm.storeUnalignedSimd128Int(in, dstAddr);
         break;
       case Scalar::Int8:
       case Scalar::Uint8:
       case Scalar::Int16:
       case Scalar::Uint16:
       case Scalar::Int32:
       case Scalar::Uint32:
+      case Scalar::Int64:
       case Scalar::Float32:
       case Scalar::Float64:
       case Scalar::Uint8Clamped:
       case Scalar::MaxTypedArrayViewType:
         MOZ_CRASH("should only handle SIMD types");
     }
 }
 
@@ -932,31 +989,33 @@ CodeGeneratorX64::emitSimdStore(LAsmJSSt
         // bounds. To avoid storing the XY before the exception is thrown, we
         // store the Z first, and record its offset in the MemoryAccess so
         // that the signal handler knows to check the bounds of the full
         // access, rather than just the Z.
         masm.vmovhlps(in, ScratchSimd128Reg, ScratchSimd128Reg);
         uint32_t before = masm.size();
         storeSimd(type, 1, ScratchSimd128Reg, dstAddrZ);
         uint32_t after = masm.size();
-        verifyHeapAccessDisassembly(before, after, /*isLoad=*/false, type, 1, dstAddrZ,
-                                    LFloatReg(ScratchSimd128Reg));
+        verifyStoreDisassembly(before, after, /* int64 */ false, type, 1, dstAddrZ,
+                               LFloatReg(ScratchSimd128Reg));
         masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::Throw, 8));
 
         // Store XY
         before = after;
         storeSimd(type, 2, in, dstAddr);
         after = masm.size();
-        verifyHeapAccessDisassembly(before, after, /*isLoad=*/false, type, 2, dstAddr, *ins->value());
+        verifyStoreDisassembly(before, after, /* int64 */ false, type, 2, dstAddr,
+                               *ins->value());
         masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::Throw));
     } else {
         uint32_t before = masm.size();
         storeSimd(type, numElems, in, dstAddr);
         uint32_t after = masm.size();
-        verifyHeapAccessDisassembly(before, after, /*isLoad=*/false, type, numElems, dstAddr, *ins->value());
+        verifyStoreDisassembly(before, after, /* int64 */ false, type, numElems, dstAddr,
+                               *ins->value());
         masm.append(AsmJSMemoryAccess(before, wasm::MemoryAccess::Throw));
     }
 
     if (hasBoundsCheck)
         cleanupAfterAsmJSBoundsCheckBranch(mir, ToRegister(ptr));
 }
 
 void
@@ -980,17 +1039,17 @@ CodeGeneratorX64::visitAsmJSStoreHeap(LA
 
     Label* rejoin;
     DebugOnly<bool> hasBoundsCheck = maybeEmitAsmJSStoreBoundsCheck(mir, ins, &rejoin);
 
     uint32_t before = masm.size();
     store(accessType, value, dstAddr);
     uint32_t after = masm.size();
 
-    verifyHeapAccessDisassembly(before, after, /*isLoad=*/false, accessType, 0, dstAddr, *value);
+    verifyStoreDisassembly(before, after, /* int64 */ false, accessType, 0, dstAddr, *value);
 
     if (rejoin) {
         MOZ_ASSERT(hasBoundsCheck);
         cleanupAfterAsmJSBoundsCheckBranch(mir, ToRegister(ptr));
         masm.bind(rejoin);
     }
 
     memoryBarrier(mir->barrierAfter());
--- a/js/src/jit/x64/CodeGenerator-x64.h
+++ b/js/src/jit/x64/CodeGenerator-x64.h
@@ -23,19 +23,23 @@ class CodeGeneratorX64 : public CodeGene
     ValueOperand ToOutValue(LInstruction* ins);
     ValueOperand ToTempValue(LInstruction* ins, size_t pos);
 
     void storeUnboxedValue(const LAllocation* value, MIRType valueType,
                            Operand dest, MIRType slotType);
     void memoryBarrier(MemoryBarrierBits barrier);
 
     void load(Scalar::Type type, const Operand& srcAddr, AnyRegister out);
-    void loadSimd(Scalar::Type type, unsigned numElems, const Operand& srcAddr, FloatRegister out);
+    void loadI64(Scalar::Type type, const Operand& srcAddr, AnyRegister out);
+    void visitWasmLoadBase(const MWasmLoad* mir, const LAllocation* ptr, const LDefinition* output,
+                           bool isInt64);
 
     void store(Scalar::Type type, const LAllocation* value, const Operand& dstAddr);
+
+    void loadSimd(Scalar::Type type, unsigned numElems, const Operand& srcAddr, FloatRegister out);
     void storeSimd(Scalar::Type type, unsigned numElems, FloatRegister in, const Operand& dstAddr);
 
     void emitSimdLoad(LAsmJSLoadHeap* ins);
     void emitSimdStore(LAsmJSStoreHeap* ins);
   public:
     CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
 
   public:
@@ -64,16 +68,17 @@ class CodeGeneratorX64 : public CodeGene
     void visitTruncateFToInt32(LTruncateFToInt32* ins);
     void visitWrapInt64ToInt32(LWrapInt64ToInt32* lir);
     void visitExtendInt32ToInt64(LExtendInt32ToInt64* lir);
     void visitWasmTruncateToInt64(LWasmTruncateToInt64* lir);
     void visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir);
     void visitLoadTypedArrayElementStatic(LLoadTypedArrayElementStatic* ins);
     void visitStoreTypedArrayElementStatic(LStoreTypedArrayElementStatic* ins);
     void visitWasmLoad(LWasmLoad* ins);
+    void visitWasmLoadI64(LWasmLoadI64* ins);
     void visitWasmStore(LWasmStore* ins);
     void visitAsmSelectI64(LAsmSelectI64* ins);
     void visitAsmJSCall(LAsmJSCall* ins);
     void visitAsmJSLoadHeap(LAsmJSLoadHeap* ins);
     void visitAsmJSStoreHeap(LAsmJSStoreHeap* ins);
     void visitAsmJSCompareExchangeHeap(LAsmJSCompareExchangeHeap* ins);
     void visitAsmJSAtomicExchangeHeap(LAsmJSAtomicExchangeHeap* ins);
     void visitAsmJSAtomicBinopHeap(LAsmJSAtomicBinopHeap* ins);
--- a/js/src/jit/x64/Lowering-x64.cpp
+++ b/js/src/jit/x64/Lowering-x64.cpp
@@ -156,41 +156,49 @@ LIRGeneratorX64::visitAsmJSUnsignedToFlo
 }
 
 void
 LIRGeneratorX64::visitWasmStore(MWasmStore* ins)
 {
     MDefinition* base = ins->base();
     MOZ_ASSERT(base->type() == MIRType::Int32);
 
-    LAllocation value;
+    MDefinition* value = ins->value();
+    LAllocation valueAlloc;
     switch (ins->accessType()) {
       case Scalar::Int8:
       case Scalar::Uint8:
       case Scalar::Int16:
       case Scalar::Uint16:
       case Scalar::Int32:
       case Scalar::Uint32:
-        value = useRegisterOrConstantAtStart(ins->value());
+        valueAlloc = useRegisterOrConstantAtStart(value);
+        break;
+      case Scalar::Int64:
+        // No way to encode an int64-to-memory move on x64.
+        if (value->isConstant() && value->type() != MIRType::Int64)
+            valueAlloc = useOrConstantAtStart(value);
+        else
+            valueAlloc = useRegisterAtStart(value);
         break;
       case Scalar::Float32:
       case Scalar::Float64:
       case Scalar::Float32x4:
       case Scalar::Int8x16:
       case Scalar::Int16x8:
       case Scalar::Int32x4:
-        value = useRegisterAtStart(ins->value());
+        valueAlloc = useRegisterAtStart(value);
         break;
       case Scalar::Uint8Clamped:
       case Scalar::MaxTypedArrayViewType:
         MOZ_CRASH("unexpected array type");
     }
 
     LAllocation baseAlloc = useRegisterOrZeroAtStart(base);
-    auto* lir = new(alloc()) LWasmStore(baseAlloc, value);
+    auto* lir = new(alloc()) LWasmStore(baseAlloc, valueAlloc);
     add(lir, ins);
 }
 
 void
 LIRGeneratorX64::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins)
 {
     MDefinition* base = ins->base();
     MOZ_ASSERT(base->type() == MIRType::Int32);
@@ -229,16 +237,17 @@ LIRGeneratorX64::visitAsmJSStoreHeap(MAs
       case Scalar::Float32:
       case Scalar::Float64:
       case Scalar::Float32x4:
       case Scalar::Int8x16:
       case Scalar::Int16x8:
       case Scalar::Int32x4:
         lir = new(alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()));
         break;
+      case Scalar::Int64:
       case Scalar::Uint8Clamped:
       case Scalar::MaxTypedArrayViewType:
         MOZ_CRASH("unexpected array type");
     }
     add(lir, ins);
 }
 
 void
--- a/js/src/jit/x86-shared/Disassembler-x86-shared.cpp
+++ b/js/src/jit/x86-shared/Disassembler-x86-shared.cpp
@@ -359,16 +359,17 @@ js::jit::Disassembler::DisassembleHeapAc
         break;
       default:
         break;
     }
 
     // Interpret the opcode.
     if (HasRIP(modrm, sib, rex))
         MOZ_CRASH("Unable to disassemble instruction");
+
     size_t memSize = 0;
     OtherOperand otherOperand(imm);
     HeapAccess::Kind kind = HeapAccess::Unknown;
     RegisterID gpr(RegisterID(Reg(modrm, sib, rex)));
     XMMRegisterID xmm(XMMRegisterID(Reg(modrm, sib, rex)));
     ComplexAddress addr(disp,
                         DecodeBase(modrm, sib, rex),
                         DecodeIndex(modrm, sib, rex),
@@ -423,24 +424,32 @@ js::jit::Disassembler::DisassembleHeapAc
         otherOperand = OtherOperand(gpr);
         memSize = 2;
         kind = HeapAccess::Load;
         break;
       case Pack2ByteOpcode(OP2_MOVSX_GvEb):
         MOZ_RELEASE_ASSERT(!haveImm);
         otherOperand = OtherOperand(gpr);
         memSize = 1;
-        kind = HeapAccess::LoadSext32;
+        kind = opsize == 8 ? HeapAccess::LoadSext64 : HeapAccess::LoadSext32;
         break;
       case Pack2ByteOpcode(OP2_MOVSX_GvEw):
         MOZ_RELEASE_ASSERT(!haveImm);
         otherOperand = OtherOperand(gpr);
         memSize = 2;
-        kind = HeapAccess::LoadSext32;
+        kind = opsize == 8 ? HeapAccess::LoadSext64 : HeapAccess::LoadSext32;
         break;
+#ifdef JS_CODEGEN_X64
+      case OP_MOVSXD_GvEv:
+        MOZ_RELEASE_ASSERT(!haveImm);
+        otherOperand = OtherOperand(gpr);
+        memSize = 4;
+        kind = HeapAccess::LoadSext64;
+        break;
+#endif // JS_CODEGEN_X64
       case Pack2ByteOpcode(OP2_MOVDQ_VdqWdq): // aka OP2_MOVDQ_VsdWsd
       case Pack2ByteOpcode(OP2_MOVAPS_VsdWsd):
         MOZ_RELEASE_ASSERT(!haveImm);
         otherOperand = OtherOperand(xmm);
         memSize = 16;
         kind = HeapAccess::Load;
         break;
       case Pack2ByteOpcode(OP2_MOVSD_VsdWsd): // aka OP2_MOVPS_VpsWps
@@ -511,24 +520,31 @@ js::jit::Disassembler::DisassembleHeapAc
 #ifdef DEBUG
 void
 js::jit::Disassembler::DumpHeapAccess(const HeapAccess& access)
 {
     switch (access.kind()) {
       case HeapAccess::Store:      fprintf(stderr, "store"); break;
       case HeapAccess::Load:       fprintf(stderr, "load"); break;
       case HeapAccess::LoadSext32: fprintf(stderr, "loadSext32"); break;
+      case HeapAccess::LoadSext64: fprintf(stderr, "loadSext64"); break;
       default:                     fprintf(stderr, "unknown"); break;
     }
     fprintf(stderr, "%u ", unsigned(access.size()));
 
     switch (access.otherOperand().kind()) {
-      case OtherOperand::Imm: fprintf(stderr, "imm %d", access.otherOperand().imm()); break;
-      case OtherOperand::GPR: fprintf(stderr, "gpr %s", X86Encoding::GPRegName(access.otherOperand().gpr())); break;
-      case OtherOperand::FPR: fprintf(stderr, "fpr %s", X86Encoding::XMMRegName(access.otherOperand().fpr())); break;
+      case OtherOperand::Imm:
+        fprintf(stderr, "imm %d", access.otherOperand().imm());
+        break;
+      case OtherOperand::GPR:
+        fprintf(stderr, "gpr %s", X86Encoding::GPRegName(access.otherOperand().gpr()));
+        break;
+      case OtherOperand::FPR:
+        fprintf(stderr, "fpr %s", X86Encoding::XMMRegName(access.otherOperand().fpr()));
+        break;
       default: fprintf(stderr, "unknown");
     }
 
     fprintf(stderr, " @ ");
 
     if (access.address().isPCRelative()) {
         fprintf(stderr, MEM_o32r " ", ADDR_o32r(access.address().disp()));
     } else if (access.address().hasIndex()) {
--- a/js/src/jit/x86-shared/Lowering-x86-shared.cpp
+++ b/js/src/jit/x86-shared/Lowering-x86-shared.cpp
@@ -334,16 +334,22 @@ LIRGeneratorX86Shared::visitWasmBoundsCh
 }
 
 void
 LIRGeneratorX86Shared::visitWasmLoad(MWasmLoad* ins)
 {
     MDefinition* base = ins->base();
     MOZ_ASSERT(base->type() == MIRType::Int32);
 
+    if (ins->type() == MIRType::Int64) {
+        auto* lir = new(alloc()) LWasmLoadI64(useRegisterOrZeroAtStart(base));
+        defineInt64(lir, ins);
+        return;
+    }
+
     auto* lir = new(alloc()) LWasmLoad(useRegisterOrZeroAtStart(base));
     define(lir, ins);
 }
 
 void
 LIRGeneratorX86Shared::lowerUDiv(MDiv* div)
 {
     if (div->rhs()->isConstant()) {