js/src/jit/x86-shared/Assembler-x86-shared.h
author Jakob Stoklund Olesen <jolesen@mozilla.com>
Tue, 31 May 2016 09:00:19 -0700
changeset 340746 8e4f48388c600edd5ee5292b014cedb0b8f7672f
parent 340740 a17bc6fab38f7beaffa3608ca8ec4a7d660a2bd4
child 342256 7af8faeaf115123fa3774221d930fa80aed1fd49
child 343799 a134a50729fadcdd7b0804862946c533a60bcd7e
permissions -rw-r--r--
Bug 1136226 - Implement compares for 8x16 and 16x8 SIMD types. r=sunfish Since SSE doesn't have unsigned comparisons, add a bias vector and use the signed comparisons instead, just like we do for the 32x4 unsigned vectors. Use 'defineReuseInput' even when SIMD input and output types differ. This is fine now since the register allocator uses a single Simd128 class for all SIMD registers.

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 * vim: set ts=8 sts=4 et sw=4 tw=99:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_x86_shared_Assembler_x86_shared_h
#define jit_x86_shared_Assembler_x86_shared_h

#include <cstddef>

#include "jit/shared/Assembler-shared.h"

#if defined(JS_CODEGEN_X86)
# include "jit/x86/BaseAssembler-x86.h"
#elif defined(JS_CODEGEN_X64)
# include "jit/x64/BaseAssembler-x64.h"
#else
# error "Unknown architecture!"
#endif

namespace js {
namespace jit {

struct ScratchFloat32Scope : public AutoFloatRegisterScope
{
    explicit ScratchFloat32Scope(MacroAssembler& masm)
      : AutoFloatRegisterScope(masm, ScratchFloat32Reg)
    { }
};

struct ScratchDoubleScope : public AutoFloatRegisterScope
{
    explicit ScratchDoubleScope(MacroAssembler& masm)
      : AutoFloatRegisterScope(masm, ScratchDoubleReg)
    { }
};

struct ScratchSimd128Scope : public AutoFloatRegisterScope
{
    explicit ScratchSimd128Scope(MacroAssembler& masm)
      : AutoFloatRegisterScope(masm, ScratchSimd128Reg)
    { }
};

class Operand
{
  public:
    enum Kind {
        REG,
        MEM_REG_DISP,
        FPREG,
        MEM_SCALE,
        MEM_ADDRESS32
    };

  private:
    Kind kind_ : 4;
    // Used as a Register::Encoding and a FloatRegister::Encoding.
    uint32_t base_ : 5;
    Scale scale_ : 3;
    Register::Encoding index_ : 5;
    int32_t disp_;

  public:
    explicit Operand(Register reg)
      : kind_(REG),
        base_(reg.encoding()),
        scale_(TimesOne),
        index_(Registers::Invalid),
        disp_(0)
    { }
    explicit Operand(FloatRegister reg)
      : kind_(FPREG),
        base_(reg.encoding()),
        scale_(TimesOne),
        index_(Registers::Invalid),
        disp_(0)
    { }
    explicit Operand(const Address& address)
      : kind_(MEM_REG_DISP),
        base_(address.base.encoding()),
        scale_(TimesOne),
        index_(Registers::Invalid),
        disp_(address.offset)
    { }
    explicit Operand(const BaseIndex& address)
      : kind_(MEM_SCALE),
        base_(address.base.encoding()),
        scale_(address.scale),
        index_(address.index.encoding()),
        disp_(address.offset)
    { }
    Operand(Register base, Register index, Scale scale, int32_t disp = 0)
      : kind_(MEM_SCALE),
        base_(base.encoding()),
        scale_(scale),
        index_(index.encoding()),
        disp_(disp)
    { }
    Operand(Register reg, int32_t disp)
      : kind_(MEM_REG_DISP),
        base_(reg.encoding()),
        scale_(TimesOne),
        index_(Registers::Invalid),
        disp_(disp)
    { }
    explicit Operand(AbsoluteAddress address)
      : kind_(MEM_ADDRESS32),
        base_(Registers::Invalid),
        scale_(TimesOne),
        index_(Registers::Invalid),
        disp_(X86Encoding::AddressImmediate(address.addr))
    { }
    explicit Operand(PatchedAbsoluteAddress address)
      : kind_(MEM_ADDRESS32),
        base_(Registers::Invalid),
        scale_(TimesOne),
        index_(Registers::Invalid),
        disp_(X86Encoding::AddressImmediate(address.addr))
    { }

    Address toAddress() const {
        MOZ_ASSERT(kind() == MEM_REG_DISP);
        return Address(Register::FromCode(base()), disp());
    }

    BaseIndex toBaseIndex() const {
        MOZ_ASSERT(kind() == MEM_SCALE);
        return BaseIndex(Register::FromCode(base()), Register::FromCode(index()), scale(), disp());
    }

    Kind kind() const {
        return kind_;
    }
    Register::Encoding reg() const {
        MOZ_ASSERT(kind() == REG);
        return Register::Encoding(base_);
    }
    Register::Encoding base() const {
        MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
        return Register::Encoding(base_);
    }
    Register::Encoding index() const {
        MOZ_ASSERT(kind() == MEM_SCALE);
        return index_;
    }
    Scale scale() const {
        MOZ_ASSERT(kind() == MEM_SCALE);
        return scale_;
    }
    FloatRegister::Encoding fpu() const {
        MOZ_ASSERT(kind() == FPREG);
        return FloatRegister::Encoding(base_);
    }
    int32_t disp() const {
        MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
        return disp_;
    }
    void* address() const {
        MOZ_ASSERT(kind() == MEM_ADDRESS32);
        return reinterpret_cast<void*>(disp_);
    }

    bool containsReg(Register r) const {
        switch (kind()) {
          case REG:          return r.encoding() == reg();
          case MEM_REG_DISP: return r.encoding() == base();
          case MEM_SCALE:    return r.encoding() == base() || r.encoding() == index();
          default: MOZ_CRASH("Unexpected Operand kind");
        }
        return false;
    }
};

inline Imm32
Imm64::firstHalf() const
{
    return low();
}

inline Imm32
Imm64::secondHalf() const
{
    return hi();
}

class CPUInfo
{
  public:
    // As the SSE's were introduced in order, the presence of a later SSE implies
    // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2 support.
    enum SSEVersion {
        UnknownSSE = 0,
        NoSSE = 1,
        SSE = 2,
        SSE2 = 3,
        SSE3 = 4,
        SSSE3 = 5,
        SSE4_1 = 6,
        SSE4_2 = 7
    };

    static SSEVersion GetSSEVersion() {
        if (maxSSEVersion == UnknownSSE)
            SetSSEVersion();

        MOZ_ASSERT(maxSSEVersion != UnknownSSE);
        MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE, maxSSEVersion <= maxEnabledSSEVersion);
        return maxSSEVersion;
    }

    static bool IsAVXPresent() {
        if (MOZ_UNLIKELY(maxSSEVersion == UnknownSSE))
            SetSSEVersion();

        MOZ_ASSERT_IF(!avxEnabled, !avxPresent);
        return avxPresent;
    }

  private:
    static SSEVersion maxSSEVersion;
    static SSEVersion maxEnabledSSEVersion;
    static bool avxPresent;
    static bool avxEnabled;
    static bool popcntPresent;

    static void SetSSEVersion();

  public:
    static bool IsSSE2Present() {
#ifdef JS_CODEGEN_X64
        return true;
#else
        return GetSSEVersion() >= SSE2;
#endif
    }
    static bool IsSSE3Present()  { return GetSSEVersion() >= SSE3; }
    static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; }
    static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; }
    static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; }
    static bool IsPOPCNTPresent() { return popcntPresent; }

#ifdef JS_CODEGEN_X86
    static void SetFloatingPointDisabled() { maxEnabledSSEVersion = NoSSE; avxEnabled = false; }
#endif
    static void SetSSE3Disabled() { maxEnabledSSEVersion = SSE2; avxEnabled = false; }
    static void SetSSE4Disabled() { maxEnabledSSEVersion = SSSE3; avxEnabled = false; }
    static void SetAVXEnabled() { avxEnabled = true; }
};

class AssemblerX86Shared : public AssemblerShared
{
  protected:
    struct RelativePatch {
        int32_t offset;
        void* target;
        Relocation::Kind kind;

        RelativePatch(int32_t offset, void* target, Relocation::Kind kind)
          : offset(offset),
            target(target),
            kind(kind)
        { }
    };

    Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
    CompactBufferWriter jumpRelocations_;
    CompactBufferWriter dataRelocations_;
    CompactBufferWriter preBarriers_;

    void writeDataRelocation(ImmGCPtr ptr) {
        if (ptr.value) {
            if (gc::IsInsideNursery(ptr.value))
                embedsNurseryPointers_ = true;
            dataRelocations_.writeUnsigned(masm.currentOffset());
        }
    }
    void writePrebarrierOffset(CodeOffset label) {
        preBarriers_.writeUnsigned(label.offset());
    }

  protected:
    X86Encoding::BaseAssemblerSpecific masm;

    typedef X86Encoding::JmpSrc JmpSrc;
    typedef X86Encoding::JmpDst JmpDst;

  public:
    AssemblerX86Shared()
    {
        if (!HasAVX())
            masm.disableVEX();
    }

    enum Condition {
        Equal = X86Encoding::ConditionE,
        NotEqual = X86Encoding::ConditionNE,
        Above = X86Encoding::ConditionA,
        AboveOrEqual = X86Encoding::ConditionAE,
        Below = X86Encoding::ConditionB,
        BelowOrEqual = X86Encoding::ConditionBE,
        GreaterThan = X86Encoding::ConditionG,
        GreaterThanOrEqual = X86Encoding::ConditionGE,
        LessThan = X86Encoding::ConditionL,
        LessThanOrEqual = X86Encoding::ConditionLE,
        Overflow = X86Encoding::ConditionO,
        Signed = X86Encoding::ConditionS,
        NotSigned = X86Encoding::ConditionNS,
        Zero = X86Encoding::ConditionE,
        NonZero = X86Encoding::ConditionNE,
        Parity = X86Encoding::ConditionP,
        NoParity = X86Encoding::ConditionNP
    };

    // If this bit is set, the vucomisd operands have to be inverted.
    static const int DoubleConditionBitInvert = 0x10;

    // Bit set when a DoubleCondition does not map to a single x86 condition.
    // The macro assembler has to special-case these conditions.
    static const int DoubleConditionBitSpecial = 0x20;
    static const int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial;

    enum DoubleCondition {
        // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN.
        DoubleOrdered = NoParity,
        DoubleEqual = Equal | DoubleConditionBitSpecial,
        DoubleNotEqual = NotEqual,
        DoubleGreaterThan = Above,
        DoubleGreaterThanOrEqual = AboveOrEqual,
        DoubleLessThan = Above | DoubleConditionBitInvert,
        DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert,
        // If either operand is NaN, these conditions always evaluate to true.
        DoubleUnordered = Parity,
        DoubleEqualOrUnordered = Equal,
        DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial,
        DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert,
        DoubleGreaterThanOrEqualOrUnordered = BelowOrEqual | DoubleConditionBitInvert,
        DoubleLessThanOrUnordered = Below,
        DoubleLessThanOrEqualOrUnordered = BelowOrEqual
    };

    enum NaNCond {
        NaN_HandledByCond,
        NaN_IsTrue,
        NaN_IsFalse
    };

    // If the primary condition returned by ConditionFromDoubleCondition doesn't
    // handle NaNs properly, return NaN_IsFalse if the comparison should be
    // overridden to return false on NaN, NaN_IsTrue if it should be overridden
    // to return true on NaN, or NaN_HandledByCond if no secondary check is
    // needed.
    static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) {
        switch (cond) {
          case DoubleOrdered:
          case DoubleNotEqual:
          case DoubleGreaterThan:
          case DoubleGreaterThanOrEqual:
          case DoubleLessThan:
          case DoubleLessThanOrEqual:
          case DoubleUnordered:
          case DoubleEqualOrUnordered:
          case DoubleGreaterThanOrUnordered:
          case DoubleGreaterThanOrEqualOrUnordered:
          case DoubleLessThanOrUnordered:
          case DoubleLessThanOrEqualOrUnordered:
            return NaN_HandledByCond;
          case DoubleEqual:
            return NaN_IsFalse;
          case DoubleNotEqualOrUnordered:
            return NaN_IsTrue;
        }

        MOZ_CRASH("Unknown double condition");
    }

    static void StaticAsserts() {
        // DoubleConditionBits should not interfere with x86 condition codes.
        JS_STATIC_ASSERT(!((Equal | NotEqual | Above | AboveOrEqual | Below |
                            BelowOrEqual | Parity | NoParity) & DoubleConditionBits));
    }

    static Condition InvertCondition(Condition cond);

    // Return the primary condition to test. Some primary conditions may not
    // handle NaNs properly and may therefore require a secondary condition.
    // Use NaNCondFromDoubleCondition to determine what else is needed.
    static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
        return static_cast<Condition>(cond & ~DoubleConditionBits);
    }

    static void TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader);

    // MacroAssemblers hold onto gcthings, so they are traced by the GC.
    void trace(JSTracer* trc);

    bool oom() const {
        return AssemblerShared::oom() ||
               masm.oom() ||
               jumpRelocations_.oom() ||
               dataRelocations_.oom() ||
               preBarriers_.oom();
    }

    void setPrinter(Sprinter* sp) {
        masm.setPrinter(sp);
    }

    static const Register getStackPointer() {
        return StackPointer;
    }

    void executableCopy(void* buffer);
    bool asmMergeWith(const AssemblerX86Shared& other) {
        MOZ_ASSERT(other.jumps_.length() == 0);
        if (!AssemblerShared::asmMergeWith(masm.size(), other))
            return false;
        return masm.appendBuffer(other.masm);
    }
    void processCodeLabels(uint8_t* rawCode);
    void copyJumpRelocationTable(uint8_t* dest);
    void copyDataRelocationTable(uint8_t* dest);
    void copyPreBarrierTable(uint8_t* dest);

    // Size of the instruction stream, in bytes.
    size_t size() const {
        return masm.size();
    }
    // Size of the jump relocation table, in bytes.
    size_t jumpRelocationTableBytes() const {
        return jumpRelocations_.length();
    }
    size_t dataRelocationTableBytes() const {
        return dataRelocations_.length();
    }
    size_t preBarrierTableBytes() const {
        return preBarriers_.length();
    }
    // Size of the data table, in bytes.
    size_t bytesNeeded() const {
        return size() +
               jumpRelocationTableBytes() +
               dataRelocationTableBytes() +
               preBarrierTableBytes();
    }

  public:
    void haltingAlign(int alignment) {
        masm.haltingAlign(alignment);
    }
    void nopAlign(int alignment) {
        masm.nopAlign(alignment);
    }
    void writeCodePointer(CodeOffset* label) {
        // A CodeOffset only has one use, bake in the "end of list" value.
        masm.jumpTablePointer(LabelBase::INVALID_OFFSET);
        label->bind(masm.size());
    }
    void cmovz(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.cmovz_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.cmovz_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.cmovz_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movl(Imm32 imm32, Register dest) {
        masm.movl_i32r(imm32.value, dest.encoding());
    }
    void movl(Register src, Register dest) {
        masm.movl_rr(src.encoding(), dest.encoding());
    }
    void movl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.movl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.movl_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.movl_mr(src.address(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movl(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.movl_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.movl_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          case Operand::MEM_ADDRESS32:
            masm.movl_rm(src.encoding(), dest.address());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movl(Imm32 imm32, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.movl_i32r(imm32.value, dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.movl_i32m(imm32.value, dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          case Operand::MEM_ADDRESS32:
            masm.movl_i32m(imm32.value, dest.address());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void xchgl(Register src, Register dest) {
        masm.xchgl_rr(src.encoding(), dest.encoding());
    }

    // Eventually vmovapd should be overloaded to support loads and
    // stores too.
    void vmovapd(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovapd_rr(src.encoding(), dest.encoding());
    }

    void vmovaps(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovaps_rr(src.encoding(), dest.encoding());
    }
    void vmovaps(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovaps_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          case Operand::FPREG:
            masm.vmovaps_rr(src.fpu(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovaps(FloatRegister src, const Operand& dest) {
        MOZ_ASSERT(HasSSE2());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovups(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovups_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovups(FloatRegister src, const Operand& dest) {
        MOZ_ASSERT(HasSSE2());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovups_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    // vmovsd is only provided in load/store form since the
    // register-to-register form has different semantics (it doesn't clobber
    // the whole output register) and isn't needed currently.
    void vmovsd(const Address& src, FloatRegister dest) {
        masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding());
    }
    void vmovsd(const BaseIndex& src, FloatRegister dest) {
        masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding());
    }
    void vmovsd(FloatRegister src, const Address& dest) {
        masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding());
    }
    void vmovsd(FloatRegister src, const BaseIndex& dest) {
        masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale);
    }
    // Although vmovss is not only provided in load/store form (for the same
    // reasons as vmovsd above), the register to register form should be only
    // used in contexts where we care about not clearing the higher lanes of
    // the FloatRegister.
    void vmovss(const Address& src, FloatRegister dest) {
        masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding());
    }
    void vmovss(const BaseIndex& src, FloatRegister dest) {
        masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding());
    }
    void vmovss(FloatRegister src, const Address& dest) {
        masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding());
    }
    void vmovss(FloatRegister src, const BaseIndex& dest) {
        masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale);
    }
    void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vmovdqu(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovdqu(FloatRegister src, const Operand& dest) {
        MOZ_ASSERT(HasSSE2());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovdqa(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::FPREG:
            masm.vmovdqa_rr(src.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovdqa(FloatRegister src, const Operand& dest) {
        MOZ_ASSERT(HasSSE2());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovdqa(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovdqa_rr(src.encoding(), dest.encoding());
    }
    void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void movzbl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movzbl_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movsbl(Register src, Register dest) {
        masm.movsbl_rr(src.encoding(), dest.encoding());
    }
    void movsbl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movsbl_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movb(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movb_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movb(Imm32 src, Register dest) {
        masm.movb_ir(src.value & 255, dest.encoding());
    }
    void movb(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movb_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movb(Imm32 src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movb_im(src.value, dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movzwl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.movzwl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.movzwl_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movzwl(Register src, Register dest) {
        masm.movzwl_rr(src.encoding(), dest.encoding());
    }
    void movw(const Operand& src, Register dest) {
        masm.prefix_16_for_32();
        movl(src, dest);
    }
    void movw(Imm32 src, Register dest) {
        masm.prefix_16_for_32();
        movl(src, dest);
    }
    void movw(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movw_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movw(Imm32 src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movw_im(src.value, dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void movswl(Register src, Register dest) {
        masm.movswl_rr(src.encoding(), dest.encoding());
    }
    void movswl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.movswl_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void leal(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.leal_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

  protected:
    void jSrc(Condition cond, Label* label) {
        if (label->bound()) {
            // The jump can be immediately encoded to the correct destination.
            masm.jCC_i(static_cast<X86Encoding::Condition>(cond), JmpDst(label->offset()));
        } else {
            // Thread the jump list through the unpatched jump targets.
            JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond));
            JmpSrc prev = JmpSrc(label->use(j.offset()));
            masm.setNextJump(j, prev);
        }
    }
    void jmpSrc(Label* label) {
        if (label->bound()) {
            // The jump can be immediately encoded to the correct destination.
            masm.jmp_i(JmpDst(label->offset()));
        } else {
            // Thread the jump list through the unpatched jump targets.
            JmpSrc j = masm.jmp();
            JmpSrc prev = JmpSrc(label->use(j.offset()));
            masm.setNextJump(j, prev);
        }
    }

    // Comparison of EAX against the address given by a Label.
    JmpSrc cmpSrc(Label* label) {
        JmpSrc j = masm.cmp_eax();
        if (label->bound()) {
            // The jump can be immediately patched to the correct destination.
            masm.linkJump(j, JmpDst(label->offset()));
        } else {
            // Thread the jump list through the unpatched jump targets.
            JmpSrc prev = JmpSrc(label->use(j.offset()));
            masm.setNextJump(j, prev);
        }
        return j;
    }

    JmpSrc jSrc(Condition cond, RepatchLabel* label) {
        JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond));
        if (label->bound()) {
            // The jump can be immediately patched to the correct destination.
            masm.linkJump(j, JmpDst(label->offset()));
        } else {
            label->use(j.offset());
        }
        return j;
    }
    JmpSrc jmpSrc(RepatchLabel* label) {
        JmpSrc j = masm.jmp();
        if (label->bound()) {
            // The jump can be immediately patched to the correct destination.
            masm.linkJump(j, JmpDst(label->offset()));
        } else {
            // Thread the jump list through the unpatched jump targets.
            label->use(j.offset());
        }
        return j;
    }

  public:
    void nop() { masm.nop(); }
    void j(Condition cond, Label* label) { jSrc(cond, label); }
    void jmp(Label* label) { jmpSrc(label); }
    void j(Condition cond, RepatchLabel* label) { jSrc(cond, label); }
    void jmp(RepatchLabel* label) { jmpSrc(label); }

    void j(Condition cond, wasm::JumpTarget target) {
        Label l;
        j(cond, &l);
        bindLater(&l, target);
    }
    void jmp(wasm::JumpTarget target) {
        Label l;
        jmp(&l);
        bindLater(&l, target);
    }

    void jmp(const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.jmp_m(op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.jmp_m(op.disp(), op.base(), op.index(), op.scale());
            break;
          case Operand::REG:
            masm.jmp_r(op.reg());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void cmpEAX(Label* label) { cmpSrc(label); }
    void bind(Label* label) {
        JmpDst dst(masm.label());
        if (label->used()) {
            bool more;
            JmpSrc jmp(label->offset());
            do {
                JmpSrc next;
                more = masm.nextJump(jmp, &next);
                masm.linkJump(jmp, dst);
                jmp = next;
            } while (more);
        }
        label->bind(dst.offset());
    }
    void bindLater(Label* label, wasm::JumpTarget target) {
        if (label->used()) {
            JmpSrc jmp(label->offset());
            do {
                append(target, jmp.offset());
            } while (masm.nextJump(jmp, &jmp));
        }
        label->reset();
    }
    void bind(RepatchLabel* label) {
        JmpDst dst(masm.label());
        if (label->used()) {
            JmpSrc jmp(label->offset());
            masm.linkJump(jmp, dst);
        }
        label->bind(dst.offset());
    }
    void use(CodeOffset* label) {
        label->bind(currentOffset());
    }
    uint32_t currentOffset() {
        return masm.label().offset();
    }

    // Re-routes pending jumps to a new label.
    void retarget(Label* label, Label* target) {
        if (!label->used())
            return;
        bool more;
        JmpSrc jmp(label->offset());
        do {
            JmpSrc next;
            more = masm.nextJump(jmp, &next);
            if (target->bound()) {
                // The jump can be immediately patched to the correct destination.
                masm.linkJump(jmp, JmpDst(target->offset()));
            } else {
                // Thread the jump list through the unpatched jump targets.
                JmpSrc prev(target->use(jmp.offset()));
                masm.setNextJump(jmp, prev);
            }
            jmp = JmpSrc(next.offset());
        } while (more);
        label->reset();
    }

    static void Bind(uint8_t* raw, CodeOffset* label, const void* address) {
        if (label->bound()) {
            intptr_t offset = label->offset();
            X86Encoding::SetPointer(raw + offset, address);
        }
    }

    // See Bind and X86Encoding::setPointer.
    size_t labelToPatchOffset(CodeOffset label) {
        return label.offset() - sizeof(void*);
    }

    void ret() {
        masm.ret();
    }
    void retn(Imm32 n) {
        // Remove the size of the return address which is included in the frame.
        masm.ret_i(n.value - sizeof(void*));
    }
    CodeOffset call(Label* label) {
        if (label->bound()) {
            masm.linkJump(masm.call(), JmpDst(label->offset()));
        } else {
            JmpSrc j = masm.call();
            JmpSrc prev = JmpSrc(label->use(j.offset()));
            masm.setNextJump(j, prev);
        }
        return CodeOffset(masm.currentOffset());
    }
    CodeOffset call(Register reg) {
        masm.call_r(reg.encoding());
        return CodeOffset(masm.currentOffset());
    }
    void call(const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.call_r(op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.call_m(op.disp(), op.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    CodeOffset callWithPatch() {
        return CodeOffset(masm.call().offset());
    }
    void patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
        unsigned char* code = masm.data();
        X86Encoding::AutoUnprotectAssemblerBufferRegion unprotect(masm, callerOffset - 4, 4);
        X86Encoding::SetRel32(code + callerOffset, code + calleeOffset);
    }
    CodeOffset thunkWithPatch() {
        return CodeOffset(masm.jmp().offset());
    }
    void patchThunk(uint32_t thunkOffset, uint32_t targetOffset) {
        unsigned char* code = masm.data();
        X86Encoding::AutoUnprotectAssemblerBufferRegion unprotect(masm, thunkOffset - 4, 4);
        X86Encoding::SetRel32(code + thunkOffset, code + targetOffset);
    }
    static void repatchThunk(uint8_t* code, uint32_t thunkOffset, uint32_t targetOffset) {
        X86Encoding::SetRel32(code + thunkOffset, code + targetOffset);
    }

    CodeOffset twoByteNop() {
        return CodeOffset(masm.twoByteNop().offset());
    }
    static void patchTwoByteNopToJump(uint8_t* jump, uint8_t* target) {
        X86Encoding::BaseAssembler::patchTwoByteNopToJump(jump, target);
    }
    static void patchJumpToTwoByteNop(uint8_t* jump) {
        X86Encoding::BaseAssembler::patchJumpToTwoByteNop(jump);
    }

    void breakpoint() {
        masm.int3();
    }

    static bool HasSSE2() { return CPUInfo::IsSSE2Present(); }
    static bool HasSSE3() { return CPUInfo::IsSSE3Present(); }
    static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); }
    static bool HasSSE41() { return CPUInfo::IsSSE41Present(); }
    static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); }
    static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); }
    static bool SupportsSimd() { return CPUInfo::IsSSE2Present(); }
    static bool HasAVX() { return CPUInfo::IsAVXPresent(); }

    void cmpl(Register rhs, Register lhs) {
        masm.cmpl_rr(rhs.encoding(), lhs.encoding());
    }
    void cmpl(const Operand& rhs, Register lhs) {
        switch (rhs.kind()) {
          case Operand::REG:
            masm.cmpl_rr(rhs.reg(), lhs.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.cmpl_mr(rhs.address(), lhs.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void cmpl(Register rhs, const Operand& lhs) {
        switch (lhs.kind()) {
          case Operand::REG:
            masm.cmpl_rr(rhs.encoding(), lhs.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base());
            break;
          case Operand::MEM_ADDRESS32:
            masm.cmpl_rm(rhs.encoding(), lhs.address());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void cmpl(Imm32 rhs, Register lhs) {
        masm.cmpl_ir(rhs.value, lhs.encoding());
    }
    void cmpl(Imm32 rhs, const Operand& lhs) {
        switch (lhs.kind()) {
          case Operand::REG:
            masm.cmpl_ir(rhs.value, lhs.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.cmpl_im(rhs.value, lhs.disp(), lhs.base());
            break;
          case Operand::MEM_SCALE:
            masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), lhs.scale());
            break;
          case Operand::MEM_ADDRESS32:
            masm.cmpl_im(rhs.value, lhs.address());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    CodeOffset cmplWithPatch(Imm32 rhs, Register lhs) {
        masm.cmpl_i32r(rhs.value, lhs.encoding());
        return CodeOffset(masm.currentOffset());
    }
    void cmpw(Register rhs, Register lhs) {
        masm.cmpw_rr(rhs.encoding(), lhs.encoding());
    }
    void setCC(Condition cond, Register r) {
        masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding());
    }
    void testb(Register rhs, Register lhs) {
        MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs));
        MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs));
        masm.testb_rr(rhs.encoding(), lhs.encoding());
    }
    void testw(Register rhs, Register lhs) {
        masm.testw_rr(lhs.encoding(), rhs.encoding());
    }
    void testl(Register rhs, Register lhs) {
        masm.testl_rr(lhs.encoding(), rhs.encoding());
    }
    void testl(Imm32 rhs, Register lhs) {
        masm.testl_ir(rhs.value, lhs.encoding());
    }
    void testl(Imm32 rhs, const Operand& lhs) {
        switch (lhs.kind()) {
          case Operand::REG:
            masm.testl_ir(rhs.value, lhs.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.testl_i32m(rhs.value, lhs.disp(), lhs.base());
            break;
          case Operand::MEM_ADDRESS32:
            masm.testl_i32m(rhs.value, lhs.address());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }

    void addl(Imm32 imm, Register dest) {
        masm.addl_ir(imm.value, dest.encoding());
    }
    CodeOffset addlWithPatch(Imm32 imm, Register dest) {
        masm.addl_i32r(imm.value, dest.encoding());
        return CodeOffset(masm.currentOffset());
    }
    void addl(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.addl_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.addl_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_ADDRESS32:
            masm.addl_im(imm.value, op.address());
            break;
          case Operand::MEM_SCALE:
            masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void addw(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.addw_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.addw_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_ADDRESS32:
            masm.addw_im(imm.value, op.address());
            break;
          case Operand::MEM_SCALE:
            masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void subl(Imm32 imm, Register dest) {
        masm.subl_ir(imm.value, dest.encoding());
    }
    void subl(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.subl_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.subl_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.subl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void subw(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.subw_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.subw_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void addl(Register src, Register dest) {
        masm.addl_rr(src.encoding(), dest.encoding());
    }
    void addl(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.addl_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.addl_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void addw(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.addw_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.addw_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void subl(Register src, Register dest) {
        masm.subl_rr(src.encoding(), dest.encoding());
    }
    void subl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.subl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.subl_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void subl(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.subl_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.subl_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void subw(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.subw_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.subw_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void orl(Register reg, Register dest) {
        masm.orl_rr(reg.encoding(), dest.encoding());
    }
    void orl(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.orl_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.orl_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void orw(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.orw_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.orw_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void orl(Imm32 imm, Register reg) {
        masm.orl_ir(imm.value, reg.encoding());
    }
    void orl(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.orl_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.orl_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void orw(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.orw_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.orw_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xorl(Register src, Register dest) {
        masm.xorl_rr(src.encoding(), dest.encoding());
    }
    void xorl(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.xorl_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.xorl_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xorw(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.xorw_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.xorw_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xorl(Imm32 imm, Register reg) {
        masm.xorl_ir(imm.value, reg.encoding());
    }
    void xorl(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.xorl_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.xorl_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xorw(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.xorw_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.xorw_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void andl(Register src, Register dest) {
        masm.andl_rr(src.encoding(), dest.encoding());
    }
    void andl(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.andl_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.andl_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void andw(Register src, const Operand& dest) {
        switch (dest.kind()) {
          case Operand::REG:
            masm.andw_rr(src.encoding(), dest.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.andw_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void andl(Imm32 imm, Register dest) {
        masm.andl_ir(imm.value, dest.encoding());
    }
    void andl(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.andl_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.andl_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void andw(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::REG:
            masm.andw_ir(imm.value, op.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.andw_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void addl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.addl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.addl_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void orl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.orl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.orl_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xorl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.xorl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.xorl_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void andl(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.andl_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.andl_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void bsrl(const Register& src, const Register& dest) {
        masm.bsrl_rr(src.encoding(), dest.encoding());
    }
    void bsfl(const Register& src, const Register& dest) {
        masm.bsfl_rr(src.encoding(), dest.encoding());
    }
    void popcntl(const Register& src, const Register& dest) {
        masm.popcntl_rr(src.encoding(), dest.encoding());
    }
    void imull(Register multiplier) {
        masm.imull_r(multiplier.encoding());
    }
    void umull(Register multiplier) {
        masm.mull_r(multiplier.encoding());
    }
    void imull(Imm32 imm, Register dest) {
        masm.imull_ir(imm.value, dest.encoding(), dest.encoding());
    }
    void imull(Register src, Register dest) {
        masm.imull_rr(src.encoding(), dest.encoding());
    }
    void imull(Imm32 imm, Register src, Register dest) {
        masm.imull_ir(imm.value, src.encoding(), dest.encoding());
    }
    void imull(const Operand& src, Register dest) {
        switch (src.kind()) {
          case Operand::REG:
            masm.imull_rr(src.reg(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.imull_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void negl(const Operand& src) {
        switch (src.kind()) {
          case Operand::REG:
            masm.negl_r(src.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.negl_m(src.disp(), src.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void negl(Register reg) {
        masm.negl_r(reg.encoding());
    }
    void notl(const Operand& src) {
        switch (src.kind()) {
          case Operand::REG:
            masm.notl_r(src.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.notl_m(src.disp(), src.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void notl(Register reg) {
        masm.notl_r(reg.encoding());
    }
    void shrl(const Imm32 imm, Register dest) {
        masm.shrl_ir(imm.value, dest.encoding());
    }
    void shll(const Imm32 imm, Register dest) {
        masm.shll_ir(imm.value, dest.encoding());
    }
    void sarl(const Imm32 imm, Register dest) {
        masm.sarl_ir(imm.value, dest.encoding());
    }
    void shrl_cl(Register dest) {
        masm.shrl_CLr(dest.encoding());
    }
    void shll_cl(Register dest) {
        masm.shll_CLr(dest.encoding());
    }
    void sarl_cl(Register dest) {
        masm.sarl_CLr(dest.encoding());
    }

    void roll(const Imm32 imm, Register dest) {
        masm.roll_ir(imm.value, dest.encoding());
    }
    void roll_cl(Register dest) {
        masm.roll_CLr(dest.encoding());
    }
    void rorl(const Imm32 imm, Register dest) {
        masm.rorl_ir(imm.value, dest.encoding());
    }
    void rorl_cl(Register dest) {
        masm.rorl_CLr(dest.encoding());
    }

    void incl(const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.incl_m32(op.disp(), op.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void lock_incl(const Operand& op) {
        masm.prefix_lock();
        incl(op);
    }

    void decl(const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.decl_m32(op.disp(), op.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void lock_decl(const Operand& op) {
        masm.prefix_lock();
        decl(op);
    }

    void addb(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.addb_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }
    void addb(Register src, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.addb_rm(src.encoding(), op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }

    void subb(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.subb_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }
    void subb(Register src, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.subb_rm(src.encoding(), op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }

    void andb(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.andb_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }
    void andb(Register src, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.andb_rm(src.encoding(), op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }

    void orb(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.orb_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }
    void orb(Register src, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.orb_rm(src.encoding(), op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }

    void xorb(Imm32 imm, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.xorb_im(imm.value, op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }
    void xorb(Register src, const Operand& op) {
        switch (op.kind()) {
          case Operand::MEM_REG_DISP:
            masm.xorb_rm(src.encoding(), op.disp(), op.base());
            break;
          case Operand::MEM_SCALE:
            masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
            break;
        }
    }

    template<typename T>
    void lock_addb(T src, const Operand& op) {
        masm.prefix_lock();
        addb(src, op);
    }
    template<typename T>
    void lock_subb(T src, const Operand& op) {
        masm.prefix_lock();
        subb(src, op);
    }
    template<typename T>
    void lock_andb(T src, const Operand& op) {
        masm.prefix_lock();
        andb(src, op);
    }
    template<typename T>
    void lock_orb(T src, const Operand& op) {
        masm.prefix_lock();
        orb(src, op);
    }
    template<typename T>
    void lock_xorb(T src, const Operand& op) {
        masm.prefix_lock();
        xorb(src, op);
    }

    template<typename T>
    void lock_addw(T src, const Operand& op) {
        masm.prefix_lock();
        addw(src, op);
    }
    template<typename T>
    void lock_subw(T src, const Operand& op) {
        masm.prefix_lock();
        subw(src, op);
    }
    template<typename T>
    void lock_andw(T src, const Operand& op) {
        masm.prefix_lock();
        andw(src, op);
    }
    template<typename T>
    void lock_orw(T src, const Operand& op) {
        masm.prefix_lock();
        orw(src, op);
    }
    template<typename T>
    void lock_xorw(T src, const Operand& op) {
        masm.prefix_lock();
        xorw(src, op);
    }

    // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems,
    // among other things.  Do not optimize, replace by XADDL, or similar.
    template<typename T>
    void lock_addl(T src, const Operand& op) {
        masm.prefix_lock();
        addl(src, op);
    }
    template<typename T>
    void lock_subl(T src, const Operand& op) {
        masm.prefix_lock();
        subl(src, op);
    }
    template<typename T>
    void lock_andl(T src, const Operand& op) {
        masm.prefix_lock();
        andl(src, op);
    }
    template<typename T>
    void lock_orl(T src, const Operand& op) {
        masm.prefix_lock();
        orl(src, op);
    }
    template<typename T>
    void lock_xorl(T src, const Operand& op) {
        masm.prefix_lock();
        xorl(src, op);
    }

    void lock_cmpxchgb(Register src, const Operand& mem) {
        masm.prefix_lock();
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.cmpxchgb(src.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void lock_cmpxchgw(Register src, const Operand& mem) {
        masm.prefix_lock();
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.cmpxchgw(src.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void lock_cmpxchgl(Register src, const Operand& mem) {
        masm.prefix_lock();
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.cmpxchgl(src.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void xchgb(Register src, const Operand& mem) {
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.xchgb_rm(src.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xchgw(Register src, const Operand& mem) {
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.xchgw_rm(src.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void xchgl(Register src, const Operand& mem) {
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.xchgl_rm(src.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void lock_xaddb(Register srcdest, const Operand& mem) {
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void lock_xaddw(Register srcdest, const Operand& mem) {
        masm.prefix_16_for_32();
        lock_xaddl(srcdest, mem);
    }
    void lock_xaddl(Register srcdest, const Operand& mem) {
        switch (mem.kind()) {
          case Operand::MEM_REG_DISP:
            masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base());
            break;
          case Operand::MEM_SCALE:
            masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void push(const Imm32 imm) {
        masm.push_i(imm.value);
    }

    void push(const Operand& src) {
        switch (src.kind()) {
          case Operand::REG:
            masm.push_r(src.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.push_m(src.disp(), src.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void push(Register src) {
        masm.push_r(src.encoding());
    }
    void push(const Address& src) {
        masm.push_m(src.offset, src.base.encoding());
    }

    void pop(const Operand& src) {
        switch (src.kind()) {
          case Operand::REG:
            masm.pop_r(src.reg());
            break;
          case Operand::MEM_REG_DISP:
            masm.pop_m(src.disp(), src.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void pop(Register src) {
        masm.pop_r(src.encoding());
    }
    void pop(const Address& src) {
        masm.pop_m(src.offset, src.base.encoding());
    }

    void pushFlags() {
        masm.push_flags();
    }
    void popFlags() {
        masm.pop_flags();
    }

#ifdef JS_CODEGEN_X86
    void pushAllRegs() {
        masm.pusha();
    }
    void popAllRegs() {
        masm.popa();
    }
#endif

    // Zero-extend byte to 32-bit integer.
    void movzbl(Register src, Register dest) {
        masm.movzbl_rr(src.encoding(), dest.encoding());
    }

    void cdq() {
        masm.cdq();
    }
    void idiv(Register divisor) {
        masm.idivl_r(divisor.encoding());
    }
    void udiv(Register divisor) {
        masm.divl_r(divisor.encoding());
    }

    void vpinsrb(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vpinsrb_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpinsrw(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) {
        masm.vpinsrw_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
    }

    void vpinsrd(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
    }

    void vpextrb(unsigned lane, FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vpextrb_irr(lane, src.encoding(), dest.encoding());
    }
    void vpextrw(unsigned lane, FloatRegister src, Register dest) {
        masm.vpextrw_irr(lane, src.encoding(), dest.encoding());
    }
    void vpextrd(unsigned lane, FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vpextrd_irr(lane, src.encoding(), dest.encoding());
    }
    void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding());
    }
    void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding());
    }
    void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding());
    }
    void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpslld_ir(count.value, src0.encoding(), dest.encoding());
    }
    void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding());
    }
    void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding());
    }

    void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding());
    }
    void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding());
    }
    void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding());
    }

    void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::REG:
            masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vcvttsd2si(FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvttsd2si_rr(src.encoding(), dest.encoding());
    }
    void vcvttss2si(FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvttss2si_rr(src.encoding(), dest.encoding());
    }
    void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::REG:
            masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vcvttps2dq(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvttps2dq_rr(src.encoding(), dest.encoding());
    }
    void vcvtdq2ps(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vcvtdq2ps_rr(src.encoding(), dest.encoding());
    }
    void vmovmskpd(FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovmskpd_rr(src.encoding(), dest.encoding());
    }
    void vmovmskps(FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovmskps_rr(src.encoding(), dest.encoding());
    }
    void vptest(FloatRegister rhs, FloatRegister lhs) {
        MOZ_ASSERT(HasSSE41());
        masm.vptest_rr(rhs.encoding(), lhs.encoding());
    }
    void vucomisd(FloatRegister rhs, FloatRegister lhs) {
        MOZ_ASSERT(HasSSE2());
        masm.vucomisd_rr(rhs.encoding(), lhs.encoding());
    }
    void vucomiss(FloatRegister rhs, FloatRegister lhs) {
        MOZ_ASSERT(HasSSE2());
        masm.vucomiss_rr(rhs.encoding(), lhs.encoding());
    }

    void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (rhs.kind()) {
          case Operand::FPREG:
            masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (rhs.kind()) {
          case Operand::FPREG:
            masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (rhs.kind()) {
          case Operand::FPREG:
            masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (rhs.kind()) {
          case Operand::FPREG:
            masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (rhs.kind()) {
          case Operand::FPREG:
            masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (rhs.kind()) {
          case Operand::FPREG:
            masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void vcmpps(uint8_t order, Operand src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        // :TODO: (Bug 1132894) See LIRGeneratorX86Shared::lowerForFPU
        // FIXME: This logic belongs in the MacroAssembler.
        if (!HasAVX() && !src0.aliases(dest)) {
            if (src1.kind() == Operand::FPREG &&
                dest.aliases(FloatRegister::FromCode(src1.fpu())))
            {
                vmovdqa(src1, ScratchSimd128Reg);
                src1 = Operand(ScratchSimd128Reg);
            }
            vmovdqa(src0, dest);
            src0 = dest;
        }
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vcmpps_rr(order, src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vcmpps_mr(order, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vcmpps_mr(order, src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vcmpeqps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        vcmpps(X86Encoding::ConditionCmp_EQ, src1, src0, dest);
    }
    void vcmpltps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        vcmpps(X86Encoding::ConditionCmp_LT, src1, src0, dest);
    }
    void vcmpleps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        vcmpps(X86Encoding::ConditionCmp_LE, src1, src0, dest);
    }
    void vcmpunordps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        vcmpps(X86Encoding::ConditionCmp_UNORD, src1, src0, dest);
    }
    void vcmpneqps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        vcmpps(X86Encoding::ConditionCmp_NEQ, src1, src0, dest);
    }
    void vcmpordps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        vcmpps(X86Encoding::ConditionCmp_ORD, src1, src0, dest);
    }
    void vrcpps(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::FPREG:
            masm.vrcpps_rr(src.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vrcpps_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vrcpps_mr(src.address(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vsqrtps(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::FPREG:
            masm.vsqrtps_rr(src.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vsqrtps_mr(src.address(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vrsqrtps(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::FPREG:
            masm.vrsqrtps_rr(src.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vrsqrtps_mr(src.address(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovd(Register src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovd_rr(src.encoding(), dest.encoding());
    }
    void vmovd(FloatRegister src, Register dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovd_rr(src.encoding(), dest.encoding());
    }
    void vmovd(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovd_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovd(FloatRegister src, const Operand& dest) {
        MOZ_ASSERT(HasSSE2());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovd_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vmovq_rm(src.encoding(), dest.address());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovq(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovq_mr(src.disp(), src.base(), dest.encoding());
            break;
          case Operand::MEM_SCALE:
            masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vmovq_mr(src.address(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovq(FloatRegister src, const Operand& dest) {
        MOZ_ASSERT(HasSSE2());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.vmovq_rm(src.encoding(), dest.disp(), dest.base());
            break;
          case Operand::MEM_SCALE:
            masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        // Negates bits of dest and then applies AND
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpshufd_irr(mask, src.encoding(), dest.encoding());
    }
    void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vpshufd_irr(mask, src1.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vpshufd_imr(mask, src1.address(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpshuflw_irr(mask, src.encoding(), dest.encoding());
    }
    void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vpshufhw_irr(mask, src.encoding(), dest.encoding());
    }
    void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSSE3());
        masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding());
    }
    void vmovddup(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE3());
        masm.vmovddup_rr(src.encoding(), dest.encoding());
    }
    void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vshufps_imr(mask, src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_ADDRESS32:
            masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vroundsd_irr(mode, src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vroundss(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vroundss_irr(mode, src1.encoding(), src0.encoding(), dest.encoding());
    }
    unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane, unsigned zeroMask = 0)
    {
        // Note that the sourceLane bits are ignored in the case of a source
        // memory operand, and the source is the given 32-bits memory location.
        MOZ_ASSERT(zeroMask < 16);
        unsigned ret = zeroMask ;
        ret |= destLane << 4;
        ret |= sourceLane << 6;
        MOZ_ASSERT(ret < 256);
        return ret;
    }
    void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    unsigned blendpsMask(bool x, bool y, bool z, bool w) {
        return (x << 0) | (y << 1) | (z << 2) | (w << 3);
    }
    void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vblendps(unsigned mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE41());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovsldup(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE3());
        masm.vmovsldup_rr(src.encoding(), dest.encoding());
    }
    void vmovsldup(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE3());
        switch (src.kind()) {
          case Operand::FPREG:
            masm.vmovsldup_rr(src.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmovshdup(FloatRegister src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE3());
        masm.vmovshdup_rr(src.encoding(), dest.encoding());
    }
    void vmovshdup(const Operand& src, FloatRegister dest) {
        MOZ_ASSERT(HasSSE3());
        switch (src.kind()) {
          case Operand::FPREG:
            masm.vmovshdup_rr(src.fpu(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        switch (src1.kind()) {
          case Operand::FPREG:
            masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
            break;
          case Operand::MEM_REG_DISP:
            masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
        MOZ_ASSERT(HasSSE2());
        masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding());
    }
    void fisttp(const Operand& dest) {
        MOZ_ASSERT(HasSSE3());
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.fisttp_m(dest.disp(), dest.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void fld(const Operand& dest) {
        switch (dest.kind()) {
          case Operand::MEM_REG_DISP:
            masm.fld_m(dest.disp(), dest.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void fstp(const Operand& src) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.fstp_m(src.disp(), src.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }
    void fstp32(const Operand& src) {
        switch (src.kind()) {
          case Operand::MEM_REG_DISP:
            masm.fstp32_m(src.disp(), src.base());
            break;
          default:
            MOZ_CRASH("unexpected operand kind");
        }
    }

    // Defined for compatibility with ARM's assembler
    uint32_t actualIndex(uint32_t x) {
        return x;
    }

    void flushBuffer() {
    }

    // Patching.

    static size_t PatchWrite_NearCallSize() {
        return 5;
    }
    static uintptr_t GetPointer(uint8_t* instPtr) {
        uintptr_t* ptr = ((uintptr_t*) instPtr) - 1;
        return *ptr;
    }
    // Write a relative call at the start location |dataLabel|.
    // Note that this DOES NOT patch data that comes before |label|.
    static void PatchWrite_NearCall(CodeLocationLabel startLabel, CodeLocationLabel target) {
        uint8_t* start = startLabel.raw();
        *start = 0xE8;
        ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize();
        MOZ_ASSERT(int32_t(offset) == offset);
        *((int32_t*) (start + 1)) = offset;
    }

    static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) {
        *((int32_t*) dataLabel.raw() - 1) = toWrite.value;
    }

    static void PatchDataWithValueCheck(CodeLocationLabel data, PatchedImmPtr newData,
                                        PatchedImmPtr expectedData) {
        // The pointer given is a pointer to *after* the data.
        uintptr_t* ptr = ((uintptr_t*) data.raw()) - 1;
        MOZ_ASSERT(*ptr == (uintptr_t)expectedData.value);
        *ptr = (uintptr_t)newData.value;
    }
    static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData, ImmPtr expectedData) {
        PatchDataWithValueCheck(data, PatchedImmPtr(newData.value), PatchedImmPtr(expectedData.value));
    }

    static void PatchInstructionImmediate(uint8_t* code, PatchedImmPtr imm) {
        MOZ_CRASH("Unused.");
    }

    static uint32_t NopSize() {
        return 1;
    }
    static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) {
        MOZ_CRASH("nextInstruction NYI on x86");
    }

    // Toggle a jmp or cmp emitted by toggledJump().
    static void ToggleToJmp(CodeLocationLabel inst) {
        uint8_t* ptr = (uint8_t*)inst.raw();
        MOZ_ASSERT(*ptr == 0x3D);
        *ptr = 0xE9;
    }
    static void ToggleToCmp(CodeLocationLabel inst) {
        uint8_t* ptr = (uint8_t*)inst.raw();
        MOZ_ASSERT(*ptr == 0xE9);
        *ptr = 0x3D;
    }
    static void ToggleCall(CodeLocationLabel inst, bool enabled) {
        uint8_t* ptr = (uint8_t*)inst.raw();
        MOZ_ASSERT(*ptr == 0x3D || // CMP
                   *ptr == 0xE8);  // CALL
        *ptr = enabled ? 0xE8 : 0x3D;
    }

    MOZ_COLD void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
                                              const Disassembler::HeapAccess& heapAccess);
};

} // namespace jit
} // namespace js

#endif /* jit_x86_shared_Assembler_x86_shared_h */