Bug 1141067 - byte operations on x86/x64. r=sunfish
authorLars T Hansen <lhansen@mozilla.com>
Wed, 11 Mar 2015 15:45:57 +0100
changeset 233144 4b16b2a84ec44241c08d3efcbb4b1cae9588d01d
parent 233143 fbe97de169968022ca4a7f0a7c95e2131338d853
child 233145 916ad7df95fd46217ee4411d011035d0e0ec6f5f
push id14463
push userryanvm@gmail.com
push dateThu, 12 Mar 2015 22:38:41 +0000
treeherderb2g-inbound@ec7746b06b6c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssunfish
bugs1141067
milestone39.0a1
Bug 1141067 - byte operations on x86/x64. r=sunfish
js/src/jit/shared/Assembler-x86-shared.h
js/src/jit/shared/BaseAssembler-x86-shared.h
js/src/jit/shared/Encoding-x86-shared.h
js/src/jit/shared/MacroAssembler-x86-shared.h
--- a/js/src/jit/shared/Assembler-x86-shared.h
+++ b/js/src/jit/shared/Assembler-x86-shared.h
@@ -1100,22 +1100,16 @@ class AssemblerX86Shared : public Assemb
             break;
           case Operand::MEM_ADDRESS32:
             masm.addl_im(imm.value, op.address());
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
-    // Note, lock_addl() is used for a memory barrier on non-SSE2 systems.
-    // Do not optimize, replace by XADDL, or similar.
-    void lock_addl(Imm32 imm, const Operand &op) {
-        masm.prefix_lock();
-        addl(imm, op);
-    }
     void subl(Imm32 imm, Register dest) {
         masm.subl_ir(imm.value, dest.code());
     }
     void subl(Imm32 imm, const Operand &op) {
         switch (op.kind()) {
           case Operand::REG:
             masm.subl_ir(imm.value, op.reg());
             break;
@@ -1124,16 +1118,31 @@ class AssemblerX86Shared : public Assemb
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
     void addl(Register src, Register dest) {
         masm.addl_rr(src.code(), dest.code());
     }
+    void addl(Register src, const Operand &dest) {
+        switch (dest.kind()) {
+          case Operand::REG:
+            masm.addl_rr(src.code(), dest.reg());
+            break;
+          case Operand::MEM_REG_DISP:
+            masm.addl_rm(src.code(), dest.disp(), dest.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.addl_rm(src.code(), dest.disp(), dest.base(), dest.index(), dest.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+        }
+    }
     void subl(Register src, Register dest) {
         masm.subl_rr(src.code(), dest.code());
     }
     void subl(const Operand &src, Register dest) {
         switch (src.kind()) {
           case Operand::REG:
             masm.subl_rr(src.reg(), dest.code());
             break;
@@ -1147,23 +1156,41 @@ class AssemblerX86Shared : public Assemb
     void subl(Register src, const Operand &dest) {
         switch (dest.kind()) {
           case Operand::REG:
             masm.subl_rr(src.code(), dest.reg());
             break;
           case Operand::MEM_REG_DISP:
             masm.subl_rm(src.code(), dest.disp(), dest.base());
             break;
+          case Operand::MEM_SCALE:
+            masm.subl_rm(src.code(), dest.disp(), dest.base(), dest.index(), dest.scale());
+            break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
     void orl(Register reg, Register dest) {
         masm.orl_rr(reg.code(), dest.code());
     }
+    void orl(Register src, const Operand &dest) {
+        switch (dest.kind()) {
+          case Operand::REG:
+            masm.orl_rr(src.code(), dest.reg());
+            break;
+          case Operand::MEM_REG_DISP:
+            masm.orl_rm(src.code(), dest.disp(), dest.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.orl_rm(src.code(), dest.disp(), dest.base(), dest.index(), dest.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+        }
+    }
     void orl(Imm32 imm, Register reg) {
         masm.orl_ir(imm.value, reg.code());
     }
     void orl(Imm32 imm, const Operand &op) {
         switch (op.kind()) {
           case Operand::REG:
             masm.orl_ir(imm.value, op.reg());
             break;
@@ -1172,16 +1199,31 @@ class AssemblerX86Shared : public Assemb
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
     void xorl(Register src, Register dest) {
         masm.xorl_rr(src.code(), dest.code());
     }
+    void xorl(Register src, const Operand &dest) {
+        switch (dest.kind()) {
+          case Operand::REG:
+            masm.xorl_rr(src.code(), dest.reg());
+            break;
+          case Operand::MEM_REG_DISP:
+            masm.xorl_rm(src.code(), dest.disp(), dest.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.xorl_rm(src.code(), dest.disp(), dest.base(), dest.index(), dest.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+        }
+    }
     void xorl(Imm32 imm, Register reg) {
         masm.xorl_ir(imm.value, reg.code());
     }
     void xorl(Imm32 imm, const Operand &op) {
         switch (op.kind()) {
           case Operand::REG:
             masm.xorl_ir(imm.value, op.reg());
             break;
@@ -1190,16 +1232,31 @@ class AssemblerX86Shared : public Assemb
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
     void andl(Register src, Register dest) {
         masm.andl_rr(src.code(), dest.code());
     }
+    void andl(Register src, const Operand &dest) {
+        switch (dest.kind()) {
+          case Operand::REG:
+            masm.andl_rr(src.code(), dest.reg());
+            break;
+          case Operand::MEM_REG_DISP:
+            masm.andl_rm(src.code(), dest.disp(), dest.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.andl_rm(src.code(), dest.disp(), dest.base(), dest.index(), dest.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+        }
+    }
     void andl(Imm32 imm, Register dest) {
         masm.andl_ir(imm.value, dest.code());
     }
     void andl(Imm32 imm, const Operand &op) {
         switch (op.kind()) {
           case Operand::REG:
             masm.andl_ir(imm.value, op.reg());
             break;
@@ -1357,50 +1414,270 @@ class AssemblerX86Shared : public Assemb
             MOZ_CRASH("unexpected operand kind");
         }
     }
     void lock_decl(const Operand &op) {
         masm.prefix_lock();
         decl(op);
     }
 
-    void lock_cmpxchg8(Register src, const Operand &mem) {
+    void addb(Imm32 imm, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.addb_im(imm.value, op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+    void addb(Register src, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.addb_rm(src.code(), op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.addb_rm(src.code(), op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+
+    void subb(Imm32 imm, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.subb_im(imm.value, op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+    void subb(Register src, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.subb_rm(src.code(), op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.subb_rm(src.code(), op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+
+    void andb(Imm32 imm, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.andb_im(imm.value, op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+    void andb(Register src, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.andb_rm(src.code(), op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.andb_rm(src.code(), op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+
+    void orb(Imm32 imm, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.orb_im(imm.value, op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+    void orb(Register src, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.orb_rm(src.code(), op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.orb_rm(src.code(), op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+
+    void xorb(Imm32 imm, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.xorb_im(imm.value, op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+    void xorb(Register src, const Operand &op) {
+        switch (op.kind()) {
+          case Operand::MEM_REG_DISP:
+            masm.xorb_rm(src.code(), op.disp(), op.base());
+            break;
+          case Operand::MEM_SCALE:
+            masm.xorb_rm(src.code(), op.disp(), op.base(), op.index(), op.scale());
+            break;
+          default:
+            MOZ_CRASH("unexpected operand kind");
+            break;
+        }
+    }
+
+    template<typename T>
+    void lock_addb(T src, const Operand &op) {
+        masm.prefix_lock();
+        addb(src, op);
+    }
+    template<typename T>
+    void lock_subb(T src, const Operand &op) {
+        masm.prefix_lock();
+        subb(src, op);
+    }
+    template<typename T>
+    void lock_andb(T src, const Operand &op) {
+        masm.prefix_lock();
+        andb(src, op);
+    }
+    template<typename T>
+    void lock_orb(T src, const Operand &op) {
+        masm.prefix_lock();
+        orb(src, op);
+    }
+    template<typename T>
+    void lock_xorb(T src, const Operand &op) {
+        masm.prefix_lock();
+        xorb(src, op);
+    }
+
+    template<typename T>
+    void lock_addw(T src, const Operand &op) {
+        masm.prefix_lock();
+        masm.prefix_16_for_32();
+        addl(src, op);
+    }
+    template<typename T>
+    void lock_subw(T src, const Operand &op) {
+        masm.prefix_lock();
+        masm.prefix_16_for_32();
+        subl(src, op);
+    }
+    template<typename T>
+    void lock_andw(T src, const Operand &op) {
+        masm.prefix_lock();
+        masm.prefix_16_for_32();
+        andl(src, op);
+    }
+    template<typename T>
+    void lock_orw(T src, const Operand &op) {
+        masm.prefix_lock();
+        masm.prefix_16_for_32();
+        orl(src, op);
+    }
+    template<typename T>
+    void lock_xorw(T src, const Operand &op) {
+        masm.prefix_lock();
+        masm.prefix_16_for_32();
+        xorl(src, op);
+    }
+
+    // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems,
+    // among other things.  Do not optimize, replace by XADDL, or similar.
+    template<typename T>
+    void lock_addl(T src, const Operand &op) {
+        masm.prefix_lock();
+        addl(src, op);
+    }
+    template<typename T>
+    void lock_subl(T src, const Operand &op) {
+        masm.prefix_lock();
+        subl(src, op);
+    }
+    template<typename T>
+    void lock_andl(T src, const Operand &op) {
+        masm.prefix_lock();
+        andl(src, op);
+    }
+    template<typename T>
+    void lock_orl(T src, const Operand &op) {
+        masm.prefix_lock();
+        orl(src, op);
+    }
+    template<typename T>
+    void lock_xorl(T src, const Operand &op) {
+        masm.prefix_lock();
+        xorl(src, op);
+    }
+
+    void lock_cmpxchgb(Register src, const Operand &mem) {
         masm.prefix_lock();
         switch (mem.kind()) {
           case Operand::MEM_REG_DISP:
-            masm.cmpxchg8(src.code(), mem.disp(), mem.base());
+            masm.cmpxchgb(src.code(), mem.disp(), mem.base());
             break;
           case Operand::MEM_SCALE:
-            masm.cmpxchg8(src.code(), mem.disp(), mem.base(), mem.index(), mem.scale());
+            masm.cmpxchgb(src.code(), mem.disp(), mem.base(), mem.index(), mem.scale());
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
-    void lock_cmpxchg16(Register src, const Operand &mem) {
+    void lock_cmpxchgw(Register src, const Operand &mem) {
         masm.prefix_lock();
         switch (mem.kind()) {
           case Operand::MEM_REG_DISP:
-            masm.cmpxchg16(src.code(), mem.disp(), mem.base());
+            masm.cmpxchgw(src.code(), mem.disp(), mem.base());
             break;
           case Operand::MEM_SCALE:
-            masm.cmpxchg16(src.code(), mem.disp(), mem.base(), mem.index(), mem.scale());
+            masm.cmpxchgw(src.code(), mem.disp(), mem.base(), mem.index(), mem.scale());
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
-    void lock_cmpxchg32(Register src, const Operand &mem) {
+    void lock_cmpxchgl(Register src, const Operand &mem) {
         masm.prefix_lock();
         switch (mem.kind()) {
           case Operand::MEM_REG_DISP:
-            masm.cmpxchg32(src.code(), mem.disp(), mem.base());
+            masm.cmpxchgl(src.code(), mem.disp(), mem.base());
             break;
           case Operand::MEM_SCALE:
-            masm.cmpxchg32(src.code(), mem.disp(), mem.base(), mem.index(), mem.scale());
+            masm.cmpxchgl(src.code(), mem.disp(), mem.base(), mem.index(), mem.scale());
             break;
           default:
             MOZ_CRASH("unexpected operand kind");
         }
     }
 
     void lock_xaddb(Register srcdest, const Operand &mem) {
         switch (mem.kind()) {
--- a/js/src/jit/shared/BaseAssembler-x86-shared.h
+++ b/js/src/jit/shared/BaseAssembler-x86-shared.h
@@ -283,16 +283,22 @@ public:
     }
 
     void addl_rm(RegisterID src, int32_t offset, RegisterID base)
     {
         spew("addl       %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_ADD_EvGv, offset, base, src);
     }
 
+    void addl_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("addl       %s, " MEM_obs, GPReg32Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_ADD_EvGv, offset, base, index, scale, src);
+    }
+
     void addl_ir(int32_t imm, RegisterID dst)
     {
         spew("addl       $%d, %s", imm, GPReg32Name(dst));
         if (CAN_SIGN_EXTEND_8_32(imm)) {
             m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_ADD);
             m_formatter.immediate8s(imm);
         } else {
             if (dst == rax)
@@ -390,28 +396,148 @@ public:
             m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_ADD);
             m_formatter.immediate8s(imm);
         } else {
             m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_ADD);
             m_formatter.immediate32(imm);
         }
     }
 
+    void addb_im(int32_t imm, int32_t offset, RegisterID base) {
+        spew("addb       $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_ADD);
+        m_formatter.immediate8(imm);
+    }
+
+    void addb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("addb       $%d, " MEM_obs, int8_t(imm), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale, GROUP1_OP_ADD);
+        m_formatter.immediate8(imm);
+    }
+
+    void addb_rm(RegisterID src, int32_t offset, RegisterID base) {
+        spew("addb       %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+        m_formatter.oneByteOp8(OP_ADD_EbGb, offset, base, src);
+    }
+
+    void addb_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("addb       %s, " MEM_obs, GPReg8Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp8(OP_ADD_EbGb, offset, base, index, scale, src);
+    }
+
+    void subb_im(int32_t imm, int32_t offset, RegisterID base) {
+        spew("subb       $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_SUB);
+        m_formatter.immediate8(imm);
+    }
+
+    void subb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("subb       $%d, " MEM_obs, int8_t(imm), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale, GROUP1_OP_SUB);
+        m_formatter.immediate8(imm);
+    }
+
+    void subb_rm(RegisterID src, int32_t offset, RegisterID base) {
+        spew("subb       %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+        m_formatter.oneByteOp8(OP_SUB_EbGb, offset, base, src);
+    }
+
+    void subb_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("subb       %s, " MEM_obs, GPReg8Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp8(OP_SUB_EbGb, offset, base, index, scale, src);
+    }
+
+    void andb_im(int32_t imm, int32_t offset, RegisterID base) {
+        spew("andb       $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_AND);
+        m_formatter.immediate8(imm);
+    }
+
+    void andb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("andb       $%d, " MEM_obs, int8_t(imm), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale, GROUP1_OP_AND);
+        m_formatter.immediate8(imm);
+    }
+
+    void andb_rm(RegisterID src, int32_t offset, RegisterID base) {
+        spew("andb       %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+        m_formatter.oneByteOp8(OP_AND_EbGb, offset, base, src);
+    }
+
+    void andb_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("andb       %s, " MEM_obs, GPReg8Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp8(OP_AND_EbGb, offset, base, index, scale, src);
+    }
+
+    void orb_im(int32_t imm, int32_t offset, RegisterID base) {
+        spew("orb       $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_OR);
+        m_formatter.immediate8(imm);
+    }
+
+    void orb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("orb        $%d, " MEM_obs, int8_t(imm), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale, GROUP1_OP_OR);
+        m_formatter.immediate8(imm);
+    }
+
+    void orb_rm(RegisterID src, int32_t offset, RegisterID base) {
+        spew("orb       %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+        m_formatter.oneByteOp8(OP_OR_EbGb, offset, base, src);
+    }
+
+    void orb_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("orb        %s, " MEM_obs, GPReg8Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp8(OP_OR_EbGb, offset, base, index, scale, src);
+    }
+
+    void xorb_im(int32_t imm, int32_t offset, RegisterID base) {
+        spew("xorb       $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_XOR);
+        m_formatter.immediate8(imm);
+    }
+
+    void xorb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("xorb       $%d, " MEM_obs, int8_t(imm), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale, GROUP1_OP_XOR);
+        m_formatter.immediate8(imm);
+    }
+
+    void xorb_rm(RegisterID src, int32_t offset, RegisterID base) {
+        spew("xorb       %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+        m_formatter.oneByteOp8(OP_XOR_EbGb, offset, base, src);
+    }
+
+    void xorb_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("xorb       %s, " MEM_obs, GPReg8Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp8(OP_XOR_EbGb, offset, base, index, scale, src);
+    }
+
     void lock_xaddb_rm(RegisterID srcdest, int32_t offset, RegisterID base)
     {
-        spew("lock xaddl %s, " MEM_ob, GPReg8Name(srcdest), ADDR_ob(offset, base));
+        spew("lock xaddb %s, " MEM_ob, GPReg8Name(srcdest), ADDR_ob(offset, base));
         m_formatter.oneByteOp(PRE_LOCK);
-        m_formatter.twoByteOp(OP2_XADD_EbGb, offset, base, srcdest);
+        m_formatter.twoByteOp8(OP2_XADD_EbGb, offset, base, srcdest);
     }
 
     void lock_xaddb_rm(RegisterID srcdest, int32_t offset, RegisterID base, RegisterID index, int scale)
     {
-        spew("lock xaddl %s, " MEM_obs, GPReg8Name(srcdest), ADDR_obs(offset, base, index, scale));
+        spew("lock xaddb %s, " MEM_obs, GPReg8Name(srcdest), ADDR_obs(offset, base, index, scale));
         m_formatter.oneByteOp(PRE_LOCK);
-        m_formatter.twoByteOp(OP2_XADD_EbGb, offset, base, index, scale, srcdest);
+        m_formatter.twoByteOp8(OP2_XADD_EbGb, offset, base, index, scale, srcdest);
     }
 
     void lock_xaddl_rm(RegisterID srcdest, int32_t offset, RegisterID base)
     {
         spew("lock xaddl %s, " MEM_ob, GPReg32Name(srcdest), ADDR_ob(offset, base));
         m_formatter.oneByteOp(PRE_LOCK);
         m_formatter.twoByteOp(OP2_XADD_EvGv, offset, base, srcdest);
     }
@@ -562,16 +688,22 @@ public:
     }
 
     void andl_rm(RegisterID src, int32_t offset, RegisterID base)
     {
         spew("andl       %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_AND_EvGv, offset, base, src);
     }
 
+    void andl_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("andl       %s, " MEM_obs, GPReg32Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_AND_EvGv, offset, base, index, scale, src);
+    }
+
     void andl_ir(int32_t imm, RegisterID dst)
     {
         spew("andl       $0x%x, %s", imm, GPReg32Name(dst));
         if (CAN_SIGN_EXTEND_8_32(imm)) {
             m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_AND);
             m_formatter.immediate8s(imm);
         } else {
             if (dst == rax)
@@ -722,16 +854,22 @@ public:
     }
 
     void orl_rm(RegisterID src, int32_t offset, RegisterID base)
     {
         spew("orl        %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_OR_EvGv, offset, base, src);
     }
 
+    void orl_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("orl        %s, " MEM_obs, GPReg32Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_OR_EvGv, offset, base, index, scale, src);
+    }
+
     void orl_ir(int32_t imm, RegisterID dst)
     {
         spew("orl        $0x%x, %s", imm, GPReg32Name(dst));
         if (CAN_SIGN_EXTEND_8_32(imm)) {
             m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_OR);
             m_formatter.immediate8s(imm);
         } else {
             if (dst == rax)
@@ -814,16 +952,22 @@ public:
     }
 
     void subl_rm(RegisterID src, int32_t offset, RegisterID base)
     {
         spew("subl       %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_SUB_EvGv, offset, base, src);
     }
 
+    void subl_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("subl       %s, " MEM_obs, GPReg32Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_SUB_EvGv, offset, base, index, scale, src);
+    }
+
     void subl_ir(int32_t imm, RegisterID dst)
     {
         spew("subl       $%d, %s", imm, GPReg32Name(dst));
         if (CAN_SIGN_EXTEND_8_32(imm)) {
             m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_SUB);
             m_formatter.immediate8s(imm);
         } else {
             if (dst == rax)
@@ -912,16 +1056,22 @@ public:
     }
 
     void xorl_rm(RegisterID src, int32_t offset, RegisterID base)
     {
         spew("xorl       %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_XOR_EvGv, offset, base, src);
     }
 
+    void xorl_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("xorl       %s, " MEM_obs, GPReg32Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.oneByteOp(OP_XOR_EvGv, offset, base, index, scale, src);
+    }
+
     void xorl_im(int32_t imm, int32_t offset, RegisterID base)
     {
         spew("xorl       $0x%x, " MEM_ob, imm, ADDR_ob(offset, base));
         if (CAN_SIGN_EXTEND_8_32(imm)) {
             m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_XOR);
             m_formatter.immediate8s(imm);
         } else {
             m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_XOR);
@@ -1131,51 +1281,51 @@ public:
     }
 
     void decl_m32(int32_t offset, RegisterID base)
     {
         spew("decl       " MEM_ob, ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, GROUP5_OP_DEC);
     }
 
-    // Note that CMPXCHG performs comparison against REG = %al/%ax/%eax.
+    // Note that CMPXCHG performs comparison against REG = %al/%ax/%eax/%rax.
     // If %REG == [%base+offset], then %src -> [%base+offset].
     // Otherwise, [%base+offset] -> %REG.
     // For the 8-bit operations src must also be an 8-bit register.
 
-    void cmpxchg8(RegisterID src, int32_t offset, RegisterID base)
-    {
-        spew("cmpxchg8   %s, " MEM_ob, GPRegName(src), ADDR_ob(offset, base));
-        m_formatter.twoByteOp(OP2_CMPXCHG_GvEb, offset, base, src);
-    }
-    void cmpxchg8(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
-    {
-        spew("cmpxchg8   %s, " MEM_obs, GPRegName(src), ADDR_obs(offset, base, index, scale));
-        m_formatter.twoByteOp(OP2_CMPXCHG_GvEb, offset, base, index, scale, src);
-    }
-    void cmpxchg16(RegisterID src, int32_t offset, RegisterID base)
-    {
-        spew("cmpxchg16  %s, " MEM_ob, GPRegName(src), ADDR_ob(offset, base));
+    void cmpxchgb(RegisterID src, int32_t offset, RegisterID base)
+    {
+        spew("cmpxchgb   %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+        m_formatter.twoByteOp8(OP2_CMPXCHG_GvEb, offset, base, src);
+    }
+    void cmpxchgb(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("cmpxchgb   %s, " MEM_obs, GPReg8Name(src), ADDR_obs(offset, base, index, scale));
+        m_formatter.twoByteOp8(OP2_CMPXCHG_GvEb, offset, base, index, scale, src);
+    }
+    void cmpxchgw(RegisterID src, int32_t offset, RegisterID base)
+    {
+        spew("cmpxchgw   %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
         m_formatter.prefix(PRE_OPERAND_SIZE);
         m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, src);
     }
-    void cmpxchg16(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
-    {
-        spew("cmpxchg16  %s, " MEM_obs, GPRegName(src), ADDR_obs(offset, base, index, scale));
+    void cmpxchgw(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("cmpxchgw   %s, " MEM_obs, GPReg16Name(src), ADDR_obs(offset, base, index, scale));
         m_formatter.prefix(PRE_OPERAND_SIZE);
         m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
     }
-    void cmpxchg32(RegisterID src, int32_t offset, RegisterID base)
-    {
-        spew("cmpxchg32  %s, " MEM_ob, GPRegName(src), ADDR_ob(offset, base));
+    void cmpxchgl(RegisterID src, int32_t offset, RegisterID base)
+    {
+        spew("cmpxchgl   %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
         m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, src);
     }
-    void cmpxchg32(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
-    {
-        spew("cmpxchg32  %s, " MEM_obs, GPRegName(src), ADDR_obs(offset, base, index, scale));
+    void cmpxchgl(RegisterID src, int32_t offset, RegisterID base, RegisterID index, int scale)
+    {
+        spew("cmpxchgl   %s, " MEM_obs, GPReg32Name(src), ADDR_obs(offset, base, index, scale));
         m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
     }
 
 
     // Comparisons:
 
     void cmpl_rr(RegisterID rhs, RegisterID lhs)
     {
@@ -1750,17 +1900,17 @@ public:
         spew("movl       $0x%x, %s", imm, GPReg32Name(dst));
         m_formatter.oneByteOp(OP_MOV_EAXIv, dst);
         m_formatter.immediate32(imm);
     }
 
     void movb_ir(int32_t imm, RegisterID reg)
     {
         spew("movb       $0x%x, %s", imm, GPReg8Name(reg));
-        m_formatter.oneByteOp(OP_MOV_EbGv, reg);
+        m_formatter.oneByteOp8(OP_MOV_EbIb, reg);
         m_formatter.immediate8(imm);
     }
 
     void movb_im(int32_t imm, int32_t offset, RegisterID base)
     {
         spew("movb       $0x%x, " MEM_ob, imm, ADDR_ob(offset, base));
         m_formatter.oneByteOp(OP_GROUP11_EvIb, offset, base, GROUP11_MOV);
         m_formatter.immediate8(imm);
@@ -4695,16 +4845,23 @@ threeByteOpImmSimd("vblendps", VEX_PD, O
         // operands.
 
         void oneByteOp8(OneByteOpcodeID opcode)
         {
             m_buffer.ensureSpace(MaxInstructionSize);
             m_buffer.putByteUnchecked(opcode);
         }
 
+        void oneByteOp8(OneByteOpcodeID opcode, RegisterID r)
+        {
+            m_buffer.ensureSpace(MaxInstructionSize);
+            emitRexIf(byteRegRequiresRex(r), 0, 0, r);
+            m_buffer.putByteUnchecked(opcode + (r & 7));
+        }
+
         void oneByteOp8(OneByteOpcodeID opcode, RegisterID rm, GroupOpcodeID groupOp)
         {
             m_buffer.ensureSpace(MaxInstructionSize);
             emitRexIf(byteRegRequiresRex(rm), 0, 0, rm);
             m_buffer.putByteUnchecked(opcode);
             registerModRM(rm, groupOp);
         }
 
@@ -4755,16 +4912,36 @@ threeByteOpImmSimd("vblendps", VEX_PD, O
         {
             m_buffer.ensureSpace(MaxInstructionSize);
             emitRexIf(byteRegRequiresRex(reg)|byteRegRequiresRex(rm), reg, 0, rm);
             m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
             m_buffer.putByteUnchecked(opcode);
             registerModRM(rm, reg);
         }
 
+        void twoByteOp8(TwoByteOpcodeID opcode, int32_t offset, RegisterID base, RegisterID reg)
+        {
+            m_buffer.ensureSpace(MaxInstructionSize);
+            emitRexIf(byteRegRequiresRex(reg)|regRequiresRex(base), reg, 0, base);
+            m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+            m_buffer.putByteUnchecked(opcode);
+            memoryModRM(offset, base, reg);
+        }
+
+        void twoByteOp8(TwoByteOpcodeID opcode, int32_t offset, RegisterID base, RegisterID index,
+                        int scale, RegisterID reg)
+        {
+            m_buffer.ensureSpace(MaxInstructionSize);
+            emitRexIf(byteRegRequiresRex(reg)|regRequiresRex(base)|regRequiresRex(index),
+                      reg, index, base);
+            m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+            m_buffer.putByteUnchecked(opcode);
+            memoryModRM(offset, base, index, scale, reg);
+        }
+
         // Like twoByteOp8 but doesn't add a REX prefix if the destination reg
         // is in esp..edi. This may be used when the destination is not an 8-bit
         // register (as in a movzbl instruction), so it doesn't need a REX
         // prefix to disambiguate it from ah..bh.
         void twoByteOp8_movx(TwoByteOpcodeID opcode, RegisterID rm, RegisterID reg)
         {
             m_buffer.ensureSpace(MaxInstructionSize);
             emitRexIf(regRequiresRex(reg)|byteRegRequiresRex(rm), reg, 0, rm);
@@ -4942,17 +5119,18 @@ threeByteOpImmSimd("vblendps", VEX_PD, O
         }
 
         // Used for operations with byte operands - use byteRegRequiresRex() to
         // check register operands, regRequiresRex() to check other registers
         // (i.e. address base & index).
         //
         // NB: WebKit's use of emitRexIf() is limited such that the
         // reqRequiresRex() checks are not needed. SpiderMonkey extends
-        // oneByteOp8 functionality such that r, x, and b can all be used.
+        // oneByteOp8 and twoByteOp8 functionality such that r, x, and b
+        // can all be used.
         void emitRexIf(bool condition, int r, int x, int b)
         {
             if (condition ||
                 regRequiresRex(RegisterID(r)) ||
                 regRequiresRex(RegisterID(x)) ||
                 regRequiresRex(RegisterID(b)))
             {
                 emitRex(false, r, x, b);
--- a/js/src/jit/shared/Encoding-x86-shared.h
+++ b/js/src/jit/shared/Encoding-x86-shared.h
@@ -84,16 +84,17 @@ enum OneByteOpcodeID {
     OP_NOP                          = 0x90,
     OP_PUSHFLAGS                    = 0x9C,
     OP_POPFLAGS                     = 0x9D,
     OP_CDQ                          = 0x99,
     OP_MOV_EAXOv                    = 0xA1,
     OP_MOV_OvEAX                    = 0xA3,
     OP_TEST_EAXIb                   = 0xA8,
     OP_TEST_EAXIv                   = 0xA9,
+    OP_MOV_EbIb                     = 0xB0,
     OP_MOV_EAXIv                    = 0xB8,
     OP_GROUP2_EvIb                  = 0xC1,
     OP_RET_Iz                       = 0xC2,
     PRE_VEX_C4                      = 0xC4,
     PRE_VEX_C5                      = 0xC5,
     OP_RET                          = 0xC3,
     OP_GROUP11_EvIb                 = 0xC6,
     OP_GROUP11_EvIz                 = 0xC7,
--- a/js/src/jit/shared/MacroAssembler-x86-shared.h
+++ b/js/src/jit/shared/MacroAssembler-x86-shared.h
@@ -11,16 +11,33 @@
 #include "mozilla/DebugOnly.h"
 
 #if defined(JS_CODEGEN_X86)
 # include "jit/x86/Assembler-x86.h"
 #elif defined(JS_CODEGEN_X64)
 # include "jit/x64/Assembler-x64.h"
 #endif
 
+#ifdef DEBUG
+  #define CHECK_BYTEREG(reg)                                    \
+      JS_BEGIN_MACRO                                            \
+        GeneralRegisterSet byteRegs(Registers::SingleByteRegs); \
+        MOZ_ASSERT(byteRegs.has(reg));                          \
+      JS_END_MACRO
+  #define CHECK_BYTEREGS(r1, r2)                                \
+      JS_BEGIN_MACRO                                            \
+        GeneralRegisterSet byteRegs(Registers::SingleByteRegs); \
+        MOZ_ASSERT(byteRegs.has(r1));                           \
+        MOZ_ASSERT(byteRegs.has(r2));                           \
+      JS_END_MACRO
+#else
+  #define CHECK_BYTEREG(reg) (void)0
+  #define CHECK_BYTEREGS(r1, r2) (void)0
+#endif
+
 namespace js {
 namespace jit {
 
 class MacroAssemblerX86Shared : public Assembler
 {
   protected:
     // Bytes pushed onto the frame by the callee; includes frameDepth_. This is
     // needed to compute offsets to stack slots while temporary space has been
@@ -206,63 +223,48 @@ class MacroAssemblerX86Shared : public A
         notl(reg);
     }
     void atomic_inc32(const Operand &addr) {
         lock_incl(addr);
     }
     void atomic_dec32(const Operand &addr) {
         lock_decl(addr);
     }
-    void atomic_cmpxchg8(Register newval, const Operand &addr, Register oldval_and_result) {
-        // %eax must be explicitly provided for calling clarity.
-        MOZ_ASSERT(oldval_and_result.code() == X86Encoding::rax);
-        lock_cmpxchg8(newval, addr);
-    }
-    void atomic_cmpxchg16(Register newval, const Operand &addr, Register oldval_and_result) {
-        // %eax must be explicitly provided for calling clarity.
-        MOZ_ASSERT(oldval_and_result.code() == X86Encoding::rax);
-        lock_cmpxchg16(newval, addr);
-    }
-    void atomic_cmpxchg32(Register newval, const Operand &addr, Register oldval_and_result) {
-        // %eax must be explicitly provided for calling clarity.
-        MOZ_ASSERT(oldval_and_result.code() == X86Encoding::rax);
-        lock_cmpxchg32(newval, addr);
-    }
 
     template <typename T>
     void atomicFetchAdd8SignExtend(Register src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREGS(src, output);
         if (src != output)
             movl(src, output);
         lock_xaddb(output, Operand(mem));
         movsbl(output, output);
     }
 
     template <typename T>
     void atomicFetchAdd8ZeroExtend(Register src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREGS(src, output);
         MOZ_ASSERT(temp == InvalidReg);
         if (src != output)
             movl(src, output);
         lock_xaddb(output, Operand(mem));
         movzbl(output, output);
     }
 
     template <typename T>
     void atomicFetchAdd8SignExtend(Imm32 src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREG(output);
         MOZ_ASSERT(temp == InvalidReg);
         movb(src, output);
         lock_xaddb(output, Operand(mem));
         movsbl(output, output);
     }
 
     template <typename T>
     void atomicFetchAdd8ZeroExtend(Imm32 src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREG(output);
         MOZ_ASSERT(temp == InvalidReg);
         movb(src, output);
         lock_xaddb(output, Operand(mem));
         movzbl(output, output);
     }
 
     template <typename T>
     void atomicFetchAdd16SignExtend(Register src, const T &mem, Register temp, Register output) {
@@ -310,48 +312,48 @@ class MacroAssemblerX86Shared : public A
     void atomicFetchAdd32(Imm32 src, const T &mem, Register temp, Register output) {
         MOZ_ASSERT(temp == InvalidReg);
         movl(src, output);
         lock_xaddl(output, Operand(mem));
     }
 
     template <typename T>
     void atomicFetchSub8SignExtend(Register src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREGS(src, output);
         MOZ_ASSERT(temp == InvalidReg);
         if (src != output)
             movl(src, output);
         negl(output);
         lock_xaddb(output, Operand(mem));
         movsbl(output, output);
     }
 
     template <typename T>
     void atomicFetchSub8ZeroExtend(Register src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREGS(src, output);
         MOZ_ASSERT(temp == InvalidReg);
         if (src != output)
             movl(src, output);
         negl(output);
         lock_xaddb(output, Operand(mem));
         movzbl(output, output);
     }
 
     template <typename T>
     void atomicFetchSub8SignExtend(Imm32 src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREG(output);
         MOZ_ASSERT(temp == InvalidReg);
         movb(Imm32(-src.value), output);
         lock_xaddb(output, Operand(mem));
         movsbl(output, output);
     }
 
     template <typename T>
     void atomicFetchSub8ZeroExtend(Imm32 src, const T &mem, Register temp, Register output) {
-        MOZ_ASSERT(output == eax);
+        CHECK_BYTEREG(output);
         MOZ_ASSERT(temp == InvalidReg);
         movb(Imm32(-src.value), output);
         lock_xaddb(output, Operand(mem));
         movzbl(output, output);
     }
 
     template <typename T>
     void atomicFetchSub16SignExtend(Register src, const T &mem, Register temp, Register output) {
@@ -400,99 +402,105 @@ class MacroAssemblerX86Shared : public A
 
     template <typename T>
     void atomicFetchSub32(Imm32 src, const T &mem, Register temp, Register output) {
         movl(Imm32(-src.value), output);
         lock_xaddl(output, Operand(mem));
     }
 
     // requires output == eax
-#define ATOMIC_BITOP_BODY(LOAD, OP, LOCK_CMPXCHG)        \
-        MOZ_ASSERT(output == eax); \
-        LOAD(Operand(mem), eax);  \
-        Label again;              \
-        bind(&again);             \
-        movl(eax, temp);          \
-        OP(src, temp);            \
-        LOCK_CMPXCHG(temp, Operand(mem)); \
+#define ATOMIC_BITOP_BODY(LOAD, OP, LOCK_CMPXCHG) \
+        MOZ_ASSERT(output == eax);                \
+        LOAD(Operand(mem), eax);                  \
+        Label again;                              \
+        bind(&again);                             \
+        movl(eax, temp);                          \
+        OP(src, temp);                            \
+        LOCK_CMPXCHG(temp, Operand(mem));         \
         j(NonZero, &again);
 
     template <typename S, typename T>
     void atomicFetchAnd8SignExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchg8)
+        ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchgb)
+        CHECK_BYTEREG(temp);
         movsbl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchAnd8ZeroExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchg8)
+        ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchgb)
+        CHECK_BYTEREG(temp);
         movzbl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchAnd16SignExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchg16)
+        ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchgw)
         movswl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchAnd16ZeroExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchg16)
+        ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchgw)
         movzwl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchAnd32(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movl, andl, lock_cmpxchg32)
+        ATOMIC_BITOP_BODY(movl, andl, lock_cmpxchgl)
     }
 
     template <typename S, typename T>
     void atomicFetchOr8SignExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchg8)
+        ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchgb)
+        CHECK_BYTEREG(temp);
         movsbl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchOr8ZeroExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchg8)
+        ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchgb)
+        CHECK_BYTEREG(temp);
         movzbl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchOr16SignExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchg16)
+        ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchgw)
         movswl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchOr16ZeroExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchg16)
+        ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchgw)
         movzwl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchOr32(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movl, orl, lock_cmpxchg32)
+        ATOMIC_BITOP_BODY(movl, orl, lock_cmpxchgl)
     }
 
     template <typename S, typename T>
     void atomicFetchXor8SignExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchg8)
+        ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchgb)
+        CHECK_BYTEREG(temp);
         movsbl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchXor8ZeroExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchg8)
+        ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchgb)
+        CHECK_BYTEREG(temp);
         movzbl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchXor16SignExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchg16)
+        ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchgw)
         movswl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchXor16ZeroExtend(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchg16)
+        ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchgw)
         movzwl(eax, eax);
     }
     template <typename S, typename T>
     void atomicFetchXor32(const S &src, const T &mem, Register temp, Register output) {
-        ATOMIC_BITOP_BODY(movl, xorl, lock_cmpxchg32)
+        ATOMIC_BITOP_BODY(movl, xorl, lock_cmpxchgl)
     }
 
 #undef ATOMIC_BITOP_BODY
 
     void storeLoadFence() {
         // This implementation follows Linux.
         if (HasSSE2())
             masm.mfence();
@@ -711,29 +719,29 @@ class MacroAssemblerX86Shared : public A
     template <typename T>
     void store8(Register src, const T &dest) {
         AutoEnsureByteRegister ensure(this, dest, src);
         movb(ensure.reg(), Operand(dest));
     }
     template <typename T>
     void compareExchange8ZeroExtend(const T &mem, Register oldval, Register newval, Register output) {
         MOZ_ASSERT(output == eax);
-        MOZ_ASSERT(newval == ebx || newval == ecx || newval == edx);
+        CHECK_BYTEREG(newval);
         if (oldval != output)
             movl(oldval, output);
-        lock_cmpxchg8(newval, Operand(mem));
+        lock_cmpxchgb(newval, Operand(mem));
         movzbl(output, output);
     }
     template <typename T>
     void compareExchange8SignExtend(const T &mem, Register oldval, Register newval, Register output) {
         MOZ_ASSERT(output == eax);
-        MOZ_ASSERT(newval == ebx || newval == ecx || newval == edx);
+        CHECK_BYTEREG(newval);
         if (oldval != output)
             movl(oldval, output);
-        lock_cmpxchg8(newval, Operand(mem));
+        lock_cmpxchgb(newval, Operand(mem));
         movsbl(output, output);
     }
     void load16ZeroExtend(const Address &src, Register dest) {
         movzwl(Operand(src), dest);
     }
     void load16ZeroExtend(const BaseIndex &src, Register dest) {
         movzwl(Operand(src), dest);
     }
@@ -741,25 +749,25 @@ class MacroAssemblerX86Shared : public A
     void store16(const S &src, const T &dest) {
         movw(src, Operand(dest));
     }
     template <typename T>
     void compareExchange16ZeroExtend(const T &mem, Register oldval, Register newval, Register output) {
         MOZ_ASSERT(output == eax);
         if (oldval != output)
             movl(oldval, output);
-        lock_cmpxchg16(newval, Operand(mem));
+        lock_cmpxchgw(newval, Operand(mem));
         movzwl(output, output);
     }
     template <typename T>
     void compareExchange16SignExtend(const T &mem, Register oldval, Register newval, Register output) {
         MOZ_ASSERT(output == eax);
         if (oldval != output)
             movl(oldval, output);
-        lock_cmpxchg16(newval, Operand(mem));
+        lock_cmpxchgw(newval, Operand(mem));
         movswl(output, output);
     }
     void load16SignExtend(const Address &src, Register dest) {
         movswl(Operand(src), dest);
     }
     void load16SignExtend(const BaseIndex &src, Register dest) {
         movswl(Operand(src), dest);
     }
@@ -776,17 +784,17 @@ class MacroAssemblerX86Shared : public A
     void store32(const S &src, const T &dest) {
         movl(src, Operand(dest));
     }
     template <typename T>
     void compareExchange32(const T &mem, Register oldval, Register newval, Register output) {
         MOZ_ASSERT(output == eax);
         if (oldval != output)
             movl(oldval, output);
-        lock_cmpxchg32(newval, Operand(mem));
+        lock_cmpxchgl(newval, Operand(mem));
     }
     template <typename S, typename T>
     void store32_NoSecondScratch(const S &src, const T &dest) {
         store32(src, dest);
     }
     void loadDouble(const Address &src, FloatRegister dest) {
         vmovsd(src, dest);
     }
@@ -1339,9 +1347,12 @@ class MacroAssemblerX86Shared : public A
 
   protected:
     bool buildOOLFakeExitFrame(void *fakeReturnAddr);
 };
 
 } // namespace jit
 } // namespace js
 
+#undef CHECK_BYTEREG
+#undef CHECK_BYTEREGS
+
 #endif /* jit_shared_MacroAssembler_x86_shared_h */