tor-browser

MacroAssembler-loong64-inl.h (79793B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_loong64_MacroAssembler_loong64_inl_h
#define jit_loong64_MacroAssembler_loong64_inl_h

#include "jit/loong64/MacroAssembler-loong64.h"

namespace js {
namespace jit {

//{{{ check_macroassembler_style

void MacroAssembler::move64(Register64 src, Register64 dest) {
 movePtr(src.reg, dest.reg);
}

void MacroAssembler::move64(Imm64 imm, Register64 dest) {
 movePtr(ImmWord(imm.value), dest.reg);
}

void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
 moveFromDouble(src, dest.reg);
}

void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
 moveToDouble(src.reg, dest);
}

void MacroAssembler::moveLowDoubleToGPR(FloatRegister src, Register dest) {
 MOZ_CRASH("Not supported for this target");
}

void MacroAssembler::move64To32(Register64 src, Register dest) {
 as_slli_w(dest, src.reg, 0);
}

void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
 as_bstrpick_d(dest.reg, src, 31, 0);
}

void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
 move32To64SignExtend(src, dest);
 move8SignExtend(dest.reg, dest.reg);
}

void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
 move32To64SignExtend(src, dest);
 move16SignExtend(dest.reg, dest.reg);
}

void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
 as_slli_w(dest.reg, src, 0);
}

void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
 moveFromFloat32(src, dest);
}

void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
 moveToFloat32(src, dest);
}

void MacroAssembler::moveFloat16ToGPR(FloatRegister src, Register dest) {
 MOZ_CRASH("Not supported for this target");
}

void MacroAssembler::moveGPRToFloat16(Register src, FloatRegister dest) {
 MOZ_CRASH("Not supported for this target");
}

void MacroAssembler::move8ZeroExtend(Register src, Register dest) {
 as_bstrpick_d(dest, src, 7, 0);
}

void MacroAssembler::move8SignExtend(Register src, Register dest) {
 as_ext_w_b(dest, src);
}

void MacroAssembler::move16SignExtend(Register src, Register dest) {
 as_ext_w_h(dest, src);
}

void MacroAssembler::move8SignExtendToPtr(Register src, Register dest) {
 move8To64SignExtend(src, Register64(dest));
}

void MacroAssembler::move16SignExtendToPtr(Register src, Register dest) {
 move16To64SignExtend(src, Register64(dest));
}

void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
 as_slli_w(dest, src, 0);
}

void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
 as_bstrpick_d(dest, src, 31, 0);
}

// ===============================================================
// Load instructions

void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
 load32(src, dest);
}

void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(ra, dest); }

// ===============================================================
// Logical instructions

void MacroAssembler::not32(Register reg) { as_nor(reg, reg, zero); }

void MacroAssembler::notPtr(Register reg) { as_nor(reg, reg, zero); }

void MacroAssembler::andPtr(Register src, Register dest) {
 as_and(dest, dest, src);
}

void MacroAssembler::andPtr(Imm32 imm, Register dest) {
 ma_and(dest, dest, imm);
}

void MacroAssembler::andPtr(Imm32 imm, Register src, Register dest) {
 ma_and(dest, src, imm);
}

void MacroAssembler::and64(Imm64 imm, Register64 dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ma_li(scratch, ImmWord(imm.value));
 as_and(dest.reg, dest.reg, scratch);
}

void MacroAssembler::and64(Register64 src, Register64 dest) {
 as_and(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::and32(Register src, Register dest) {
 as_and(dest, dest, src);
}

void MacroAssembler::and32(Imm32 imm, Register dest) {
 ma_and(dest, dest, imm);
}

void MacroAssembler::and32(Imm32 imm, Register src, Register dest) {
 ma_and(dest, src, imm);
}

void MacroAssembler::and32(Imm32 imm, const Address& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(dest, scratch);
 and32(imm, scratch);
 store32(scratch, dest);
}

void MacroAssembler::and32(const Address& src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(src, scratch);
 as_and(dest, dest, scratch);
}

void MacroAssembler::or64(Imm64 imm, Register64 dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ma_li(scratch, ImmWord(imm.value));
 as_or(dest.reg, dest.reg, scratch);
}

void MacroAssembler::or32(Register src, Register dest) {
 as_or(dest, dest, src);
}

void MacroAssembler::or32(Imm32 imm, Register dest) { ma_or(dest, dest, imm); }

void MacroAssembler::or32(Imm32 imm, Register src, Register dest) {
 ma_or(dest, src, imm);
}

void MacroAssembler::or32(Imm32 imm, const Address& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(dest, scratch);
 or32(imm, scratch);
 store32(scratch, dest);
}

void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ma_li(scratch, ImmWord(imm.value));
 as_xor(dest.reg, dest.reg, scratch);
}

void MacroAssembler::orPtr(Register src, Register dest) {
 as_or(dest, dest, src);
}

void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, dest, imm); }

void MacroAssembler::orPtr(Imm32 imm, Register src, Register dest) {
 ma_or(dest, src, imm);
}

void MacroAssembler::or64(Register64 src, Register64 dest) {
 as_or(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::xor64(Register64 src, Register64 dest) {
 as_xor(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::xorPtr(Register src, Register dest) {
 as_xor(dest, dest, src);
}

void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
 ma_xor(dest, dest, imm);
}

void MacroAssembler::xorPtr(Imm32 imm, Register src, Register dest) {
 ma_xor(dest, src, imm);
}

void MacroAssembler::xor32(Register src, Register dest) {
 as_xor(dest, dest, src);
}

void MacroAssembler::xor32(Imm32 imm, Register dest) {
 ma_xor(dest, dest, imm);
}

void MacroAssembler::xor32(Imm32 imm, Register src, Register dest) {
 ma_xor(dest, src, imm);
}

void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(dest, scratch);
 xor32(imm, scratch);
 store32(scratch, dest);
}

void MacroAssembler::xor32(const Address& src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(src, scratch);
 xor32(scratch, dest);
}

// ===============================================================
// Swap instructions

void MacroAssembler::byteSwap16SignExtend(Register reg) {
 as_revb_2h(reg, reg);
 as_ext_w_h(reg, reg);
}

void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
 as_revb_2h(reg, reg);
 as_bstrpick_d(reg, reg, 15, 0);
}

void MacroAssembler::byteSwap32(Register reg) {
 as_revb_2w(reg, reg);
 as_slli_w(reg, reg, 0);
}

void MacroAssembler::byteSwap64(Register64 reg64) {
 as_revb_d(reg64.reg, reg64.reg);
}

// ===============================================================
// Arithmetic functions

void MacroAssembler::addPtr(Register src, Register dest) {
 as_add_d(dest, dest, src);
}

void MacroAssembler::addPtr(Imm32 imm, Register dest) {
 ma_add_d(dest, dest, imm);
}

void MacroAssembler::addPtr(ImmWord imm, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 movePtr(imm, scratch);
 addPtr(scratch, dest);
}

void MacroAssembler::add64(Register64 src, Register64 dest) {
 addPtr(src.reg, dest.reg);
}

void MacroAssembler::add64(Imm32 imm, Register64 dest) {
 ma_add_d(dest.reg, dest.reg, imm);
}

void MacroAssembler::add64(Imm64 imm, Register64 dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(dest.reg != scratch);
 mov(ImmWord(imm.value), scratch);
 as_add_d(dest.reg, dest.reg, scratch);
}

void MacroAssembler::add32(Register src, Register dest) {
 as_add_w(dest, dest, src);
}

void MacroAssembler::add32(Imm32 imm, Register dest) {
 ma_add_w(dest, dest, imm);
}

void MacroAssembler::add32(Imm32 imm, Register src, Register dest) {
 ma_add_w(dest, src, imm);
}

void MacroAssembler::add32(Imm32 imm, const Address& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(dest, scratch);
 ma_add_w(scratch, scratch, imm);
 store32(scratch, dest);
}

void MacroAssembler::add32(const Address& src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(src, scratch);
 as_add_w(dest, dest, scratch);
}

void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(dest, scratch);
 addPtr(imm, scratch);
 storePtr(scratch, dest);
}

void MacroAssembler::addPtr(const Address& src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(src, scratch);
 addPtr(scratch, dest);
}

void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
 as_fadd_d(dest, dest, src);
}

void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
 as_fadd_s(dest, dest, src);
}

CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
 CodeOffset offset = CodeOffset(currentOffset());
 MacroAssemblerLOONG64::ma_liPatchable(dest, Imm32(0));
 as_sub_d(dest, StackPointer, dest);
 return offset;
}

void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
 // TODO: by wangqing
 InstImm* lu12i_w = (InstImm*)m_buffer.getInst(BufferOffset(offset.offset()));
 InstImm* ori = (InstImm*)m_buffer.getInst(BufferOffset(offset.offset() + 4));

 MOZ_ASSERT((lu12i_w->extractBitField(31, 25)) ==
            ((uint32_t)op_lu12i_w >> 25));
 MOZ_ASSERT((ori->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22));

 *lu12i_w = InstImm(op_lu12i_w, (int32_t)((imm.value >> 12) & 0xfffff),
                    Register::FromCode(lu12i_w->extractRD()), false);
 *ori = InstImm(op_ori, (int32_t)(imm.value & 0xfff),
                Register::FromCode(ori->extractRJ()),
                Register::FromCode(ori->extractRD()), 12);
}

void MacroAssembler::subPtr(Register src, Register dest) {
 as_sub_d(dest, dest, src);
}

void MacroAssembler::subPtr(Imm32 imm, Register dest) {
 ma_sub_d(dest, dest, imm);
}

void MacroAssembler::sub64(Register64 src, Register64 dest) {
 as_sub_d(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(dest.reg != scratch);
 mov(ImmWord(imm.value), scratch);
 as_sub_d(dest.reg, dest.reg, scratch);
}

void MacroAssembler::sub32(Register src, Register dest) {
 as_sub_w(dest, dest, src);
}

void MacroAssembler::sub32(Imm32 imm, Register dest) {
 ma_sub_w(dest, dest, imm);
}

void MacroAssembler::sub32(const Address& src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(src, scratch);
 as_sub_w(dest, dest, scratch);
}

void MacroAssembler::subPtr(Register src, const Address& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(dest, scratch);
 subPtr(src, scratch);
 storePtr(scratch, dest);
}

void MacroAssembler::subPtr(const Address& addr, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(addr, scratch);
 subPtr(scratch, dest);
}

void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
 as_fsub_d(dest, dest, src);
}

void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
 as_fsub_s(dest, dest, src);
}

void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(dest.reg != scratch);
 mov(ImmWord(imm.value), scratch);
 as_mul_d(dest.reg, dest.reg, scratch);
}

void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
                          const Register temp) {
 MOZ_ASSERT(temp == Register::Invalid());
 mul64(imm, dest);
}

void MacroAssembler::mul64(const Register64& src, const Register64& dest,
                          const Register temp) {
 MOZ_ASSERT(temp == Register::Invalid());
 as_mul_d(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
 as_mul_d(srcDest, srcDest, rhs);
}

void MacroAssembler::mulPtr(ImmWord rhs, Register srcDest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(srcDest != scratch);
 mov(rhs, scratch);
 mulPtr(scratch, srcDest);
}

void MacroAssembler::mulBy3(Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(src != scratch);
 as_add_d(scratch, src, src);
 as_add_d(dest, scratch, src);
}

void MacroAssembler::mul32(Register rhs, Register srcDest) {
 as_mul_w(srcDest, srcDest, rhs);
}

void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 move32(imm, scratch);
 mul32(scratch, srcDest);
}

void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(src != scratch);
 move32(imm, scratch);
 as_mulh_wu(dest, src, scratch);
}

void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
 as_fmul_s(dest, dest, src);
}

void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
 as_fmul_d(dest, dest, src);
}

void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
                                 FloatRegister dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ScratchDoubleScope fpscratch(asMasm());
 movePtr(imm, scratch);
 loadDouble(Address(scratch, 0), fpscratch);
 mulDouble(fpscratch, dest);
}

void MacroAssembler::inc64(AbsoluteAddress dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 Register scratch2 = temps.Acquire();
 ma_li(scratch, ImmWord(uintptr_t(dest.addr)));
 as_ld_d(scratch2, scratch, 0);
 as_addi_d(scratch2, scratch2, 1);
 as_st_d(scratch2, scratch, 0);
}

void MacroAssembler::quotient32(Register lhs, Register rhs, Register dest,
                               bool isUnsigned) {
 if (isUnsigned) {
   as_div_wu(dest, lhs, rhs);
 } else {
   as_div_w(dest, lhs, rhs);
 }
}

void MacroAssembler::quotient64(Register lhs, Register rhs, Register dest,
                               bool isUnsigned) {
 if (isUnsigned) {
   as_div_du(dest, lhs, rhs);
 } else {
   as_div_d(dest, lhs, rhs);
 }
}

void MacroAssembler::remainder32(Register lhs, Register rhs, Register dest,
                                bool isUnsigned) {
 if (isUnsigned) {
   as_mod_wu(dest, lhs, rhs);
 } else {
   as_mod_w(dest, lhs, rhs);
 }
}

void MacroAssembler::remainder64(Register lhs, Register rhs, Register dest,
                                bool isUnsigned) {
 if (isUnsigned) {
   as_mod_du(dest, lhs, rhs);
 } else {
   as_mod_d(dest, lhs, rhs);
 }
}

void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
 as_fdiv_s(dest, dest, src);
}

void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
 as_fdiv_d(dest, dest, src);
}

void MacroAssembler::neg64(Register64 reg) { as_sub_d(reg.reg, zero, reg.reg); }

void MacroAssembler::negPtr(Register reg) { as_sub_d(reg, zero, reg); }

void MacroAssembler::neg32(Register reg) { as_sub_w(reg, zero, reg); }

void MacroAssembler::negateDouble(FloatRegister reg) { as_fneg_d(reg, reg); }

void MacroAssembler::negateFloat(FloatRegister reg) { as_fneg_s(reg, reg); }

void MacroAssembler::abs32(Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 as_srai_w(scratch, src, 31);
 as_xor(dest, src, scratch);
 as_sub_w(dest, dest, scratch);
}

void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
 as_fabs_s(dest, src);
}

void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
 as_fabs_d(dest, src);
}

void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
 as_fsqrt_s(dest, src);
}

void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
 as_fsqrt_d(dest, src);
}

void MacroAssembler::min32(Register lhs, Register rhs, Register dest) {
 minMaxPtr(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::min32(Register lhs, Imm32 rhs, Register dest) {
 minMaxPtr(lhs, ImmWord(rhs.value), dest, /* isMax = */ false);
}

void MacroAssembler::max32(Register lhs, Register rhs, Register dest) {
 minMaxPtr(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::max32(Register lhs, Imm32 rhs, Register dest) {
 minMaxPtr(lhs, ImmWord(rhs.value), dest, /* isMax = */ true);
}

void MacroAssembler::minPtr(Register lhs, Register rhs, Register dest) {
 minMaxPtr(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::minPtr(Register lhs, ImmWord rhs, Register dest) {
 minMaxPtr(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::maxPtr(Register lhs, Register rhs, Register dest) {
 minMaxPtr(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::maxPtr(Register lhs, ImmWord rhs, Register dest) {
 minMaxPtr(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
                               bool handleNaN) {
 minMaxFloat32(srcDest, other, handleNaN, false);
}

void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
                              bool handleNaN) {
 minMaxDouble(srcDest, other, handleNaN, false);
}

void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
                               bool handleNaN) {
 minMaxFloat32(srcDest, other, handleNaN, true);
}

void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
                              bool handleNaN) {
 minMaxDouble(srcDest, other, handleNaN, true);
}

// ===============================================================
// Shift functions

void MacroAssembler::lshift32(Register src, Register dest) {
 as_sll_w(dest, dest, src);
}

void MacroAssembler::lshift32(Imm32 imm, Register dest) {
 lshift32(imm, dest, dest);
}

void MacroAssembler::lshift32(Imm32 imm, Register src, Register dest) {
 as_slli_w(dest, src, imm.value & 0x1f);
}

void MacroAssembler::flexibleLshift32(Register src, Register dest) {
 lshift32(src, dest);
}

void MacroAssembler::lshift64(Register shift, Register64 dest) {
 as_sll_d(dest.reg, dest.reg, shift);
}

void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
 as_slli_d(dest.reg, dest.reg, imm.value);
}

void MacroAssembler::lshiftPtr(Register shift, Register dest) {
 as_sll_d(dest, dest, shift);
}

void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
 lshiftPtr(imm, dest, dest);
}

void MacroAssembler::lshiftPtr(Imm32 imm, Register src, Register dest) {
 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
 as_slli_d(dest, src, imm.value);
}

void MacroAssembler::flexibleLshiftPtr(Register shift, Register srcDest) {
 lshiftPtr(shift, srcDest);
}

void MacroAssembler::rshift32(Register src, Register dest) {
 as_srl_w(dest, dest, src);
}

void MacroAssembler::rshift32(Imm32 imm, Register dest) {
 rshift32(imm, dest, dest);
}

void MacroAssembler::rshift32(Imm32 imm, Register src, Register dest) {
 as_srli_w(dest, src, imm.value & 0x1f);
}

void MacroAssembler::flexibleRshift32(Register src, Register dest) {
 rshift32(src, dest);
}

void MacroAssembler::rshift32Arithmetic(Register src, Register dest) {
 as_sra_w(dest, dest, src);
}

void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
 rshift32Arithmetic(imm, dest, dest);
}

void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register src,
                                       Register dest) {
 as_srai_w(dest, src, imm.value & 0x1f);
}

void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
 rshift32Arithmetic(src, dest);
}

void MacroAssembler::rshift64(Register shift, Register64 dest) {
 as_srl_d(dest.reg, dest.reg, shift);
}

void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
 as_srli_d(dest.reg, dest.reg, imm.value);
}

void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
 as_srai_d(dest.reg, dest.reg, imm.value);
}

void MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest) {
 as_sra_d(dest.reg, dest.reg, shift);
}

void MacroAssembler::rshiftPtr(Register shift, Register dest) {
 as_srl_d(dest, dest, shift);
}

void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
 rshiftPtr(imm, dest, dest);
}

void MacroAssembler::rshiftPtr(Imm32 imm, Register src, Register dest) {
 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
 as_srli_d(dest, src, imm.value);
}

void MacroAssembler::flexibleRshiftPtr(Register shift, Register srcDest) {
 rshiftPtr(shift, srcDest);
}

void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
 rshiftPtrArithmetic(imm, dest, dest);
}

void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register src,
                                        Register dest) {
 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
 as_srai_d(dest, src, imm.value);
}

void MacroAssembler::rshiftPtrArithmetic(Register shift, Register dest) {
 as_sra_d(dest, dest, shift);
}

void MacroAssembler::flexibleRshiftPtrArithmetic(Register shift,
                                                Register srcDest) {
 rshiftPtrArithmetic(shift, srcDest);
}

// ===============================================================
// Rotation functions

void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 as_sub_w(scratch, zero, count);
 as_rotr_w(dest, input, scratch);
}

void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
 as_rotri_w(dest, input, (32 - count.value) & 31);
}

void MacroAssembler::rotateLeft64(Register count, Register64 src,
                                 Register64 dest, Register temp) {
 MOZ_ASSERT(temp == Register::Invalid());
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 as_sub_d(scratch, zero, count);
 as_rotr_d(dest.reg, src.reg, scratch);
}

void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
                                 Register temp) {
 MOZ_ASSERT(temp == Register::Invalid());
 as_rotri_d(dest.reg, src.reg, (64 - count.value) & 63);
}

void MacroAssembler::rotateRight(Register count, Register input,
                                Register dest) {
 as_rotr_w(dest, input, count);
}

void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
 as_rotri_w(dest, input, count.value & 31);
}

void MacroAssembler::rotateRight64(Register count, Register64 src,
                                  Register64 dest, Register temp) {
 MOZ_ASSERT(temp == Register::Invalid());
 as_rotr_d(dest.reg, src.reg, count);
}

void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
                                  Register temp) {
 MOZ_ASSERT(temp == Register::Invalid());
 as_rotri_d(dest.reg, src.reg, count.value & 63);
}

// Bit counting functions

void MacroAssembler::clz64(Register64 src, Register64 dest) {
 as_clz_d(dest.reg, src.reg);
}

void MacroAssembler::ctz64(Register64 src, Register64 dest) {
 as_ctz_d(dest.reg, src.reg);
}

void MacroAssembler::popcnt64(Register64 input, Register64 output,
                             Register tmp) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 as_or(output.reg, input.reg, zero);
 as_srai_d(tmp, input.reg, 1);
 ma_li(scratch, Imm32(0x55555555));
 as_bstrins_d(scratch, scratch, 63, 32);
 as_and(tmp, tmp, scratch);
 as_sub_d(output.reg, output.reg, tmp);
 as_srai_d(tmp, output.reg, 2);
 ma_li(scratch, Imm32(0x33333333));
 as_bstrins_d(scratch, scratch, 63, 32);
 as_and(output.reg, output.reg, scratch);
 as_and(tmp, tmp, scratch);
 as_add_d(output.reg, output.reg, tmp);
 as_srli_d(tmp, output.reg, 4);
 as_add_d(output.reg, output.reg, tmp);
 ma_li(scratch, Imm32(0xF0F0F0F));
 as_bstrins_d(scratch, scratch, 63, 32);
 as_and(output.reg, output.reg, scratch);
 ma_li(tmp, Imm32(0x1010101));
 as_bstrins_d(tmp, tmp, 63, 32);
 as_mul_d(output.reg, output.reg, tmp);
 as_srai_d(output.reg, output.reg, 56);
}

void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
 as_clz_w(dest, src);
}

void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
 as_ctz_w(dest, src);
}

void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
 // Equivalent to GCC output of mozilla::CountPopulation32()
 as_or(output, input, zero);
 as_srai_w(tmp, input, 1);
 ma_and(tmp, tmp, Imm32(0x55555555));
 as_sub_w(output, output, tmp);
 as_srai_w(tmp, output, 2);
 ma_and(output, output, Imm32(0x33333333));
 ma_and(tmp, tmp, Imm32(0x33333333));
 as_add_w(output, output, tmp);
 as_srli_w(tmp, output, 4);
 as_add_w(output, output, tmp);
 ma_and(output, output, Imm32(0xF0F0F0F));
 as_slli_w(tmp, output, 8);
 as_add_w(output, output, tmp);
 as_slli_w(tmp, output, 16);
 as_add_w(output, output, tmp);
 as_srai_w(output, output, 24);
}

// ===============================================================
// Condition functions

void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
                            Register dest) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(scratch != lhs.base);

 switch (cond) {
   case Assembler::Equal:
   case Assembler::NotEqual:
   case Assembler::Above:
   case Assembler::AboveOrEqual:
   case Assembler::Below:
   case Assembler::BelowOrEqual:
     load8ZeroExtend(lhs, scratch);
     ma_cmp_set(dest, scratch, Imm32(uint8_t(rhs.value)), cond);
     break;

   case Assembler::GreaterThan:
   case Assembler::GreaterThanOrEqual:
   case Assembler::LessThan:
   case Assembler::LessThanOrEqual:
     load8SignExtend(lhs, scratch);
     ma_cmp_set(dest, scratch, Imm32(int8_t(rhs.value)), cond);
     break;

   default:
     MOZ_CRASH("unexpected condition");
 }
}

void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
                             Register dest) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(scratch != lhs.base);

 switch (cond) {
   case Assembler::Equal:
   case Assembler::NotEqual:
   case Assembler::Above:
   case Assembler::AboveOrEqual:
   case Assembler::Below:
   case Assembler::BelowOrEqual:
     load16ZeroExtend(lhs, scratch);
     ma_cmp_set(dest, scratch, Imm32(uint16_t(rhs.value)), cond);
     break;

   case Assembler::GreaterThan:
   case Assembler::GreaterThanOrEqual:
   case Assembler::LessThan:
   case Assembler::LessThanOrEqual:
     load16SignExtend(lhs, scratch);
     ma_cmp_set(dest, scratch, Imm32(int16_t(rhs.value)), cond);
     break;

   default:
     MOZ_CRASH("unexpected condition");
 }
}

// Also see below for specializations of cmpPtrSet.
template <typename T1, typename T2>
void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
 ma_cmp_set(dest, lhs, rhs, cond);
}

void MacroAssembler::cmp64Set(Condition cond, Register64 lhs, Register64 rhs,
                             Register dest) {
 ma_cmp_set(dest, lhs.reg, rhs.reg, cond);
}

void MacroAssembler::cmp64Set(Condition cond, Register64 lhs, Imm64 rhs,
                             Register dest) {
 ma_cmp_set(dest, lhs.reg, ImmWord(uint64_t(rhs.value)), cond);
}

void MacroAssembler::cmp64Set(Condition cond, Address lhs, Register64 rhs,
                             Register dest) {
 ma_cmp_set(dest, lhs, rhs.reg, cond);
}

void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
                             Register dest) {
 ma_cmp_set(dest, lhs, ImmWord(uint64_t(rhs.value)), cond);
}

template <typename T1, typename T2>
void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
 ma_cmp_set(dest, lhs, rhs, cond);
}

// ===============================================================
// Branch functions

void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
                            Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(scratch != lhs.base);

 switch (cond) {
   case Assembler::Equal:
   case Assembler::NotEqual:
   case Assembler::Above:
   case Assembler::AboveOrEqual:
   case Assembler::Below:
   case Assembler::BelowOrEqual:
     load8ZeroExtend(lhs, scratch);
     branch32(cond, scratch, Imm32(uint8_t(rhs.value)), label);
     break;

   case Assembler::GreaterThan:
   case Assembler::GreaterThanOrEqual:
   case Assembler::LessThan:
   case Assembler::LessThanOrEqual:
     load8SignExtend(lhs, scratch);
     branch32(cond, scratch, Imm32(int8_t(rhs.value)), label);
     break;

   default:
     MOZ_CRASH("unexpected condition");
 }
}

void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
                            Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(scratch != lhs.base);

 computeScaledAddress(lhs, scratch);

 switch (cond) {
   case Assembler::Equal:
   case Assembler::NotEqual:
   case Assembler::Above:
   case Assembler::AboveOrEqual:
   case Assembler::Below:
   case Assembler::BelowOrEqual:
     load8ZeroExtend(Address(scratch, lhs.offset), scratch);
     branch32(cond, scratch, rhs, label);
     break;

   case Assembler::GreaterThan:
   case Assembler::GreaterThanOrEqual:
   case Assembler::LessThan:
   case Assembler::LessThanOrEqual:
     load8SignExtend(Address(scratch, lhs.offset), scratch);
     branch32(cond, scratch, rhs, label);
     break;

   default:
     MOZ_CRASH("unexpected condition");
 }
}

void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
                             Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(scratch != lhs.base);

 switch (cond) {
   case Assembler::Equal:
   case Assembler::NotEqual:
   case Assembler::Above:
   case Assembler::AboveOrEqual:
   case Assembler::Below:
   case Assembler::BelowOrEqual:
     load16ZeroExtend(lhs, scratch);
     branch32(cond, scratch, Imm32(uint16_t(rhs.value)), label);
     break;

   case Assembler::GreaterThan:
   case Assembler::GreaterThanOrEqual:
   case Assembler::LessThan:
   case Assembler::LessThanOrEqual:
     load16SignExtend(lhs, scratch);
     branch32(cond, scratch, Imm32(int16_t(rhs.value)), label);
     break;

   default:
     MOZ_CRASH("unexpected condition");
 }
}

void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
                             Label* label) {
 ma_b(lhs, rhs, label, cond);
}

void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm,
                             Label* label) {
 ma_b(lhs, imm, label, cond);
}

void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
                             Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 ma_b(scratch, rhs, label, cond);
}

void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
                             Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 ma_b(scratch, rhs, label, cond);
}

void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
                             Register rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 ma_b(scratch, rhs, label, cond);
}

void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
                             Imm32 rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 ma_b(scratch, rhs, label, cond);
}

void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
                             Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 ma_b(scratch, rhs, label, cond);
}

void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr,
                             Imm32 imm, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(addr, scratch);
 ma_b(scratch, imm, label, cond);
}

void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
                             Label* success, Label* fail) {
 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                cond == Assembler::LessThan ||
                cond == Assembler::LessThanOrEqual ||
                cond == Assembler::GreaterThan ||
                cond == Assembler::GreaterThanOrEqual ||
                cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                cond == Assembler::Above || cond == Assembler::AboveOrEqual,
            "other condition codes not supported");

 branchPtr(cond, lhs.reg, ImmWord(val.value), success);
 if (fail) {
   jump(fail);
 }
}

void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
                             Label* success, Label* fail) {
 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                cond == Assembler::LessThan ||
                cond == Assembler::LessThanOrEqual ||
                cond == Assembler::GreaterThan ||
                cond == Assembler::GreaterThanOrEqual ||
                cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                cond == Assembler::Above || cond == Assembler::AboveOrEqual,
            "other condition codes not supported");

 branchPtr(cond, lhs.reg, rhs.reg, success);
 if (fail) {
   jump(fail);
 }
}

void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
                             Label* success, Label* fail) {
 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                cond == Assembler::LessThan ||
                cond == Assembler::LessThanOrEqual ||
                cond == Assembler::GreaterThan ||
                cond == Assembler::GreaterThanOrEqual ||
                cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                cond == Assembler::Above || cond == Assembler::AboveOrEqual,
            "other condition codes not supported");

 branchPtr(cond, lhs, ImmWord(val.value), success);
 if (fail) {
   jump(fail);
 }
}

void MacroAssembler::branch64(Condition cond, const Address& lhs,
                             Register64 rhs, Label* success, Label* fail) {
 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                cond == Assembler::LessThan ||
                cond == Assembler::LessThanOrEqual ||
                cond == Assembler::GreaterThan ||
                cond == Assembler::GreaterThanOrEqual ||
                cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                cond == Assembler::Above || cond == Assembler::AboveOrEqual,
            "other condition codes not supported");

 branchPtr(cond, lhs, rhs.reg, success);
 if (fail) {
   jump(fail);
 }
}

void MacroAssembler::branch64(Condition cond, const Address& lhs,
                             const Address& rhs, Register scratch,
                             Label* label) {
 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
            "other condition codes not supported");
 MOZ_ASSERT(lhs.base != scratch);
 MOZ_ASSERT(rhs.base != scratch);

 loadPtr(rhs, scratch);
 branchPtr(cond, lhs, scratch, label);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
                              Label* label) {
 ma_b(lhs, rhs, label, cond);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
                              Label* label) {
 ma_b(lhs, rhs, label, cond);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
                              Label* label) {
 ma_b(lhs, rhs, label, cond);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
                              Label* label) {
 ma_b(lhs, rhs, label, cond);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
                              Label* label) {
 ma_b(lhs, rhs, label, cond);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
                              Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
                              Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
                              Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
                              Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
                              Register rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
                              ImmWord rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
                              Register rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
                              Register rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
                              ImmWord rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
                                     Register rhs, Label* label) {
 branchPtr(cond, lhs, rhs, label);
}

void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
                                FloatRegister rhs, Label* label) {
 ma_bc_s(lhs, rhs, label, cond);
}

void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
                                                        Register dest,
                                                        Label* fail) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ScratchDoubleScope fpscratch(asMasm());
 as_ftintrz_l_s(fpscratch, src);
 as_movfcsr2gr(scratch);
 moveFromDouble(fpscratch, dest);
 MOZ_ASSERT(Assembler::CauseV < 32);
 as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
 ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);

 as_slli_w(dest, dest, 0);
}

void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
                                                 Register dest, Label* fail) {
 convertFloat32ToInt32(src, dest, fail, false);
}

void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
                                 FloatRegister rhs, Label* label) {
 ma_bc_d(lhs, rhs, label, cond);
}

void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
                                                       Register dest,
                                                       Label* fail) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ScratchDoubleScope fpscratch(asMasm());
 as_ftintrz_l_d(fpscratch, src);
 as_movfcsr2gr(scratch);
 moveFromDouble(fpscratch, dest);
 MOZ_ASSERT(Assembler::CauseV < 32);
 as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
 ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);

 as_slli_w(dest, dest, 0);
}

void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
                                                Register dest, Label* fail) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ScratchDoubleScope fpscratch(asMasm());

 // Convert scalar to signed 64-bit fixed-point, rounding toward zero.
 // In the case of overflow, the output is saturated.
 // In the case of NaN and -0, the output is zero.
 as_ftintrz_l_d(fpscratch, src);
 moveFromDouble(fpscratch, dest);

 // Fail on overflow cases.
 as_slli_w(scratch, dest, 0);
 ma_b(dest, scratch, fail, Assembler::NotEqual);
}

void MacroAssembler::branchInt64NotInPtrRange(Register64 src, Label* label) {
 // No-op on 64-bit platforms.
}

void MacroAssembler::branchUInt64NotInPtrRange(Register64 src, Label* label) {
 branchTest64(Assembler::Signed, src, src, label);
}

template <typename T>
void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
                                Label* overflow) {
 switch (cond) {
   case Overflow:
     ma_add32TestOverflow(dest, dest, src, overflow);
     break;
   case CarryClear:
   case CarrySet:
     ma_add32TestCarry(cond, dest, dest, src, overflow);
     break;
   default:
     MOZ_CRASH("NYI");
 }
}

// the type of 'T src' maybe a Register, maybe a Imm32,depends on who call it.
template <typename T>
void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
                                Label* overflow) {
 switch (cond) {
   case Overflow:
     ma_sub32TestOverflow(dest, dest, src, overflow);
     break;
   case NonZero:
   case Zero:
   case Signed:
   case NotSigned:
     ma_sub_w(dest, dest, src);
     ma_b(dest, dest, overflow, cond);
     break;
   default:
     MOZ_CRASH("NYI");
 }
}

template <typename T>
void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
                                Label* overflow) {
 MOZ_ASSERT(cond == Assembler::Overflow);
 ma_mul32TestOverflow(dest, dest, src, overflow);
}

template <typename T>
void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
                                   Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero);
 rshift32(src, dest);
 branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
}

void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
 MOZ_ASSERT(cond == Overflow);
 neg32(reg);
 branch32(Assembler::Equal, reg, Imm32(INT32_MIN), label);
}

template <typename T>
void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
                                 Label* label) {
 switch (cond) {
   case Overflow:
     ma_addPtrTestOverflow(dest, dest, src, label);
     break;
   case CarryClear:
   case CarrySet:
     ma_addPtrTestCarry(cond, dest, dest, src, label);
     break;
   case Signed:
   case NotSigned:
     ma_addPtrTestSigned(cond, dest, dest, src, label);
     break;
   default:
     MOZ_CRASH("NYI");
 }
}

template <typename T>
void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
                                 Label* label) {
 switch (cond) {
   case Overflow:
     ma_subPtrTestOverflow(dest, dest, src, label);
     break;
   case NonZero:
   case Zero:
   case Signed:
   case NotSigned:
     subPtr(src, dest);
     ma_b(dest, dest, label, cond);
     break;
   default:
     MOZ_CRASH("NYI");
 }
}

void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
                                 Label* label) {
 MOZ_ASSERT(cond == Assembler::Overflow);
 ma_mulPtrTestOverflow(dest, dest, src, label);
}

void MacroAssembler::branchNegPtr(Condition cond, Register reg, Label* label) {
 MOZ_ASSERT(cond == Overflow);
 negPtr(reg);
 branchPtr(Assembler::Equal, reg, ImmWord(INTPTR_MIN), label);
}

void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
                                 Label* label) {
 subPtr(rhs, lhs);
 branchPtr(cond, lhs, Imm32(0), label);
}

void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
                                 Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 if (lhs == rhs) {
   ma_b(lhs, rhs, label, cond);
 } else {
   UseScratchRegisterScope temps(asMasm());
   Register scratch = temps.Acquire();
   as_and(scratch, lhs, rhs);
   ma_b(scratch, scratch, label, cond);
 }
}

void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
                                 Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ma_and(scratch, lhs, rhs);
 ma_b(scratch, scratch, label, cond);
}

void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
                                 Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 and32(rhs, scratch);
 ma_b(scratch, scratch, label, cond);
}

void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
                                 Imm32 rhs, Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 load32(lhs, scratch);
 and32(rhs, scratch);
 ma_b(scratch, scratch, label, cond);
}

void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
                                  Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 if (lhs == rhs) {
   ma_b(lhs, rhs, label, cond);
 } else {
   UseScratchRegisterScope temps(asMasm());
   Register scratch = temps.Acquire();
   as_and(scratch, lhs, rhs);
   ma_b(scratch, scratch, label, cond);
 }
}

void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
                                  Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ma_and(scratch, lhs, rhs);
 ma_b(scratch, scratch, label, cond);
}

void MacroAssembler::branchTestPtr(Condition cond, Register lhs, ImmWord rhs,
                                  Label* label) {
 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
            cond == NotSigned);
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 ma_li(scratch, rhs);
 as_and(scratch, lhs, scratch);
 ma_b(scratch, scratch, label, cond);
}

void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
                                  Imm32 rhs, Label* label) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 branchTestPtr(cond, scratch, rhs, label);
}

void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
                                 Register64 rhs, Register temp, Label* success,
                                 Label* fail) {
 branchTestPtr(cond, lhs.reg, rhs.reg, success);
 if (fail) {
   jump(fail);
 }
}

void MacroAssembler::branchTest64(Condition cond, Register64 lhs, Imm64 rhs,
                                 Label* success, Label* fail) {
 branchTestPtr(cond, lhs.reg, ImmWord(rhs.value), success);
 if (fail) {
   jump(fail);
 }
}

void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
                                        Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_UNDEFINED), label, cond);
}

void MacroAssembler::branchTestUndefined(Condition cond,
                                        const ValueOperand& value,
                                        Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestUndefined(cond, scratch, label);
}

void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
                                        Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestUndefined(cond, tag, label);
}

void MacroAssembler::branchTestUndefined(Condition cond,
                                        const BaseIndex& address,
                                        Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestUndefined(cond, tag, label);
}

void MacroAssembler::branchTestInt32(Condition cond, Register tag,
                                    Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_INT32), label, cond);
}

void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
                                    Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestInt32(cond, scratch, label);
}

void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
                                    Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestInt32(cond, tag, label);
}

void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
                                    Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestInt32(cond, tag, label);
}

void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value,
                                          Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 as_bstrpick_d(scratch, value.valueReg(), 31, 0);
 ma_b(scratch, scratch, label, b ? NonZero : Zero);
}

void MacroAssembler::branchTestDouble(Condition cond, Register tag,
                                     Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 Condition actual = (cond == Equal) ? BelowOrEqual : Above;
 ma_b(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE), label, actual);
}

void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestDouble(cond, scratch, label);
}

void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestDouble(cond, tag, label);
}

void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestDouble(cond, tag, label);
}

void MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value,
                                           Label* label) {
 ScratchDoubleScope fpscratch(*this);
 ma_lid(fpscratch, 0.0);
 DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered;
 ma_bc_d(value, fpscratch, label, cond);
}

void MacroAssembler::branchTestNumber(Condition cond, Register tag,
                                     Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 Condition actual = cond == Equal ? BelowOrEqual : Above;
 ma_b(tag, ImmTag(JS::detail::ValueUpperInclNumberTag), label, actual);
}

void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestNumber(cond, scratch, label);
}

void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
                                      Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_BOOLEAN), label, cond);
}

void MacroAssembler::branchTestBoolean(Condition cond,
                                      const ValueOperand& value,
                                      Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestBoolean(cond, scratch, label);
}

void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
                                      Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestBoolean(cond, tag, label);
}

void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
                                      Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestBoolean(cond, tag, label);
}

void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value,
                                            Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 unboxBoolean(value, scratch);
 ma_b(scratch, scratch, label, b ? NonZero : Zero);
}

void MacroAssembler::branchTestString(Condition cond, Register tag,
                                     Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_STRING), label, cond);
}

void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestString(cond, scratch, label);
}

void MacroAssembler::branchTestString(Condition cond, const Address& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestString(cond, tag, label);
}

void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestString(cond, tag, label);
}

void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value,
                                           Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 unboxString(value, scratch);
 load32(Address(scratch, JSString::offsetOfLength()), scratch);
 ma_b(scratch, Imm32(0), label, b ? NotEqual : Equal);
}

void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
                                     Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_SYMBOL), label, cond);
}

void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestSymbol(cond, scratch, label);
}

void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestSymbol(cond, tag, label);
}

void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestSymbol(cond, tag, label);
}

void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
                                     Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_BIGINT), label, cond);
}

void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestBigInt(cond, scratch, label);
}

void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestBigInt(cond, tag, label);
}

void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 computeEffectiveAddress(address, scratch);
 splitTag(scratch, scratch);
 branchTestBigInt(cond, scratch, label);
}

void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value,
                                           Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 unboxBigInt(value, scratch);
 load32(Address(scratch, BigInt::offsetOfDigitLength()), scratch);
 ma_b(scratch, Imm32(0), label, b ? NotEqual : Equal);
}

void MacroAssembler::branchTestNull(Condition cond, Register tag,
                                   Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_NULL), label, cond);
}

void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
                                   Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestNull(cond, scratch, label);
}

void MacroAssembler::branchTestNull(Condition cond, const Address& address,
                                   Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestNull(cond, tag, label);
}

void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
                                   Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestNull(cond, tag, label);
}

void MacroAssembler::branchTestObject(Condition cond, Register tag,
                                     Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_OBJECT), label, cond);
}

void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestObject(cond, scratch, label);
}

void MacroAssembler::branchTestObject(Condition cond, const Address& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestObject(cond, tag, label);
}

void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
                                     Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestObject(cond, tag, label);
}

void MacroAssembler::branchTestPrimitive(Condition cond,
                                        const ValueOperand& value,
                                        Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 branchTestPrimitive(cond, scratch, label);
}

void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
                                      Label* label) {
 branchTestGCThingImpl(cond, address, label);
}

void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
                                      Label* label) {
 branchTestGCThingImpl(cond, address, label);
}

void MacroAssembler::branchTestGCThing(Condition cond,
                                      const ValueOperand& address,
                                      Label* label) {
 branchTestGCThingImpl(cond, address, label);
}

template <typename T>
void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& address,
                                          Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 ma_b(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag), label,
      (cond == Equal) ? AboveOrEqual : Below);
}

void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
                                        Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag), label,
      (cond == Equal) ? Below : AboveOrEqual);
}

void MacroAssembler::branchTestMagic(Condition cond, Register tag,
                                    Label* label) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 ma_b(tag, ImmTag(JSVAL_TAG_MAGIC), label, cond);
}

void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
                                    Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestMagic(cond, tag, label);
}

void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
                                    Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(address, scratch);
 branchTestMagic(cond, tag, label);
}

void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
                                    Label* label) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 splitTag(value, scratch);
 ma_b(scratch, ImmTag(JSVAL_TAG_MAGIC), label, cond);
}

void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
                                    JSWhyMagic why, Label* label) {
 uint64_t magic = MagicValue(why).asRawBits();
 UseScratchRegisterScope temp(*this);
 Register scratch = temp.Acquire();
 loadPtr(valaddr, scratch);
 ma_b(scratch, ImmWord(magic), label, cond);
}

template <typename T>
void MacroAssembler::branchTestValue(Condition cond, const T& lhs,
                                    const ValueOperand& rhs, Label* label) {
 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
 branchPtr(cond, lhs, rhs.valueReg(), label);
}

template <typename T>
void MacroAssembler::testNumberSet(Condition cond, const T& src,
                                  Register dest) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(src, scratch);
 ma_cmp_set(dest, tag, ImmTag(JS::detail::ValueUpperInclNumberTag),
            cond == Equal ? BelowOrEqual : Above);
}

template <typename T>
void MacroAssembler::testBooleanSet(Condition cond, const T& src,
                                   Register dest) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(src, scratch);
 ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_BOOLEAN), cond);
}

template <typename T>
void MacroAssembler::testStringSet(Condition cond, const T& src,
                                  Register dest) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(src, scratch);
 ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_STRING), cond);
}

template <typename T>
void MacroAssembler::testSymbolSet(Condition cond, const T& src,
                                  Register dest) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(src, scratch);
 ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_SYMBOL), cond);
}

template <typename T>
void MacroAssembler::testBigIntSet(Condition cond, const T& src,
                                  Register dest) {
 MOZ_ASSERT(cond == Equal || cond == NotEqual);
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 Register tag = extractTag(src, scratch);
 ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_BIGINT), cond);
}

void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(addr, scratch);
 branch(scratch);
}

void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Imm32 rhs,
                                Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 cmp32Set(cond, lhs, rhs, scratch);
 moveIfNotZero(dest, src, scratch);
}

void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
                                Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 cmp32Set(cond, lhs, rhs, scratch);
 moveIfNotZero(dest, src, scratch);
}

void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
                                const Address& rhs, Register src,
                                Register dest) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(lhs != scratch && src != scratch && dest != scratch);
 load32(rhs, scratch);
 cmp32Move32(cond, lhs, scratch, src, dest);
}

void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
                                 Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 cmp32Set(cond, lhs, rhs, scratch);
 moveIfNotZero(dest, src, scratch);
}

void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Imm32 rhs,
                                  Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 cmpPtrSet(cond, lhs, rhs, scratch);
 moveIfNotZero(dest, src, scratch);
}

void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
                                  Register src, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 cmpPtrSet(cond, lhs, rhs, scratch);
 moveIfNotZero(dest, src, scratch);
}

void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
                                  const Address& rhs, Register src,
                                  Register dest) {
 MOZ_CRASH("NYI");
}

void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
                                const Address& rhs, const Address& src,
                                Register dest) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 MOZ_ASSERT(lhs != scratch && dest != scratch);
 load32(rhs, scratch);
 cmp32Load32(cond, lhs, scratch, src, dest);
}

void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
                                const Address& src, Register dest) {
 Label skip;
 branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
 load32(src, dest);
 bind(&skip);
}

void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Imm32 rhs,
                                const Address& src, Register dest) {
 Label skip;
 branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
 load32(src, dest);
 bind(&skip);
}

void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
                                 const Address& src, Register dest) {
 Label skip;
 branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
 loadPtr(src, dest);
 bind(&skip);
}

void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
                                  Imm32 mask, const Address& src,
                                  Register dest) {
 MOZ_RELEASE_ASSERT(!JitOptions.spectreStringMitigations);
 Label skip;
 branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip);
 loadPtr(src, dest);
 bind(&skip);
}

void MacroAssembler::test32MovePtr(Condition cond, Register operand, Imm32 mask,
                                  Register src, Register dest) {
 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
 Label skip;
 branchTest32(Assembler::InvertCondition(cond), operand, mask, &skip);
 movePtr(src, dest);
 bind(&skip);
}

void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
                                  Imm32 mask, Register src, Register dest) {
 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
 Label skip;
 branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip);
 movePtr(src, dest);
 bind(&skip);
}

void MacroAssembler::spectreMovePtr(Condition cond, Register src,
                                   Register dest) {
 MOZ_CRASH();
}

void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
                                        Register dest) {
 MOZ_CRASH();
}

void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
                                         Register maybeScratch,
                                         Label* failure) {
 MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
 branch32(Assembler::BelowOrEqual, length, index, failure);
}

void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
                                         Register maybeScratch,
                                         Label* failure) {
 MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
 branch32(Assembler::BelowOrEqual, length, index, failure);
}

void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
                                          Register maybeScratch,
                                          Label* failure) {
 MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
 branchPtr(Assembler::BelowOrEqual, length, index, failure);
}

void MacroAssembler::spectreBoundsCheckPtr(Register index,
                                          const Address& length,
                                          Register maybeScratch,
                                          Label* failure) {
 MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
 branchPtr(Assembler::BelowOrEqual, length, index, failure);
}

// ========================================================================
// Memory access primitives.

FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
                                               const Address& addr) {
 return ma_fst_s(src, addr);
}
FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
                                               const BaseIndex& addr) {
 return ma_fst_s(src, addr);
}

FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
                                              const Address& addr) {
 return ma_fst_d(src, addr);
}
FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
                                              const BaseIndex& addr) {
 return ma_fst_d(src, addr);
}

FaultingCodeOffset MacroAssembler::storeFloat16(FloatRegister src,
                                               const Address& dest, Register) {
 MOZ_CRASH("Not supported for this target");
}
FaultingCodeOffset MacroAssembler::storeFloat16(FloatRegister src,
                                               const BaseIndex& dest,
                                               Register) {
 MOZ_CRASH("Not supported for this target");
}

void MacroAssembler::memoryBarrier(MemoryBarrier barrier) {
 if (!barrier.isNone()) {
   as_dbar(0);
 }
}

// ===============================================================
// Clamping functions.

void MacroAssembler::clampIntToUint8(Register reg) {
 UseScratchRegisterScope temps(*this);
 Register scratch = temps.Acquire();
 // If reg is < 0, then we want to clamp to 0.
 as_slti(scratch, reg, 0);
 as_masknez(reg, reg, scratch);

 // If reg is >= 255, then we want to clamp to 255.
 as_addi_d(reg, reg, -255);
 as_slt(scratch, reg, zero);
 as_maskeqz(reg, reg, scratch);
 as_addi_d(reg, reg, 255);
}

void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
                                     JSValueType type, Label* fail) {
 MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
            type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
 // dest := src XOR mask
 // scratch := dest >> JSVAL_TAG_SHIFT
 // fail if scratch != 0
 //
 // Note: src and dest can be the same register
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(src.valueReg() != scratch);
 mov(ImmShiftedTag(type), scratch);
 as_xor(dest, src.valueReg(), scratch);
 as_srli_d(scratch, dest, JSVAL_TAG_SHIFT);
 ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);
}

void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
                                     JSValueType type, Label* fail) {
 loadValue(src, ValueOperand(dest));
 fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
}

void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
                                     JSValueType type, Label* fail) {
 loadValue(src, ValueOperand(dest));
 fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
}

//}}} check_macroassembler_style
// ===============================================================

// The specializations for cmpPtrSet are outside the braces because
// check_macroassembler_style can't yet deal with specializations.

template <>
inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
                                     ImmPtr rhs, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 loadPtr(lhs, scratch);
 cmpPtrSet(cond, Register(scratch), rhs, dest);
}

template <>
inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs,
                                     Address rhs, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(lhs != scratch);
 loadPtr(rhs, scratch);
 cmpPtrSet(cond, lhs, Register(scratch), dest);
}

template <>
inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
                                     Register rhs, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(rhs != scratch);
 loadPtr(lhs, scratch);
 cmpPtrSet(cond, Register(scratch), rhs, dest);
}

template <>
inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs,
                                    Address rhs, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(lhs != scratch);
 load32(rhs, scratch);
 cmp32Set(cond, lhs, Register(scratch), dest);
}

template <>
inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Address lhs,
                                    Register rhs, Register dest) {
 UseScratchRegisterScope temps(asMasm());
 Register scratch = temps.Acquire();
 MOZ_ASSERT(rhs != scratch);
 load32(lhs, scratch);
 cmp32Set(cond, Register(scratch), rhs, dest);
}

void MacroAssemblerLOONG64Compat::incrementInt32Value(const Address& addr) {
 asMasm().add32(Imm32(1), addr);
}

void MacroAssemblerLOONG64Compat::retn(Imm32 n) {
 // pc <- [sp]; sp += n
 loadPtr(Address(StackPointer, 0), ra);
 asMasm().addPtr(n, StackPointer);
 as_jirl(zero, ra, BOffImm16(0));
}

}  // namespace jit
}  // namespace js

#endif /* jit_loong64_MacroAssembler_loong64_inl_h */