tor-browser

MacroAssembler.h (278127B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_MacroAssembler_h
#define jit_MacroAssembler_h

#include "mozilla/MacroForEach.h"
#include "mozilla/Maybe.h"
#include "mozilla/Variant.h"

#if defined(JS_CODEGEN_X86)
#  include "jit/x86/MacroAssembler-x86.h"
#elif defined(JS_CODEGEN_X64)
#  include "jit/x64/MacroAssembler-x64.h"
#elif defined(JS_CODEGEN_ARM)
#  include "jit/arm/MacroAssembler-arm.h"
#elif defined(JS_CODEGEN_ARM64)
#  include "jit/arm64/MacroAssembler-arm64.h"
#elif defined(JS_CODEGEN_MIPS64)
#  include "jit/mips64/MacroAssembler-mips64.h"
#elif defined(JS_CODEGEN_LOONG64)
#  include "jit/loong64/MacroAssembler-loong64.h"
#elif defined(JS_CODEGEN_RISCV64)
#  include "jit/riscv64/MacroAssembler-riscv64.h"
#elif defined(JS_CODEGEN_WASM32)
#  include "jit/wasm32/MacroAssembler-wasm32.h"
#elif defined(JS_CODEGEN_NONE)
#  include "jit/none/MacroAssembler-none.h"
#else
#  error "Unknown architecture!"
#endif
#include "jit/ABIArgGenerator.h"
#include "jit/ABIFunctions.h"
#include "jit/AtomicOp.h"
#include "jit/IonTypes.h"
#include "jit/MoveResolver.h"
#include "jit/VMFunctions.h"
#include "js/ScalarType.h"  // js::Scalar::Type
#include "util/Memory.h"
#include "vm/FunctionFlags.h"
#include "vm/Opcodes.h"
#include "vm/RealmFuses.h"
#include "vm/RuntimeFuses.h"
#include "wasm/WasmAnyRef.h"

// [SMDOC] MacroAssembler multi-platform overview
//
// * How to read/write MacroAssembler method declarations:
//
// The following macros are made to avoid #ifdef around each method declarations
// of the Macro Assembler, and they are also used as an hint on the location of
// the implementations of each method.  For example, the following declaration
//
//   void Pop(FloatRegister t) DEFINED_ON(x86_shared, arm);
//
// suggests the MacroAssembler::Pop(FloatRegister) method is implemented in
// x86-shared/MacroAssembler-x86-shared.h, and also in arm/MacroAssembler-arm.h.
//
// - If there is no annotation, then there is only one generic definition in
//   MacroAssembler.cpp.
//
// - If the declaration is "inline", then the method definition(s) would be in
//   the "-inl.h" variant of the same file(s).
//
// The script check_macroassembler_style.py (which runs on every build) is
// used to verify that method definitions match the annotation on the method
// declarations.  If there is any difference, then you either forgot to define
// the method in one of the macro assembler, or you forgot to update the
// annotation of the macro assembler declaration.
//
// Some convenient short-cuts are used to avoid repeating the same list of
// architectures on each method declaration, such as PER_ARCH and
// PER_SHARED_ARCH.
//
// Functions that are architecture-agnostic and are the same for all
// architectures, that it's necessary to define inline *in this header* to
// avoid used-before-defined warnings/errors that would occur if the
// definitions were in MacroAssembler-inl.h, should use the OOL_IN_HEADER
// marker at end of the declaration:
//
//   inline uint32_t framePushed() const OOL_IN_HEADER;
//
// Such functions should then be defined immediately after MacroAssembler's
// definition, for example:
//
//   //{{{ check_macroassembler_style
//   inline uint32_t
//   MacroAssembler::framePushed() const
//   {
//       return framePushed_;
//   }
//   ////}}} check_macroassembler_style

#define ALL_ARCH mips64, arm, arm64, x86, x64, loong64, riscv64, wasm32
#define ALL_SHARED_ARCH arm, arm64, loong64, mips64, riscv64, x86_shared, wasm32

// * How this macro works:
//
// DEFINED_ON is a macro which check if, for the current architecture, the
// method is defined on the macro assembler or not.
//
// For each architecture, we have a macro named DEFINED_ON_arch.  This macro is
// empty if this is not the current architecture.  Otherwise it must be either
// set to "define" or "crash" (only used for the none target so far).
//
// The DEFINED_ON macro maps the list of architecture names given as arguments
// to a list of macro names.  For example,
//
//   DEFINED_ON(arm, x86_shared)
//
// is expanded to
//
//   DEFINED_ON_none DEFINED_ON_arm DEFINED_ON_x86_shared
//
// which are later expanded on ARM, x86, x64 by DEFINED_ON_EXPAND_ARCH_RESULTS
// to
//
//   define
//
// or if the JIT is disabled or set to no architecture to
//
//   crash
//
// or to nothing, if the current architecture is not listed in the list of
// arguments of DEFINED_ON.  Note, only one of the DEFINED_ON_arch macro
// contributes to the non-empty result, which is the macro of the current
// architecture if it is listed in the arguments of DEFINED_ON.
//
// This result is appended to DEFINED_ON_RESULT_ before expanding the macro,
// which results in either no annotation, a MOZ_CRASH(), or a "= delete"
// annotation on the method declaration.

#define DEFINED_ON_x86
#define DEFINED_ON_x64
#define DEFINED_ON_x86_shared
#define DEFINED_ON_arm
#define DEFINED_ON_arm64
#define DEFINED_ON_mips64
#define DEFINED_ON_loong64
#define DEFINED_ON_riscv64
#define DEFINED_ON_wasm32
#define DEFINED_ON_none

// Specialize for each architecture.
#if defined(JS_CODEGEN_X86)
#  undef DEFINED_ON_x86
#  define DEFINED_ON_x86 define
#  undef DEFINED_ON_x86_shared
#  define DEFINED_ON_x86_shared define
#elif defined(JS_CODEGEN_X64)
#  undef DEFINED_ON_x64
#  define DEFINED_ON_x64 define
#  undef DEFINED_ON_x86_shared
#  define DEFINED_ON_x86_shared define
#elif defined(JS_CODEGEN_ARM)
#  undef DEFINED_ON_arm
#  define DEFINED_ON_arm define
#elif defined(JS_CODEGEN_ARM64)
#  undef DEFINED_ON_arm64
#  define DEFINED_ON_arm64 define
#elif defined(JS_CODEGEN_MIPS64)
#  undef DEFINED_ON_mips64
#  define DEFINED_ON_mips64 define
#elif defined(JS_CODEGEN_LOONG64)
#  undef DEFINED_ON_loong64
#  define DEFINED_ON_loong64 define
#elif defined(JS_CODEGEN_RISCV64)
#  undef DEFINED_ON_riscv64
#  define DEFINED_ON_riscv64 define
#elif defined(JS_CODEGEN_WASM32)
#  undef DEFINED_ON_wasm32
#  define DEFINED_ON_wasm32 define
#elif defined(JS_CODEGEN_NONE)
#  undef DEFINED_ON_none
#  define DEFINED_ON_none crash
#else
#  error "Unknown architecture!"
#endif

#define DEFINED_ON_RESULT_crash \
 {                             \
   MOZ_CRASH();                \
 }
#define DEFINED_ON_RESULT_define
#define DEFINED_ON_RESULT_ = delete

#define DEFINED_ON_DISPATCH_RESULT_2(Macro, Result) Macro##Result
#define DEFINED_ON_DISPATCH_RESULT(...) \
 DEFINED_ON_DISPATCH_RESULT_2(DEFINED_ON_RESULT_, __VA_ARGS__)

// We need to let the evaluation of MOZ_FOR_EACH terminates.
#define DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult) \
 DEFINED_ON_DISPATCH_RESULT ParenResult
#define DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult) \
 DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult)
#define DEFINED_ON_EXPAND_ARCH_RESULTS(ParenResult) \
 DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult)

#define DEFINED_ON_FWDARCH(Arch) DEFINED_ON_##Arch
#define DEFINED_ON_MAP_ON_ARCHS(ArchList) \
 DEFINED_ON_EXPAND_ARCH_RESULTS(         \
     (MOZ_FOR_EACH(DEFINED_ON_FWDARCH, (), ArchList)))

#define DEFINED_ON(...) DEFINED_ON_MAP_ON_ARCHS((none, __VA_ARGS__))

#define PER_ARCH DEFINED_ON(ALL_ARCH)
#define PER_SHARED_ARCH DEFINED_ON(ALL_SHARED_ARCH)
#define OOL_IN_HEADER

namespace JS {
struct ExpandoAndGeneration;
}

namespace js {

class StaticStrings;
class FixedLengthTypedArrayObject;

namespace wasm {
class CalleeDesc;
class CallSiteDesc;
class BytecodeOffset;
class MemoryAccessDesc;

enum class FailureMode : uint8_t;
enum class SimdOp;
enum class SymbolicAddress;
enum class Trap;
}  // namespace wasm

namespace jit {

// Defined in JitFrames.h
class FrameDescriptor;
enum class ExitFrameType : uint8_t;

class AutoSaveLiveRegisters;
class CompileZone;
class TemplateNativeObject;
class TemplateObject;

enum class CheckUnsafeCallWithABI {
 // Require the callee to use AutoUnsafeCallWithABI.
 Check,

 // We pushed an exit frame so this callWithABI can safely GC and walk the
 // stack.
 DontCheckHasExitFrame,

 // Don't check this callWithABI uses AutoUnsafeCallWithABI, for instance
 // because we're calling a simple helper function (like malloc or js_free)
 // that we can't change and/or that we know won't GC.
 DontCheckOther,
};

// This is a global function made to create the DynFn type in a controlled
// environment which would check if the function signature has been registered
// as an ABI function signature.
template <typename Sig>
static inline DynFn DynamicFunction(Sig fun);

enum class CharEncoding { Latin1, TwoByte };

constexpr uint32_t WasmCallerInstanceOffsetBeforeCall =
   wasm::FrameWithInstances::callerInstanceOffsetWithoutFrame();
constexpr uint32_t WasmCalleeInstanceOffsetBeforeCall =
   wasm::FrameWithInstances::calleeInstanceOffsetWithoutFrame();

// Allocation sites may be passed to GC thing allocation methods either via a
// register (for baseline compilation) or an enum indicating one of the
// catch-all allocation sites (for optimized compilation).
struct AllocSiteInput
   : public mozilla::Variant<Register, gc::CatchAllAllocSite> {
 using Base = mozilla::Variant<Register, gc::CatchAllAllocSite>;
 AllocSiteInput() : Base(gc::CatchAllAllocSite::Unknown) {}
 explicit AllocSiteInput(gc::CatchAllAllocSite catchAll) : Base(catchAll) {}
 explicit AllocSiteInput(Register reg) : Base(reg) {}
};

// Instance slots (including ShadowStackArea) and arguments size information
// from two neighboring frames.
// Used in Wasm tail calls to remove frame.
struct ReturnCallAdjustmentInfo {
 uint32_t newSlotsAndStackArgBytes;
 uint32_t oldSlotsAndStackArgBytes;

 ReturnCallAdjustmentInfo(uint32_t newSlotsAndStackArgBytes,
                          uint32_t oldSlotsAndStackArgBytes)
     : newSlotsAndStackArgBytes(newSlotsAndStackArgBytes),
       oldSlotsAndStackArgBytes(oldSlotsAndStackArgBytes) {}
};

struct BranchWasmRefIsSubtypeRegisters {
 bool needSuperSTV;
 bool needScratch1;
 bool needScratch2;
};

// [SMDOC] Code generation invariants (incomplete)
//
// ## 64-bit GPRs carrying 32-bit values
//
// At least at the end of every JS or Wasm operation (= SpiderMonkey bytecode or
// Wasm bytecode; this is necessarily a little vague), if a 64-bit GPR has a
// 32-bit value, then the upper 32 bits of the register may be predictable in
// accordance with platform-specific rules, as follows.
//
// - On x64 and arm64, the upper bits are zero
// - On mips64 and loongarch64 the upper bits are the sign extension of the
//   lower bits
// - (On risc-v we have no rule, having no port yet.  Sign extension is the most
//   likely rule, but "unpredictable" is an option.)
//
// In most cases no extra work needs to be done to maintain the invariant:
//
// - 32-bit operations on x64 and arm64 zero-extend the result to 64 bits.
//   These operations ignore the upper bits of the inputs.
// - 32-bit operations on mips64 sign-extend the result to 64 bits (even many
//   that are labeled as "unsigned", eg ADDU, though not all, eg LU).
//   Additionally, the inputs to many 32-bit operations must be properly
//   sign-extended to avoid "unpredictable" behavior, and our simulators check
//   that inputs conform.
// - (32-bit operations on risc-v and loongarch64 sign-extend, much as mips, but
//   appear to ignore the upper bits of the inputs.)
//
// The upshot of these invariants is, among other things, that:
//
// - No code needs to be generated when a 32-bit value is extended to 64 bits
//   or a 64-bit value is wrapped to 32 bits, if the upper bits are known to be
//   correct because they resulted from an operation that produced them
//   predictably.
// - Literal loads must be careful to avoid instructions that might extend the
//   literal in the wrong way.
// - Code that produces values using intermediate values with non-canonical
//   extensions must extend according to platform conventions before being
//   "done".
//
// All optimizations are necessarily platform-specific and should only be used
// in platform-specific code.  We may add architectures in the future that do
// not follow the patterns of the few architectures we already have.
//
// Also see MacroAssembler::debugAssertCanonicalInt32().

// The public entrypoint for emitting assembly. Note that a MacroAssembler can
// use cx->lifoAlloc, so take care not to interleave masm use with other
// lifoAlloc use if one will be destroyed before the other.
class MacroAssembler : public MacroAssemblerSpecific {
private:
 // Information about the current JSRuntime. This is nullptr only for Wasm
 // compilations.
 CompileRuntime* maybeRuntime_ = nullptr;

 // Information about the current Realm. This is nullptr for Wasm compilations
 // and when compiling runtime-wide jitcode that will live in the Atom zone:
 // for example, trampolines, the baseline interpreter, and (if
 // self_hosted_cache is enabled) self-hosted baseline code.
 CompileRealm* maybeRealm_ = nullptr;

 // Labels for handling exceptions and failures.
 NonAssertingLabel failureLabel_;

protected:
 // Constructor is protected. Use one of the derived classes!
 explicit MacroAssembler(TempAllocator& alloc,
                         CompileRuntime* maybeRuntime = nullptr,
                         CompileRealm* maybeRealm = nullptr);

public:
 MoveResolver& moveResolver() {
   // As an optimization, the MoveResolver is a persistent data structure
   // shared between visitors in the CodeGenerator. This assertion
   // checks that state is not leaking from visitor to visitor
   // via an unresolved addMove().
   MOZ_ASSERT(moveResolver_.hasNoPendingMoves());
   return moveResolver_;
 }

 size_t instructionsSize() const { return size(); }

 CompileRealm* realm() const {
   MOZ_ASSERT(maybeRealm());
   return maybeRealm();
 }
 CompileRealm* maybeRealm() const { return maybeRealm_; }
 CompileRuntime* runtime() const {
   MOZ_ASSERT(maybeRuntime_);
   return maybeRuntime_;
 }

#ifdef JS_HAS_HIDDEN_SP
 void Push(RegisterOrSP reg);
#endif

#ifdef ENABLE_WASM_SIMD
 // `op` should be a shift operation. Return true if a variable-width shift
 // operation on this architecture should pre-mask the shift count, and if so,
 // return the mask in `*mask`.
 static bool MustMaskShiftCountSimd128(wasm::SimdOp op, int32_t* mask);
#endif

 //{{{ check_macroassembler_decl_style
public:
 // ===============================================================
 // MacroAssembler high-level usage.

 // Flushes the assembly buffer, on platforms that need it.
 void flush() PER_SHARED_ARCH;

 // Add a comment that is visible in the pretty printed assembly code.
 void comment(const char* msg) PER_SHARED_ARCH;

 // ===============================================================
 // Frame manipulation functions.

 inline uint32_t framePushed() const OOL_IN_HEADER;
 inline void setFramePushed(uint32_t framePushed) OOL_IN_HEADER;
 inline void adjustFrame(int32_t value) OOL_IN_HEADER;

 // Adjust the frame, to account for implicit modification of the stack
 // pointer, such that callee can remove arguments on the behalf of the
 // caller.
 inline void implicitPop(uint32_t bytes) OOL_IN_HEADER;

private:
 // This field is used to statically (at compilation time) emulate a frame
 // pointer by keeping track of stack manipulations.
 //
 // It is maintained by all stack manipulation functions below.
 uint32_t framePushed_;

public:
 // ===============================================================
 // Stack manipulation functions -- sets of registers.

 // Approximately speaking, the following routines must use the same memory
 // layout.  Any inconsistencies will certainly lead to crashing in generated
 // code:
 //
 //   MacroAssembler::PushRegsInMaskSizeInBytes
 //   MacroAssembler::PushRegsInMask
 //   MacroAssembler::storeRegsInMask
 //   MacroAssembler::PopRegsInMask
 //   MacroAssembler::PopRegsInMaskIgnore
 //   FloatRegister::getRegisterDumpOffsetInBytes
 //   (no class) PushRegisterDump
 //   (union) RegisterContent
 //   JitRuntime::generateInvalidator
 //   JitRuntime::generateBailoutHandler
 //   JSJitFrameIter::machineState
 //
 // To be more exact, the invariants are:
 //
 // * The save area is conceptually viewed as starting at a highest address
 //   (really, at "highest address - 1") and working down to some lower
 //   address.
 //
 // * PushRegsInMask, storeRegsInMask and PopRegsInMask{Ignore} must use
 //   exactly the same memory layout, when starting from the abovementioned
 //   highest address.
 //
 // * PushRegsInMaskSizeInBytes must produce a value which is exactly equal
 //   to the change in the machine's stack pointer register as a result of
 //   calling PushRegsInMask or PopRegsInMask{Ignore}.  This value must be at
 //   least uintptr_t-aligned on the target, and may be more aligned than that.
 //
 // * PushRegsInMaskSizeInBytes must produce a value which is greater than or
 //   equal to the amount of space used by storeRegsInMask.
 //
 // * Hence, regardless of whether the save area is created with
 //   storeRegsInMask or PushRegsInMask, it is guaranteed to fit inside an
 //   area of size calculated by PushRegsInMaskSizeInBytes.
 //
 // * For the `ignore` argument of PopRegsInMaskIgnore, equality checking
 //   for the floating point/SIMD registers is done on the basis of the
 //   underlying physical register, regardless of width.  For example, if the
 //   to-restore set contains v17 (the SIMD register with encoding 17) and
 //   the ignore set contains d17 (the double register with encoding 17) then
 //   no part of the physical register with encoding 17 will be restored.
 //   (This is probably not true on arm32, since that has aliased float32
 //   registers; but none of our other targets do.)
 //
 // * {Push,store}RegsInMask/storeRegsInMask are further constrained as
 //   follows: when given the argument AllFloatRegisters, the resulting
 //   memory area must contain exactly all the SIMD/FP registers for the
 //   target at their widest width (that we care about).  [We have no targets
 //   where the SIMD registers and FP register sets are disjoint.]  They must
 //   be packed end-to-end with no holes, with the register with the lowest
 //   encoding number (0), as returned by FloatRegister::encoding(), at the
 //   abovementioned highest address, register 1 just below that, etc.
 //
 //   Furthermore the sizeof(RegisterContent) must equal the size of a SIMD
 //   register in the abovementioned array.
 //
 //   Furthermore the value returned by
 //   FloatRegister::getRegisterDumpOffsetInBytes must be a correct index
 //   into the abovementioned array.  Given the constraints, the only correct
 //   value is `reg.encoding() * sizeof(RegisterContent)`.
 //
 // Note that some of the routines listed above are JS-only, and do not support
 // SIMD registers. They are otherwise part of the same equivalence class.
 // Register spilling for e.g. OOL VM calls is implemented using
 // PushRegsInMask, and recovered on bailout using machineState. This requires
 // the same layout to be used in machineState, and therefore in all other code
 // that can spill registers that are recovered on bailout. Implementations of
 // JitRuntime::generate{Invalidator,BailoutHandler} should either call
 // PushRegsInMask, or check carefully to be sure that they generate the same
 // layout.

 // The size of the area used by PushRegsInMask.
 static size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set) PER_SHARED_ARCH;

 void PushRegsInMask(LiveRegisterSet set) PER_SHARED_ARCH;
 void PushRegsInMask(LiveGeneralRegisterSet set);

 // Like PushRegsInMask, but instead of pushing the registers, store them to
 // |dest|. |dest| should point to the end of the reserved space, so the
 // first register will be stored at |dest.offset - sizeof(register)|.  It is
 // required that |dest.offset| is at least as large as the value computed by
 // PushRegsInMaskSizeInBytes for this |set|.  In other words, |dest.base|
 // must point to either the lowest address in the save area, or some address
 // below that.
 void storeRegsInMask(LiveRegisterSet set, Address dest,
                      Register scratch) PER_SHARED_ARCH;

 void PopRegsInMask(LiveRegisterSet set);
 void PopRegsInMask(LiveGeneralRegisterSet set);
 void PopRegsInMaskIgnore(LiveRegisterSet set,
                          LiveRegisterSet ignore) PER_SHARED_ARCH;

 // ===============================================================
 // Stack manipulation functions -- single registers/values.

 void Push(const Operand op) DEFINED_ON(x86_shared);
 void Push(Register reg) PER_SHARED_ARCH;
 void Push(Register reg1, Register reg2, Register reg3, Register reg4)
     DEFINED_ON(arm64);
 void Push(const Imm32 imm) PER_SHARED_ARCH;
 void Push(const ImmWord imm) PER_SHARED_ARCH;
 void Push(const ImmPtr imm) PER_SHARED_ARCH;
 void Push(const ImmGCPtr ptr) PER_SHARED_ARCH;
 void Push(FloatRegister reg) PER_SHARED_ARCH;
 void PushBoxed(FloatRegister reg) PER_ARCH;
 void PushFlags() DEFINED_ON(x86_shared);
 void Push(PropertyKey key, Register scratchReg);
 void Push(const Address& addr);
 void Push(TypedOrValueRegister v);
 void Push(const ConstantOrRegister& v);
 void Push(const ValueOperand& val);
 void Push(const Value& val);
 void Push(JSValueType type, Register reg);
 void Push(const Register64 reg);
 void PushEmptyRooted(VMFunctionData::RootType rootType);
 inline CodeOffset PushWithPatch(ImmWord word);
 inline CodeOffset PushWithPatch(ImmPtr imm);

 using MacroAssemblerSpecific::push;

 void Pop(const Operand op) DEFINED_ON(x86_shared);
 void Pop(Register reg) PER_SHARED_ARCH;
 void Pop(FloatRegister t) PER_SHARED_ARCH;
 void Pop(const ValueOperand& val) PER_SHARED_ARCH;
 void Pop(const Register64 reg);
 void PopFlags() DEFINED_ON(x86_shared);
 void PopStackPtr()
     DEFINED_ON(arm, mips64, x86_shared, loong64, riscv64, wasm32);

 // Move the stack pointer based on the requested amount.
 void adjustStack(int amount);
 void freeStack(uint32_t amount);

 // Move the stack pointer to the specified position. It assumes the SP
 // register is not valid -- it uses FP to set the position.
 void freeStackTo(uint32_t framePushed) PER_SHARED_ARCH;

private:
 // ===============================================================
 // Register allocation fields.
#ifdef DEBUG
 friend AutoRegisterScope;
 friend AutoFloatRegisterScope;
 // Used to track register scopes for debug builds.
 // Manipulated by the AutoGenericRegisterScope class.
 AllocatableRegisterSet debugTrackedRegisters_;
#endif  // DEBUG

public:
 // ===============================================================
 // Simple call functions.

 // The returned CodeOffset is the assembler offset for the instruction
 // immediately following the call; that is, for the return point.
 CodeOffset call(Register reg) PER_SHARED_ARCH;
 CodeOffset call(Label* label) PER_SHARED_ARCH;
 CodeOffset call(const Address& addr) PER_SHARED_ARCH;

 void call(ImmWord imm) PER_SHARED_ARCH;
 // Call a target native function, which is neither traceable nor movable.
 void call(ImmPtr imm) PER_SHARED_ARCH;
 CodeOffset call(wasm::SymbolicAddress imm) PER_SHARED_ARCH;
 inline CodeOffset call(const wasm::CallSiteDesc& desc,
                        wasm::SymbolicAddress imm);

 // Call a target JitCode, which must be traceable, and may be movable.
 void call(JitCode* c) PER_SHARED_ARCH;

 inline void call(TrampolinePtr code);

 inline CodeOffset call(const wasm::CallSiteDesc& desc, const Register reg);
 inline CodeOffset call(const wasm::CallSiteDesc& desc, uint32_t funcDefIndex);
 inline void call(const wasm::CallSiteDesc& desc, wasm::Trap trap);

 CodeOffset callWithPatch() PER_SHARED_ARCH;
 void patchCall(uint32_t callerOffset, uint32_t calleeOffset) PER_SHARED_ARCH;

 // Push the return address and make a call. On platforms where this function
 // is not defined, push the link register (pushReturnAddress) at the entry
 // point of the callee.
 void callAndPushReturnAddress(Register reg) DEFINED_ON(x86_shared);
 void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared);

 // These do not adjust framePushed().
 void pushReturnAddress()
     DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32);
 void popReturnAddress()
     DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32);

 // Useful for dealing with two-valued returns.
 void moveRegPair(Register src0, Register src1, Register dst0, Register dst1,
                  MoveOp::Type type = MoveOp::GENERAL);

 void reserveVMFunctionOutParamSpace(const VMFunctionData& f);
 void loadVMFunctionOutParam(const VMFunctionData& f, const Address& addr);

public:
 // ===============================================================
 // Patchable near/far jumps.

 // "Far jumps" provide the ability to jump to any uint32_t offset from any
 // other uint32_t offset without using a constant pool (thus returning a
 // simple CodeOffset instead of a CodeOffsetJump).
 CodeOffset farJumpWithPatch() PER_SHARED_ARCH;
 void patchFarJump(CodeOffset farJump, uint32_t targetOffset) PER_SHARED_ARCH;
 static void patchFarJump(uint8_t* farJump, uint8_t* target)
     DEFINED_ON(arm, arm64, x86_shared, loong64, mips64, riscv64);

 // Emit a nop that can be patched to and from a nop and a call with int32
 // relative displacement.
 CodeOffset nopPatchableToCall() PER_SHARED_ARCH;
 void nopPatchableToCall(const wasm::CallSiteDesc& desc);
 static void patchNopToCall(uint8_t* callsite,
                            uint8_t* target) PER_SHARED_ARCH;
 static void patchCallToNop(uint8_t* callsite) PER_SHARED_ARCH;

 // These methods are like movWithPatch/PatchDataWithValueCheck but allow
 // using pc-relative addressing on certain platforms (RIP-relative LEA on x64,
 // ADR instruction on arm64).
 //
 // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is
 // release-asserted).
 CodeOffset moveNearAddressWithPatch(Register dest) PER_ARCH;
 static void patchNearAddressMove(CodeLocationLabel loc,
                                  CodeLocationLabel target) PER_ARCH;

 // Creates a move of a patchable 32-bit value into `dest`.  On 64-bit
 // targets, the value (`n`) is extended to 64 bits using the target
 // architecture's standard 32-to-64 extension rule.  Hence consistent cross
 // target behaviour is only provided for `n` in the range 0 .. 2^31-1
 // inclusive.
 CodeOffset move32WithPatch(Register dest)
     DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64);
 void patchMove32(CodeOffset offset, Imm32 n)
     DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64);

public:
 // ===============================================================
 // [SMDOC] JIT-to-C++ Function Calls (callWithABI)
 //
 // callWithABI is used to make a call using the standard C/C++ system ABI.
 //
 // callWithABI is a low level interface for making calls, as such every call
 // made with callWithABI should be organized with 6 steps: spilling live
 // registers, aligning the stack, listing arguments of the called function,
 // calling a function pointer, extracting the returned value and restoring
 // live registers.
 //
 // A more detailed example of the six stages:
 //
 // 1) Saving of registers that are live. This will vary depending on which
 //    SpiderMonkey compiler you are working on. Registers that shouldn't be
 //    restored can be excluded.
 //
 //      LiveRegisterSet volatileRegs(...);
 //      volatileRegs.take(scratch);
 //      masm.PushRegsInMask(volatileRegs);
 //
 // 2) Align the stack to perform the call with the correct stack alignment.
 //
 //    When the stack pointer alignment is unknown and cannot be corrected
 //    when generating the code, setupUnalignedABICall must be used to
 //    dynamically align the stack pointer to the expectation of the ABI.
 //    When the stack pointer is known at JIT compilation time, the stack can
 //    be fixed manually and setupAlignedABICall and setupWasmABICall can be
 //    used.
 //
 //    setupWasmABICall is a special case of setupAlignedABICall as
 //    SpiderMonkey's WebAssembly implementation mostly follow the system
 //    ABI, except for float/double arguments, which always use floating
 //    point registers, even if this is not supported by the system ABI.
 //
 //      masm.setupUnalignedABICall(scratch);
 //
 // 3) Passing arguments. Arguments are passed left-to-right.
 //
 //      masm.passABIArg(scratch);
 //      masm.passABIArg(FloatOp0, ABIType::Float64);
 //
 //    Note how float register arguments are annotated with ABIType::Float64.
 //
 //    Concerning stack-relative address, see the note on passABIArg.
 //
 // 4) Make the call:
 //
 //      using Fn = int32_t (*)(int32_t)
 //      masm.callWithABI<Fn, Callee>();
 //
 //    In the case where the call returns a double, that needs to be
 //    indicated to the callWithABI like this:
 //
 //      using Fn = double (*)(int32_t)
 //      masm.callWithABI<Fn, Callee>(ABIType::Float64);
 //
 //    There are overloads to allow calls to registers and addresses.
 //
 // 5) Take care of the result
 //
 //      masm.storeCallPointerResult(scratch1);
 //      masm.storeCallBoolResult(scratch1);
 //      masm.storeCallInt32Result(scratch1);
 //      masm.storeCallFloatResult(scratch1);
 //
 // 6) Restore the potentially clobbered volatile registers
 //
 //      masm.PopRegsInMask(volatileRegs);
 //
 //    If expecting a returned value, this call should use
 //    PopRegsInMaskIgnore to filter out the registers which are containing
 //    the returned value.
 //
 // Unless an exit frame is pushed prior to the setupABICall, the callee
 // should not GC. To ensure this is the case callWithABI is instrumented to
 // make sure that in the default case callees are annotated with an
 // AutoUnsafeCallWithABI on the stack.
 //
 // A callWithABI can opt out of checking, if for example it is known there
 // is an exit frame, or the callee is known not to GC.
 //
 // If your callee needs to be able to GC, consider using a VMFunction, or
 // create a fake exit frame, and instrument the TraceJitExitFrame
 // accordingly.

 // Setup a call to C/C++ code, given the assumption that the framePushed
 // accurately defines the state of the stack, and that the top of the stack
 // was properly aligned. Note that this only supports cdecl.
 //
 // As a rule of thumb, this can be used in CodeGenerator but not in CacheIR or
 // Baseline code (because the stack is not aligned to ABIStackAlignment).
 void setupAlignedABICall();

 // As setupAlignedABICall, but for WebAssembly native ABI calls, which pass
 // through a builtin thunk that uses the wasm ABI. All the wasm ABI calls
 // can be native, since we always know the stack alignment a priori.
 void setupWasmABICall(wasm::SymbolicAddress builtin);

 // Setup an ABI call for when the alignment is not known. This may need a
 // scratch register.
 void setupUnalignedABICall(Register scratch) PER_ARCH;

 // Like setupUnalignedABICall, but more efficient because it doesn't push/pop
 // the unaligned stack pointer. The caller is responsible for restoring SP
 // after the callWithABI, for example using the frame pointer register.
 void setupUnalignedABICallDontSaveRestoreSP();

 // Arguments must be assigned to a C/C++ call in order. They are moved
 // in parallel immediately before performing the call. This process may
 // temporarily use more stack, in which case esp-relative addresses will be
 // automatically adjusted. It is extremely important that esp-relative
 // addresses are computed *after* setupABICall(). Furthermore, no
 // operations should be emitted while setting arguments.
 void passABIArg(const MoveOperand& from, ABIType type);
 inline void passABIArg(Register reg);
 void passABIArg(Register64 reg);
 inline void passABIArg(FloatRegister reg, ABIType type);

 inline void callWithABI(
     DynFn fun, ABIType result = ABIType::General,
     CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check);
 template <typename Sig, Sig fun>
 inline void callWithABI(
     ABIType result = ABIType::General,
     CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check);
 inline void callWithABI(Register fun, ABIType result = ABIType::General);
 inline void callWithABI(const Address& fun,
                         ABIType result = ABIType::General);

 CodeOffset callWithABI(wasm::BytecodeOffset offset, wasm::SymbolicAddress fun,
                        mozilla::Maybe<int32_t> instanceOffset,
                        ABIType result = ABIType::General);
 void callDebugWithABI(wasm::SymbolicAddress fun,
                       ABIType result = ABIType::General);

private:
 // Reinitialize the variables which have to be cleared before making a call
 // with callWithABI.
 void setupABICallHelper(ABIKind kind);

 // Reinitialize the variables which have to be cleared before making a call
 // with native abi.
 void setupNativeABICall();

 // Reserve the stack and resolve the arguments move.
 void callWithABIPre(uint32_t* stackAdjust,
                     bool callFromWasm = false) PER_ARCH;

 // Emits a call to a C/C++ function, resolving all argument moves.
 void callWithABINoProfiler(void* fun, ABIType result,
                            CheckUnsafeCallWithABI check);
 void callWithABINoProfiler(Register fun, ABIType result) PER_ARCH;
 void callWithABINoProfiler(const Address& fun, ABIType result) PER_ARCH;

 // Restore the stack to its state before the setup function call.
 void callWithABIPost(uint32_t stackAdjust, ABIType result) PER_ARCH;

#ifdef JS_CHECK_UNSAFE_CALL_WITH_ABI
 // Set the JSContext::inUnsafeCallWithABI flag using InstanceReg.
 void wasmCheckUnsafeCallWithABIPre();
 // Check JSContext::inUnsafeCallWithABI was cleared as expected.
 void wasmCheckUnsafeCallWithABIPost();
#endif

 // Create the signature to be able to decode the arguments of a native
 // function, when calling a function within the simulator.
 inline void appendSignatureType(ABIType type);
 inline ABIFunctionType signature() const;

 // Private variables used to handle moves between registers given as
 // arguments to passABIArg and the list of ABI registers expected for the
 // signature of the function.
 MoveResolver moveResolver_;

 // Architecture specific implementation which specify how registers & stack
 // offsets are used for calling a function.
 ABIArgGenerator abiArgs_;

#ifdef DEBUG
 // Flag use to assert that we use ABI function in the right context.
 bool inCall_;
#endif

 // If set by setupUnalignedABICall then callWithABI will pop the stack
 // register which is on the stack.
 bool dynamicAlignment_;

#ifdef JS_SIMULATOR
 // The signature is used to accumulate all types of arguments which are used
 // by the caller. This is used by the simulators to decode the arguments
 // properly, and cast the function pointer to the right type.
 uint32_t signature_;
#endif

public:
 // ===============================================================
 // Jit Frames.
 //
 // These functions are used to build the content of the Jit frames.  See
 // CommonFrameLayout class, and all its derivatives. The content should be
 // pushed in the opposite order as the fields of the structures, such that
 // the structures can be used to interpret the content of the stack.

 // Call the Jit function, and push the return address (or let the callee
 // push the return address).
 //
 // These functions return the offset of the return address, in order to use
 // the return address to index the safepoints, which are used to list all
 // live registers.
 inline uint32_t callJitNoProfiler(Register callee);
 inline uint32_t callJit(Register callee);
 inline uint32_t callJit(JitCode* code);
 inline uint32_t callJit(TrampolinePtr code);
 inline uint32_t callJit(ImmPtr callee);

 // The frame descriptor is the second field of all Jit frames, pushed before
 // calling the Jit function. See CommonFrameLayout::descriptor_.
 inline void push(FrameDescriptor descriptor);
 inline void Push(FrameDescriptor descriptor);

 // For JitFrameLayout, the descriptor also stores the number of arguments
 // passed by the caller. See MakeFrameDescriptorForJitCall.
 inline void pushFrameDescriptorForJitCall(FrameType type, Register argc,
                                           Register scratch,
                                           bool hasInlineICScript = false);
 inline void PushFrameDescriptorForJitCall(FrameType type, Register argc,
                                           Register scratch,
                                           bool hasInlineICScript = false);
 inline void makeFrameDescriptorForJitCall(FrameType type, Register argc,
                                           Register dest,
                                           bool hasInlineICScript = false);

 // Load the number of actual arguments from the frame's JitFrameLayout.
 inline void loadNumActualArgs(Register framePtr, Register dest);

 // Push the callee token of a JSFunction which pointer is stored in the
 // |callee| register. The callee token is packed with a |constructing| flag
 // which correspond to the fact that the JS function is called with "new" or
 // not.
 inline void PushCalleeToken(Register callee, bool constructing);

 // Unpack a callee token located at the |token| address, and return the
 // JSFunction pointer in the |dest| register.
 inline void loadFunctionFromCalleeToken(Address token, Register dest);

 // This function emulates a call by pushing an exit frame on the stack,
 // except that the fake-function is inlined within the body of the caller.
 //
 // This function assumes that the current frame is an IonJS frame.
 //
 // This function returns the offset of the /fake/ return address, in order to
 // use the return address to index the safepoints, which are used to list all
 // live registers.
 //
 // This function should be balanced with a call to adjustStack, to pop the
 // exit frame and emulate the return statement of the inlined function.
 inline uint32_t buildFakeExitFrame(Register scratch);

private:
 // This function is used by buildFakeExitFrame to push a fake return address
 // on the stack. This fake return address should never be used for resuming
 // any execution, and can even be an invalid pointer into the instruction
 // stream, as long as it does not alias any other.
 uint32_t pushFakeReturnAddress(Register scratch) PER_SHARED_ARCH;

public:
 // ===============================================================
 // Exit frame footer.
 //
 // When calling outside the Jit we push an exit frame. To mark the stack
 // correctly, we have to push additional information, called the Exit frame
 // footer, which is used to identify how the stack is marked.
 //
 // See JitFrames.h, and TraceJitExitFrame in JitFrames.cpp.

 // Links the exit frame and pushes the ExitFooterFrame.
 inline void enterExitFrame(Register cxreg, Register scratch, VMFunctionId f);

 // Push an exit frame token to identify which fake exit frame this footer
 // corresponds to.
 inline void enterFakeExitFrame(Register cxreg, Register scratch,
                                ExitFrameType type);

 // Push an exit frame token for a native call.
 inline void enterFakeExitFrameForNative(Register cxreg, Register scratch,
                                         bool isConstructing);

 // Pop ExitFrame footer in addition to the extra frame.
 inline void leaveExitFrame(size_t extraFrame = 0);

private:
 // Save the top of the stack into JitActivation::packedExitFP of the
 // current thread, which should be the location of the latest exit frame.
 void linkExitFrame(Register cxreg, Register scratch);

public:
 // ===============================================================
 // Move instructions

 inline void move64(Imm64 imm, Register64 dest) PER_ARCH;
 inline void move64(Register64 src, Register64 dest) PER_ARCH;

 inline void moveFloat16ToGPR(FloatRegister src,
                              Register dest) PER_SHARED_ARCH;
 // Clears the high words of `src`.
 inline void moveGPRToFloat16(Register src,
                              FloatRegister dest) PER_SHARED_ARCH;

 inline void moveFloat32ToGPR(FloatRegister src,
                              Register dest) PER_SHARED_ARCH;
 inline void moveGPRToFloat32(Register src,
                              FloatRegister dest) PER_SHARED_ARCH;

 inline void moveDoubleToGPR64(FloatRegister src, Register64 dest) PER_ARCH;
 inline void moveGPR64ToDouble(Register64 src, FloatRegister dest) PER_ARCH;

 // Move the low 32-bits of a double.
 inline void moveLowDoubleToGPR(FloatRegister src,
                                Register dest) PER_SHARED_ARCH;

 inline void move8ZeroExtend(Register src, Register dest) PER_SHARED_ARCH;

 inline void move8SignExtend(Register src, Register dest) PER_SHARED_ARCH;
 inline void move16SignExtend(Register src, Register dest) PER_SHARED_ARCH;

 // move64To32 will clear the high bits of `dest` on 64-bit systems.
 inline void move64To32(Register64 src, Register dest) PER_ARCH;

 inline void move32To64ZeroExtend(Register src, Register64 dest) PER_ARCH;

 inline void move8To64SignExtend(Register src, Register64 dest) PER_ARCH;
 inline void move16To64SignExtend(Register src, Register64 dest) PER_ARCH;
 inline void move32To64SignExtend(Register src, Register64 dest) PER_ARCH;

 inline void move8SignExtendToPtr(Register src, Register dest) PER_ARCH;
 inline void move16SignExtendToPtr(Register src, Register dest) PER_ARCH;
 inline void move32SignExtendToPtr(Register src, Register dest) PER_ARCH;

 inline void move32ZeroExtendToPtr(Register src, Register dest) PER_ARCH;

 // Copy a constant, typed-register, or a ValueOperand into a ValueOperand
 // destination.
 inline void moveValue(const ConstantOrRegister& src,
                       const ValueOperand& dest);
 void moveValue(const TypedOrValueRegister& src, const ValueOperand& dest);
 void moveValue(const ValueOperand& src, const ValueOperand& dest) PER_ARCH;
 void moveValue(const Value& src, const ValueOperand& dest) PER_ARCH;

 void movePropertyKey(PropertyKey key, Register dest);

 // ===============================================================
 // Load instructions

 inline void load32SignExtendToPtr(const Address& src, Register dest) PER_ARCH;

 inline void loadAbiReturnAddress(Register dest) PER_SHARED_ARCH;

 // ===============================================================
 // Copy instructions

 inline void copy64(const Address& src, const Address& dest, Register scratch);

public:
 // ===============================================================
 // Logical instructions

 inline void not32(Register reg) PER_SHARED_ARCH;
 inline void notPtr(Register reg) PER_ARCH;

 inline void and32(Register src, Register dest) PER_SHARED_ARCH;
 inline void and32(Imm32 imm, Register dest) PER_SHARED_ARCH;
 inline void and32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH;
 inline void and32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
 inline void and32(const Address& src, Register dest) PER_SHARED_ARCH;

 inline void andPtr(Register src, Register dest) PER_ARCH;
 inline void andPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void andPtr(Imm32 imm, Register src, Register dest) PER_ARCH;

 inline void and64(Imm64 imm, Register64 dest) PER_ARCH;
 inline void or64(Imm64 imm, Register64 dest) PER_ARCH;
 inline void xor64(Imm64 imm, Register64 dest) PER_ARCH;

 inline void or32(Register src, Register dest) PER_SHARED_ARCH;
 inline void or32(Imm32 imm, Register dest) PER_SHARED_ARCH;
 inline void or32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH;
 inline void or32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;

 inline void orPtr(Register src, Register dest) PER_ARCH;
 inline void orPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void orPtr(Imm32 imm, Register src, Register dest) PER_ARCH;

 inline void and64(Register64 src, Register64 dest) PER_ARCH;
 inline void or64(Register64 src, Register64 dest) PER_ARCH;
 inline void xor64(Register64 src, Register64 dest) PER_ARCH;

 inline void xor32(Register src, Register dest) PER_SHARED_ARCH;
 inline void xor32(Imm32 imm, Register dest) PER_SHARED_ARCH;
 inline void xor32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH;
 inline void xor32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
 inline void xor32(const Address& src, Register dest) PER_SHARED_ARCH;

 inline void xorPtr(Register src, Register dest) PER_ARCH;
 inline void xorPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void xorPtr(Imm32 imm, Register src, Register dest) PER_ARCH;

 inline void and64(const Operand& src, Register64 dest) DEFINED_ON(x64);
 inline void or64(const Operand& src, Register64 dest) DEFINED_ON(x64);
 inline void xor64(const Operand& src, Register64 dest) DEFINED_ON(x64);

 // ===============================================================
 // Swap instructions

 // Swap the two lower bytes and sign extend the result to 32-bit.
 inline void byteSwap16SignExtend(Register reg) PER_SHARED_ARCH;

 // Swap the two lower bytes and zero extend the result to 32-bit.
 inline void byteSwap16ZeroExtend(Register reg) PER_SHARED_ARCH;

 // Swap all four bytes in a 32-bit integer.
 inline void byteSwap32(Register reg) PER_SHARED_ARCH;

 // Swap all eight bytes in a 64-bit integer.
 inline void byteSwap64(Register64 reg) PER_ARCH;

 // ===============================================================
 // Arithmetic functions

 // Condition flags aren't guaranteed to be set by these functions, for example
 // x86 will always set condition flags, but ARM64 won't do it unless
 // explicitly requested. Instead use branch(Add|Sub|Mul|Neg) to test for
 // condition flags after performing arithmetic operations.

 inline void add32(const Address& src, Register dest) PER_SHARED_ARCH;
 inline void add32(Register src, Register dest) PER_SHARED_ARCH;
 inline void add32(Imm32 imm, Register dest) PER_SHARED_ARCH;
 inline void add32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH;
 inline void add32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
 inline void add32(Imm32 imm, const AbsoluteAddress& dest)
     DEFINED_ON(x86_shared);

 inline void addPtr(Register src, Register dest) PER_ARCH;
 inline void addPtr(Register src1, Register src2, Register dest)
     DEFINED_ON(arm64);
 inline void addPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void addPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64);
 inline void addPtr(ImmWord imm, Register dest) PER_ARCH;
 inline void addPtr(ImmPtr imm, Register dest);
 inline void addPtr(Imm32 imm, const Address& dest) PER_ARCH;
 inline void addPtr(Imm32 imm, const AbsoluteAddress& dest)
     DEFINED_ON(x86, x64);
 inline void addPtr(const Address& src, Register dest) PER_ARCH;

 inline void add64(Register64 src, Register64 dest) PER_ARCH;
 inline void add64(Imm32 imm, Register64 dest) PER_ARCH;
 inline void add64(Imm64 imm, Register64 dest) PER_ARCH;
 inline void add64(const Operand& src, Register64 dest) DEFINED_ON(x64);

 inline void addFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 // Compute dest=SP-imm where dest is a pointer registers and not SP.  The
 // offset returned from sub32FromStackPtrWithPatch() must be passed to
 // patchSub32FromStackPtr().
 inline CodeOffset sub32FromStackPtrWithPatch(Register dest) PER_ARCH;
 inline void patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) PER_ARCH;

 inline void addDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
 inline void addConstantDouble(double d, FloatRegister dest) DEFINED_ON(x86);

 inline void sub32(const Address& src, Register dest) PER_SHARED_ARCH;
 inline void sub32(Register src, Register dest) PER_SHARED_ARCH;
 inline void sub32(Imm32 imm, Register dest) PER_SHARED_ARCH;

 inline void subPtr(Register src, Register dest) PER_ARCH;
 inline void subPtr(Register src, const Address& dest) PER_ARCH;
 inline void subPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void subPtr(ImmWord imm, Register dest) DEFINED_ON(x86, x64);
 inline void subPtr(const Address& addr, Register dest) PER_ARCH;

 inline void sub64(Register64 src, Register64 dest) PER_ARCH;
 inline void sub64(Imm64 imm, Register64 dest) PER_ARCH;
 inline void sub64(const Operand& src, Register64 dest) DEFINED_ON(x64);

 inline void subFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 inline void subDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 inline void mul32(Register rhs, Register srcDest) PER_SHARED_ARCH;
 inline void mul32(Imm32 imm, Register srcDest) PER_SHARED_ARCH;

 inline void mul32(Register src1, Register src2, Register dest, Label* onOver)
     DEFINED_ON(arm64);

 // Return the high word of the unsigned multiplication into |dest|.
 inline void mulHighUnsigned32(Imm32 imm, Register src,
                               Register dest) PER_ARCH;

 inline void mulPtr(Register rhs, Register srcDest) PER_ARCH;
 inline void mulPtr(ImmWord rhs, Register srcDest) PER_ARCH;

 inline void mul64(const Operand& src, const Register64& dest) DEFINED_ON(x64);
 inline void mul64(const Operand& src, const Register64& dest,
                   const Register temp) DEFINED_ON(x64);
 inline void mul64(Imm64 imm, const Register64& dest) PER_ARCH;
 inline void mul64(Imm64 imm, const Register64& dest, const Register temp)
     DEFINED_ON(x86, x64, arm, mips64, loong64, riscv64);
 inline void mul64(const Register64& src, const Register64& dest,
                   const Register temp) PER_ARCH;
 inline void mul64(const Register64& src1, const Register64& src2,
                   const Register64& dest) DEFINED_ON(arm64);
 inline void mul64(Imm64 src1, const Register64& src2, const Register64& dest)
     DEFINED_ON(arm64);

 inline void mulBy3(Register src, Register dest) PER_ARCH;

 inline void mulFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
 inline void mulDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 inline void mulDoublePtr(ImmPtr imm, Register temp,
                          FloatRegister dest) PER_ARCH;

 // Perform an integer division, returning the integer part rounded toward
 // zero. rhs must not be zero, and the division must not overflow.
 //
 // On ARM, the chip must have hardware division instructions.
 inline void quotient32(Register lhs, Register rhs, Register dest,
                        bool isUnsigned)
     DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32);

 inline void quotient64(Register lhs, Register rhs, Register dest,
                        bool isUnsigned)
     DEFINED_ON(arm64, loong64, mips64, riscv64);

 // As above, but lhs and dest must be eax and tempEdx must be edx.
 inline void quotient32(Register lhs, Register rhs, Register dest,
                        Register tempEdx, bool isUnsigned)
     DEFINED_ON(x86_shared);

 // Perform an integer division, returning the remainder part.
 // rhs must not be zero, and the division must not overflow.
 //
 // On ARM, the chip must have hardware division instructions.
 inline void remainder32(Register lhs, Register rhs, Register dest,
                         bool isUnsigned)
     DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32);

 inline void remainder64(Register lhs, Register rhs, Register dest,
                         bool isUnsigned)
     DEFINED_ON(arm64, loong64, mips64, riscv64);

 // As above, but lhs and dest must be eax and tempEdx must be edx.
 inline void remainder32(Register lhs, Register rhs, Register dest,
                         Register tempEdx, bool isUnsigned)
     DEFINED_ON(x86_shared);

 // Perform an integer division, returning the integer part rounded toward
 // zero. rhs must not be zero, and the division must not overflow.
 //
 // This variant preserves registers, and doesn't require hardware division
 // instructions on ARM (will call out to a runtime routine).
 void flexibleRemainder32(
     Register lhs, Register rhs, Register dest, bool isUnsigned,
     const LiveRegisterSet& volatileLiveRegs) PER_SHARED_ARCH;
 void flexibleRemainderPtr(Register lhs, Register rhs, Register dest,
                           bool isUnsigned,
                           const LiveRegisterSet& volatileLiveRegs) PER_ARCH;

 // Perform an integer division, returning the integer part rounded toward
 // zero. rhs must not be zero, and the division must not overflow.
 //
 // This variant preserves registers, and doesn't require hardware division
 // instructions on ARM (will call out to a runtime routine).
 void flexibleQuotient32(
     Register lhs, Register rhs, Register dest, bool isUnsigned,
     const LiveRegisterSet& volatileLiveRegs) PER_SHARED_ARCH;
 void flexibleQuotientPtr(Register lhs, Register rhs, Register dest,
                          bool isUnsigned,
                          const LiveRegisterSet& volatileLiveRegs) PER_ARCH;

 // Perform an integer division, returning the integer part rounded toward
 // zero in the third argument register. rhs must not be zero, and the division
 // must not overflow. The remainder is stored into the fourth argument
 // register here.
 //
 // This variant preserves registers, and doesn't require hardware division
 // instructions on ARM (will call out to a runtime routine).
 //
 // lhs and rhs are preserved, divOutput and remOutput are clobbered.
 void flexibleDivMod32(
     Register lhs, Register rhs, Register divOutput, Register remOutput,
     bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) PER_SHARED_ARCH;

 inline void divFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
 inline void divDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 inline void inc64(AbsoluteAddress dest) PER_ARCH;

 inline void neg32(Register reg) PER_SHARED_ARCH;
 inline void neg64(Register64 reg) PER_ARCH;
 inline void negPtr(Register reg) PER_ARCH;

 inline void negateFloat(FloatRegister reg) PER_SHARED_ARCH;

 inline void negateDouble(FloatRegister reg) PER_SHARED_ARCH;

 inline void abs32(Register src, Register dest) PER_SHARED_ARCH;
 inline void absFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
 inline void absDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 inline void sqrtFloat32(FloatRegister src,
                         FloatRegister dest) PER_SHARED_ARCH;
 inline void sqrtDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;

 void floorFloat32ToInt32(FloatRegister src, Register dest,
                          Label* fail) PER_SHARED_ARCH;
 void floorDoubleToInt32(FloatRegister src, Register dest,
                         Label* fail) PER_SHARED_ARCH;

 void ceilFloat32ToInt32(FloatRegister src, Register dest,
                         Label* fail) PER_SHARED_ARCH;
 void ceilDoubleToInt32(FloatRegister src, Register dest,
                        Label* fail) PER_SHARED_ARCH;

 void roundFloat32ToInt32(FloatRegister src, Register dest, FloatRegister temp,
                          Label* fail) PER_SHARED_ARCH;
 void roundDoubleToInt32(FloatRegister src, Register dest, FloatRegister temp,
                         Label* fail) PER_SHARED_ARCH;

 void truncFloat32ToInt32(FloatRegister src, Register dest,
                          Label* fail) PER_SHARED_ARCH;
 void truncDoubleToInt32(FloatRegister src, Register dest,
                         Label* fail) PER_SHARED_ARCH;

 void nearbyIntDouble(RoundingMode mode, FloatRegister src,
                      FloatRegister dest) PER_SHARED_ARCH;
 void nearbyIntFloat32(RoundingMode mode, FloatRegister src,
                       FloatRegister dest) PER_SHARED_ARCH;

 void signInt32(Register input, Register output);
 void signDouble(FloatRegister input, FloatRegister output);
 void signDoubleToInt32(FloatRegister input, Register output,
                        FloatRegister temp, Label* fail);

 void copySignDouble(FloatRegister lhs, FloatRegister rhs,
                     FloatRegister output) PER_SHARED_ARCH;
 void copySignFloat32(FloatRegister lhs, FloatRegister rhs,
                      FloatRegister output) PER_SHARED_ARCH;

 // Returns a random double in range [0, 1) in |dest|. The |rng| register must
 // hold a pointer to a mozilla::non_crypto::XorShift128PlusRNG.
 void randomDouble(Register rng, FloatRegister dest, Register64 temp0,
                   Register64 temp1);

 inline void min32(Register lhs, Register rhs, Register result) PER_ARCH;
 inline void min32(Register lhs, Imm32 rhs, Register result) PER_ARCH;

 inline void max32(Register lhs, Register rhs, Register result) PER_ARCH;
 inline void max32(Register lhs, Imm32 rhs, Register result) PER_ARCH;

 inline void minPtr(Register lhs, Register rhs, Register result) PER_ARCH;
 inline void minPtr(Register lhs, ImmWord rhs, Register result) PER_ARCH;

 inline void maxPtr(Register lhs, Register rhs, Register result) PER_ARCH;
 inline void maxPtr(Register lhs, ImmWord rhs, Register result) PER_ARCH;

 // srcDest = {min,max}{Float32,Double}(srcDest, other)
 // For min and max, handle NaN specially if handleNaN is true.

 inline void minFloat32(FloatRegister other, FloatRegister srcDest,
                        bool handleNaN) PER_SHARED_ARCH;
 inline void minDouble(FloatRegister other, FloatRegister srcDest,
                       bool handleNaN) PER_SHARED_ARCH;

 inline void maxFloat32(FloatRegister other, FloatRegister srcDest,
                        bool handleNaN) PER_SHARED_ARCH;
 inline void maxDouble(FloatRegister other, FloatRegister srcDest,
                       bool handleNaN) PER_SHARED_ARCH;

 void minMaxArrayInt32(Register array, Register result, Register temp1,
                       Register temp2, Register temp3, bool isMax,
                       Label* fail);
 void minMaxArrayNumber(Register array, FloatRegister result,
                        FloatRegister floatTemp, Register temp1,
                        Register temp2, bool isMax, Label* fail);

 // Compute |pow(base, power)| and store the result in |dest|. If the result
 // exceeds the int32 range, jumps to |onOver|.
 // |base| and |power| are preserved, the other input registers are clobbered.
 void pow32(Register base, Register power, Register dest, Register temp1,
            Register temp2, Label* onOver);
 void powPtr(Register base, Register power, Register dest, Register temp1,
             Register temp2, Label* onOver);

 // Inline implementation of Math.round.
 void roundFloat32(FloatRegister src, FloatRegister dest);
 void roundDouble(FloatRegister src, FloatRegister dest);

 void sameValueDouble(FloatRegister left, FloatRegister right,
                      FloatRegister temp, Register dest);

 void loadRegExpLastIndex(Register regexp, Register string, Register lastIndex,
                          Label* notFoundZeroLastIndex);

 void loadAndClearRegExpSearcherLastLimit(Register result, Register scratch);

 void loadParsedRegExpShared(Register regexp, Register result,
                             Label* unparsed);

 // ===============================================================
 // Shift functions

 // For shift-by-register there may be platform-specific variations, for
 // example, x86 will perform the shift mod 32 but ARM will perform the shift
 // mod 256.
 //
 // For shift-by-immediate the platform assembler may restrict the immediate,
 // for example, the ARM assembler requires the count for 32-bit shifts to be
 // in the range [0,31].

 inline void lshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
 inline void lshift32(Imm32 shift, Register src,
                      Register dest) PER_SHARED_ARCH;
 inline void rshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
 inline void rshift32(Imm32 shift, Register src,
                      Register dest) PER_SHARED_ARCH;
 inline void rshift32Arithmetic(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
 inline void rshift32Arithmetic(Imm32 shift, Register src,
                                Register dest) PER_SHARED_ARCH;

 inline void lshiftPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void lshiftPtr(Imm32 imm, Register src, Register dest) PER_ARCH;
 inline void rshiftPtr(Imm32 imm, Register dest) PER_ARCH;
 inline void rshiftPtr(Imm32 imm, Register src, Register dest) PER_ARCH;
 inline void rshiftPtrArithmetic(Imm32 imm, Register dest) PER_ARCH;
 inline void rshiftPtrArithmetic(Imm32 imm, Register src,
                                 Register dest) PER_ARCH;

 inline void lshift64(Imm32 imm, Register64 dest) PER_ARCH;
 inline void rshift64(Imm32 imm, Register64 dest) PER_ARCH;
 inline void rshift64Arithmetic(Imm32 imm, Register64 dest) PER_ARCH;

 // On x86_shared these have the constraint that shift must be in CL.
 inline void lshift32(Register shift, Register srcDest) PER_SHARED_ARCH;
 inline void rshift32(Register shift, Register srcDest) PER_SHARED_ARCH;
 inline void rshift32Arithmetic(Register shift,
                                Register srcDest) PER_SHARED_ARCH;
 inline void lshiftPtr(Register shift, Register srcDest) PER_ARCH;
 inline void rshiftPtr(Register shift, Register srcDest) PER_ARCH;
 inline void rshiftPtrArithmetic(Register shift, Register srcDest) PER_ARCH;

 // These variants do not have the above constraint, but may emit some extra
 // instructions on x86_shared. They also handle shift >= 32 consistently by
 // masking with 0x1F (either explicitly or relying on the hardware to do
 // that).
 inline void flexibleLshift32(Register shift,
                              Register srcDest) PER_SHARED_ARCH;
 inline void flexibleRshift32(Register shift,
                              Register srcDest) PER_SHARED_ARCH;
 inline void flexibleRshift32Arithmetic(Register shift,
                                        Register srcDest) PER_SHARED_ARCH;
 inline void flexibleLshiftPtr(Register shift, Register srcDest) PER_ARCH;
 inline void flexibleRshiftPtr(Register shift, Register srcDest) PER_ARCH;
 inline void flexibleRshiftPtrArithmetic(Register shift,
                                         Register srcDest) PER_ARCH;

 inline void lshift64(Register shift, Register64 srcDest) PER_ARCH;
 inline void rshift64(Register shift, Register64 srcDest) PER_ARCH;
 inline void rshift64Arithmetic(Register shift, Register64 srcDest) PER_ARCH;

 // ===============================================================
 // Rotation functions
 // Note: - on x86 and x64 the count register must be in CL.
 //       - on x64 the temp register should be InvalidReg.

 inline void rotateLeft(Imm32 count, Register input,
                        Register dest) PER_SHARED_ARCH;
 inline void rotateLeft(Register count, Register input,
                        Register dest) PER_SHARED_ARCH;
 inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest)
     DEFINED_ON(x64);
 inline void rotateLeft64(Register count, Register64 input, Register64 dest)
     DEFINED_ON(x64);
 inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest,
                          Register temp) PER_ARCH;
 inline void rotateLeft64(Register count, Register64 input, Register64 dest,
                          Register temp) PER_ARCH;

 inline void rotateRight(Imm32 count, Register input,
                         Register dest) PER_SHARED_ARCH;
 inline void rotateRight(Register count, Register input,
                         Register dest) PER_SHARED_ARCH;
 inline void rotateRight64(Imm32 count, Register64 input, Register64 dest)
     DEFINED_ON(x64);
 inline void rotateRight64(Register count, Register64 input, Register64 dest)
     DEFINED_ON(x64);
 inline void rotateRight64(Imm32 count, Register64 input, Register64 dest,
                           Register temp) PER_ARCH;
 inline void rotateRight64(Register count, Register64 input, Register64 dest,
                           Register temp) PER_ARCH;

 // ===============================================================
 // Bit counting functions

 // knownNotZero may be true only if the src is known not to be zero.
 inline void clz32(Register src, Register dest,
                   bool knownNotZero) PER_SHARED_ARCH;
 inline void ctz32(Register src, Register dest,
                   bool knownNotZero) PER_SHARED_ARCH;

 inline void clz64(Register64 src, Register64 dest) PER_ARCH;
 inline void ctz64(Register64 src, Register64 dest) PER_ARCH;

 // On x86_shared, temp may be Invalid only if the chip has the POPCNT
 // instruction. On ARM, temp may never be Invalid.
 inline void popcnt32(Register src, Register dest,
                      Register temp) PER_SHARED_ARCH;

 // temp may be invalid only if the chip has the POPCNT instruction.
 inline void popcnt64(Register64 src, Register64 dest, Register temp) PER_ARCH;

 // ===============================================================
 // Condition functions

 inline void cmp8Set(Condition cond, Address lhs, Imm32 rhs,
                     Register dest) PER_SHARED_ARCH;

 inline void cmp16Set(Condition cond, Address lhs, Imm32 rhs,
                      Register dest) PER_SHARED_ARCH;

 template <typename T1, typename T2>
 inline void cmp32Set(Condition cond, T1 lhs, T2 rhs,
                      Register dest) PER_SHARED_ARCH;

 inline void cmp64Set(Condition cond, Register64 lhs, Register64 rhs,
                      Register dest) PER_ARCH;

 inline void cmp64Set(Condition cond, Register64 lhs, Imm64 rhs,
                      Register dest) PER_ARCH;

 inline void cmp64Set(Condition cond, Address lhs, Register64 rhs,
                      Register dest) PER_ARCH;

 inline void cmp64Set(Condition cond, Address lhs, Imm64 rhs,
                      Register dest) PER_ARCH;

 template <typename T1, typename T2>
 inline void cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) PER_ARCH;

 // ===============================================================
 // Branch functions

 inline void branch8(Condition cond, const Address& lhs, Imm32 rhs,
                     Label* label) PER_SHARED_ARCH;

 // Compares the byte in |lhs| against |rhs| using a 8-bit comparison on
 // x86/x64 or a 32-bit comparison (all other platforms). The caller should
 // ensure |rhs| is a zero- resp. sign-extended byte value for cross-platform
 // compatible code.
 inline void branch8(Condition cond, const BaseIndex& lhs, Register rhs,
                     Label* label) PER_SHARED_ARCH;

 inline void branch16(Condition cond, const Address& lhs, Imm32 rhs,
                      Label* label) PER_SHARED_ARCH;

 inline void branch32(Condition cond, Register lhs, Register rhs,
                      Label* label) PER_SHARED_ARCH;
 inline void branch32(Condition cond, Register lhs, Imm32 rhs,
                      Label* label) PER_SHARED_ARCH;

 inline void branch32(Condition cond, Register lhs, const Address& rhs,
                      Label* label) DEFINED_ON(arm64);

 inline void branch32(Condition cond, const Address& lhs, Register rhs,
                      Label* label) PER_SHARED_ARCH;
 inline void branch32(Condition cond, const Address& lhs, Imm32 rhs,
                      Label* label) PER_SHARED_ARCH;

 inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs,
                      Label* label) PER_ARCH;
 inline void branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs,
                      Label* label) PER_ARCH;

 inline void branch32(Condition cond, const BaseIndex& lhs, Register rhs,
                      Label* label) DEFINED_ON(arm, x86_shared);
 inline void branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
                      Label* label) PER_SHARED_ARCH;

 inline void branch32(Condition cond, const Operand& lhs, Register rhs,
                      Label* label) DEFINED_ON(x86_shared);
 inline void branch32(Condition cond, const Operand& lhs, Imm32 rhs,
                      Label* label) DEFINED_ON(x86_shared);

 inline void branch32(Condition cond, wasm::SymbolicAddress lhs, Imm32 rhs,
                      Label* label) PER_ARCH;

 // The supported condition are Equal, NotEqual, LessThan(orEqual),
 // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label
 // is not defined it will fall through to next instruction, else jump to the
 // fail label.
 inline void branch64(Condition cond, Register64 lhs, Imm64 val,
                      Label* success, Label* fail = nullptr) PER_ARCH;
 inline void branch64(Condition cond, Register64 lhs, Register64 rhs,
                      Label* success, Label* fail = nullptr) PER_ARCH;
 inline void branch64(Condition cond, const Address& lhs, Imm64 val,
                      Label* success, Label* fail = nullptr) PER_ARCH;
 inline void branch64(Condition cond, const Address& lhs, Register64 rhs,
                      Label* success, Label* fail = nullptr) PER_ARCH;

 // Compare the value at |lhs| with the value at |rhs|.  The scratch
 // register *must not* be the base of |lhs| or |rhs|.
 // Only the NotEqual and Equal conditions are allowed.
 inline void branch64(Condition cond, const Address& lhs, const Address& rhs,
                      Register scratch, Label* label) PER_ARCH;

 inline void branchPtr(Condition cond, Register lhs, Register rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, Register lhs, Imm32 rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, Register lhs, ImmPtr rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, Register lhs, ImmWord rhs,
                       Label* label) PER_SHARED_ARCH;

 inline void branchPtr(Condition cond, const Address& lhs, Register rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
                       Label* label) PER_SHARED_ARCH;

 inline void branchPtr(Condition cond, const BaseIndex& lhs, ImmWord rhs,
                       Label* label) PER_SHARED_ARCH;
 inline void branchPtr(Condition cond, const BaseIndex& lhs, Register rhs,
                       Label* label) PER_SHARED_ARCH;

 inline void branchPtr(Condition cond, const AbsoluteAddress& lhs,
                       Register rhs, Label* label) PER_ARCH;
 inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs,
                       Label* label) PER_ARCH;

 inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs,
                       Label* label) PER_ARCH;

 // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its
 // chunk header, or nullptr if it is in the tenured heap.
 void loadStoreBuffer(Register ptr, Register buffer) PER_ARCH;

 void branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
                              Label* label) PER_ARCH;
 void branchPtrInNurseryChunk(Condition cond, const Address& address,
                              Register temp, Label* label) DEFINED_ON(x86);
 void branchValueIsNurseryCell(Condition cond, const Address& address,
                               Register temp, Label* label) PER_ARCH;
 void branchValueIsNurseryCell(Condition cond, ValueOperand value,
                               Register temp, Label* label) PER_ARCH;

 // This function compares a Value (lhs) which is having a private pointer
 // boxed inside a js::Value, with a raw pointer (rhs).
 inline void branchPrivatePtr(Condition cond, const Address& lhs, Register rhs,
                              Label* label) PER_ARCH;

 inline void branchFloat(DoubleCondition cond, FloatRegister lhs,
                         FloatRegister rhs, Label* label) PER_SHARED_ARCH;

 // Truncate a double/float32 to int32 and when it doesn't fit an int32 it will
 // jump to the failure label. This particular variant is allowed to return the
 // value module 2**32, which isn't implemented on all architectures.
 inline void branchTruncateFloat32MaybeModUint32(FloatRegister src,
                                                 Register dest,
                                                 Label* fail) PER_ARCH;
 inline void branchTruncateDoubleMaybeModUint32(FloatRegister src,
                                                Register dest,
                                                Label* fail) PER_ARCH;

 // Truncate a double/float32 to intptr and when it doesn't fit jump to the
 // failure label.
 inline void branchTruncateFloat32ToPtr(FloatRegister src, Register dest,
                                        Label* fail) DEFINED_ON(x86, x64);
 inline void branchTruncateDoubleToPtr(FloatRegister src, Register dest,
                                       Label* fail) DEFINED_ON(x86, x64);

 // Truncate a double/float32 to int32 and when it doesn't fit jump to the
 // failure label.
 inline void branchTruncateFloat32ToInt32(FloatRegister src, Register dest,
                                          Label* fail) PER_ARCH;
 inline void branchTruncateDoubleToInt32(FloatRegister src, Register dest,
                                         Label* fail) PER_ARCH;

 inline void branchDouble(DoubleCondition cond, FloatRegister lhs,
                          FloatRegister rhs, Label* label) PER_SHARED_ARCH;

 inline void branchDoubleNotInInt64Range(Address src, Register temp,
                                         Label* fail);
 inline void branchDoubleNotInUInt64Range(Address src, Register temp,
                                          Label* fail);
 inline void branchFloat32NotInInt64Range(Address src, Register temp,
                                          Label* fail);
 inline void branchFloat32NotInUInt64Range(Address src, Register temp,
                                           Label* fail);

 // Branch if the (un)signed int64 is outside the range of a signed intptr.
 inline void branchInt64NotInPtrRange(Register64 src, Label* label) PER_ARCH;
 inline void branchUInt64NotInPtrRange(Register64 src, Label* label) PER_ARCH;

 template <typename T>
 inline void branchAdd32(Condition cond, T src, Register dest,
                         Label* label) PER_SHARED_ARCH;
 template <typename T>
 inline void branchSub32(Condition cond, T src, Register dest,
                         Label* label) PER_SHARED_ARCH;
 template <typename T>
 inline void branchMul32(Condition cond, T src, Register dest,
                         Label* label) PER_SHARED_ARCH;
 template <typename T>
 inline void branchRshift32(Condition cond, T src, Register dest,
                            Label* label) PER_SHARED_ARCH;

 inline void branchNeg32(Condition cond, Register reg,
                         Label* label) PER_SHARED_ARCH;

 inline void branchAdd64(Condition cond, Imm64 imm, Register64 dest,
                         Label* label) DEFINED_ON(x86, arm, wasm32);

 template <typename T>
 inline void branchAddPtr(Condition cond, T src, Register dest,
                          Label* label) PER_SHARED_ARCH;

 template <typename T>
 inline void branchSubPtr(Condition cond, T src, Register dest,
                          Label* label) PER_SHARED_ARCH;

 inline void branchMulPtr(Condition cond, Register src, Register dest,
                          Label* label) PER_SHARED_ARCH;

 inline void branchNegPtr(Condition cond, Register reg,
                          Label* label) PER_SHARED_ARCH;

 inline void decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
                          Label* label) PER_SHARED_ARCH;

 inline void branchTest32(Condition cond, Register lhs, Register rhs,
                          Label* label) PER_SHARED_ARCH;
 inline void branchTest32(Condition cond, Register lhs, Imm32 rhs,
                          Label* label) PER_SHARED_ARCH;
 inline void branchTest32(Condition cond, const Address& lhs, Imm32 rhh,
                          Label* label) PER_SHARED_ARCH;
 inline void branchTest32(Condition cond, const AbsoluteAddress& lhs,
                          Imm32 rhs, Label* label) PER_ARCH;

 inline void branchTestPtr(Condition cond, Register lhs, Register rhs,
                           Label* label) PER_SHARED_ARCH;
 inline void branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
                           Label* label) PER_SHARED_ARCH;
 inline void branchTestPtr(Condition cond, Register lhs, ImmWord rhs,
                           Label* label) PER_ARCH;
 inline void branchTestPtr(Condition cond, const Address& lhs, Imm32 rhs,
                           Label* label) PER_SHARED_ARCH;

 // When a fail label is not defined it will fall through to next instruction,
 // else jump to the fail label.
 //
 // On x86 if |lhs == rhs|, |temp| is used to generate a single branch
 // instruction. Otherwise |temp| is unused and can be |InvalidReg|.
 inline void branchTest64(Condition cond, Register64 lhs, Register64 rhs,
                          Register temp, Label* success,
                          Label* fail = nullptr) PER_ARCH;
 inline void branchTest64(Condition cond, Register64 lhs, Register64 rhs,
                          Label* success, Label* fail = nullptr);
 inline void branchTest64(Condition cond, Register64 lhs, Imm64 rhs,
                          Label* success, Label* fail = nullptr) PER_ARCH;

 // Branches to |label| if |reg| is false. |reg| should be a C++ bool.
 inline void branchIfFalseBool(Register reg, Label* label);

 // Branches to |label| if |reg| is true. |reg| should be a C++ bool.
 inline void branchIfTrueBool(Register reg, Label* label);

 inline void branchIfNotNullOrUndefined(ValueOperand val, Label* label);

 inline void branchIfRope(Register str, Label* label);
 inline void branchIfNotRope(Register str, Label* label);

 inline void branchLatin1String(Register string, Label* label);
 inline void branchTwoByteString(Register string, Label* label);

 inline void branchIfBigIntIsNegative(Register bigInt, Label* label);
 inline void branchIfBigIntIsNonNegative(Register bigInt, Label* label);
 inline void branchIfBigIntIsZero(Register bigInt, Label* label);
 inline void branchIfBigIntIsNonZero(Register bigInt, Label* label);

 inline void branchTestFunctionFlags(Register fun, uint32_t flags,
                                     Condition cond, Label* label);

 inline void branchIfNotFunctionIsNonBuiltinCtor(Register fun,
                                                 Register scratch,
                                                 Label* label);

 inline void branchIfFunctionHasNoJitEntry(Register fun, Label* label);
 inline void branchIfFunctionHasJitEntry(Register fun, Label* label);

 inline void branchIfScriptHasJitScript(Register script, Label* label);
 inline void branchIfScriptHasNoJitScript(Register script, Label* label);
 inline void loadJitScript(Register script, Register dest);

 // Loads the function's argument count.
 inline void loadFunctionArgCount(Register func, Register output);

 // Loads the function length. This handles interpreted, native, and bound
 // functions. The caller is responsible for checking that INTERPRETED_LAZY and
 // RESOLVED_LENGTH flags are not set.
 void loadFunctionLength(Register func, Register funFlagsAndArgCount,
                         Register output, Label* slowPath);

 // Loads the function name. This handles interpreted, native, and bound
 // functions.
 void loadFunctionName(Register func, Register output, ImmGCPtr emptyString,
                       Label* slowPath);

 void assertFunctionIsExtended(Register func);

 inline void branchFunctionKind(Condition cond,
                                FunctionFlags::FunctionKind kind, Register fun,
                                Register scratch, Label* label);

 inline void branchIfObjectEmulatesUndefined(Register objReg, Register scratch,
                                             Label* slowCheck, Label* label);

 // For all methods below: spectreRegToZero is a register that will be zeroed
 // on speculatively executed code paths (when the branch should be taken but
 // branch prediction speculates it isn't). Usually this will be the object
 // register but the caller may pass a different register.

 inline void branchTestObjClass(Condition cond, Register obj,
                                const JSClass* clasp, Register scratch,
                                Register spectreRegToZero, Label* label);
 inline void branchTestObjClassNoSpectreMitigations(Condition cond,
                                                    Register obj,
                                                    const JSClass* clasp,
                                                    Register scratch,
                                                    Label* label);

 inline void branchTestObjClass(Condition cond, Register obj,
                                const Address& clasp, Register scratch,
                                Register spectreRegToZero, Label* label);
 inline void branchTestObjClassNoSpectreMitigations(Condition cond,
                                                    Register obj,
                                                    const Address& clasp,
                                                    Register scratch,
                                                    Label* label);

 inline void branchTestObjClass(Condition cond, Register obj, Register clasp,
                                Register scratch, Register spectreRegToZero,
                                Label* label);

 inline void branchTestObjShape(Condition cond, Register obj,
                                const Shape* shape, Register scratch,
                                Register spectreRegToZero, Label* label);
 inline void branchTestObjShapeNoSpectreMitigations(Condition cond,
                                                    Register obj,
                                                    const Shape* shape,
                                                    Label* label);

private:
 void branchTestObjShapeListImpl(Register obj, Register shapeElements,
                                 size_t itemSize, Register shapeScratch,
                                 Register endScratch, Register spectreScratch,
                                 Label* fail);

public:
 void branchTestObjShapeList(Register obj, Register shapeElements,
                             Register shapeScratch, Register endScratch,
                             Register spectreScratch, Label* fail);

 void branchTestObjShapeListSetOffset(Register obj, Register shapeElements,
                                      Register offset, Register shapeScratch,
                                      Register endScratch,
                                      Register spectreScratch, Label* fail);

 inline void branchTestClassIsFunction(Condition cond, Register clasp,
                                       Label* label);
 inline void branchTestObjIsFunction(Condition cond, Register obj,
                                     Register scratch,
                                     Register spectreRegToZero, Label* label);
 inline void branchTestObjIsFunctionNoSpectreMitigations(Condition cond,
                                                         Register obj,
                                                         Register scratch,
                                                         Label* label);

 inline void branchTestObjShape(Condition cond, Register obj, Register shape,
                                Register scratch, Register spectreRegToZero,
                                Label* label);
 inline void branchTestObjShapeNoSpectreMitigations(Condition cond,
                                                    Register obj,
                                                    Register shape,
                                                    Label* label);

 // TODO: audit/fix callers to be Spectre safe.
 inline void branchTestObjShapeUnsafe(Condition cond, Register obj,
                                      Register shape, Label* label);

 void branchTestObjCompartment(Condition cond, Register obj,
                               const Address& compartment, Register scratch,
                               Label* label);
 void branchTestObjCompartment(Condition cond, Register obj,
                               const JS::Compartment* compartment,
                               Register scratch, Label* label);

 void branchIfNonNativeObj(Register obj, Register scratch, Label* label);

 void branchIfObjectNotExtensible(Register obj, Register scratch,
                                  Label* label);

 void branchTestObjectNeedsProxyResultValidation(Condition condition,
                                                 Register obj,
                                                 Register scratch,
                                                 Label* label);

 inline void branchTestClassIsProxy(bool proxy, Register clasp, Label* label);

 inline void branchTestObjectIsProxy(bool proxy, Register object,
                                     Register scratch, Label* label);

 inline void branchTestProxyHandlerFamily(Condition cond, Register proxy,
                                          Register scratch,
                                          const void* handlerp, Label* label);

 inline void branchTestNeedsIncrementalBarrier(Condition cond, Label* label);
 inline void branchTestNeedsIncrementalBarrierAnyZone(Condition cond,
                                                      Label* label,
                                                      Register scratch);

 // Perform a type-test on a tag of a Value (32bits boxing), or the tagged
 // value (64bits boxing).
 inline void branchTestUndefined(Condition cond, Register tag,
                                 Label* label) PER_SHARED_ARCH;
 inline void branchTestInt32(Condition cond, Register tag,
                             Label* label) PER_SHARED_ARCH;
 inline void branchTestDouble(Condition cond, Register tag,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestNumber(Condition cond, Register tag,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestBoolean(Condition cond, Register tag,
                               Label* label) PER_SHARED_ARCH;
 inline void branchTestString(Condition cond, Register tag,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestSymbol(Condition cond, Register tag,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestBigInt(Condition cond, Register tag,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestNull(Condition cond, Register tag,
                            Label* label) PER_SHARED_ARCH;
 inline void branchTestObject(Condition cond, Register tag,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestPrimitive(Condition cond, Register tag,
                                 Label* label) PER_SHARED_ARCH;
 inline void branchTestMagic(Condition cond, Register tag,
                             Label* label) PER_SHARED_ARCH;
 void branchTestType(Condition cond, Register tag, JSValueType type,
                     Label* label);

 // Perform a type-test on a Value, addressed by Address or BaseIndex, or
 // loaded into ValueOperand.
 // BaseIndex and ValueOperand variants clobber the ScratchReg on x64.
 // All Variants clobber the ScratchReg on arm64.
 inline void branchTestUndefined(Condition cond, const Address& address,
                                 Label* label) PER_SHARED_ARCH;
 inline void branchTestUndefined(Condition cond, const BaseIndex& address,
                                 Label* label) PER_SHARED_ARCH;
 inline void branchTestUndefined(Condition cond, const ValueOperand& value,
                                 Label* label) PER_SHARED_ARCH;

 inline void branchTestInt32(Condition cond, const Address& address,
                             Label* label) PER_SHARED_ARCH;
 inline void branchTestInt32(Condition cond, const BaseIndex& address,
                             Label* label) PER_SHARED_ARCH;
 inline void branchTestInt32(Condition cond, const ValueOperand& value,
                             Label* label) PER_SHARED_ARCH;

 inline void branchTestDouble(Condition cond, const Address& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestDouble(Condition cond, const BaseIndex& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestDouble(Condition cond, const ValueOperand& value,
                              Label* label) PER_SHARED_ARCH;

 inline void branchTestNumber(Condition cond, const ValueOperand& value,
                              Label* label) PER_SHARED_ARCH;

 inline void branchTestBoolean(Condition cond, const Address& address,
                               Label* label) PER_SHARED_ARCH;
 inline void branchTestBoolean(Condition cond, const BaseIndex& address,
                               Label* label) PER_SHARED_ARCH;
 inline void branchTestBoolean(Condition cond, const ValueOperand& value,
                               Label* label) PER_SHARED_ARCH;

 inline void branchTestString(Condition cond, const Address& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestString(Condition cond, const BaseIndex& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestString(Condition cond, const ValueOperand& value,
                              Label* label) PER_SHARED_ARCH;

 inline void branchTestSymbol(Condition cond, const Address& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestSymbol(Condition cond, const BaseIndex& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestSymbol(Condition cond, const ValueOperand& value,
                              Label* label) PER_SHARED_ARCH;

 inline void branchTestBigInt(Condition cond, const Address& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestBigInt(Condition cond, const BaseIndex& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestBigInt(Condition cond, const ValueOperand& value,
                              Label* label) PER_SHARED_ARCH;

 inline void branchTestNull(Condition cond, const Address& address,
                            Label* label) PER_SHARED_ARCH;
 inline void branchTestNull(Condition cond, const BaseIndex& address,
                            Label* label) PER_SHARED_ARCH;
 inline void branchTestNull(Condition cond, const ValueOperand& value,
                            Label* label) PER_SHARED_ARCH;

 // Clobbers the ScratchReg on x64.
 inline void branchTestObject(Condition cond, const Address& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestObject(Condition cond, const BaseIndex& address,
                              Label* label) PER_SHARED_ARCH;
 inline void branchTestObject(Condition cond, const ValueOperand& value,
                              Label* label) PER_SHARED_ARCH;

 inline void branchTestGCThing(Condition cond, const Address& address,
                               Label* label) PER_SHARED_ARCH;
 inline void branchTestGCThing(Condition cond, const BaseIndex& address,
                               Label* label) PER_SHARED_ARCH;
 inline void branchTestGCThing(Condition cond, const ValueOperand& value,
                               Label* label) PER_SHARED_ARCH;

 inline void branchTestPrimitive(Condition cond, const ValueOperand& value,
                                 Label* label) PER_SHARED_ARCH;

 inline void branchTestMagic(Condition cond, const Address& address,
                             Label* label) PER_SHARED_ARCH;
 inline void branchTestMagic(Condition cond, const BaseIndex& address,
                             Label* label) PER_SHARED_ARCH;
 inline void branchTestMagic(Condition cond, const ValueOperand& value,
                             Label* label) PER_SHARED_ARCH;

 inline void branchTestMagic(Condition cond, const Address& valaddr,
                             JSWhyMagic why, Label* label) PER_ARCH;

 inline void branchTestMagicValue(Condition cond, const ValueOperand& val,
                                  JSWhyMagic why, Label* label);

 void branchTestValue(Condition cond, const ValueOperand& lhs,
                      const Value& rhs, Label* label) PER_ARCH;
 void branchTestNaNValue(Condition cond, const ValueOperand& val,
                         Register temp, Label* label) PER_ARCH;

 template <typename T>
 inline void branchTestValue(Condition cond, const T& lhs,
                             const ValueOperand& rhs, Label* label) PER_ARCH;

 // Checks if given Value is evaluated to true or false in a condition.
 // The type of the value should match the type of the method.
 inline void branchTestInt32Truthy(bool truthy, const ValueOperand& value,
                                   Label* label) PER_SHARED_ARCH;
 inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg,
                                    Label* label) PER_SHARED_ARCH;
 inline void branchTestBooleanTruthy(bool truthy, const ValueOperand& value,
                                     Label* label) PER_ARCH;
 inline void branchTestStringTruthy(bool truthy, const ValueOperand& value,
                                    Label* label) PER_SHARED_ARCH;
 inline void branchTestBigIntTruthy(bool truthy, const ValueOperand& value,
                                    Label* label) PER_SHARED_ARCH;

 // Create an unconditional branch to the address given as argument.
 inline void branchToComputedAddress(const BaseIndex& address) PER_ARCH;

 // Subtract a constant in the range 1 .. 127 inclusive from the value stored
 // at `address`, write the result back to `address`, and jump to `label` if
 // the updated value is negative.  The subtract is a 32-bit operation even
 // though the value to be subtracted must fit in 7 bits.
 CodeOffset sub32FromMemAndBranchIfNegativeWithPatch(
     Address address, Label* label) PER_SHARED_ARCH;

 // Patch in the value to be subtracted.  Must be 1 .. 127 inclusive.
 void patchSub32FromMemAndBranchIfNegative(CodeOffset offset,
                                           Imm32 imm) PER_SHARED_ARCH;

private:
 template <typename T, typename S>
 inline void branchPtrImpl(Condition cond, const T& lhs, const S& rhs,
                           Label* label) DEFINED_ON(x86_shared);

 void branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label* label)
     DEFINED_ON(x86);
 template <typename T>
 void branchValueIsNurseryCellImpl(Condition cond, const T& value,
                                   Register temp, Label* label)
     DEFINED_ON(arm64, x64, mips64, loong64, riscv64);

 template <typename T>
 inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestInt32Impl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestDoubleImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestNumberImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestBooleanImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestStringImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestSymbolImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestBigIntImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestNullImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestObjectImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestGCThingImpl(Condition cond, const T& t,
                                   Label* label) PER_SHARED_ARCH;
 template <typename T>
 inline void branchTestPrimitiveImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);
 template <typename T>
 inline void branchTestMagicImpl(Condition cond, const T& t, Label* label)
     DEFINED_ON(arm, arm64, x86_shared);

public:
 template <typename T>
 inline void testNumberSet(Condition cond, const T& src,
                           Register dest) PER_SHARED_ARCH;
 template <typename T>
 inline void testBooleanSet(Condition cond, const T& src,
                            Register dest) PER_SHARED_ARCH;
 template <typename T>
 inline void testStringSet(Condition cond, const T& src,
                           Register dest) PER_SHARED_ARCH;
 template <typename T>
 inline void testSymbolSet(Condition cond, const T& src,
                           Register dest) PER_SHARED_ARCH;
 template <typename T>
 inline void testBigIntSet(Condition cond, const T& src,
                           Register dest) PER_SHARED_ARCH;

public:
 // The fallibleUnbox* methods below combine a Value type check with an unbox.
 // Especially on 64-bit platforms this can be implemented more efficiently
 // than a separate branch + unbox.
 //
 // |src| and |dest| can be the same register, but |dest| may hold garbage on
 // failure.
 inline void fallibleUnboxPtr(const ValueOperand& src, Register dest,
                              JSValueType type, Label* fail) PER_ARCH;
 inline void fallibleUnboxPtr(const Address& src, Register dest,
                              JSValueType type, Label* fail) PER_ARCH;
 inline void fallibleUnboxPtr(const BaseIndex& src, Register dest,
                              JSValueType type, Label* fail) PER_ARCH;
 template <typename T>
 inline void fallibleUnboxInt32(const T& src, Register dest, Label* fail);
 template <typename T>
 inline void fallibleUnboxBoolean(const T& src, Register dest, Label* fail);
 template <typename T>
 inline void fallibleUnboxObject(const T& src, Register dest, Label* fail);
 template <typename T>
 inline void fallibleUnboxString(const T& src, Register dest, Label* fail);
 template <typename T>
 inline void fallibleUnboxSymbol(const T& src, Register dest, Label* fail);
 template <typename T>
 inline void fallibleUnboxBigInt(const T& src, Register dest, Label* fail);

 inline void cmp32Move32(Condition cond, Register lhs, Imm32 rhs, Register src,
                         Register dest) PER_SHARED_ARCH;

 inline void cmp32Move32(Condition cond, Register lhs, Register rhs,
                         Register src, Register dest) PER_SHARED_ARCH;

 inline void cmp32Move32(Condition cond, Register lhs, const Address& rhs,
                         Register src, Register dest) PER_SHARED_ARCH;

 inline void cmpPtrMovePtr(Condition cond, Register lhs, Imm32 rhs,
                           Register src, Register dest) PER_ARCH;

 inline void cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
                           Register src, Register dest) PER_ARCH;

 inline void cmpPtrMovePtr(Condition cond, Register lhs, const Address& rhs,
                           Register src, Register dest) PER_ARCH;

 inline void cmp32Load32(Condition cond, Register lhs, const Address& rhs,
                         const Address& src, Register dest) PER_SHARED_ARCH;

 inline void cmp32Load32(Condition cond, Register lhs, Register rhs,
                         const Address& src, Register dest) PER_SHARED_ARCH;

 inline void cmp32Load32(Condition cond, Register lhs, Imm32 rhs,
                         const Address& src, Register dest) PER_SHARED_ARCH;

 inline void cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
                          const Address& src, Register dest) PER_ARCH;

 inline void cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
                          Register src, Register dest) PER_ARCH;

 inline void test32LoadPtr(Condition cond, const Address& addr, Imm32 mask,
                           const Address& src, Register dest) PER_ARCH;

 inline void test32MovePtr(Condition cond, Register operand, Imm32 mask,
                           Register src, Register dest) PER_ARCH;

 inline void test32MovePtr(Condition cond, const Address& addr, Imm32 mask,
                           Register src, Register dest) PER_ARCH;

 // Conditional move for Spectre mitigations.
 inline void spectreMovePtr(Condition cond, Register src,
                            Register dest) PER_ARCH;

 // Zeroes dest if the condition is true.
 inline void spectreZeroRegister(Condition cond, Register scratch,
                                 Register dest) PER_SHARED_ARCH;

 // Performs a bounds check and zeroes the index register if out-of-bounds
 // (to mitigate Spectre).
private:
 inline void spectreBoundsCheck32(Register index, const Operand& length,
                                  Register maybeScratch, Label* failure)
     DEFINED_ON(x86);

public:
 inline void spectreBoundsCheck32(Register index, Register length,
                                  Register maybeScratch,
                                  Label* failure) PER_ARCH;
 inline void spectreBoundsCheck32(Register index, const Address& length,
                                  Register maybeScratch,
                                  Label* failure) PER_ARCH;

 inline void spectreBoundsCheckPtr(Register index, Register length,
                                   Register maybeScratch,
                                   Label* failure) PER_ARCH;
 inline void spectreBoundsCheckPtr(Register index, const Address& length,
                                   Register maybeScratch,
                                   Label* failure) PER_ARCH;

 // ========================================================================
 // Canonicalization primitives.
 inline void canonicalizeDouble(FloatRegister reg);

 inline void canonicalizeFloat(FloatRegister reg);

public:
 // ========================================================================
 // Memory access primitives.
 inline FaultingCodeOffset storeDouble(FloatRegister src,
                                       const Address& dest) PER_SHARED_ARCH;
 inline FaultingCodeOffset storeDouble(FloatRegister src,
                                       const BaseIndex& dest) PER_SHARED_ARCH;
 inline FaultingCodeOffset storeDouble(FloatRegister src, const Operand& dest)
     DEFINED_ON(x86_shared);

 template <class T>
 inline void boxDouble(FloatRegister src, const T& dest);

 using MacroAssemblerSpecific::boxDouble;

 inline FaultingCodeOffset storeFloat32(FloatRegister src,
                                        const Address& dest) PER_SHARED_ARCH;
 inline FaultingCodeOffset storeFloat32(FloatRegister src,
                                        const BaseIndex& dest) PER_SHARED_ARCH;
 inline FaultingCodeOffset storeFloat32(FloatRegister src, const Operand& dest)
     DEFINED_ON(x86_shared);

 inline FaultingCodeOffset storeFloat16(FloatRegister src, const Address& dest,
                                        Register scratch) PER_SHARED_ARCH;
 inline FaultingCodeOffset storeFloat16(FloatRegister src,
                                        const BaseIndex& dest,
                                        Register scratch) PER_SHARED_ARCH;

 template <typename T>
 void storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType,
                        const T& dest) PER_ARCH;

 inline void memoryBarrier(MemoryBarrier barrier) PER_SHARED_ARCH;

public:
 // ========================================================================
 // Wasm SIMD
 //
 // Naming is "operationSimd128" when operate on the whole vector, otherwise
 // it's "operation<Type><Size>x<Lanes>".
 //
 // For microarchitectural reasons we can in principle get a performance win by
 // using int or float specific instructions in the operationSimd128 case when
 // we know that subsequent operations on the result are int or float oriented.
 // In practice, we don't care about that yet.
 //
 // The order of operations here follows those in the SIMD overview document,
 // https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md.
 //
 // Since we must target Intel SSE indefinitely and SSE is one-address or
 // two-address, the x86 porting interfaces are nearly all one-address or
 // two-address.  Likewise there are two-address ARM64 interfaces to support
 // the baseline compiler.  But there are also three-address ARM64 interfaces
 // as the ARM64 Ion back-end can use those.  In the future, they may support
 // AVX2 or similar for x86.
 //
 // Conventions for argument order and naming and semantics:
 //  - Condition codes come first.
 //  - Other immediates (masks, shift counts) come next.
 //  - Operands come next:
 //    - For a binary two-address operator where the left-hand-side has the
 //      same type as the result, one register parameter is normally named
 //      `lhsDest` and is both the left-hand side and destination; the other
 //      parameter is named `rhs` and is the right-hand side.  `rhs` comes
 //      first, `lhsDest` second.  `rhs` and `lhsDest` may be the same register
 //      (if rhs is a register).
 //    - For a binary three-address operator the order is `lhs`, `rhs`, `dest`,
 //      and generally these registers may be the same.
 //    - For a unary operator, the input is named `src` and the output is named
 //      `dest`.  `src` comes first, `dest` second.  `src` and `dest` may be
 //      the same register (if `src` is a register).
 //  - Temp registers follow operands and are named `temp` if there's only one,
 //    otherwise `temp1`, `temp2`, etc regardless of type.  GPR temps precede
 //    FPU temps.  If there are several temps then they must be distinct
 //    registers, and they must be distinct from the operand registers unless
 //    noted.

 // Moves

 inline void moveSimd128(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Constants

 inline void loadConstantSimd128(const SimdConstant& v, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Splat

 inline void splatX16(Register src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void splatX16(uint32_t srcLane, FloatRegister src, FloatRegister dest)
     DEFINED_ON(arm64);

 inline void splatX8(Register src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void splatX8(uint32_t srcLane, FloatRegister src, FloatRegister dest)
     DEFINED_ON(arm64);

 inline void splatX4(Register src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void splatX4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void splatX2(Register64 src, FloatRegister dest)
     DEFINED_ON(x86, x64, arm64);

 inline void splatX2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Extract lane as scalar.  Float extraction does not canonicalize the value.

 inline void extractLaneInt8x16(uint32_t lane, FloatRegister src,
                                Register dest) DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtractLaneInt8x16(uint32_t lane, FloatRegister src,
                                        Register dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extractLaneInt16x8(uint32_t lane, FloatRegister src,
                                Register dest) DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtractLaneInt16x8(uint32_t lane, FloatRegister src,
                                        Register dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extractLaneInt32x4(uint32_t lane, FloatRegister src,
                                Register dest) DEFINED_ON(x86_shared, arm64);

 inline void extractLaneInt64x2(uint32_t lane, FloatRegister src,
                                Register64 dest) DEFINED_ON(x86, x64, arm64);

 inline void extractLaneFloat32x4(uint32_t lane, FloatRegister src,
                                  FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extractLaneFloat64x2(uint32_t lane, FloatRegister src,
                                  FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Replace lane value

 inline void replaceLaneInt8x16(unsigned lane, FloatRegister lhs, Register rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void replaceLaneInt8x16(unsigned lane, Register rhs,
                                FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void replaceLaneInt16x8(unsigned lane, FloatRegister lhs, Register rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void replaceLaneInt16x8(unsigned lane, Register rhs,
                                FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void replaceLaneInt32x4(unsigned lane, FloatRegister lhs, Register rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void replaceLaneInt32x4(unsigned lane, Register rhs,
                                FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
                                Register64 rhs, FloatRegister dest)
     DEFINED_ON(x86, x64);

 inline void replaceLaneInt64x2(unsigned lane, Register64 rhs,
                                FloatRegister lhsDest)
     DEFINED_ON(x86, x64, arm64);

 inline void replaceLaneFloat32x4(unsigned lane, FloatRegister lhs,
                                  FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
                                  FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void replaceLaneFloat64x2(unsigned lane, FloatRegister lhs,
                                  FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
                                  FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 // Shuffle - blend and permute with immediate indices, and its many
 // specializations.  Lane values other than those mentioned are illegal.

 // lane values 0..31
 inline void shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
                            FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
                            FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Lane values must be 0 (select from lhs) or FF (select from rhs).
 // The behavior is undefined for lane values that are neither 0 nor FF.
 // on x86_shared: it is required that lhs == dest.
 inline void blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
                          FloatRegister rhs, FloatRegister dest,
                          FloatRegister temp) DEFINED_ON(x86_shared);

 // Lane values must be 0 (select from lhs) or FF (select from rhs).
 // The behavior is undefined for lane values that are neither 0 nor FF.
 inline void blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
                          FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(arm64);

 // Lane values must be 0 (select from lhs) or FFFF (select from rhs).
 // The behavior is undefined for lane values that are neither 0 nor FFFF.
 // on x86_shared: it is required that lhs == dest.
 inline void blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
                          FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Mask lane values must be ~0 or 0. The former selects from lhs and the
 // latter from rhs.
 // The implementation works effectively for I8x16, I16x8, I32x4, and I64x2.
 inline void laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
                               FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
                                  FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
                                  FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
                                  FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
                                  FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Permute - permute with immediate indices.

 // lane values 0..15
 inline void permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
                            FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 // lane values 0..7
 inline void permuteInt16x8(const uint16_t lanes[8], FloatRegister src,
                            FloatRegister dest) DEFINED_ON(arm64);

 // lane values 0..3 [sic].
 inline void permuteHighInt16x8(const uint16_t lanes[4], FloatRegister src,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 // lane values 0..3.
 inline void permuteLowInt16x8(const uint16_t lanes[4], FloatRegister src,
                               FloatRegister dest) DEFINED_ON(x86_shared);

 // lane values 0..3
 inline void permuteInt32x4(const uint32_t lanes[4], FloatRegister src,
                            FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 // Funnel shift by immediate count:
 //   low_16_bytes_of((lhs ++ rhs) >> shift*8), shift must be < 16
 inline void concatAndRightShiftSimd128(FloatRegister lhs, FloatRegister rhs,
                                        FloatRegister dest, uint32_t shift)
     DEFINED_ON(x86_shared, arm64);

 // Rotate right by immediate count:
 //   low_16_bytes_of((src ++ src) >> shift*8), shift must be < 16
 inline void rotateRightSimd128(FloatRegister src, FloatRegister dest,
                                uint32_t shift) DEFINED_ON(arm64);

 // Shift bytes with immediate count, shifting in zeroes.  Shift count 0..15.

 inline void leftShiftSimd128(Imm32 count, FloatRegister src,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void rightShiftSimd128(Imm32 count, FloatRegister src,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Zero extend int values.

 inline void zeroExtend8x16To16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);
 inline void zeroExtend8x16To32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);
 inline void zeroExtend8x16To64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);
 inline void zeroExtend16x8To32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);
 inline void zeroExtend16x8To64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);
 inline void zeroExtend32x4To64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Reverse bytes in lanes.

 inline void reverseInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void reverseInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void reverseInt64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Swizzle - permute with variable indices.  `rhs` holds the lanes parameter.

 inline void swizzleInt8x16(FloatRegister lhs, FloatRegister rhs,
                            FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void swizzleInt8x16Relaxed(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Integer Add

 inline void addInt8x16(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void addInt16x8(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void addInt32x4(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void addInt64x2(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addInt64x2(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 // Integer Subtract

 inline void subInt8x16(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void subInt16x8(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void subInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void subInt32x4(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subInt64x2(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void subInt64x2(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 // Integer Multiply

 inline void mulInt16x8(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void mulInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void mulInt32x4(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void mulInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 // On x86_shared, it is required lhs == dest
 inline void mulInt64x2(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest, FloatRegister temp)
     DEFINED_ON(x86_shared);

 inline void mulInt64x2(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest, FloatRegister temp)
     DEFINED_ON(x86_shared);

 inline void mulInt64x2(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest, FloatRegister temp1,
                        FloatRegister temp2) DEFINED_ON(arm64);

 // Note for the extMul opcodes, the NxM designation is for the input lanes;
 // the output lanes are twice as wide.
 inline void extMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
                                      FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
                                      FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
                                      FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void q15MulrSatInt16x8(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Integer Negate

 inline void negInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void negInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void negInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void negInt64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Saturating integer add

 inline void addSatInt8x16(FloatRegister lhs, FloatRegister rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedAddSatInt8x16(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedAddSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                                   FloatRegister dest) DEFINED_ON(x86_shared);

 inline void addSatInt16x8(FloatRegister lhs, FloatRegister rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedAddSatInt16x8(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedAddSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                                   FloatRegister dest) DEFINED_ON(x86_shared);

 // Saturating integer subtract

 inline void subSatInt8x16(FloatRegister lhs, FloatRegister rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedSubSatInt8x16(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedSubSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                                   FloatRegister dest) DEFINED_ON(x86_shared);

 inline void subSatInt16x8(FloatRegister lhs, FloatRegister rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedSubSatInt16x8(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedSubSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                                   FloatRegister dest) DEFINED_ON(x86_shared);

 // Lane-wise integer minimum

 inline void minInt8x16(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void minInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedMinInt8x16(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedMinInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void minInt16x8(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void minInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedMinInt16x8(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedMinInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void minInt32x4(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void minInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedMinInt32x4(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedMinInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 // Lane-wise integer maximum

 inline void maxInt8x16(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void maxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedMaxInt8x16(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedMaxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void maxInt16x8(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void maxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedMaxInt16x8(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedMaxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 inline void maxInt32x4(FloatRegister lhs, FloatRegister rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void maxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                        FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedMaxInt32x4(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedMaxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                                FloatRegister dest) DEFINED_ON(x86_shared);

 // Lane-wise integer rounding average

 inline void unsignedAverageInt8x16(FloatRegister lhs, FloatRegister rhs,
                                    FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedAverageInt16x8(FloatRegister lhs, FloatRegister rhs,
                                    FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Lane-wise integer absolute value

 inline void absInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void absInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void absInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void absInt64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Left shift by scalar. Immediates and variable shifts must have been
 // masked; shifts of zero will work but may or may not generate code.

 inline void leftShiftInt8x16(Register rhs, FloatRegister lhsDest,
                              FloatRegister temp) DEFINED_ON(x86_shared);

 inline void leftShiftInt8x16(FloatRegister lhs, Register rhs,
                              FloatRegister dest) DEFINED_ON(arm64);

 inline void leftShiftInt8x16(Imm32 count, FloatRegister src,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void leftShiftInt16x8(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void leftShiftInt16x8(FloatRegister lhs, Register rhs,
                              FloatRegister dest) DEFINED_ON(arm64);

 inline void leftShiftInt16x8(Imm32 count, FloatRegister src,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void leftShiftInt32x4(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void leftShiftInt32x4(FloatRegister lhs, Register rhs,
                              FloatRegister dest) DEFINED_ON(arm64);

 inline void leftShiftInt32x4(Imm32 count, FloatRegister src,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void leftShiftInt64x2(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void leftShiftInt64x2(FloatRegister lhs, Register rhs,
                              FloatRegister dest) DEFINED_ON(arm64);

 inline void leftShiftInt64x2(Imm32 count, FloatRegister src,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Right shift by scalar. Immediates and variable shifts must have been
 // masked; shifts of zero will work but may or may not generate code.

 inline void rightShiftInt8x16(Register rhs, FloatRegister lhsDest,
                               FloatRegister temp) DEFINED_ON(x86_shared);

 inline void rightShiftInt8x16(FloatRegister lhs, Register rhs,
                               FloatRegister dest) DEFINED_ON(arm64);

 inline void rightShiftInt8x16(Imm32 count, FloatRegister src,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedRightShiftInt8x16(Register rhs, FloatRegister lhsDest,
                                       FloatRegister temp)
     DEFINED_ON(x86_shared);

 inline void unsignedRightShiftInt8x16(FloatRegister lhs, Register rhs,
                                       FloatRegister dest) DEFINED_ON(arm64);

 inline void unsignedRightShiftInt8x16(Imm32 count, FloatRegister src,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void rightShiftInt16x8(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void rightShiftInt16x8(FloatRegister lhs, Register rhs,
                               FloatRegister dest) DEFINED_ON(arm64);

 inline void rightShiftInt16x8(Imm32 count, FloatRegister src,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedRightShiftInt16x8(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void unsignedRightShiftInt16x8(FloatRegister lhs, Register rhs,
                                       FloatRegister dest) DEFINED_ON(arm64);

 inline void unsignedRightShiftInt16x8(Imm32 count, FloatRegister src,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void rightShiftInt32x4(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void rightShiftInt32x4(FloatRegister lhs, Register rhs,
                               FloatRegister dest) DEFINED_ON(arm64);

 inline void rightShiftInt32x4(Imm32 count, FloatRegister src,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedRightShiftInt32x4(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void unsignedRightShiftInt32x4(FloatRegister lhs, Register rhs,
                                       FloatRegister dest) DEFINED_ON(arm64);

 inline void unsignedRightShiftInt32x4(Imm32 count, FloatRegister src,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void rightShiftInt64x2(Register rhs, FloatRegister lhsDest,
                               FloatRegister temp) DEFINED_ON(x86_shared);

 inline void rightShiftInt64x2(Imm32 count, FloatRegister src,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void rightShiftInt64x2(FloatRegister lhs, Register rhs,
                               FloatRegister dest) DEFINED_ON(arm64);

 inline void unsignedRightShiftInt64x2(Register rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 inline void unsignedRightShiftInt64x2(FloatRegister lhs, Register rhs,
                                       FloatRegister dest) DEFINED_ON(arm64);

 inline void unsignedRightShiftInt64x2(Imm32 count, FloatRegister src,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Sign replication operation

 inline void signReplicationInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void signReplicationInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void signReplicationInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void signReplicationInt64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared);

 // Bitwise and, or, xor, not

 inline void bitwiseAndSimd128(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseAndSimd128(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseAndSimd128(FloatRegister lhs, const SimdConstant& rhs,
                               FloatRegister dest) DEFINED_ON(x86_shared);

 inline void bitwiseOrSimd128(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseOrSimd128(FloatRegister lhs, FloatRegister rhs,
                              FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseOrSimd128(FloatRegister lhs, const SimdConstant& rhs,
                              FloatRegister dest) DEFINED_ON(x86_shared);

 inline void bitwiseXorSimd128(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseXorSimd128(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseXorSimd128(FloatRegister lhs, const SimdConstant& rhs,
                               FloatRegister dest) DEFINED_ON(x86_shared);

 inline void bitwiseNotSimd128(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Bitwise AND with compliment: dest = lhs & ~rhs, note only arm64 can do it.
 inline void bitwiseAndNotSimd128(FloatRegister lhs, FloatRegister rhs,
                                  FloatRegister lhsDest) DEFINED_ON(arm64);

 // Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
 // wants but what the x86 hardware offers.  Hence the name.

 inline void bitwiseNotAndSimd128(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void bitwiseNotAndSimd128(FloatRegister lhs, FloatRegister rhs,
                                  FloatRegister lhsDest)
     DEFINED_ON(x86_shared);

 // Bitwise select

 inline void bitwiseSelectSimd128(FloatRegister mask, FloatRegister onTrue,
                                  FloatRegister onFalse, FloatRegister dest,
                                  FloatRegister temp) DEFINED_ON(x86_shared);

 inline void bitwiseSelectSimd128(FloatRegister onTrue, FloatRegister onFalse,
                                  FloatRegister maskDest) DEFINED_ON(arm64);

 // Population count

 inline void popcntInt8x16(FloatRegister src, FloatRegister dest,
                           FloatRegister temp) DEFINED_ON(x86_shared);

 inline void popcntInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(arm64);

 // Any lane true, ie, any bit set

 inline void anyTrueSimd128(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared, arm64);

 // All lanes true

 inline void allTrueInt8x16(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared, arm64);

 inline void allTrueInt16x8(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared, arm64);

 inline void allTrueInt32x4(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared, arm64);

 inline void allTrueInt64x2(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared, arm64);

 // Bitmask, ie extract and compress high bits of all lanes

 inline void bitmaskInt8x16(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared);

 inline void bitmaskInt8x16(FloatRegister src, Register dest,
                            FloatRegister temp) DEFINED_ON(arm64);

 inline void bitmaskInt16x8(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared);

 inline void bitmaskInt16x8(FloatRegister src, Register dest,
                            FloatRegister temp) DEFINED_ON(arm64);

 inline void bitmaskInt32x4(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared);

 inline void bitmaskInt32x4(FloatRegister src, Register dest,
                            FloatRegister temp) DEFINED_ON(arm64);

 inline void bitmaskInt64x2(FloatRegister src, Register dest)
     DEFINED_ON(x86_shared);

 inline void bitmaskInt64x2(FloatRegister src, Register dest,
                            FloatRegister temp) DEFINED_ON(arm64);

 // Comparisons (integer and floating-point)

 inline void compareInt8x16(Assembler::Condition cond, FloatRegister rhs,
                            FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 // On x86_shared, limited to !=, ==, <=, >
 inline void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
                            const SimdConstant& rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 // On arm64, use any integer comparison condition.
 inline void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
                            FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void compareInt16x8(Assembler::Condition cond, FloatRegister rhs,
                            FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
                            FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // On x86_shared, limited to !=, ==, <=, >
 inline void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
                            const SimdConstant& rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 // On x86_shared, limited to !=, ==, <=, >
 inline void compareInt32x4(Assembler::Condition cond, FloatRegister rhs,
                            FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
                            const SimdConstant& rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 // On arm64, use any integer comparison condition.
 inline void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
                            FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void compareForEqualityInt64x2(Assembler::Condition cond,
                                       FloatRegister lhs, FloatRegister rhs,
                                       FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void compareForOrderingInt64x2(Assembler::Condition cond,
                                       FloatRegister lhs, FloatRegister rhs,
                                       FloatRegister dest, FloatRegister temp1,
                                       FloatRegister temp2)
     DEFINED_ON(x86_shared);

 inline void compareInt64x2(Assembler::Condition cond, FloatRegister rhs,
                            FloatRegister lhsDest) DEFINED_ON(arm64);

 inline void compareInt64x2(Assembler::Condition cond, FloatRegister lhs,
                            FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(arm64);

 inline void compareFloat32x4(Assembler::Condition cond, FloatRegister rhs,
                              FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 // On x86_shared, limited to ==, !=, <, <=
 inline void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
                              const SimdConstant& rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 // On x86_shared, limited to ==, !=, <, <=
 // On arm64, use any float-point comparison condition.
 inline void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
                              FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void compareFloat64x2(Assembler::Condition cond, FloatRegister rhs,
                              FloatRegister lhsDest)
     DEFINED_ON(x86_shared, arm64);

 // On x86_shared, limited to ==, !=, <, <=
 inline void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
                              const SimdConstant& rhs, FloatRegister dest)
     DEFINED_ON(x86_shared);

 // On x86_shared, limited to ==, !=, <, <=
 // On arm64, use any float-point comparison condition.
 inline void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
                              FloatRegister rhs, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Load

 inline void loadUnalignedSimd128(const Operand& src, FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline FaultingCodeOffset loadUnalignedSimd128(const Address& src,
                                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline FaultingCodeOffset loadUnalignedSimd128(const BaseIndex& src,
                                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Store

 inline FaultingCodeOffset storeUnalignedSimd128(FloatRegister src,
                                                 const Address& dest)
     DEFINED_ON(x86_shared, arm64);

 inline FaultingCodeOffset storeUnalignedSimd128(FloatRegister src,
                                                 const BaseIndex& dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating point negation

 inline void negFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void negFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating point absolute value

 inline void absFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void absFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // NaN-propagating minimum

 inline void minFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest, FloatRegister temp1,
                          FloatRegister temp2) DEFINED_ON(x86_shared);

 inline void minFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(arm64);

 inline void minFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(arm64);

 inline void minFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest, FloatRegister temp1,
                          FloatRegister temp2) DEFINED_ON(x86_shared);

 inline void minFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(arm64);

 inline void minFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(arm64);

 // NaN-propagating maximum

 inline void maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest, FloatRegister temp1,
                          FloatRegister temp2) DEFINED_ON(x86_shared);

 inline void maxFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(arm64);

 inline void maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(arm64);

 inline void maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest, FloatRegister temp1,
                          FloatRegister temp2) DEFINED_ON(x86_shared);

 inline void maxFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
     DEFINED_ON(arm64);

 inline void maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(arm64);

 // Floating add

 inline void addFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 inline void addFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void addFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 // Floating subtract

 inline void subFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 inline void subFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void subFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 // Floating division

 inline void divFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void divFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 inline void divFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void divFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 // Floating Multiply

 inline void mulFloat32x4(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void mulFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 inline void mulFloat64x2(FloatRegister lhs, FloatRegister rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void mulFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
                          FloatRegister dest) DEFINED_ON(x86_shared);

 // Pairwise add

 inline void extAddPairwiseInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtAddPairwiseInt8x16(FloatRegister src,
                                           FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void extAddPairwiseInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedExtAddPairwiseInt16x8(FloatRegister src,
                                           FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating square root

 inline void sqrtFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void sqrtFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Integer to floating point with rounding

 inline void convertInt32x4ToFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedConvertInt32x4ToFloat32x4(FloatRegister src,
                                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void convertInt32x4ToFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedConvertInt32x4ToFloat64x2(FloatRegister src,
                                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating point to integer with saturation

 inline void truncSatFloat32x4ToInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
                                                FloatRegister dest,
                                                FloatRegister temp)
     DEFINED_ON(x86_shared);

 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
                                                FloatRegister dest)
     DEFINED_ON(arm64);

 inline void truncSatFloat64x2ToInt32x4(FloatRegister src, FloatRegister dest,
                                        FloatRegister temp)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,
                                                FloatRegister dest,
                                                FloatRegister temp)
     DEFINED_ON(x86_shared, arm64);

 inline void truncFloat32x4ToInt32x4Relaxed(FloatRegister src,
                                            FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedTruncFloat32x4ToInt32x4Relaxed(FloatRegister src,
                                                    FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void truncFloat64x2ToInt32x4Relaxed(FloatRegister src,
                                            FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedTruncFloat64x2ToInt32x4Relaxed(FloatRegister src,
                                                    FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating point narrowing

 inline void convertFloat64x2ToFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating point widening

 inline void convertFloat32x4ToFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Integer to integer narrowing

 inline void narrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared);

 inline void narrowInt16x8(FloatRegister lhs, FloatRegister rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void unsignedNarrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                                   FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedNarrowInt16x8(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void narrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared);

 inline void narrowInt32x4(FloatRegister lhs, FloatRegister rhs,
                           FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void unsignedNarrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
                                   FloatRegister dest) DEFINED_ON(x86_shared);

 inline void unsignedNarrowInt32x4(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Integer to integer widening

 inline void widenLowInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void widenHighInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedWidenLowInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedWidenHighInt8x16(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void widenLowInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void widenHighInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedWidenLowInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedWidenHighInt16x8(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void widenLowInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedWidenLowInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void widenHighInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void unsignedWidenHighInt32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Compare-based minimum/maximum
 //
 // On x86, the signature is (rhsDest, lhs); on arm64 it is (rhs, lhsDest).
 //
 // The masm preprocessor can't deal with multiple declarations with identical
 // signatures even if they are on different platforms, hence the weird
 // argument names.

 inline void pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,
                                FloatRegister lhsOrLhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMinFloat32x4(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,
                                FloatRegister lhsOrLhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMinFloat64x2(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,
                                FloatRegister lhsOrLhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMaxFloat32x4(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,
                                FloatRegister lhsOrLhsDest)
     DEFINED_ON(x86_shared, arm64);

 inline void pseudoMaxFloat64x2(FloatRegister lhs, FloatRegister rhs,
                                FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Widening/pairwise integer dot product

 inline void widenDotInt16x8(FloatRegister lhs, FloatRegister rhs,
                             FloatRegister dest) DEFINED_ON(x86_shared, arm64);

 inline void widenDotInt16x8(FloatRegister lhs, const SimdConstant& rhs,
                             FloatRegister dest) DEFINED_ON(x86_shared);

 inline void dotInt8x16Int7x16(FloatRegister lhs, FloatRegister rhs,
                               FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs, FloatRegister rhs,
                                      FloatRegister dest)
     DEFINED_ON(x86_shared);

 inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs, FloatRegister rhs,
                                      FloatRegister dest, FloatRegister temp)
     DEFINED_ON(arm64);

 // Floating point rounding

 inline void ceilFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void ceilFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void floorFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void floorFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void truncFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void truncFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void nearestFloat32x4(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void nearestFloat64x2(FloatRegister src, FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 // Floating multiply-accumulate: srcDest [+-]= src1 * src2

 inline void fmaFloat32x4(FloatRegister src1, FloatRegister src2,
                          FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);

 inline void fnmaFloat32x4(FloatRegister src1, FloatRegister src2,
                           FloatRegister srcDest)
     DEFINED_ON(x86_shared, arm64);

 inline void fmaFloat64x2(FloatRegister src1, FloatRegister src2,
                          FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);

 inline void fnmaFloat64x2(FloatRegister src1, FloatRegister src2,
                           FloatRegister srcDest)
     DEFINED_ON(x86_shared, arm64);

 inline void minFloat32x4Relaxed(FloatRegister src, FloatRegister srcDest)
     DEFINED_ON(x86_shared, arm64);

 inline void minFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
                                 FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void maxFloat32x4Relaxed(FloatRegister src, FloatRegister srcDest)
     DEFINED_ON(x86_shared, arm64);

 inline void maxFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
                                 FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void minFloat64x2Relaxed(FloatRegister src, FloatRegister srcDest)
     DEFINED_ON(x86_shared, arm64);

 inline void minFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
                                 FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void maxFloat64x2Relaxed(FloatRegister src, FloatRegister srcDest)
     DEFINED_ON(x86_shared, arm64);

 inline void maxFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
                                 FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

 inline void q15MulrInt16x8Relaxed(FloatRegister lhs, FloatRegister rhs,
                                   FloatRegister dest)
     DEFINED_ON(x86_shared, arm64);

public:
 // ========================================================================
 // Truncate floating point.

 // Undefined behaviour when truncation is outside Int64 range.
 // Needs a temp register if SSE3 is not present.
 inline void truncateFloat32ToInt64(Address src, Address dest, Register temp)
     DEFINED_ON(x86_shared);
 inline void truncateFloat32ToUInt64(Address src, Address dest, Register temp,
                                     FloatRegister floatTemp)
     DEFINED_ON(x86, x64);
 inline void truncateDoubleToInt64(Address src, Address dest, Register temp)
     DEFINED_ON(x86_shared);
 inline void truncateDoubleToUInt64(Address src, Address dest, Register temp,
                                    FloatRegister floatTemp)
     DEFINED_ON(x86, x64);

public:
 // ========================================================================
 // Convert floating point.

 // temp required on x86 and x64; must be undefined on mips64 and loong64.
 void convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp)
     DEFINED_ON(arm64, mips64, loong64, riscv64, wasm32, x64, x86);

 void convertInt64ToFloat32(Register64 src, FloatRegister dest)
     DEFINED_ON(arm64, mips64, loong64, riscv64, wasm32, x64, x86);

 bool convertUInt64ToDoubleNeedsTemp() PER_ARCH;

 // temp required when convertUInt64ToDoubleNeedsTemp() returns true.
 void convertUInt64ToDouble(Register64 src, FloatRegister dest,
                            Register temp) PER_ARCH;

 void convertInt64ToDouble(Register64 src, FloatRegister dest) PER_ARCH;

 void convertIntPtrToDouble(Register src, FloatRegister dest) PER_ARCH;

public:
 // ========================================================================
 // wasm support

 FaultingCodeOffset wasmTrapInstruction() PER_SHARED_ARCH;

 void wasmTrap(wasm::Trap trap, const wasm::TrapSiteDesc& trapSiteDesc);

 // Load all pinned regs via InstanceReg.  If the trapOffset is something,
 // give the first load a trap descriptor with type IndirectCallToNull, so that
 // a null instance will cause a trap.
 void loadWasmPinnedRegsFromInstance(
     const wasm::MaybeTrapSiteDesc& trapSiteDesc);

 // Branches to the fail label if the stack would overflow the current stack
 // limit. Returns the number of extra bytes of stack allocated prior to
 // branching to the fail label.
 uint32_t wasmReserveStackChecked(uint32_t amount, Label* fail);

 // Emit a bounds check against the wasm heap limit, jumping to 'ok' if 'cond'
 // holds; this can be the label either of the access or of the trap.  The
 // label should name a code position greater than the position of the bounds
 // check.
 //
 // If JitOptions.spectreMaskIndex is true, a no-op speculation barrier is
 // emitted in the code stream after the check to prevent an OOB access from
 // being executed speculatively.  (On current tier-1 platforms the barrier is
 // a conditional saturation of 'index' to 'boundsCheckLimit', using the same
 // condition as the check.)  If the condition is such that the bounds check
 // branches out of line to the trap, the barrier will actually be executed
 // when the bounds check passes.
 //
 // On 32-bit systems for both wasm and asm.js, and on 64-bit systems for
 // asm.js, heap lengths are limited to 2GB.  On 64-bit systems for wasm,
 // 32-bit heap lengths are limited to 4GB, and 64-bit heap lengths will be
 // limited to something much larger.

 void wasmBoundsCheck32(Condition cond, Register index,
                        Register boundsCheckLimit,
                        Label* label) PER_SHARED_ARCH;

 void wasmBoundsCheck32(Condition cond, Register index,
                        Address boundsCheckLimit,
                        Label* label) PER_SHARED_ARCH;

 void wasmBoundsCheck64(Condition cond, Register64 index,
                        Register64 boundsCheckLimit, Label* label) PER_ARCH;

 void wasmBoundsCheck64(Condition cond, Register64 index,
                        Address boundsCheckLimit, Label* label) PER_ARCH;

 // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds.
 void wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr,
               AnyRegister out) DEFINED_ON(x86, x64);
 void wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAddr,
                  Register64 out) DEFINED_ON(x86, x64);
 void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
                Operand dstAddr) DEFINED_ON(x86, x64);
 void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
                   Operand dstAddr) DEFINED_ON(x86);

 // For all the ARM/MIPS/LOONG64 wasmLoad and wasmStore functions below, `ptr`
 // MUST equal `ptrScratch`, and that register will be updated based on
 // conditions listed below (where it is only mentioned as `ptr`).

 // `ptr` will be updated if access.offset32() != 0 or access.type() ==
 // Scalar::Int64.
 void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
               Register ptr, Register ptrScratch, AnyRegister output)
     DEFINED_ON(arm, loong64, riscv64, mips64);
 void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
                  Register ptr, Register ptrScratch, Register64 output)
     DEFINED_ON(arm, mips64, loong64, riscv64);
 void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
                Register memoryBase, Register ptr, Register ptrScratch)
     DEFINED_ON(arm, loong64, riscv64, mips64);
 void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
                   Register memoryBase, Register ptr, Register ptrScratch)
     DEFINED_ON(arm, mips64, loong64, riscv64);

 // These accept general memoryBase + ptr + offset (in `access`); the offset is
 // always smaller than the guard region.  They will insert an additional add
 // if the offset is nonzero, and of course that add may require a temporary
 // register for the offset if the offset is large, and instructions to set it
 // up.
 void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
               Register ptr, AnyRegister output) DEFINED_ON(arm64);
 void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
                  Register ptr, Register64 output) DEFINED_ON(arm64);
 void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
                Register memoryBase, Register ptr) DEFINED_ON(arm64);
 void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
                   Register memoryBase, Register ptr) DEFINED_ON(arm64);

 // `ptr` will always be updated.
 void wasmUnalignedLoad(const wasm::MemoryAccessDesc& access,
                        Register memoryBase, Register ptr, Register ptrScratch,
                        Register output, Register tmp) DEFINED_ON(mips64);

 // MIPS: `ptr` will always be updated.
 void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access,
                          Register memoryBase, Register ptr,
                          Register ptrScratch, FloatRegister output,
                          Register tmp1) DEFINED_ON(mips64);

 // `ptr` will always be updated.
 void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
                           Register memoryBase, Register ptr,
                           Register ptrScratch, Register64 output,
                           Register tmp) DEFINED_ON(mips64);

 // MIPS: `ptr` will always be updated.
 void wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value,
                         Register memoryBase, Register ptr,
                         Register ptrScratch, Register tmp) DEFINED_ON(mips64);

 // `ptr` will always be updated.
 void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access,
                           FloatRegister floatValue, Register memoryBase,
                           Register ptr, Register ptrScratch, Register tmp)
     DEFINED_ON(mips64);

 // `ptr` will always be updated.
 void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
                            Register64 value, Register memoryBase,
                            Register ptr, Register ptrScratch, Register tmp)
     DEFINED_ON(mips64);

 // wasm specific methods, used in both the wasm baseline compiler and ion.

 // The truncate-to-int32 methods do not bind the rejoin label; clients must
 // do so if oolWasmTruncateCheckF64ToI32() can jump to it.
 void wasmTruncateDoubleToUInt32(FloatRegister input, Register output,
                                 bool isSaturating, Label* oolEntry) PER_ARCH;
 void wasmTruncateDoubleToInt32(FloatRegister input, Register output,
                                bool isSaturating,
                                Label* oolEntry) PER_SHARED_ARCH;
 void oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output,
                                   TruncFlags flags,
                                   const wasm::TrapSiteDesc& trapSiteDesc,
                                   Label* rejoin) PER_SHARED_ARCH;

 void wasmTruncateFloat32ToUInt32(FloatRegister input, Register output,
                                  bool isSaturating, Label* oolEntry) PER_ARCH;
 void wasmTruncateFloat32ToInt32(FloatRegister input, Register output,
                                 bool isSaturating,
                                 Label* oolEntry) PER_SHARED_ARCH;
 void oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output,
                                   TruncFlags flags,
                                   const wasm::TrapSiteDesc& trapSiteDesc,
                                   Label* rejoin) PER_SHARED_ARCH;

 // The truncate-to-int64 methods will always bind the `oolRejoin` label
 // after the last emitted instruction.
 void wasmTruncateDoubleToInt64(FloatRegister input, Register64 output,
                                bool isSaturating, Label* oolEntry,
                                Label* oolRejoin, FloatRegister tempDouble)
     DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32);
 void wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output,
                                 bool isSaturating, Label* oolEntry,
                                 Label* oolRejoin, FloatRegister tempDouble)
     DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32);
 void oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output,
                                   TruncFlags flags,
                                   const wasm::TrapSiteDesc& trapSiteDesc,
                                   Label* rejoin) PER_SHARED_ARCH;

 void wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output,
                                 bool isSaturating, Label* oolEntry,
                                 Label* oolRejoin, FloatRegister tempDouble)
     DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32);
 void wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output,
                                  bool isSaturating, Label* oolEntry,
                                  Label* oolRejoin, FloatRegister tempDouble)
     DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32);
 void oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output,
                                   TruncFlags flags,
                                   const wasm::TrapSiteDesc& trapSiteDesc,
                                   Label* rejoin) PER_SHARED_ARCH;

 // This function takes care of loading the callee's instance and pinned regs
 // but it is the caller's responsibility to save/restore instance or pinned
 // regs.
 CodeOffset wasmCallImport(const wasm::CallSiteDesc& desc,
                           const wasm::CalleeDesc& callee);

 CodeOffset wasmReturnCallImport(const wasm::CallSiteDesc& desc,
                                 const wasm::CalleeDesc& callee,
                                 const ReturnCallAdjustmentInfo& retCallInfo);

 CodeOffset wasmReturnCall(const wasm::CallSiteDesc& desc,
                           uint32_t funcDefIndex,
                           const ReturnCallAdjustmentInfo& retCallInfo);

 void wasmCollapseFrameSlow(const ReturnCallAdjustmentInfo& retCallInfo,
                            wasm::CallSiteDesc desc);

 void wasmCollapseFrameFast(const ReturnCallAdjustmentInfo& retCallInfo);

 void wasmCheckSlowCallsite(Register ra, Label* notSlow, Register temp1,
                            Register temp2) PER_ARCH;

 // Places slow class marker for tail calls.
 void wasmMarkCallAsSlow() PER_ARCH;

 // Combines slow class marker with actual assembler call.
 CodeOffset wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
                               const Register reg) PER_SHARED_ARCH;

 void wasmClampTable64Address(Register64 address, Register out);

 // WasmTableCallIndexReg must contain the index of the indirect call.  This is
 // for wasm calls only.
 //
 // Indirect calls use a dual-path mechanism where a run-time test determines
 // whether a context switch is needed (slow path) or not (fast path).  This
 // gives rise to two call instructions, both of which need safe points.  As
 // per normal, the call offsets are the code offsets at the end of the call
 // instructions (the return points).
 //
 // `boundsCheckFailedLabel` is non-null iff a bounds check is required.
 // `nullCheckFailedLabel` is non-null only on platforms that can't fold the
 // null check into the rest of the call instructions.
 void wasmCallIndirect(const wasm::CallSiteDesc& desc,
                       const wasm::CalleeDesc& callee,
                       Label* nullCheckFailedLabel, CodeOffset* fastCallOffset,
                       CodeOffset* slowCallOffset);

 // WasmTableCallIndexReg must contain the index of the indirect call.  This is
 // for wasm calls only.
 //
 // `boundsCheckFailedLabel` is non-null iff a bounds check is required.
 // `nullCheckFailedLabel` is non-null only on platforms that can't fold the
 // null check into the rest of the call instructions.
 void wasmReturnCallIndirect(const wasm::CallSiteDesc& desc,
                             const wasm::CalleeDesc& callee,
                             Label* nullCheckFailedLabel,
                             const ReturnCallAdjustmentInfo& retCallInfo);

 // This function takes care of loading the callee's instance and address from
 // pinned reg.
 void wasmCallRef(const wasm::CallSiteDesc& desc,
                  const wasm::CalleeDesc& callee, CodeOffset* fastCallOffset,
                  CodeOffset* slowCallOffset);

 void wasmReturnCallRef(const wasm::CallSiteDesc& desc,
                        const wasm::CalleeDesc& callee,
                        const ReturnCallAdjustmentInfo& retCallInfo);

 // WasmTableCallIndexReg must contain the index of the indirect call.
 // This is for asm.js calls only.
 CodeOffset asmCallIndirect(const wasm::CallSiteDesc& desc,
                            const wasm::CalleeDesc& callee);

 // This function takes care of loading the pointer to the current instance
 // as the implicit first argument. It preserves instance and pinned registers.
 // (instance & pinned regs are non-volatile registers in the system ABI).
 CodeOffset wasmCallBuiltinInstanceMethod(const wasm::CallSiteDesc& desc,
                                          const ABIArg& instanceArg,
                                          wasm::SymbolicAddress builtin,
                                          wasm::FailureMode failureMode,
                                          wasm::Trap failureTrap);

 // Performs the appropriate check based on the instance call's FailureMode,
 // and traps if the check fails. The resultRegister should likely be
 // ReturnReg, but this depends on whatever you do with registers immediately
 // after the call.
 void wasmTrapOnFailedInstanceCall(Register resultRegister,
                                   wasm::FailureMode failureMode,
                                   wasm::Trap failureTrap,
                                   const wasm::TrapSiteDesc& trapSiteDesc);

 // Performs a bounds check for ranged wasm operations like memory.fill or
 // array.fill. This handles the bizarre edge case in the wasm spec where a
 // write to index N is valid as long as the length is zero - despite the index
 // itself being out of bounds.
 //
 // `length` and `limit` will be unchanged.
 void wasmBoundsCheckRange32(Register index, Register length, Register limit,
                             Register tmp,
                             const wasm::TrapSiteDesc& trapSiteDesc);

 // Returns information about which registers are necessary for a
 // branchWasmRefIsSubtype call.
 static BranchWasmRefIsSubtypeRegisters regsForBranchWasmRefIsSubtype(
     wasm::RefType type);

 // Perform a subtype check that `ref` is a subtype of `type`, branching to
 // `label` depending on `onSuccess`.
 //
 // Will select one of the other branchWasmRefIsSubtype* functions depending on
 // destType. See each function for the register allocation requirements, as
 // well as which registers will be preserved.
 //
 // If this function returns a valid FaultingCodeOffset, then you must emit a
 // trap site to catch the bad cast. It will never return a valid
 // FaultingCodeOffset when signalNullChecks is false.
 FaultingCodeOffset branchWasmRefIsSubtype(
     Register ref, wasm::MaybeRefType sourceType, wasm::RefType destType,
     Label* label, bool onSuccess, bool signalNullChecks, Register superSTV,
     Register scratch1, Register scratch2);

 // Perform a subtype check that `ref` is a subtype of `type`, branching to
 // `label` depending on `onSuccess`. `type` must be in the `any` hierarchy.
 //
 // `superSTV` is required iff the destination type is a concrete
 // type. `scratch1` is required iff the destination type is eq or lower and
 // not none. `scratch2` is required iff the destination type is a concrete
 // type and its `subTypingDepth` is >= wasm::MinSuperTypeVectorLength. See
 // regsForBranchWasmRefIsSubtype.
 //
 // `ref` and `superSTV` are preserved. Scratch registers are
 // clobbered.
 //
 // If this function returns a valid FaultingCodeOffset, then you must emit a
 // trap site to catch the bad cast. It will never return a valid
 // FaultingCodeOffset when signalNullChecks is false.
 FaultingCodeOffset branchWasmRefIsSubtypeAny(
     Register ref, wasm::RefType sourceType, wasm::RefType destType,
     Label* label, bool onSuccess, bool signalNullChecks, Register superSTV,
     Register scratch1, Register scratch2);

 // Perform a subtype check that `ref` is a subtype of `type`, branching to
 // `label` depending on `onSuccess`. `type` must be in the `func` hierarchy.
 //
 // `superSTV` and `scratch1` are required iff the destination type
 // is a concrete type (not func and not nofunc). `scratch2` is required iff
 // the destination type is a concrete type and its `subTypingDepth` is >=
 // wasm::MinSuperTypeVectorLength. See regsForBranchWasmRefIsSubtype.
 //
 // `ref` and `superSTV` are preserved. Scratch registers are
 // clobbered.
 void branchWasmRefIsSubtypeFunc(Register ref, wasm::RefType sourceType,
                                 wasm::RefType destType, Label* label,
                                 bool onSuccess, Register superSTV,
                                 Register scratch1, Register scratch2);

 // Perform a subtype check that `ref` is a subtype of `destType`, branching to
 // `label` depending on `onSuccess`. `type` must be in the `extern` hierarchy.
 void branchWasmRefIsSubtypeExtern(Register ref, wasm::RefType sourceType,
                                   wasm::RefType destType, Label* label,
                                   bool onSuccess);

 // Perform a subtype check that `ref` is a subtype of `destType`, branching to
 // `label` depending on `onSuccess`. `type` must be in the `exn` hierarchy.
 void branchWasmRefIsSubtypeExn(Register ref, wasm::RefType sourceType,
                                wasm::RefType destType, Label* label,
                                bool onSuccess);

 // Perform a subtype check that `subSTV` is a subtype of `superSTV`, branching
 // to `label` depending on `onSuccess`. This method is a specialization of the
 // general `wasm::TypeDef::isSubTypeOf` method for the case where the
 // `superSTV` is statically known, which is the case for all wasm
 // instructions.
 //
 // `scratch` is required iff the destination type is not final and the
 // `superDepth` is >= wasm::MinSuperTypeVectorLength. `subSTV` is clobbered by
 // this method if the destination type is not final. `superSTV` is always
 // preserved.
 void branchWasmSTVIsSubtype(Register subSTV, Register superSTV,
                             Register scratch, const wasm::TypeDef* destType,
                             Label* label, bool onSuccess);

 // Same as branchWasmSTVIsSubtype, but looks up a dynamic position in the
 // super type vector.
 //
 // `scratch` is always required. `subSTV` and `superDepth` are clobbered.
 // `superSTV` is preserved.
 void branchWasmSTVIsSubtypeDynamicDepth(Register subSTV, Register superSTV,
                                         Register superDepth, Register scratch,
                                         Label* label, bool onSuccess);

 // Extract the tag of wasm anyref `src`.
 void extractWasmAnyRefTag(Register src, Register dest);

 // Remove the known tag of wasm anyref `src`.
 void untagWasmAnyRef(Register src, Register dest, wasm::AnyRefTag tag);

 // Branch if the wasm anyref `src` is or is not the null value.
 void branchWasmAnyRefIsNull(bool isNull, Register src, Label* label);
 // Branch if the wasm anyref `src` is or is not an I31.
 void branchWasmAnyRefIsI31(bool isI31, Register src, Label* label);
 // Branch if the wasm anyref `src` is or is not a JSObject*.
 void branchWasmAnyRefIsObjectOrNull(bool isObject, Register src,
                                     Label* label);
 // Branch if the wasm anyref `src` is or is not a JSString.
 void branchWasmAnyRefIsJSString(bool isJSString, Register src, Register temp,
                                 Label* label);
 // Branch if the wasm anyref `src` is or is not a GC thing.
 void branchWasmAnyRefIsGCThing(bool isGCThing, Register src, Label* label);
 // Branch if the wasm anyref `src` is or is not pointing to a nursery cell.
 void branchWasmAnyRefIsNurseryCell(bool isNurseryCell, Register src,
                                    Register scratch, Label* label);

 // Create a wasm i31ref by truncating the 32-bit integer.
 void truncate32ToWasmI31Ref(Register src, Register dest);
 // Convert a wasm i31ref to a signed 32-bit integer.
 void convertWasmI31RefTo32Signed(Register src, Register dest);
 // Convert a wasm i31ref to an unsigned 32-bit integer.
 void convertWasmI31RefTo32Unsigned(Register src, Register dest);

 // Branch if the JS value `src` would need to be boxed out of line to be
 // converted to a wasm anyref.
 void branchValueConvertsToWasmAnyRefInline(ValueOperand src,
                                            Register scratchInt,
                                            FloatRegister scratchFloat,
                                            Label* label);
 // Convert a JS value to a wasm anyref. If the value requires boxing, this
 // will branch to `oolConvert`.
 void convertValueToWasmAnyRef(ValueOperand src, Register dest,
                               FloatRegister scratchFloat, Label* oolConvert);
 // Convert a JS object to a wasm anyref. This cannot fail.
 void convertObjectToWasmAnyRef(Register src, Register dest);
 // Convert a JS string to a wasm anyref. This cannot fail.
 void convertStringToWasmAnyRef(Register src, Register dest);

 // Convert a wasm anyref to a JS value. This cannot fail.
 //
 // Due to spectre mitigations, these methods may clobber src.
 void convertWasmAnyRefToValue(Register instance, Register src,
                               ValueOperand dst, Register scratch);
 void convertWasmAnyRefToValue(Register instance, Register src,
                               const Address& dst, Register scratch);

 // Branch if the object `src` is or is not a WasmGcObject.
 FaultingCodeOffset branchObjectIsWasmGcObject(bool isGcObject, Register src,
                                               Register scratch, Label* label);

 // `typeDefData` will be preserved. `instance` and `result` may be the same
 // register, in which case `instance` will be clobbered.
 void wasmNewStructObject(Register instance, Register result,
                          Register allocSite, Register temp1,
                          size_t offsetOfTypeDefData, Label* fail,
                          gc::AllocKind allocKind, bool zeroFields);
 // Allocates a wasm array with a dynamic number of elements.
 //
 // `numElements` and `typeDefData` will be preserved. `instance` and `result`
 // may be the same register, in which case `instance` will be clobbered.
 void wasmNewArrayObject(Register instance, Register result,
                         Register numElements, Register allocSite,
                         Register temp, size_t offsetOfTypeDefData,
                         Label* fail, uint32_t elemSize, bool zeroFields);
 // Allocates a wasm array with a fixed number of elements.
 //
 // `typeDefData` will be preserved. `instance` and `result` may be the same
 // register, in which case `instance` will be clobbered.
 void wasmNewArrayObjectFixed(Register instance, Register result,
                              Register allocSite, Register temp1,
                              Register temp2, size_t offsetOfTypeDefData,
                              Label* fail, uint32_t numElements,
                              uint32_t storageBytes, bool zeroFields);

 // This function handles nursery allocations for wasm. For JS, see
 // MacroAssembler::bumpPointerAllocate.
 //
 // `typeDefData` will be preserved. `instance` and `result` may be the same
 // register, in which case `instance` will be clobbered.
 //
 // See also the dynamically-sized version,
 // MacroAssembler::wasmBumpPointerAllocateDynamic.
 void wasmBumpPointerAllocate(Register instance, Register result,
                              Register allocSite, Register temp1, Label* fail,
                              uint32_t size);
 // This function handles nursery allocations for wasm of dynamic size. For
 // fixed-size allocations, see MacroAssembler::wasmBumpPointerAllocate.
 //
 // `typeDefData` and `size` will be preserved. `instance` and `result` may be
 // the same register, in which case `instance` will be clobbered.
 void wasmBumpPointerAllocateDynamic(Register instance, Register result,
                                     Register allocSite, Register size,
                                     Register temp1, Label* fail);

 // Compute ptr += (indexTemp32 << shift) where shift can be any value < 32.
 // May destroy indexTemp32.  The value of indexTemp32 must be positive, and it
 // is implementation-defined what happens if bits are lost or the value
 // becomes negative through the shift.  On 64-bit systems, the high 32 bits of
 // indexTemp32 must be zero, not garbage.
 void shiftIndex32AndAdd(Register indexTemp32, int shift,
                         Register pointer) PER_SHARED_ARCH;

 // The System ABI frequently states that the high bits of a 64-bit register
 // that holds a 32-bit return value are unpredictable, and C++ compilers will
 // indeed generate code that leaves garbage in the upper bits.
 //
 // Adjust the contents of the 64-bit register `r` to conform to our internal
 // convention, which requires predictable high bits.  In practice, this means
 // that the 32-bit value will be zero-extended or sign-extended to 64 bits as
 // appropriate for the platform.
 void widenInt32(Register r) DEFINED_ON(arm64, x64, mips64, loong64, riscv64);

 // As enterFakeExitFrame(), but using register conventions appropriate for
 // wasm stubs.
 void enterFakeExitFrameForWasm(Register cxreg, Register scratch,
                                ExitFrameType type) PER_SHARED_ARCH;

public:
 // ========================================================================
 // Barrier functions.

 void emitPreBarrierFastPath(MIRType type, Register temp1, Register temp2,
                             Register temp3, Label* noBarrier);
 void emitValueReadBarrierFastPath(ValueOperand value, Register cell,
                                   Register temp1, Register temp2,
                                   Register temp3, Register temp4,
                                   Label* barrier);

private:
 void loadMarkBits(Register cell, Register chunk, Register markWord,
                   Register bitIndex, Register temp, gc::ColorBit color);

public:
 // ========================================================================
 // Clamping functions.

 inline void clampIntToUint8(Register reg) PER_SHARED_ARCH;

public:
 // ========================================================================
 // Primitive atomic operations.
 //
 // If the access is from JS and the eventual destination of the result is a
 // js::Value, it's probably best to use the JS-specific versions of these,
 // see further below.
 //
 // Temp registers must be defined unless otherwise noted in the per-function
 // constraints.

 // 8-bit, 16-bit, and 32-bit wide operations.
 //
 // The 8-bit and 16-bit operations zero-extend or sign-extend the result to
 // 32 bits, according to `type`. On 64-bit systems, the upper 32 bits of the
 // result will be zero on some platforms (eg, on x64) and will be the sign
 // extension of the lower bits on other platforms (eg, MIPS).

 // CompareExchange with memory.  Return the value that was in memory,
 // whether we wrote or not.
 //
 // x86-shared: `output` must be eax.
 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
 // and 16-bit wide operations.

 void compareExchange(Scalar::Type type, Synchronization sync,
                      const Address& mem, Register expected,
                      Register replacement, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void compareExchange(Scalar::Type type, Synchronization sync,
                      const BaseIndex& mem, Register expected,
                      Register replacement, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void compareExchange(Scalar::Type type, Synchronization sync,
                      const Address& mem, Register expected,
                      Register replacement, Register valueTemp,
                      Register offsetTemp, Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void compareExchange(Scalar::Type type, Synchronization sync,
                      const BaseIndex& mem, Register expected,
                      Register replacement, Register valueTemp,
                      Register offsetTemp, Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 // x86: `expected` and `output` must be edx:eax; `replacement` is ecx:ebx.
 // x64: `output` must be rax.
 // ARM: Registers must be distinct; `replacement` and `output` must be
 // (even,odd) pairs.

 void compareExchange64(Synchronization sync, const Address& mem,
                        Register64 expected, Register64 replacement,
                        Register64 output)
     DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64);

 void compareExchange64(Synchronization sync, const BaseIndex& mem,
                        Register64 expected, Register64 replacement,
                        Register64 output)
     DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64);

 // Exchange with memory.  Return the value initially in memory.
 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
 // and 16-bit wide operations.

 void atomicExchange(Scalar::Type type, Synchronization sync,
                     const Address& mem, Register value, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void atomicExchange(Scalar::Type type, Synchronization sync,
                     const BaseIndex& mem, Register value, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void atomicExchange(Scalar::Type type, Synchronization sync,
                     const Address& mem, Register value, Register valueTemp,
                     Register offsetTemp, Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicExchange(Scalar::Type type, Synchronization sync,
                     const BaseIndex& mem, Register value, Register valueTemp,
                     Register offsetTemp, Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
 // ARM: `value` and `output` must be distinct and (even,odd) pairs.
 // ARM64: `value` and `output` must be distinct.

 void atomicExchange64(Synchronization sync, const Address& mem,
                       Register64 value, Register64 output)
     DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64);

 void atomicExchange64(Synchronization sync, const BaseIndex& mem,
                       Register64 value, Register64 output)
     DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64);

 // Read-modify-write with memory.  Return the value in memory before the
 // operation.
 //
 // x86-shared:
 //   For 8-bit operations, `value` and `output` must have a byte subregister.
 //   For Add and Sub, `temp` must be invalid.
 //   For And, Or, and Xor, `output` must be eax and `temp` must have a byte
 //   subregister.
 //
 // ARM: Registers `value` and `output` must differ.
 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
 // and 16-bit wide operations; `value` and `output` must differ.

 void atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
                    Register value, const Address& mem, Register temp,
                    Register output) DEFINED_ON(arm, arm64, x86_shared);

 void atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
                    Imm32 value, const Address& mem, Register temp,
                    Register output) DEFINED_ON(x86_shared);

 void atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
                    Register value, const BaseIndex& mem, Register temp,
                    Register output) DEFINED_ON(arm, arm64, x86_shared);

 void atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
                    Imm32 value, const BaseIndex& mem, Register temp,
                    Register output) DEFINED_ON(x86_shared);

 void atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
                    Register value, const Address& mem, Register valueTemp,
                    Register offsetTemp, Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
                    Register value, const BaseIndex& mem, Register valueTemp,
                    Register offsetTemp, Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 // x86:
 //   `temp` must be ecx:ebx; `output` must be edx:eax.
 // x64:
 //   For Add and Sub, `temp` is ignored.
 //   For And, Or, and Xor, `output` must be rax.
 // ARM:
 //   `temp` and `output` must be (even,odd) pairs and distinct from `value`.
 // ARM64:
 //   Registers `value`, `temp`, and `output` must all differ.

 void atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value,
                      const Address& mem, Register64 temp, Register64 output)
     DEFINED_ON(arm, arm64, x64, mips64, loong64, riscv64);

 void atomicFetchOp64(Synchronization sync, AtomicOp op, const Address& value,
                      const Address& mem, Register64 temp, Register64 output)
     DEFINED_ON(x86);

 void atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value,
                      const BaseIndex& mem, Register64 temp, Register64 output)
     DEFINED_ON(arm, arm64, x64, mips64, loong64, riscv64);

 void atomicFetchOp64(Synchronization sync, AtomicOp op, const Address& value,
                      const BaseIndex& mem, Register64 temp, Register64 output)
     DEFINED_ON(x86);

 // x64:
 //   `value` can be any register.
 // ARM:
 //   `temp` must be an (even,odd) pair and distinct from `value`.
 // ARM64:
 //   Registers `value` and `temp` must differ.

 void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value,
                       const Address& mem) DEFINED_ON(x64);

 void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value,
                       const Address& mem, Register64 temp)
     DEFINED_ON(arm, arm64, mips64, loong64, riscv64);

 void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value,
                       const BaseIndex& mem) DEFINED_ON(x64);

 void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value,
                       const BaseIndex& mem, Register64 temp)
     DEFINED_ON(arm, arm64, mips64, loong64, riscv64);

 // 64-bit atomic load. On 64-bit systems, use regular load with
 // Synchronization::Load, not this method.
 //
 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
 // ARM: `output` must be (even,odd) pair.

 void atomicLoad64(Synchronization sync, const Address& mem, Register64 temp,
                   Register64 output) DEFINED_ON(x86);

 void atomicLoad64(Synchronization sync, const BaseIndex& mem, Register64 temp,
                   Register64 output) DEFINED_ON(x86);

 void atomicLoad64(Synchronization sync, const Address& mem, Register64 output)
     DEFINED_ON(arm);

 void atomicLoad64(Synchronization sync, const BaseIndex& mem,
                   Register64 output) DEFINED_ON(arm);

 // 64-bit atomic store. On 64-bit systems, use regular store with
 // Synchronization::Store, not this method.
 //
 // x86: `value` must be ecx:ebx; `temp` must be edx:eax.
 // ARM: `value` and `temp` must be (even,odd) pairs.

 void atomicStore64(Synchronization sync, const Address& mem, Register64 value,
                    Register64 temp) DEFINED_ON(x86, arm);

 void atomicStore64(Synchronization sync, const BaseIndex& mem,
                    Register64 value, Register64 temp) DEFINED_ON(x86, arm);

 // ========================================================================
 // Wasm atomic operations.
 //
 // Constraints, when omitted, are exactly as for the primitive operations
 // above.

 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
                          const Address& mem, Register expected,
                          Register replacement, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
                          const BaseIndex& mem, Register expected,
                          Register replacement, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
                          const Address& mem, Register expected,
                          Register replacement, Register valueTemp,
                          Register offsetTemp, Register maskTemp,
                          Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
                          const BaseIndex& mem, Register expected,
                          Register replacement, Register valueTemp,
                          Register offsetTemp, Register maskTemp,
                          Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
                         const Address& mem, Register value, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
                         const BaseIndex& mem, Register value, Register output)
     DEFINED_ON(arm, arm64, x86_shared);

 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
                         const Address& mem, Register value,
                         Register valueTemp, Register offsetTemp,
                         Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
                         const BaseIndex& mem, Register value,
                         Register valueTemp, Register offsetTemp,
                         Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                        Register value, const Address& mem, Register temp,
                        Register output) DEFINED_ON(arm, arm64, x86_shared);

 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                        Imm32 value, const Address& mem, Register temp,
                        Register output) DEFINED_ON(x86_shared);

 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                        Register value, const BaseIndex& mem, Register temp,
                        Register output) DEFINED_ON(arm, arm64, x86_shared);

 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                        Imm32 value, const BaseIndex& mem, Register temp,
                        Register output) DEFINED_ON(x86_shared);

 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                        Register value, const Address& mem, Register valueTemp,
                        Register offsetTemp, Register maskTemp,
                        Register output) DEFINED_ON(mips64, loong64, riscv64);

 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                        Register value, const BaseIndex& mem,
                        Register valueTemp, Register offsetTemp,
                        Register maskTemp, Register output)
     DEFINED_ON(mips64, loong64, riscv64);

 // Read-modify-write with memory.  Return no value.
 //
 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
 // and 16-bit wide operations.

 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                         Register value, const Address& mem, Register temp)
     DEFINED_ON(arm, arm64, x86_shared);

 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                         Imm32 value, const Address& mem, Register temp)
     DEFINED_ON(x86_shared);

 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                         Register value, const BaseIndex& mem, Register temp)
     DEFINED_ON(arm, arm64, x86_shared);

 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                         Imm32 value, const BaseIndex& mem, Register temp)
     DEFINED_ON(x86_shared);

 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                         Register value, const Address& mem,
                         Register valueTemp, Register offsetTemp,
                         Register maskTemp)
     DEFINED_ON(mips64, loong64, riscv64);

 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
                         Register value, const BaseIndex& mem,
                         Register valueTemp, Register offsetTemp,
                         Register maskTemp)
     DEFINED_ON(mips64, loong64, riscv64);

 // 64-bit wide operations.

 // 64-bit atomic load.  On 64-bit systems, use regular wasm load with
 // Synchronization::Load, not this method.
 //
 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
 // ARM: `temp` should be invalid; `output` must be (even,odd) pair.

 void wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
                       const Address& mem, Register64 temp, Register64 output)
     DEFINED_ON(arm, x86, wasm32);

 void wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
                       const BaseIndex& mem, Register64 temp,
                       Register64 output) DEFINED_ON(arm, x86, wasm32);

 // x86: `expected` must be the same as `output`, and must be edx:eax.
 // x86: `replacement` must be ecx:ebx.
 // x64: `output` must be rax.
 // ARM: Registers must be distinct; `replacement` and `output` must be
 // (even,odd) pairs.
 // ARM64: The base register in `mem` must not overlap `output`.
 // MIPS: Registers must be distinct.

 void wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
                            const Address& mem, Register64 expected,
                            Register64 replacement,
                            Register64 output) PER_ARCH;

 void wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
                            const BaseIndex& mem, Register64 expected,
                            Register64 replacement,
                            Register64 output) PER_ARCH;

 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
 // ARM: Registers must be distinct; `value` and `output` must be (even,odd)
 // pairs.
 // MIPS: Registers must be distinct.

 void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
                           const Address& mem, Register64 value,
                           Register64 output) PER_ARCH;

 void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
                           const BaseIndex& mem, Register64 value,
                           Register64 output) PER_ARCH;

 // x86: `output` must be edx:eax, `temp` must be ecx:ebx.
 // x64: For And, Or, and Xor `output` must be rax.
 // ARM: Registers must be distinct; `temp` and `output` must be (even,odd)
 // pairs.
 // MIPS: Registers must be distinct.

 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
                          Register64 value, const Address& mem,
                          Register64 temp, Register64 output)
     DEFINED_ON(arm, arm64, mips64, loong64, riscv64, x64);

 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
                          Register64 value, const BaseIndex& mem,
                          Register64 temp, Register64 output)
     DEFINED_ON(arm, arm64, mips64, loong64, riscv64, x64);

 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
                          const Address& value, const Address& mem,
                          Register64 temp, Register64 output) DEFINED_ON(x86);

 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
                          const Address& value, const BaseIndex& mem,
                          Register64 temp, Register64 output) DEFINED_ON(x86);

 // Here `value` can be any register.

 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
                           Register64 value, const BaseIndex& mem)
     DEFINED_ON(x64);

 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
                           Register64 value, const BaseIndex& mem,
                           Register64 temp) DEFINED_ON(arm64);

 // ========================================================================
 // JS atomic operations.
 //
 // Here the arrayType must be a type that is valid for JS.  As of 2017 that
 // is an 8-bit, 16-bit, or 32-bit integer type.
 //
 // If arrayType is Scalar::Uint32 then:
 //
 //   - `output` must be a float register
 //   - if the operation takes one temp register then `temp` must be defined
 //   - if the operation takes two temp registers then `temp2` must be defined.
 //
 // Otherwise `output` must be a GPR and `temp`/`temp2` should be InvalidReg.
 // (`temp1` must always be valid.)
 //
 // For additional register constraints, see the primitive 32-bit operations
 // and/or wasm operations above.

 void compareExchangeJS(Scalar::Type arrayType, Synchronization sync,
                        const Address& mem, Register expected,
                        Register replacement, Register temp,
                        AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);

 void compareExchangeJS(Scalar::Type arrayType, Synchronization sync,
                        const BaseIndex& mem, Register expected,
                        Register replacement, Register temp,
                        AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);

 void compareExchangeJS(Scalar::Type arrayType, Synchronization sync,
                        const Address& mem, Register expected,
                        Register replacement, Register valueTemp,
                        Register offsetTemp, Register maskTemp, Register temp,
                        AnyRegister output)
     DEFINED_ON(mips64, loong64, riscv64);

 void compareExchangeJS(Scalar::Type arrayType, Synchronization sync,
                        const BaseIndex& mem, Register expected,
                        Register replacement, Register valueTemp,
                        Register offsetTemp, Register maskTemp, Register temp,
                        AnyRegister output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync,
                       const Address& mem, Register value, Register temp,
                       AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);

 void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync,
                       const BaseIndex& mem, Register value, Register temp,
                       AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);

 void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync,
                       const Address& mem, Register value, Register valueTemp,
                       Register offsetTemp, Register maskTemp, Register temp,
                       AnyRegister output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync,
                       const BaseIndex& mem, Register value,
                       Register valueTemp, Register offsetTemp,
                       Register maskTemp, Register temp, AnyRegister output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync,
                      AtomicOp op, Register value, const Address& mem,
                      Register temp1, Register temp2, AnyRegister output)
     DEFINED_ON(arm, arm64, x86_shared);

 void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync,
                      AtomicOp op, Register value, const BaseIndex& mem,
                      Register temp1, Register temp2, AnyRegister output)
     DEFINED_ON(arm, arm64, x86_shared);

 void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync,
                      AtomicOp op, Imm32 value, const Address& mem,
                      Register temp1, Register temp2, AnyRegister output)
     DEFINED_ON(x86_shared);

 void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync,
                      AtomicOp op, Imm32 value, const BaseIndex& mem,
                      Register temp1, Register temp2, AnyRegister output)
     DEFINED_ON(x86_shared);

 void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync,
                      AtomicOp op, Register value, const Address& mem,
                      Register valueTemp, Register offsetTemp,
                      Register maskTemp, Register temp, AnyRegister output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync,
                      AtomicOp op, Register value, const BaseIndex& mem,
                      Register valueTemp, Register offsetTemp,
                      Register maskTemp, Register temp, AnyRegister output)
     DEFINED_ON(mips64, loong64, riscv64);

 void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync,
                       AtomicOp op, Register value, const Address& mem,
                       Register temp) DEFINED_ON(arm, arm64, x86_shared);

 void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync,
                       AtomicOp op, Register value, const BaseIndex& mem,
                       Register temp) DEFINED_ON(arm, arm64, x86_shared);

 void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync,
                       AtomicOp op, Imm32 value, const Address& mem,
                       Register temp) DEFINED_ON(x86_shared);

 void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync,
                       AtomicOp op, Imm32 value, const BaseIndex& mem,
                       Register temp) DEFINED_ON(x86_shared);

 void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync,
                       AtomicOp op, Register value, const Address& mem,
                       Register valueTemp, Register offsetTemp,
                       Register maskTemp) DEFINED_ON(mips64, loong64, riscv64);

 void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync,
                       AtomicOp op, Register value, const BaseIndex& mem,
                       Register valueTemp, Register offsetTemp,
                       Register maskTemp) DEFINED_ON(mips64, loong64, riscv64);

 void atomicIsLockFreeJS(Register value, Register output);

 void atomicPause() PER_SHARED_ARCH;

 // ========================================================================
 // Spectre Mitigations.
 //
 // Spectre attacks are side-channel attacks based on cache pollution or
 // slow-execution of some instructions. We have multiple spectre mitigations
 // possible:
 //
 //   - Stop speculative executions, with memory barriers. Memory barriers
 //     force all branches depending on loads to be resolved, and thus
 //     resolve all miss-speculated paths.
 //
 //   - Use conditional move instructions. Some CPUs have a branch predictor,
 //     and not a flag predictor. In such cases, using a conditional move
 //     instruction to zero some pointer/index is enough to add a
 //     data-dependency which prevents any futher executions until the load is
 //     resolved.

 void spectreMaskIndex32(Register index, Register length, Register output);
 void spectreMaskIndex32(Register index, const Address& length,
                         Register output);
 void spectreMaskIndexPtr(Register index, Register length, Register output);
 void spectreMaskIndexPtr(Register index, const Address& length,
                          Register output);

 // The length must be a power of two. Performs a bounds check and Spectre
 // index masking.
 void boundsCheck32PowerOfTwo(Register index, uint32_t length, Label* failure);

 void speculationBarrier() PER_SHARED_ARCH;

 //}}} check_macroassembler_decl_style
public:
 // Unsafe here means the caller is responsible for Spectre mitigations if
 // needed. Prefer branchTestObjClass or one of the other masm helpers!
 inline void loadObjClassUnsafe(Register obj, Register dest);
 inline void loadObjShapeUnsafe(Register obj, Register dest);

 template <typename EmitPreBarrier>
 inline void storeObjShape(Register shape, Register obj,
                           EmitPreBarrier emitPreBarrier);
 template <typename EmitPreBarrier>
 inline void storeObjShape(Shape* shape, Register obj,
                           EmitPreBarrier emitPreBarrier);

 inline void loadObjProto(Register obj, Register dest);

 inline void loadStringLength(Register str, Register dest);

 void loadStringChars(Register str, Register dest, CharEncoding encoding);

 void loadNonInlineStringChars(Register str, Register dest,
                               CharEncoding encoding);
 void loadNonInlineStringCharsForStore(Register str, Register dest);
 void storeNonInlineStringChars(Register chars, Register str);

 void loadInlineStringChars(Register str, Register dest,
                            CharEncoding encoding);
 void loadInlineStringCharsForStore(Register str, Register dest);

private:
 enum class CharKind { CharCode, CodePoint };

 void branchIfMaybeSplitSurrogatePair(Register leftChild, Register index,
                                      Register scratch, Label* maybeSplit,
                                      Label* notSplit);

 void loadRopeChild(CharKind kind, Register str, Register index,
                    Register output, Register maybeScratch, Label* isLinear,
                    Label* splitSurrogate);

 void branchIfCanLoadStringChar(CharKind kind, Register str, Register index,
                                Register scratch, Register maybeScratch,
                                Label* label);
 void branchIfNotCanLoadStringChar(CharKind kind, Register str, Register index,
                                   Register scratch, Register maybeScratch,
                                   Label* label);

 void loadStringChar(CharKind kind, Register str, Register index,
                     Register output, Register scratch1, Register scratch2,
                     Label* fail);

public:
 void branchIfCanLoadStringChar(Register str, Register index, Register scratch,
                                Label* label) {
   branchIfCanLoadStringChar(CharKind::CharCode, str, index, scratch,
                             InvalidReg, label);
 }
 void branchIfNotCanLoadStringChar(Register str, Register index,
                                   Register scratch, Label* label) {
   branchIfNotCanLoadStringChar(CharKind::CharCode, str, index, scratch,
                                InvalidReg, label);
 }

 void branchIfCanLoadStringCodePoint(Register str, Register index,
                                     Register scratch1, Register scratch2,
                                     Label* label) {
   branchIfCanLoadStringChar(CharKind::CodePoint, str, index, scratch1,
                             scratch2, label);
 }
 void branchIfNotCanLoadStringCodePoint(Register str, Register index,
                                        Register scratch1, Register scratch2,
                                        Label* label) {
   branchIfNotCanLoadStringChar(CharKind::CodePoint, str, index, scratch1,
                                scratch2, label);
 }

 void loadStringChar(Register str, Register index, Register output,
                     Register scratch1, Register scratch2, Label* fail) {
   loadStringChar(CharKind::CharCode, str, index, output, scratch1, scratch2,
                  fail);
 }

 void loadStringChar(Register str, int32_t index, Register output,
                     Register scratch1, Register scratch2, Label* fail);

 void loadStringCodePoint(Register str, Register index, Register output,
                          Register scratch1, Register scratch2, Label* fail) {
   loadStringChar(CharKind::CodePoint, str, index, output, scratch1, scratch2,
                  fail);
 }

 void loadRopeLeftChild(Register str, Register dest);
 void loadRopeRightChild(Register str, Register dest);
 void storeRopeChildren(Register left, Register right, Register str);

 void loadDependentStringBase(Register str, Register dest);
 void storeDependentStringBase(Register base, Register str);

 void loadStringIndexValue(Register str, Register dest, Label* fail);

 /**
  * Store the character in |src| to |dest|.
  */
 template <typename T>
 void storeChar(const T& src, Address dest, CharEncoding encoding) {
   if (encoding == CharEncoding::Latin1) {
     store8(src, dest);
   } else {
     store16(src, dest);
   }
 }

 /**
  * Load the character at |src| into |dest|.
  */
 template <typename T>
 void loadChar(const T& src, Register dest, CharEncoding encoding) {
   if (encoding == CharEncoding::Latin1) {
     load8ZeroExtend(src, dest);
   } else {
     load16ZeroExtend(src, dest);
   }
 }

 /**
  * Load the character at |chars[index + offset]| into |dest|. The optional
  * offset argument is not scaled to the character encoding.
  */
 void loadChar(Register chars, Register index, Register dest,
               CharEncoding encoding, int32_t offset = 0);

 /**
  * Add |index| to |chars| so that |chars| now points at |chars[index]|.
  */
 void addToCharPtr(Register chars, Register index, CharEncoding encoding);

 /**
  * Branch if |src| is not a lead surrogate character.
  */
 void branchIfNotLeadSurrogate(Register src, Label* label);

private:
 enum class SurrogateChar { Lead, Trail };
 void branchSurrogate(Assembler::Condition cond, Register src,
                      Register scratch, Label* label,
                      SurrogateChar surrogateChar);

public:
 /**
  * Branch if |src| is a lead surrogate character.
  */
 void branchIfLeadSurrogate(Register src, Register scratch, Label* label) {
   branchSurrogate(Assembler::Equal, src, scratch, label, SurrogateChar::Lead);
 }

 /**
  * Branch if |src| is not a lead surrogate character.
  */
 void branchIfNotLeadSurrogate(Register src, Register scratch, Label* label) {
   branchSurrogate(Assembler::NotEqual, src, scratch, label,
                   SurrogateChar::Lead);
 }

 /**
  * Branch if |src| is not a trail surrogate character.
  */
 void branchIfNotTrailSurrogate(Register src, Register scratch, Label* label) {
   branchSurrogate(Assembler::NotEqual, src, scratch, label,
                   SurrogateChar::Trail);
 }

private:
 void loadStringFromUnit(Register unit, Register dest,
                         const StaticStrings& staticStrings);
 void loadLengthTwoString(Register c1, Register c2, Register dest,
                          const StaticStrings& staticStrings);

public:
 /**
  * Lookup the length-one string from the static strings cache.
  */
 void lookupStaticString(Register ch, Register dest,
                         const StaticStrings& staticStrings);

 /**
  * Lookup the length-one string from the static strings cache. Jumps to |fail|
  * when the string wasn't found in the strings cache.
  */
 void lookupStaticString(Register ch, Register dest,
                         const StaticStrings& staticStrings, Label* fail);

 /**
  * Lookup the length-two string from the static strings cache. Jumps to |fail|
  * when the string wasn't found in the strings cache.
  *
  * Clobbers |ch1| and |ch2|.
  */
 void lookupStaticString(Register ch1, Register ch2, Register dest,
                         const StaticStrings& staticStrings, Label* fail);

 /**
  * Lookup the integer string from the static integer strings cache. Jumps to
  * |fail| when the string wasn't found in the strings cache.
  */
 void lookupStaticIntString(Register integer, Register dest, Register scratch,
                            const StaticStrings& staticStrings, Label* fail);
 void lookupStaticIntString(Register integer, Register dest,
                            const StaticStrings& staticStrings, Label* fail) {
   lookupStaticIntString(integer, dest, dest, staticStrings, fail);
 }

 /**
  * Load the string representation of |input| in base |base|. Jumps to |fail|
  * when the string representation needs to be allocated dynamically.
  */
 void loadInt32ToStringWithBase(Register input, Register base, Register dest,
                                Register scratch1, Register scratch2,
                                const StaticStrings& staticStrings,
                                const LiveRegisterSet& volatileRegs,
                                bool lowerCase, Label* fail);
 void loadInt32ToStringWithBase(Register input, int32_t base, Register dest,
                                Register scratch1, Register scratch2,
                                const StaticStrings& staticStrings,
                                bool lowerCase, Label* fail);

 /**
  * Load the BigInt digits from |bigInt| into |digits|.
  */
 void loadBigIntDigits(Register bigInt, Register digits);

 /**
  * Load the first [u]int64 value from |bigInt| into |dest|.
  */
 void loadBigInt64(Register bigInt, Register64 dest);

 /**
  * Load the first digit from |bigInt| into |dest|.
  *
  * Note: A BigInt digit is a pointer-sized value storing an unsigned number.
  */
 void loadBigIntDigit(Register bigInt, Register dest);

 /**
  * Load the first digit from |bigInt| into |dest|. Jumps to |fail| when there
  * is more than one BigInt digit.
  *
  * Note: A BigInt digit is a pointer-sized value storing an unsigned number.
  */
 void loadBigIntDigit(Register bigInt, Register dest, Label* fail);

 /**
  * Load the number stored in |bigInt| into |dest|. Jumps to |fail| when the
  * number can't be saved into a single pointer-sized register.
  */
 void loadBigIntPtr(Register bigInt, Register dest, Label* fail);

 /**
  * Initialize a BigInt from |val|. Clobbers |val| when |temp| is invalid and
  * |type == Scalar::BigInt64|!
  */
 void initializeBigInt64(Scalar::Type type, Register bigInt, Register64 val,
                         Register64 temp = Register64::Invalid());

 /**
  * Initialize a BigInt from the signed, pointer-sized register |val|.
  * Clobbers |val|!
  */
 void initializeBigIntPtr(Register bigInt, Register val);

 /**
  * Copy a BigInt. Jumps to |fail| on allocation failure or when the BigInt
  * digits need to be heap allocated.
  */
 void copyBigIntWithInlineDigits(Register src, Register dest, Register temp,
                                 gc::Heap initialHeap, Label* fail);

 /**
  * Compare a BigInt and an Int32 value. Falls through to the false case.
  */
 void compareBigIntAndInt32(JSOp op, Register bigInt, Register int32,
                            Register scratch1, Register scratch2,
                            Label* ifTrue, Label* ifFalse);

 /**
  * Compare a BigInt and an Int32 value. Falls through to the false case.
  */
 void compareBigIntAndInt32(JSOp op, Register bigInt, Imm32 int32,
                            Register scratch, Label* ifTrue, Label* ifFalse);

 /**
  * Compare two BigInts for equality. Falls through if both BigInts are equal
  * to each other.
  *
  * - When we jump to |notSameLength|, |temp1| holds the length of the right
  *   operand.
  * - When we jump to |notSameDigit|, |temp2| points to the current digit of
  *   the left operand and |temp4| holds the current digit of the right
  *   operand.
  */
 void equalBigInts(Register left, Register right, Register temp1,
                   Register temp2, Register temp3, Register temp4,
                   Label* notSameSign, Label* notSameLength,
                   Label* notSameDigit);

 void loadJSContext(Register dest);

 void loadGlobalObjectData(Register dest);

 void loadRealmFuse(RealmFuses::FuseIndex index, Register dest);

 void loadRuntimeFuse(RuntimeFuses::FuseIndex index, Register dest);

 void guardRuntimeFuse(RuntimeFuses::FuseIndex index, Label* fail);

 void switchToRealm(Register realm);
 void switchToRealm(const void* realm, Register scratch);
 void switchToObjectRealm(Register obj, Register scratch);
 void switchToBaselineFrameRealm(Register scratch);
 void switchToWasmInstanceRealm(Register scratch1, Register scratch2);
 void debugAssertContextRealm(const void* realm, Register scratch);

 void guardObjectHasSameRealm(Register obj, Register scratch, Label* fail);

 template <typename ValueType>
 void storeLocalAllocSite(ValueType value, Register scratch);

 void loadBaselineCompileQueue(Register dest);

 void loadJitActivation(Register dest);

 void guardSpecificAtom(Register str, JSOffThreadAtom* atom, Register scratch,
                        const LiveRegisterSet& volatileRegs, Label* fail);

 void guardStringToInt32(Register str, Register output, Register scratch,
                         LiveRegisterSet volatileRegs, Label* fail);

 template <typename T>
 void loadTypedOrValue(const T& src, TypedOrValueRegister dest) {
   if (dest.hasValue()) {
     loadValue(src, dest.valueReg());
   } else {
     loadUnboxedValue(src, dest.type(), dest.typedReg());
   }
 }

 template <typename T>
 void storeTypedOrValue(TypedOrValueRegister src, const T& dest) {
   if (src.hasValue()) {
     storeValue(src.valueReg(), dest);
   } else if (IsFloatingPointType(src.type())) {
     FloatRegister reg = src.typedReg().fpu();
     if (src.type() == MIRType::Float32) {
       ScratchDoubleScope fpscratch(*this);
       convertFloat32ToDouble(reg, fpscratch);
       boxDouble(fpscratch, dest);
     } else {
       boxDouble(reg, dest);
     }
   } else {
     storeValue(ValueTypeFromMIRType(src.type()), src.typedReg().gpr(), dest);
   }
 }

 template <typename T>
 void storeConstantOrRegister(const ConstantOrRegister& src, const T& dest) {
   if (src.constant()) {
     storeValue(src.value(), dest);
   } else {
     storeTypedOrValue(src.reg(), dest);
   }
 }

 void storeCallPointerResult(Register reg) {
   if (reg != ReturnReg) {
     mov(ReturnReg, reg);
   }
 }

 inline void storeCallBoolResult(Register reg);
 inline void storeCallInt32Result(Register reg);

 void storeCallFloatResult(FloatRegister reg) {
   if (reg.isSingle()) {
     if (reg != ReturnFloat32Reg) {
       moveFloat32(ReturnFloat32Reg, reg);
     }
   } else {
     if (reg != ReturnDoubleReg) {
       moveDouble(ReturnDoubleReg, reg);
     }
   }
 }

 inline void storeCallResultValue(AnyRegister dest, JSValueType type);

 void storeCallResultValue(ValueOperand dest) {
#if defined(JS_NUNBOX32)
   // reshuffle the return registers used for a call result to store into
   // dest, using ReturnReg as a scratch register if necessary. This must
   // only be called after returning from a call, at a point when the
   // return register is not live. XXX would be better to allow wrappers
   // to store the return value to different places.
   if (dest.typeReg() == JSReturnReg_Data) {
     if (dest.payloadReg() == JSReturnReg_Type) {
       // swap the two registers.
       mov(JSReturnReg_Type, ReturnReg);
       mov(JSReturnReg_Data, JSReturnReg_Type);
       mov(ReturnReg, JSReturnReg_Data);
     } else {
       mov(JSReturnReg_Data, dest.payloadReg());
       mov(JSReturnReg_Type, dest.typeReg());
     }
   } else {
     mov(JSReturnReg_Type, dest.typeReg());
     mov(JSReturnReg_Data, dest.payloadReg());
   }
#elif defined(JS_PUNBOX64)
   if (dest.valueReg() != JSReturnReg) {
     mov(JSReturnReg, dest.valueReg());
   }
#else
#  error "Bad architecture"
#endif
 }

 inline void storeCallResultValue(TypedOrValueRegister dest);

private:
 TrampolinePtr preBarrierTrampoline(MIRType type);

 template <typename T>
 void unguardedCallPreBarrier(const T& address, MIRType type) {
   Label done;
   if (type == MIRType::Value) {
     branchTestGCThing(Assembler::NotEqual, address, &done);
   } else if (type == MIRType::Object || type == MIRType::String) {
     branchPtr(Assembler::Equal, address, ImmWord(0), &done);
   }

   Push(PreBarrierReg);
   computeEffectiveAddress(address, PreBarrierReg);

   TrampolinePtr preBarrier = preBarrierTrampoline(type);

   call(preBarrier);
   Pop(PreBarrierReg);
   // On arm64, SP may be < PSP now (that's OK).
   // eg testcase: tests/auto-regress/bug702915.js
   bind(&done);
 }

public:
 template <typename T>
 void guardedCallPreBarrier(const T& address, MIRType type) {
   Label done;
   branchTestNeedsIncrementalBarrier(Assembler::Zero, &done);
   unguardedCallPreBarrier(address, type);
   bind(&done);
 }

 // Like guardedCallPreBarrier, but unlike guardedCallPreBarrier this can be
 // called from runtime-wide trampolines because it loads cx->zone (instead of
 // baking in the current Zone) if JitContext::realm is nullptr.
 template <typename T>
 void guardedCallPreBarrierAnyZone(const T& address, MIRType type,
                                   Register scratch) {
   Label done;
   branchTestNeedsIncrementalBarrierAnyZone(Assembler::Zero, &done, scratch);
   unguardedCallPreBarrier(address, type);
   bind(&done);
 }

 enum class Uint32Mode { FailOnDouble, ForceDouble };

 void boxUint32(Register source, ValueOperand dest, Uint32Mode uint32Mode,
                Label* fail);

 static bool LoadRequiresCall(Scalar::Type type) {
   return type == Scalar::Float16 && !MacroAssembler::SupportsFloat32To16();
 }

 static bool StoreRequiresCall(Scalar::Type type) {
   return type == Scalar::Float16 && !MacroAssembler::SupportsFloat32To16();
 }

 template <typename T>
 void loadFromTypedArray(Scalar::Type arrayType, const T& src,
                         AnyRegister dest, Register temp1, Register temp2,
                         Label* fail, LiveRegisterSet volatileLiveReg);

 void loadFromTypedArray(Scalar::Type arrayType, const BaseIndex& src,
                         const ValueOperand& dest, Uint32Mode uint32Mode,
                         Register temp, Label* fail,
                         LiveRegisterSet volatileLiveReg);

 void loadFromTypedBigIntArray(Scalar::Type arrayType, const BaseIndex& src,
                               const ValueOperand& dest, Register bigInt,
                               Register64 temp);

 template <typename S, typename T>
 void storeToTypedIntArray(Scalar::Type arrayType, const S& value,
                           const T& dest) {
   switch (arrayType) {
     case Scalar::Int8:
     case Scalar::Uint8:
     case Scalar::Uint8Clamped:
       store8(value, dest);
       break;
     case Scalar::Int16:
     case Scalar::Uint16:
       store16(value, dest);
       break;
     case Scalar::Int32:
     case Scalar::Uint32:
       store32(value, dest);
       break;
     default:
       MOZ_CRASH("Invalid typed array type");
   }
 }

 template <typename T>
 void storeToTypedFloatArray(Scalar::Type arrayType, FloatRegister value,
                             const T& dest, Register temp,
                             LiveRegisterSet volatileLiveRegs);

 template <typename S, typename T>
 void storeToTypedBigIntArray(const S& value, const T& dest) {
   store64(value, dest);
 }

 void memoryBarrierBefore(Synchronization sync);
 void memoryBarrierAfter(Synchronization sync);

 using MacroAssemblerSpecific::convertDoubleToFloat16;
 using MacroAssemblerSpecific::convertFloat32ToFloat16;
 using MacroAssemblerSpecific::convertInt32ToFloat16;
 using MacroAssemblerSpecific::loadFloat16;

 void convertDoubleToFloat16(FloatRegister src, FloatRegister dest,
                             Register temp, LiveRegisterSet volatileLiveRegs);

 void convertDoubleToFloat16(FloatRegister src, FloatRegister dest,
                             Register temp1, Register temp2);

 void convertFloat32ToFloat16(FloatRegister src, FloatRegister dest,
                              Register temp, LiveRegisterSet volatileLiveRegs);

 void convertInt32ToFloat16(Register src, FloatRegister dest, Register temp,
                            LiveRegisterSet volatileLiveRegs);

 template <typename T>
 void loadFloat16(const T& src, FloatRegister dest, Register temp1,
                  Register temp2, LiveRegisterSet volatileLiveRegs);

 template <typename T>
 void storeFloat16(FloatRegister src, const T& dest, Register temp,
                   LiveRegisterSet volatileLiveRegs);

 void moveFloat16ToGPR(FloatRegister src, Register dest,
                       LiveRegisterSet volatileLiveRegs);

 void moveGPRToFloat16(Register src, FloatRegister dest, Register temp,
                       LiveRegisterSet volatileLiveRegs);

 void debugAssertIsObject(const ValueOperand& val);
 void debugAssertObjHasFixedSlots(Register obj, Register scratch);

 void debugAssertObjectHasClass(Register obj, Register scratch,
                                const JSClass* clasp);

 void debugAssertGCThingIsTenured(Register ptr, Register temp);

 void branchArrayIsNotPacked(Register array, Register temp1, Register temp2,
                             Label* label);

 void setIsPackedArray(Register obj, Register output, Register temp);

 void packedArrayPop(Register array, ValueOperand output, Register temp1,
                     Register temp2, Label* fail);
 void packedArrayShift(Register array, ValueOperand output, Register temp1,
                       Register temp2, LiveRegisterSet volatileRegs,
                       Label* fail);

 void loadArgumentsObjectElement(Register obj, Register index,
                                 ValueOperand output, Register temp,
                                 Label* fail);
 void loadArgumentsObjectElementHole(Register obj, Register index,
                                     ValueOperand output, Register temp,
                                     Label* fail);
 void loadArgumentsObjectElementExists(Register obj, Register index,
                                       Register output, Register temp,
                                       Label* fail);

 void loadArgumentsObjectLength(Register obj, Register output, Label* fail);
 void loadArgumentsObjectLength(Register obj, Register output);

 void branchTestArgumentsObjectFlags(Register obj, Register temp,
                                     uint32_t flags, Condition cond,
                                     Label* label);

 void typedArrayElementSize(Register obj, Register output);

private:
 // Shift |output| by the element shift of the ResizableTypedArray in |obj|.
 void resizableTypedArrayElementShiftBy(Register obj, Register output,
                                        Register scratch);

public:
 void branchIfClassIsNotTypedArray(Register clasp, Label* notTypedArray);
 void branchIfClassIsNotNonResizableTypedArray(Register clasp,
                                               Label* notTypedArray);
 void branchIfClassIsNotResizableTypedArray(Register clasp,
                                            Label* notTypedArray);

 void branchIfIsNotArrayBuffer(Register obj, Register temp, Label* label);
 void branchIfIsNotSharedArrayBuffer(Register obj, Register temp,
                                     Label* label);
 void branchIfIsArrayBufferMaybeShared(Register obj, Register temp,
                                       Label* label);

private:
 enum class BranchIfDetached { No, Yes };

 void branchIfHasDetachedArrayBuffer(BranchIfDetached branchIf, Register obj,
                                     Register temp, Label* label);

public:
 void branchIfHasDetachedArrayBuffer(Register obj, Register temp,
                                     Label* label) {
   branchIfHasDetachedArrayBuffer(BranchIfDetached::Yes, obj, temp, label);
 }

 void branchIfHasAttachedArrayBuffer(Register obj, Register temp,
                                     Label* label) {
   branchIfHasDetachedArrayBuffer(BranchIfDetached::No, obj, temp, label);
 }

 void branchIfResizableArrayBufferViewOutOfBounds(Register obj, Register temp,
                                                  Label* label);

 void branchIfResizableArrayBufferViewInBounds(Register obj, Register temp,
                                               Label* label);

 void branchIfNativeIteratorNotReusable(Register ni, Label* notReusable);

 void maybeLoadIteratorFromShape(Register obj, Register dest, Register temp,
                                 Register temp2, Register temp3,
                                 Label* failure, bool exclusive);

 void iteratorMore(Register obj, ValueOperand output, Register temp);
 void iteratorClose(Register obj, Register temp1, Register temp2,
                    Register temp3);
 void iteratorLength(Register obj, Register output);
 void iteratorLoadElement(Register obj, Register index, Register output);
 void iteratorLoadElement(Register obj, int32_t index, Register output);
 void registerIterator(Register enumeratorsList, Register iter, Register temp);

 void prepareOOBStoreElement(Register object, Register index,
                             Register elements, Register spectreTemp,
                             Label* failure, LiveRegisterSet volatileLiveRegs);

 void toHashableNonGCThing(ValueOperand value, ValueOperand result,
                           FloatRegister tempFloat);

 void toHashableValue(ValueOperand value, ValueOperand result,
                      FloatRegister tempFloat, Label* atomizeString,
                      Label* tagString);

private:
 void scrambleHashCode(Register result);

public:
 void hashAndScrambleValue(ValueOperand value, Register result, Register temp);
 void prepareHashNonGCThing(ValueOperand value, Register result,
                            Register temp);
 void prepareHashString(Register str, Register result, Register temp);
 void prepareHashSymbol(Register sym, Register result);
 void prepareHashBigInt(Register bigInt, Register result, Register temp1,
                        Register temp2, Register temp3);
 void prepareHashObject(Register setObj, ValueOperand value, Register result,
                        Register temp1, Register temp2, Register temp3,
                        Register temp4);
 void prepareHashValue(Register setObj, ValueOperand value, Register result,
                       Register temp1, Register temp2, Register temp3,
                       Register temp4);

 // Helper functions used to implement mozilla::HashTable lookup inline
 // in jitcode.
 void prepareHashMFBT(Register hashCode, bool alreadyScrambled);
 template <typename Table>
 void computeHash1MFBT(Register hashTable, Register hashCode, Register hash1,
                       Register scratch);
 template <typename Table>
 void computeHash2MFBT(Register hashTable, Register hashCode, Register hash2,
                       Register sizeMask, Register scratch);
 void applyDoubleHashMFBT(Register hash1, Register hash2, Register sizeMask);
 template <typename Table>
 void checkForMatchMFBT(Register hashTable, Register hashIndex,
                        Register hashCode, Register scratch, Register scratch2,
                        Label* missing, Label* collision);

public:
 // This generates an inlined version of mozilla::detail::HashTable::lookup
 // (ForNonAdd).
 // Inputs/requirements:
 // - hashTable: A register containing a pointer to a Table. The Table type
 //              must define:
 //              - Table::Entry
 //              - Table::offsetOfHashShift()
 //              - Table::offsetOfTable()
 // - hashCode:  The 32-bit hash of the key to look up. This should already
 //              have been scrambled using prepareHashMFBT.
 // - match:     A lambda to generate code to compare keys. The code that it
 //              generates can assume that `scratch` contains the address of
 //              a Table::Entry with a matching hash value. `scratch2` can be
 //              safely used without clobbering anything. If the keys don't
 //              match, the generated code should fall through. If the keys
 //              match, the generated code is responsible for jumping to the
 //              correct continuation.
 // - missing:   A label to jump to if the key does not exist in the table.
 template <typename Table, typename Match>
 void lookupMFBT(Register hashTable, Register hashCode, Register scratch,
                 Register scratch2, Register scratch3, Register scratch4,
                 Register scratch5, Label* missing, Match match);

private:
 enum class IsBigInt { No, Yes, Maybe };

 /**
  * Search for a value in a MapObject or SetObject.
  *
  * When we jump to |found|, |entryTemp| holds the found hashtable entry.
  */
 template <typename TableObject>
 void orderedHashTableLookup(Register setOrMapObj, ValueOperand value,
                             Register hash, Register entryTemp, Register temp1,
                             Register temp3, Register temp4, Register temp5,
                             Label* found, IsBigInt isBigInt);

 void setObjectHas(Register setObj, ValueOperand value, Register hash,
                   Register result, Register temp1, Register temp2,
                   Register temp3, Register temp4, IsBigInt isBigInt);

 void mapObjectHas(Register mapObj, ValueOperand value, Register hash,
                   Register result, Register temp1, Register temp2,
                   Register temp3, Register temp4, IsBigInt isBigInt);

 void mapObjectGet(Register mapObj, ValueOperand value, Register hash,
                   ValueOperand result, Register temp1, Register temp2,
                   Register temp3, Register temp4, Register temp5,
                   IsBigInt isBigInt);

public:
 void setObjectHasNonBigInt(Register setObj, ValueOperand value, Register hash,
                            Register result, Register temp1, Register temp2) {
   return setObjectHas(setObj, value, hash, result, temp1, temp2, InvalidReg,
                       InvalidReg, IsBigInt::No);
 }
 void setObjectHasBigInt(Register setObj, ValueOperand value, Register hash,
                         Register result, Register temp1, Register temp2,
                         Register temp3, Register temp4) {
   return setObjectHas(setObj, value, hash, result, temp1, temp2, temp3, temp4,
                       IsBigInt::Yes);
 }
 void setObjectHasValue(Register setObj, ValueOperand value, Register hash,
                        Register result, Register temp1, Register temp2,
                        Register temp3, Register temp4) {
   return setObjectHas(setObj, value, hash, result, temp1, temp2, temp3, temp4,
                       IsBigInt::Maybe);
 }

 void mapObjectHasNonBigInt(Register mapObj, ValueOperand value, Register hash,
                            Register result, Register temp1, Register temp2) {
   return mapObjectHas(mapObj, value, hash, result, temp1, temp2, InvalidReg,
                       InvalidReg, IsBigInt::No);
 }
 void mapObjectHasBigInt(Register mapObj, ValueOperand value, Register hash,
                         Register result, Register temp1, Register temp2,
                         Register temp3, Register temp4) {
   return mapObjectHas(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
                       IsBigInt::Yes);
 }
 void mapObjectHasValue(Register mapObj, ValueOperand value, Register hash,
                        Register result, Register temp1, Register temp2,
                        Register temp3, Register temp4) {
   return mapObjectHas(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
                       IsBigInt::Maybe);
 }

 void mapObjectGetNonBigInt(Register mapObj, ValueOperand value, Register hash,
                            ValueOperand result, Register temp1,
                            Register temp2, Register temp3) {
   return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3,
                       InvalidReg, InvalidReg, IsBigInt::No);
 }
 void mapObjectGetBigInt(Register mapObj, ValueOperand value, Register hash,
                         ValueOperand result, Register temp1, Register temp2,
                         Register temp3, Register temp4, Register temp5) {
   return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
                       temp5, IsBigInt::Yes);
 }
 void mapObjectGetValue(Register mapObj, ValueOperand value, Register hash,
                        ValueOperand result, Register temp1, Register temp2,
                        Register temp3, Register temp4, Register temp5) {
   return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
                       temp5, IsBigInt::Maybe);
 }

private:
 template <typename TableObject>
 void loadOrderedHashTableCount(Register setOrMapObj, Register result);

public:
 void loadSetObjectSize(Register setObj, Register result);
 void loadMapObjectSize(Register mapObj, Register result);

 // Inline version of js::ClampDoubleToUint8.
 // This function clobbers the input register.
 void clampDoubleToUint8(FloatRegister input, Register output) PER_ARCH;

 // If source is a double, load into dest.
 // If source is int32, convert to double and store in dest.
 // Else, branch to failure.
 inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
                          Label* failure);

 template <typename S>
 void ensureDouble(const S& source, FloatRegister dest, Label* failure) {
   Label isDouble, done;
   branchTestDouble(Assembler::Equal, source, &isDouble);
   branchTestInt32(Assembler::NotEqual, source, failure);

   convertInt32ToDouble(source, dest);
   jump(&done);

   bind(&isDouble);
   unboxDouble(source, dest);

   bind(&done);
 }

 // Inline allocation.
private:
 void checkAllocatorState(Register temp, gc::AllocKind allocKind, Label* fail);
 bool shouldNurseryAllocate(gc::AllocKind allocKind, gc::Heap initialHeap);
 void nurseryAllocateObject(
     Register result, Register temp, gc::AllocKind allocKind,
     size_t nDynamicSlots, Label* fail,
     const AllocSiteInput& allocSite = AllocSiteInput());
 void bumpPointerAllocate(Register result, Register temp, Label* fail,
                          CompileZone* zone, JS::TraceKind traceKind,
                          uint32_t size,
                          const AllocSiteInput& allocSite = AllocSiteInput());
 void updateAllocSite(Register temp, Register result, CompileZone* zone,
                      Register site);

 void freeListAllocate(Register result, Register temp, gc::AllocKind allocKind,
                       Label* fail);
 void allocateObject(Register result, Register temp, gc::AllocKind allocKind,
                     uint32_t nDynamicSlots, gc::Heap initialHeap, Label* fail,
                     const AllocSiteInput& allocSite = AllocSiteInput());
 void nurseryAllocateString(Register result, Register temp,
                            gc::AllocKind allocKind, Label* fail);
 void allocateString(Register result, Register temp, gc::AllocKind allocKind,
                     gc::Heap initialHeap, Label* fail);
 void nurseryAllocateBigInt(Register result, Register temp, Label* fail);
 void copySlotsFromTemplate(Register obj,
                            const TemplateNativeObject& templateObj,
                            uint32_t start, uint32_t end);
 void fillSlotsWithConstantValue(Address addr, Register temp, uint32_t start,
                                 uint32_t end, const Value& v);
 void fillSlotsWithUndefined(Address addr, Register temp, uint32_t start,
                             uint32_t end);
 void fillSlotsWithUninitialized(Address addr, Register temp, uint32_t start,
                                 uint32_t end);

 void initGCSlots(Register obj, Register temp,
                  const TemplateNativeObject& templateObj);

public:
 void createGCObject(Register result, Register temp,
                     const TemplateObject& templateObj, gc::Heap initialHeap,
                     Label* fail, bool initContents = true,
                     const AllocSiteInput& allocSite = AllocSiteInput());

 void createPlainGCObject(Register result, Register shape, Register temp,
                          Register temp2, uint32_t numFixedSlots,
                          uint32_t numDynamicSlots, gc::AllocKind allocKind,
                          gc::Heap initialHeap, Label* fail,
                          const AllocSiteInput& allocSite,
                          bool initContents = true);

 // dynamicSlotsTemp is used to initialize the dynamic slots after allocating
 // the object. If numUsedDynamicSlots == 0, it may be InvalidReg.
 void createArrayWithFixedElements(
     Register result, Register shape, Register temp, Register dynamicSlotsTemp,
     uint32_t arrayLength, uint32_t arrayCapacity,
     uint32_t numUsedDynamicSlots, uint32_t numDynamicSlots,
     gc::AllocKind allocKind, gc::Heap initialHeap, Label* fail,
     const AllocSiteInput& allocSite = AllocSiteInput());

 void createFunctionClone(Register result, Register canonical,
                          Register envChain, Register temp,
                          gc::AllocKind allocKind, Label* fail,
                          const AllocSiteInput& allocSite);

 void initGCThing(Register obj, Register temp,
                  const TemplateObject& templateObj, bool initContents = true);

 void initTypedArraySlots(Register obj, Register length, Register temp1,
                          Register temp2, Label* fail,
                          const FixedLengthTypedArrayObject* templateObj);

 void initTypedArraySlotsInline(
     Register obj, Register temp,
     const FixedLengthTypedArrayObject* templateObj);

 void newGCString(Register result, Register temp, gc::Heap initialHeap,
                  Label* fail);
 void newGCFatInlineString(Register result, Register temp,
                           gc::Heap initialHeap, Label* fail);

 void newGCBigInt(Register result, Register temp, gc::Heap initialHeap,
                  Label* fail);

 void preserveWrapper(Register wrapper, Register temp1, Register temp2,
                      const LiveRegisterSet& liveRegs);

private:
 void branchIfNotStringCharsEquals(Register stringChars,
                                   const JSOffThreadAtom* str, Label* label);

public:
 // Returns true if |str| is a (non-empty) string which can be compared
 // using |compareStringChars|.
 static bool canCompareStringCharsInline(const JSOffThreadAtom* str);

 // Load the string characters in preparation for |compareStringChars|.
 void loadStringCharsForCompare(Register input, const JSOffThreadAtom* str,
                                Register stringChars, Label* fail);

 // Compare string characters based on the equality operator. The string
 // characters must be at least as long as the length of |str|.
 void compareStringChars(JSOp op, Register stringChars,
                         const JSOffThreadAtom* str, Register result);

 // Compares two strings for equality based on the JSOP.
 // This checks for identical pointers, atoms and length and fails for
 // everything else.
 void compareStrings(JSOp op, Register left, Register right, Register result,
                     Label* fail);

 // Result of the typeof operation. Falls back to slow-path for proxies.
 void typeOfObject(Register objReg, Register scratch, Label* slow,
                   Label* isObject, Label* isCallable, Label* isUndefined);

 // Implementation of IsCallable. Doesn't handle proxies.
 void isCallable(Register obj, Register output, Label* isProxy) {
   isCallableOrConstructor(true, obj, output, isProxy);
 }
 void isConstructor(Register obj, Register output, Label* isProxy) {
   isCallableOrConstructor(false, obj, output, isProxy);
 }

 void setIsCrossRealmArrayConstructor(Register obj, Register output);

 void setIsDefinitelyTypedArrayConstructor(Register obj, Register output);

 void loadMegamorphicCache(Register dest);
 void tryFastAtomize(Register str, Register scratch, Register output,
                     Label* fail);
 void loadMegamorphicSetPropCache(Register dest);

 void loadAtomOrSymbolAndHash(ValueOperand value, Register outId,
                              Register outHash, Label* cacheMiss);

 void loadAtomHash(Register id, Register hash, Label* done);

 void emitExtractValueFromMegamorphicCacheEntry(
     Register obj, Register entry, Register scratch1, Register scratch2,
     ValueOperand output, Label* cacheHit, Label* cacheMiss,
     Label* cacheHitGetter);

 template <typename IdOperandType>
 void emitMegamorphicCacheLookupByValueCommon(
     IdOperandType id, Register obj, Register scratch1, Register scratch2,
     Register outEntryPtr, Label* cacheMiss, Label* cacheMissWithEntry);

 void emitMegamorphicCacheLookup(PropertyKey id, Register obj,
                                 Register scratch1, Register scratch2,
                                 Register outEntryPtr, ValueOperand output,
                                 Label* cacheHit,
                                 Label* cacheHitGetter = nullptr);

 // NOTE: |id| must either be a ValueOperand or a Register. If it is a
 // Register, we assume that it is an atom.
 template <typename IdOperandType>
 void emitMegamorphicCacheLookupByValue(IdOperandType id, Register obj,
                                        Register scratch1, Register scratch2,
                                        Register outEntryPtr,
                                        ValueOperand output, Label* cacheHit,
                                        Label* cacheHitGetter = nullptr);

 void emitMegamorphicCacheLookupExists(ValueOperand id, Register obj,
                                       Register scratch1, Register scratch2,
                                       Register outEntryPtr, Register output,
                                       Label* cacheHit, bool hasOwn);

 // Given a PropertyIteratorObject with valid indices, extract the current
 // PropertyIndex, storing the index in |outIndex| and the kind in |outKind|
 void extractCurrentIndexAndKindFromIterator(Register iterator,
                                             Register outIndex,
                                             Register outKind);
 void extractIndexAndKindFromIteratorByIterIndex(Register iterator,
                                                 Register inOutIndex,
                                                 Register outKind,
                                                 Register scratch);

 template <typename IdType>
#ifdef JS_CODEGEN_X86
 // See MegamorphicSetElement in LIROps.yaml
 void emitMegamorphicCachedSetSlot(
     IdType id, Register obj, Register scratch1, ValueOperand value,
     const LiveRegisterSet& liveRegs, Label* cacheHit,
     void (*emitPreBarrier)(MacroAssembler&, const Address&, MIRType));
#else
 void emitMegamorphicCachedSetSlot(
     IdType id, Register obj, Register scratch1, Register scratch2,
     Register scratch3, ValueOperand value, const LiveRegisterSet& liveRegs,
     Label* cacheHit,
     void (*emitPreBarrier)(MacroAssembler&, const Address&, MIRType));
#endif

 void loadDOMExpandoValueGuardGeneration(
     Register obj, ValueOperand output,
     JS::ExpandoAndGeneration* expandoAndGeneration, uint64_t generation,
     Label* fail);

 void guardNonNegativeIntPtrToInt32(Register reg, Label* fail);

 void loadArrayBufferByteLengthIntPtr(Register obj, Register output);
 void loadArrayBufferViewByteOffsetIntPtr(Register obj, Register output);
 void loadArrayBufferViewLengthIntPtr(Register obj, Register output);

 void loadGrowableSharedArrayBufferByteLengthIntPtr(Synchronization sync,
                                                    Register obj,
                                                    Register output);

private:
 enum class ResizableArrayBufferView { TypedArray, DataView };

 void loadResizableArrayBufferViewLengthIntPtr(ResizableArrayBufferView view,
                                               Synchronization sync,
                                               Register obj, Register output,
                                               Register scratch);

public:
 void loadResizableTypedArrayLengthIntPtr(Synchronization sync, Register obj,
                                          Register output, Register scratch) {
   loadResizableArrayBufferViewLengthIntPtr(
       ResizableArrayBufferView::TypedArray, sync, obj, output, scratch);
 }

 void loadResizableDataViewByteLengthIntPtr(Synchronization sync, Register obj,
                                            Register output,
                                            Register scratch) {
   loadResizableArrayBufferViewLengthIntPtr(ResizableArrayBufferView::DataView,
                                            sync, obj, output, scratch);
 }

 void dateFillLocalTimeSlots(Register obj, Register scratch,
                             const LiveRegisterSet& volatileRegs);

private:
 void udiv32ByConstant(Register src, uint32_t divisor, Register dest);

 void umod32ByConstant(Register src, uint32_t divisor, Register dest,
                       Register scratch);

 template <typename GetTimeFn>
 void dateTimeFromSecondsIntoYear(ValueOperand secondsIntoYear,
                                  ValueOperand output, Register scratch1,
                                  Register scratch2, GetTimeFn getTimeFn);

public:
 void dateHoursFromSecondsIntoYear(ValueOperand secondsIntoYear,
                                   ValueOperand output, Register scratch1,
                                   Register scratch2);

 void dateMinutesFromSecondsIntoYear(ValueOperand secondsIntoYear,
                                     ValueOperand output, Register scratch1,
                                     Register scratch2);

 void dateSecondsFromSecondsIntoYear(ValueOperand secondsIntoYear,
                                     ValueOperand output, Register scratch1,
                                     Register scratch2);

 void computeImplicitThis(Register env, ValueOperand output, Label* slowPath);

private:
 void isCallableOrConstructor(bool isCallable, Register obj, Register output,
                              Label* isProxy);

public:
 // Generates code used to complete a bailout.
 void generateBailoutTail(Register scratch, Register bailoutInfo);

public:
#ifndef JS_CODEGEN_ARM64
 // StackPointer manipulation functions.
 // On ARM64, the StackPointer is implemented as two synchronized registers.
 // Code shared across platforms must use these functions to be valid.
 template <typename T>
 inline void addToStackPtr(T t);
 template <typename T>
 inline void addStackPtrTo(T t);

 void subFromStackPtr(Imm32 imm32)
     DEFINED_ON(mips64, loong64, riscv64, wasm32, arm, x86, x64);
 void subFromStackPtr(Register reg);

 template <typename T>
 void subStackPtrFrom(T t) {
   subPtr(getStackPointer(), t);
 }

 template <typename T>
 void andToStackPtr(T t) {
   andPtr(t, getStackPointer());
 }

 template <typename T>
 void moveToStackPtr(T t) {
   movePtr(t, getStackPointer());
 }
 template <typename T>
 void moveStackPtrTo(T t) {
   movePtr(getStackPointer(), t);
 }

 template <typename T>
 void loadStackPtr(T t) {
   loadPtr(t, getStackPointer());
 }
 template <typename T>
 void storeStackPtr(T t) {
   storePtr(getStackPointer(), t);
 }

 template <typename T>
 void loadStackPtrFromPrivateValue(T t) {
   loadStackPtr(t);
 }
 template <typename T>
 void storeStackPtrToPrivateValue(T t) {
   storeStackPtr(t);
 }

 // StackPointer testing functions.
 // On ARM64, sp can function as the zero register depending on context.
 // Code shared across platforms must use these functions to be valid.
 template <typename T>
 inline void branchTestStackPtr(Condition cond, T t, Label* label);
 template <typename T>
 inline void branchStackPtr(Condition cond, T rhs, Label* label);
 template <typename T>
 inline void branchStackPtrRhs(Condition cond, T lhs, Label* label);

 // Move the stack pointer based on the requested amount.
 inline void reserveStack(uint32_t amount);
#else  // !JS_CODEGEN_ARM64
 void reserveStack(uint32_t amount);
#endif

public:
 void enableProfilingInstrumentation() {
   emitProfilingInstrumentation_ = true;
 }

private:
 // This class is used to surround call sites throughout the assembler. This
 // is used by callWithABI, and callJit functions, except if suffixed by
 // NoProfiler.
 class MOZ_RAII AutoProfilerCallInstrumentation {
  public:
   explicit AutoProfilerCallInstrumentation(MacroAssembler& masm);
   ~AutoProfilerCallInstrumentation() = default;
 };
 friend class AutoProfilerCallInstrumentation;

 void appendProfilerCallSite(CodeOffset label) {
   propagateOOM(profilerCallSites_.append(label));
 }

 // Fix up the code pointers to be written for locations where profilerCallSite
 // emitted moves of RIP to a register.
 void linkProfilerCallSites(JitCode* code);

 // This field is used to manage profiling instrumentation output. If
 // provided and enabled, then instrumentation will be emitted around call
 // sites.
 bool emitProfilingInstrumentation_;

 // Record locations of the call sites.
 Vector<CodeOffset, 0, SystemAllocPolicy> profilerCallSites_;

public:
 void loadJitCodeRaw(Register func, Register dest);
 void loadJitCodeRawNoIon(Register func, Register dest, Register scratch);

 void loadBaselineFramePtr(Register framePtr, Register dest);

 void pushBaselineFramePtr(Register framePtr, Register scratch) {
   loadBaselineFramePtr(framePtr, scratch);
   push(scratch);
 }

 void PushBaselineFramePtr(Register framePtr, Register scratch) {
   loadBaselineFramePtr(framePtr, scratch);
   Push(scratch);
 }

 using MacroAssemblerSpecific::movePtr;

 void movePtr(TrampolinePtr ptr, Register dest) {
   movePtr(ImmPtr(ptr.value), dest);
 }

private:
 void handleFailure();

public:
 Label* exceptionLabel() {
   // Exceptions are currently handled the same way as sequential failures.
   return &failureLabel_;
 }

 Label* failureLabel() { return &failureLabel_; }

 void finish();
 void link(JitCode* code);

 void assumeUnreachable(const char* output);

 void printf(const char* output);
 void printf(const char* output, Register value);

 void outOfLineTruncateSlow(FloatRegister src, Register dest,
                            bool widenFloatToDouble, bool compilingWasm,
                            wasm::BytecodeOffset callOffset);

 void convertInt32ValueToDouble(ValueOperand val);

private:
 enum class FloatingPointType { Double, Float32, Float16 };

 void convertValueToFloatingPoint(ValueOperand value, FloatRegister output,
                                  Register maybeTemp,
                                  LiveRegisterSet volatileLiveRegs,
                                  Label* fail, FloatingPointType outputType);

public:
 void convertValueToDouble(ValueOperand value, FloatRegister output,
                           Label* fail) {
   convertValueToFloatingPoint(value, output, InvalidReg, LiveRegisterSet{},
                               fail, FloatingPointType::Double);
 }

 void convertValueToFloat32(ValueOperand value, FloatRegister output,
                            Label* fail) {
   convertValueToFloatingPoint(value, output, InvalidReg, LiveRegisterSet{},
                               fail, FloatingPointType::Float32);
 }

 void convertValueToFloat16(ValueOperand value, FloatRegister output,
                            Register maybeTemp,
                            LiveRegisterSet volatileLiveRegs, Label* fail) {
   convertValueToFloatingPoint(value, output, maybeTemp, volatileLiveRegs,
                               fail, FloatingPointType::Float16);
 }

 //
 // Functions for converting values to int.
 //

 // This carries over the MToNumberInt32 operation on the ValueOperand
 // input; see comment at the top of this class.
 void convertValueToInt32(ValueOperand value, FloatRegister temp,
                          Register output, Label* fail, bool negativeZeroCheck,
                          IntConversionInputKind conversion);

 // This carries over the MTruncateToInt32 operation on the ValueOperand
 // input; see the comment at the top of this class.
 //
 // Strings may be handled by providing labels to jump to. The subroutine,
 // usually an OOL call, is passed the unboxed string in |stringReg| and should
 // convert it to a double store into |temp|.
 void truncateValueToInt32(ValueOperand value, Label* handleStringEntry,
                           Label* handleStringRejoin,
                           Label* truncateDoubleSlow, Register stringReg,
                           FloatRegister temp, Register output, Label* fail);

 void truncateValueToInt32(ValueOperand value, FloatRegister temp,
                           Register output, Label* fail) {
   truncateValueToInt32(value, nullptr, nullptr, nullptr, InvalidReg, temp,
                        output, fail);
 }

 // Convenience functions for clamping values to uint8.
 //
 // Strings are handled by providing labels to jump to. The subroutine, usually
 // an OOL call, is passed the unboxed string in |stringReg| and should convert
 // it to a double store into |temp|.
 void clampValueToUint8(ValueOperand value, Label* handleStringEntry,
                        Label* handleStringRejoin, Register stringReg,
                        FloatRegister temp, Register output, Label* fail);

 [[nodiscard]] bool icBuildOOLFakeExitFrame(void* fakeReturnAddr,
                                            AutoSaveLiveRegisters& save);

 // Align the stack pointer based on the number of arguments which are pushed
 // on the stack, such that the JitFrameLayout would be correctly aligned on
 // the JitStackAlignment.
 void alignJitStackBasedOnNArgs(Register nargs, bool countIncludesThis);
 void alignJitStackBasedOnNArgs(uint32_t argc, bool countIncludesThis);

 inline void assertStackAlignment(uint32_t alignment, int32_t offset = 0);

 void touchFrameValues(Register numStackValues, Register scratch1,
                       Register scratch2);

#ifdef JS_64BIT
 // See comment block "64-bit GPRs carrying 32-bit values" above.  This asserts
 // that the high bits of the register are appropriate for the architecture and
 // the value in the low bits.
 void debugAssertCanonicalInt32(Register r);
#endif

#ifdef FUZZING_JS_FUZZILLI
 void fuzzilliHashDouble(FloatRegister src, Register result, Register temp);

 void fuzzilliStoreHash(Register value, Register temp1, Register temp2);
#endif
};

// StackMacroAssembler checks no GC will happen while it's on the stack.
class MOZ_RAII StackMacroAssembler : public MacroAssembler {
 JS::AutoCheckCannotGC nogc;

public:
 StackMacroAssembler(JSContext* cx, TempAllocator& alloc);
};

// WasmMacroAssembler does not contain GC pointers, so it doesn't need the no-GC
// checking StackMacroAssembler has.
class MOZ_RAII WasmMacroAssembler : public MacroAssembler {
public:
 explicit WasmMacroAssembler(TempAllocator& alloc, bool limitedSize = true);
 ~WasmMacroAssembler() { assertNoGCThings(); }
};

// Heap-allocated MacroAssembler used for off-thread code generation.
// GC cancels off-thread compilations.
class OffThreadMacroAssembler : public MacroAssembler {
public:
 OffThreadMacroAssembler(TempAllocator& alloc, CompileRealm* realm);
};

//{{{ check_macroassembler_style
inline uint32_t MacroAssembler::framePushed() const { return framePushed_; }

inline void MacroAssembler::setFramePushed(uint32_t framePushed) {
 framePushed_ = framePushed;
}

inline void MacroAssembler::adjustFrame(int32_t value) {
 MOZ_ASSERT_IF(value < 0, framePushed_ >= uint32_t(-value));
 setFramePushed(framePushed_ + value);
}

inline void MacroAssembler::implicitPop(uint32_t bytes) {
 MOZ_ASSERT(bytes % sizeof(intptr_t) == 0);
 MOZ_ASSERT(bytes <= INT32_MAX);
 adjustFrame(-int32_t(bytes));
}
//}}} check_macroassembler_style

static inline Assembler::DoubleCondition JSOpToDoubleCondition(JSOp op) {
 switch (op) {
   case JSOp::Eq:
   case JSOp::StrictEq:
     return Assembler::DoubleEqual;
   case JSOp::Ne:
   case JSOp::StrictNe:
     return Assembler::DoubleNotEqualOrUnordered;
   case JSOp::Lt:
     return Assembler::DoubleLessThan;
   case JSOp::Le:
     return Assembler::DoubleLessThanOrEqual;
   case JSOp::Gt:
     return Assembler::DoubleGreaterThan;
   case JSOp::Ge:
     return Assembler::DoubleGreaterThanOrEqual;
   default:
     MOZ_CRASH("Unexpected comparison operation");
 }
}

// Note: the op may have been inverted during lowering (to put constants in a
// position where they can be immediates), so it is important to use the
// lir->jsop() instead of the mir->jsop() when it is present.
static inline Assembler::Condition JSOpToCondition(JSOp op, bool isSigned) {
 if (isSigned) {
   switch (op) {
     case JSOp::Eq:
     case JSOp::StrictEq:
       return Assembler::Equal;
     case JSOp::Ne:
     case JSOp::StrictNe:
       return Assembler::NotEqual;
     case JSOp::Lt:
       return Assembler::LessThan;
     case JSOp::Le:
       return Assembler::LessThanOrEqual;
     case JSOp::Gt:
       return Assembler::GreaterThan;
     case JSOp::Ge:
       return Assembler::GreaterThanOrEqual;
     default:
       MOZ_CRASH("Unrecognized comparison operation");
   }
 } else {
   switch (op) {
     case JSOp::Eq:
     case JSOp::StrictEq:
       return Assembler::Equal;
     case JSOp::Ne:
     case JSOp::StrictNe:
       return Assembler::NotEqual;
     case JSOp::Lt:
       return Assembler::Below;
     case JSOp::Le:
       return Assembler::BelowOrEqual;
     case JSOp::Gt:
       return Assembler::Above;
     case JSOp::Ge:
       return Assembler::AboveOrEqual;
     default:
       MOZ_CRASH("Unrecognized comparison operation");
   }
 }
}

static inline size_t StackDecrementForCall(uint32_t alignment,
                                          size_t bytesAlreadyPushed,
                                          size_t bytesToPush) {
 return bytesToPush +
        ComputeByteAlignment(bytesAlreadyPushed + bytesToPush, alignment);
}

// Helper for generatePreBarrier.
inline DynFn JitPreWriteBarrier(MIRType type);
}  // namespace jit

}  // namespace js

#endif /* jit_MacroAssembler_h */