tor-browser

Assembler-arm64.h (32990B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef A64_ASSEMBLER_A64_H_
#define A64_ASSEMBLER_A64_H_

#include "jit/arm64/vixl/Assembler-vixl.h"

#include "jit/CompactBuffer.h"
#include "jit/shared/Disassembler-shared.h"
#include "wasm/WasmTypeDecls.h"

namespace js {
namespace jit {

// VIXL imports.
using ARMRegister = vixl::Register;
using ARMFPRegister = vixl::FPRegister;
using vixl::ARMBuffer;
using vixl::Instruction;

using LabelDoc = DisassemblerSpew::LabelDoc;
using LiteralDoc = DisassemblerSpew::LiteralDoc;

static const uint32_t AlignmentAtPrologue = 0;
static const uint32_t AlignmentMidPrologue = 8;
static const Scale ScalePointer = TimesEight;

// The MacroAssembler uses scratch registers extensively and unexpectedly.
// For safety, scratch registers should always be acquired using
// vixl::UseScratchRegisterScope.
static constexpr Register ScratchReg{Registers::ip0};
static constexpr ARMRegister ScratchReg64 = {ScratchReg, 64};

static constexpr Register ScratchReg2{Registers::ip1};
static constexpr ARMRegister ScratchReg2_64 = {ScratchReg2, 64};

static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::d0,
                                                 FloatRegisters::Double};
static constexpr FloatRegister ScratchDoubleReg_ = {FloatRegisters::d31,
                                                   FloatRegisters::Double};
struct ScratchDoubleScope : public AutoFloatRegisterScope {
 explicit ScratchDoubleScope(MacroAssembler& masm)
     : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
};

static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::s0,
                                                  FloatRegisters::Single};
static constexpr FloatRegister ScratchFloat32Reg_ = {FloatRegisters::s31,
                                                    FloatRegisters::Single};
struct ScratchFloat32Scope : public AutoFloatRegisterScope {
 explicit ScratchFloat32Scope(MacroAssembler& masm)
     : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
};

#ifdef ENABLE_WASM_SIMD
static constexpr FloatRegister ReturnSimd128Reg = {FloatRegisters::v0,
                                                  FloatRegisters::Simd128};
static constexpr FloatRegister ScratchSimd128Reg = {FloatRegisters::v31,
                                                   FloatRegisters::Simd128};
struct ScratchSimd128Scope : public AutoFloatRegisterScope {
 explicit ScratchSimd128Scope(MacroAssembler& masm)
     : AutoFloatRegisterScope(masm, ScratchSimd128Reg) {}
};
#else
struct ScratchSimd128Scope : public AutoFloatRegisterScope {
 explicit ScratchSimd128Scope(MacroAssembler& masm)
     : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {
   MOZ_CRASH("SIMD not enabled");
 }
};
#endif

static constexpr Register InvalidReg{Registers::Invalid};
static constexpr FloatRegister InvalidFloatReg = {};

static constexpr Register OsrFrameReg{Registers::x3};
static constexpr Register CallTempReg0{Registers::x9};
static constexpr Register CallTempReg1{Registers::x10};
static constexpr Register CallTempReg2{Registers::x11};
static constexpr Register CallTempReg3{Registers::x12};
static constexpr Register CallTempReg4{Registers::x13};
static constexpr Register CallTempReg5{Registers::x14};

static constexpr Register PreBarrierReg{Registers::x1};

static constexpr Register InterpreterPCReg{Registers::x9};

static constexpr Register ReturnReg{Registers::x0};
static constexpr Register64 ReturnReg64(ReturnReg);
static constexpr Register JSReturnReg{Registers::x2};
static constexpr Register FramePointer{Registers::fp};
static constexpr ARMRegister FramePointer64{FramePointer, 64};
static constexpr Register ZeroRegister{Registers::sp};
static constexpr ARMRegister ZeroRegister64{Registers::sp, 64};
static constexpr ARMRegister ZeroRegister32{Registers::sp, 32};

// [SMDOC] AArch64 Stack Pointer and Pseudo Stack Pointer conventions
//
//                               ================
//
// Stack pointer (SP), PseudoStackPointer (PSP), and RealStackPointer:
//
// The ARM64 real SP has a constraint: it must be 16-byte aligned whenever it
// is used as the base pointer for a memory access.  (SP+offset need not be
// 16-byte aligned, but the SP value itself must be.)  The SP register may
// take on unaligned values but may not be used for a memory access while it
// is unaligned.
//
// Stack-alignment checking can be enabled or disabled by a control register;
// however that register cannot be modified by user space.  We have to assume
// stack alignment checking is enabled, and that does usually appear to be the
// case.  See the ARM Architecture Reference Manual, "D1.8.2 SP alignment
// checking", for further details.
//
// A second constraint is forced upon us by the ARM64 ABI.  This requires that
// all accesses to the stack must be at or above SP.  Accesses below SP are
// strictly forbidden, presumably because the kernel might use that area of
// memory for its own purposes -- in particular, signal delivery -- and hence
// it may get trashed at any time.
//
// Note this doesn't mean that accesses to the stack must be based off
// register SP.  Only that the effective addresses must be >= SP, regardless
// of how the address is formed.
//
// In order to allow word-wise pushes and pops, some of our ARM64 jits
// (JS-Baseline, JS-Ion, and Wasm-Ion, but not Wasm-Baseline) dedicate x28 to
// be used as a PseudoStackPointer (PSP).
//
// Initially the PSP will have the same value as the SP.  Code can, if it
// wants, push a single word by subtracting 8 from the PSP, doing SP := PSP,
// then storing the value at PSP+0.  Given other constraints on the alignment
// of the SP at function call boundaries, this works out OK, at the cost of
// the two extra instructions per push / pop.
//
// This is all a bit messy, and is probably not robustly adhered to.  However,
// the following appear to be the intended, and mostly implemented, current
// invariants:
//
// (1) PSP is "primary", SP is "secondary".  Most stack refs are
//     PSP-relative. SP-relative is rare and (obviously) only done when we
//     know that SP is aligned.
//
// (2) At all times, the relationship SP <= PSP is maintained.  The fact that
//     SP may validly be less than PSP means that pushes on the stack force
//     the two values to become equal, by copying PSP into SP.  However, pops
//     behave differently: PSP moves back up and SP stays the same, since that
//     doesn't break the SP <= PSP invariant.
//
// (3) However, immediately before a call instruction, SP and PSP must be the
//     same.  To enforce this, PSP is copied into SP by the arm64-specific
//     MacroAssembler::call routines.
//
// (4) Also, after a function has returned, it is expected that SP holds the
//     "primary" value.  How exactly this is implemented remains not entirely
//     clear and merits further investigation.  The following points are
//     believed to be relevant:
//
//     - For calls to functions observing the system AArch64 ABI, PSP (x28) is
//       callee-saved.  That, combined with (3) above, implies SP == PSP
//       immediately after the call returns.
//
//     - JIT-generated routines return using MacroAssemblerCompat::retn, and
//       that copies PSP into SP (bizarrely; this would make more sense if it
//       copied SP into PSP); but in any case, the point is that they are the
//       same at the point that the return instruction executes.
//
//     - MacroAssembler::callWithABIPost copies PSP into SP after the return
//       of a call requiring dynamic alignment.
//
//     Given the above, it is unclear exactly where in the return sequence it
//     is expected that SP == PSP, and also whether it is the callee or caller
//     that is expected to enforce it.
//
// In general it would be nice to be able to move (at some time in the future,
// not now) to a world where *every* assignment to PSP or SP is followed
// immediately by a copy into the other register.  That would make all
// required correctness proofs trivial in the sense that it would require only
// local inspection of code immediately following (dominated by) any such
// assignment.  For the moment, however, this is a guideline, not a hard
// requirement.
//
//                               ================
//
// Mechanics of keeping the stack pointers in sync:
//
// The following two methods require that the masm's SP has been set to the PSP
// with MacroAssembler::SetStackPointer64(PseudoStackPointer64), or they will be
// no-ops.  The setup is performed manually by the jits after creating the masm.
//
// * MacroAssembler::syncStackPtr() performs SP := PSP, presumably after PSP has
//   been updated, so SP needs to move too.  This is used pretty liberally
//   throughout the code base.
//
// * MacroAssembler::initPseudoStackPtr() performs PSP := SP.  This can be used
//   after calls to non-ABI compliant code; it's not used much.
//
// In the ARM64 assembler there is a function Instruction::IsStackPtrSync() that
// recognizes the instruction emitted by syncStackPtr(), and this is used to
// skip that instruction a few places, should it be present, in the JS JIT where
// code is generated to deal with toggled calls.
//
// In various places there are calls to MacroAssembler::syncStackPtr() which
// appear to be redundant.  Investigation shows that they often are redundant,
// but not always.  Finding and removing such redundancies would be quite some
// work, so we live for now with the occasional redundant update.  Perusal of
// the Cortex-A55 and -A72 optimization guides shows no evidence that such
// assignments are any more expensive than assignments between vanilla integer
// registers, so the costs of such redundant updates are assumed to be small.
//
// Invariants on the PSP at function call boundaries:
//
// It *appears* that the following invariants exist:
//
// * On entry to JIT code, PSP == SP, ie the stack pointer is transmitted via
//   both registers.
//
// * On entry to C++ code, PSP == SP.  Certainly it appears that all calls
//   created by the MacroAssembler::call(..) routines perform 'syncStackPtr'
//   immediately before the call, and all ABI calls are routed through the
//   MacroAssembler::call layer.
//
// * The stubs generated by WasmStubs.cpp assume that, on entry, SP is the
//   active stack pointer and that PSP is dead.
//
// * The PSP is non-volatile (callee-saved).  Along a normal return path from
//   JIT code, simply having PSP == SP on exit is correct, since the exit SP is
//   the same as the entry SP by the JIT ABI.
//
// * Call-outs to non-JIT C++ code do not need to set up the PSP (it won't be
//   used), and will not need to restore the PSP on return because x28 is
//   non-volatile in the ARM64 ABI.
//
//                               ================
//
// Future cleanups to the SP-vs-PSP machinery:
//
// Currently we have somewhat unclear invariants, which are not obviously
// always enforced, and which may require complex non-local reasoning.
// Auditing the code to ensure that the invariants always hold, whilst not
// generating duplicate syncs, is close to impossible.  A future rework to
// tidy this might be as follows.  (This suggestion pertains the the entire
// JIT complex: all of the JS compilers, wasm compilers, stub generators,
// regexp compilers, etc).
//
// Currently we have that, in JIT-generated code, PSP is "primary" and SP is
// "secondary", meaning that PSP has the "real" stack pointer value and SP is
// updated whenever PSP acquires a lower value, so as to ensure that SP <= PSP.
// An exception to this scheme is the stubs code generated by WasmStubs.cpp,
// which assumes that SP is "primary" and PSP is dead.
//
// It might give us an easier incremental path to eventually removing PSP
// entirely if we switched to having SP always be the primary.  That is:
//
// (1) SP is primary, PSP is secondary
// (2) After any assignment to SP, it is copied into PSP
// (3) All (non-frame-pointer-based) stack accesses are PSP-relative
//     (as at present)
//
// This would have the effect that:
//
// * It would reinstate the invariant that on all targets, the "real" SP value
//   is in the ABI-and-or-hardware-mandated stack pointer register.
//
// * It would give us a simple story about calls and returns:
//   - for calls to non-JIT generated code (viz, C++ etc), we need no extra
//     copies, because PSP (x28) is callee-saved
//   - for calls to JIT-generated code, we need no extra copies, because of (2)
//     above
//
// * We could incrementally migrate those parts of the code generator where we
//   know that SP is 16-aligned, to use SP- rather than PSP-relative accesses
//
// * The consistent use of (2) would remove the requirement to have to perform
//   path-dependent reasoning (for paths in the generated code, not in the
//   compiler) when reading/understanding the code.
//
// * x28 would become free for use by stubs and the baseline compiler without
//   having to worry about interoperating with code that expects x28 to hold a
//   valid PSP.
//
// One might ask what mechanical checks we can add to ensure correctness, rather
// than having to verify these invariants by hand indefinitely.  Maybe some
// combination of:
//
// * In debug builds, compiling-in assert(SP == PSP) at critical places.  This
//   can be done using the existing `assertStackPtrsSynced` function.
//
// * In debug builds, scanning sections of generated code to ensure no
//   SP-relative stack accesses have been created -- for some sections, at
//   least every assignment to SP is immediately followed by a copy to x28.
//   This would also facilitate detection of duplicate syncs.
//
//                               ================
//
// Other investigative notes, for the code base at present:
//
// * Some disassembly dumps suggest that we sync the stack pointer too often.
//   This could be the result of various pieces of code working at cross
//   purposes when syncing the stack pointer, or of not paying attention to the
//   precise invariants.
//
// * As documented in RegExpNativeMacroAssembler.cpp, function
//   SMRegExpMacroAssembler::createStackFrame:
//
//   // ARM64 communicates stack address via SP, but uses a pseudo-sp (PSP) for
//   // addressing.  The register we use for PSP may however also be used by
//   // calling code, and it is nonvolatile, so save it.  Do this as a special
//   // case first because the generic save/restore code needs the PSP to be
//   // initialized already.
//
//   and also in function SMRegExpMacroAssembler::exitHandler:
//
//   // Restore the saved value of the PSP register, this value is whatever the
//   // caller had saved in it, not any actual SP value, and it must not be
//   // overwritten subsequently.
//
//   The original source for these comments was a patch for bug 1445907.
//
// * MacroAssembler-arm64.h has an interesting comment in the retn()
//   function:
//
//   syncStackPtr();  // SP is always used to transmit the stack between calls.
//
//   Same comment at abiret() in that file, and in MacroAssembler-arm64.cpp,
//   at callWithABIPre and callWithABIPost.
//
// * In Trampoline-arm64.cpp function JitRuntime::generateVMWrapper we find
//
//   // SP is used to transfer stack across call boundaries.
//   masm.initPseudoStackPtr();
//
//   after the return point of a callWithVMWrapper.  The only reasonable
//   conclusion from all those (assuming they are right) is that SP == PSP.
//
// * Wasm-Baseline does not use the PSP, but as Wasm-Ion code requires SP==PSP
//   and tiered code can have Baseline->Ion calls, Baseline will set PSP=SP
//   before a call to wasm code.
//
//                               ================

// StackPointer is intentionally undefined on ARM64 to prevent misuse: using
// sp as a base register is only valid if sp % 16 == 0.
static constexpr Register RealStackPointer{Registers::sp};

static constexpr Register PseudoStackPointer{Registers::x28};
static constexpr ARMRegister PseudoStackPointer64 = {Registers::x28, 64};
static constexpr ARMRegister PseudoStackPointer32 = {Registers::x28, 32};

static constexpr Register IntArgReg0{Registers::x0};
static constexpr Register IntArgReg1{Registers::x1};
static constexpr Register IntArgReg2{Registers::x2};
static constexpr Register IntArgReg3{Registers::x3};
static constexpr Register IntArgReg4{Registers::x4};
static constexpr Register IntArgReg5{Registers::x5};
static constexpr Register IntArgReg6{Registers::x6};
static constexpr Register IntArgReg7{Registers::x7};

// Define unsized Registers.
#define DEFINE_UNSIZED_REGISTERS(N) \
 static constexpr Register r##N{Registers::x##N};
REGISTER_CODE_LIST(DEFINE_UNSIZED_REGISTERS)
#undef DEFINE_UNSIZED_REGISTERS
static constexpr Register ip0{Registers::x16};
static constexpr Register ip1{Registers::x17};
static constexpr Register fp{Registers::x29};
static constexpr Register lr{Registers::x30};
static constexpr Register rzr{Registers::xzr};

// Import VIXL registers into the js::jit namespace.
#define IMPORT_VIXL_REGISTERS(N)                  \
 static constexpr ARMRegister w##N = vixl::w##N; \
 static constexpr ARMRegister x##N = vixl::x##N;
REGISTER_CODE_LIST(IMPORT_VIXL_REGISTERS)
#undef IMPORT_VIXL_REGISTERS
static constexpr ARMRegister wzr = vixl::wzr;
static constexpr ARMRegister xzr = vixl::xzr;
static constexpr ARMRegister wsp = vixl::wsp;
static constexpr ARMRegister sp = vixl::sp;

// Import VIXL VRegisters into the js::jit namespace.
#define IMPORT_VIXL_VREGISTERS(N)                   \
 static constexpr ARMFPRegister s##N = vixl::s##N; \
 static constexpr ARMFPRegister d##N = vixl::d##N;
REGISTER_CODE_LIST(IMPORT_VIXL_VREGISTERS)
#undef IMPORT_VIXL_VREGISTERS

static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);

// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use
// JSReturnOperand).
static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
static constexpr Register RegExpMatcherStringReg = CallTempReg1;
static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;

// Registers used by RegExpExecTest stub (do not use ReturnReg).
static constexpr Register RegExpExecTestRegExpReg = CallTempReg0;
static constexpr Register RegExpExecTestStringReg = CallTempReg1;

// Registers used by RegExpSearcher stub (do not use ReturnReg).
static constexpr Register RegExpSearcherRegExpReg = CallTempReg0;
static constexpr Register RegExpSearcherStringReg = CallTempReg1;
static constexpr Register RegExpSearcherLastIndexReg = CallTempReg2;

static constexpr Register JSReturnReg_Type = r3;
static constexpr Register JSReturnReg_Data = r2;

static constexpr FloatRegister NANReg = {FloatRegisters::d14,
                                        FloatRegisters::Single};
// N.B. r8 isn't listed as an aapcs temp register, but we can use it as such
// because we never use return-structs.
static constexpr Register CallTempNonArgRegs[] = {r8,  r9,  r10, r11,
                                                 r12, r13, r14, r15};
static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);

static constexpr uint32_t JitStackAlignment = 16;

static constexpr uint32_t JitStackValueAlignment =
   JitStackAlignment / sizeof(Value);
static_assert(JitStackAlignment % sizeof(Value) == 0 &&
                 JitStackValueAlignment >= 1,
             "Stack alignment should be a non-zero multiple of sizeof(Value)");

static constexpr uint32_t SimdMemoryAlignment = 16;

static_assert(CodeAlignment % SimdMemoryAlignment == 0,
             "Code alignment should be larger than any of the alignments "
             "which are used for "
             "the constant sections of the code buffer.  Thus it should be "
             "larger than the "
             "alignment for SIMD constants.");

static const uint32_t WasmStackAlignment = SimdMemoryAlignment;
static const uint32_t WasmTrapInstructionLength = 4;

// See comments in wasm::GenerateFunctionPrologue.  The difference between these
// is the size of the largest callable prologue on the platform.
static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;

class Assembler : public vixl::Assembler {
public:
 Assembler() : vixl::Assembler() {}

 using Condition = vixl::Condition;

 void finish();
 bool appendRawCode(const uint8_t* code, size_t numBytes);
 bool reserve(size_t size);
 bool swapBuffer(wasm::Bytes& bytes);

 // Emit the jump table, returning the BufferOffset to the first entry in the
 // table.
 BufferOffset emitExtendedJumpTable();
 BufferOffset ExtendedJumpTable_;
 void executableCopy(uint8_t* buffer);

 BufferOffset immPool(ARMRegister dest, uint8_t* value, vixl::LoadLiteralOp op,
                      const LiteralDoc& doc,
                      ARMBuffer::PoolEntry* pe = nullptr);
 BufferOffset immPool64(ARMRegister dest, uint64_t value,
                        ARMBuffer::PoolEntry* pe = nullptr);
 BufferOffset fImmPool(ARMFPRegister dest, uint8_t* value,
                       vixl::LoadLiteralOp op, const LiteralDoc& doc);
 BufferOffset fImmPool64(ARMFPRegister dest, double value);
 BufferOffset fImmPool32(ARMFPRegister dest, float value);

 uint32_t currentOffset() const { return nextOffset().getOffset(); }

 void bind(Label* label) { bind(label, nextOffset()); }
 void bind(Label* label, BufferOffset boff);
 void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }

 void setUnlimitedBuffer() { armbuffer_.setUnlimited(); }
 bool oom() const {
   return AssemblerShared::oom() || armbuffer_.oom() ||
          jumpRelocations_.oom() || dataRelocations_.oom();
 }

 void copyJumpRelocationTable(uint8_t* dest) const {
   if (jumpRelocations_.length()) {
     memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
   }
 }
 void copyDataRelocationTable(uint8_t* dest) const {
   if (dataRelocations_.length()) {
     memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
   }
 }

 size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); }
 size_t dataRelocationTableBytes() const { return dataRelocations_.length(); }
 size_t bytesNeeded() const {
   return SizeOfCodeGenerated() + jumpRelocationTableBytes() +
          dataRelocationTableBytes();
 }

 void processCodeLabels(uint8_t* rawCode) {
   for (const CodeLabel& label : codeLabels_) {
     Bind(rawCode, label);
   }
 }

 static void UpdateLoad64Value(Instruction* inst0, uint64_t value);

 static void Bind(uint8_t* rawCode, const CodeLabel& label) {
   auto mode = label.linkMode();
   size_t patchAtOffset = label.patchAt().offset();
   size_t targetOffset = label.target().offset();

   if (mode == CodeLabel::MoveImmediate) {
     Instruction* inst = (Instruction*)(rawCode + patchAtOffset);
     Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + targetOffset));
   } else {
     *reinterpret_cast<const void**>(rawCode + patchAtOffset) =
         rawCode + targetOffset;
   }
 }

 void retarget(Label* cur, Label* next);

 // The buffer is about to be linked. Ensure any constant pools or
 // excess bookkeeping has been flushed to the instruction stream.
 void flush() { armbuffer_.flushPool(); }

 void comment(const char* msg) {
#ifdef JS_DISASM_ARM64
   spew_.spew("; %s", msg);
#endif
 }

 void setPrinter(Sprinter* sp) {
#ifdef JS_DISASM_ARM64
   spew_.setPrinter(sp);
#endif
 }

 static bool SupportsFloatingPoint() { return true; }
 static bool SupportsUnalignedAccesses() { return true; }
 static bool SupportsFastUnalignedFPAccesses() { return true; }
 static bool SupportsWasmSimd() { return true; }
 static bool SupportsFloat64To16() { return true; }
 static bool SupportsFloat32To16() { return true; }

 static bool HasRoundInstruction(RoundingMode mode) {
   switch (mode) {
     case RoundingMode::Up:
     case RoundingMode::Down:
     case RoundingMode::NearestTiesToEven:
     case RoundingMode::TowardsZero:
       return true;
   }
   MOZ_CRASH("unexpected mode");
 }

protected:
 // Add a jump whose target is unknown until finalization.
 // The jump may not be patched at runtime.
 void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind);

public:
 static uint32_t PatchWrite_NearCallSize() { return 4; }

 static uint32_t NopSize() { return 4; }

 static void PatchWrite_NearCall(CodeLocationLabel start,
                                 CodeLocationLabel toCall);
 static void PatchDataWithValueCheck(CodeLocationLabel label,
                                     PatchedImmPtr newValue,
                                     PatchedImmPtr expected);

 static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
                                     ImmPtr expected);

 static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
   // Raw is going to be the return address.
   uint32_t* raw = (uint32_t*)label.raw();
   // Overwrite the 4 bytes before the return address, which will end up being
   // the call instruction.
   *(raw - 1) = imm.value;
 }
 static uint32_t AlignDoubleArg(uint32_t offset) {
   MOZ_CRASH("AlignDoubleArg()");
 }
 static uintptr_t GetPointer(uint8_t* ptr) {
   Instruction* i = reinterpret_cast<Instruction*>(ptr);
   uint64_t ret = i->Literal64();
   return ret;
 }

 // Toggle a jmp or cmp emitted by toggledJump().
 static void ToggleToJmp(CodeLocationLabel inst_);
 static void ToggleToCmp(CodeLocationLabel inst_);
 static void ToggleCall(CodeLocationLabel inst_, bool enabled);

 static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
                                  CompactBufferReader& reader);
 static void TraceDataRelocations(JSTracer* trc, JitCode* code,
                                  CompactBufferReader& reader);

 void assertNoGCThings() const {
#ifdef DEBUG
   MOZ_ASSERT(dataRelocations_.length() == 0);
   for (auto& j : pendingJumps_) {
     MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
   }
#endif
 }

public:
 // A Jump table entry is 2 instructions, with 8 bytes of raw data
 static const size_t SizeOfJumpTableEntry = 16;

 struct JumpTableEntry {
   uint32_t ldr;
   uint32_t br;
   void* data;

   Instruction* getLdr() { return reinterpret_cast<Instruction*>(&ldr); }
 };

 // Offset of the patchable target for the given entry.
 static const size_t OffsetOfJumpTableEntryPointer = 8;

public:
 void writeCodePointer(CodeLabel* label) {
   armbuffer_.assertNoPoolAndNoNops();
   uintptr_t x = uintptr_t(-1);
   BufferOffset off = EmitData(&x, sizeof(uintptr_t));
   label->patchAt()->bind(off.getOffset());
 }

 void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
                                  const Disassembler::HeapAccess& heapAccess) {
   MOZ_CRASH("verifyHeapAccessDisassembly");
 }

protected:
 // Structure for fixing up pc-relative loads/jumps when the machine
 // code gets moved (executable copy, gc, etc.).
 struct RelativePatch {
   BufferOffset offset;
   void* target;
   RelocationKind kind;

   RelativePatch(BufferOffset offset, void* target, RelocationKind kind)
       : offset(offset), target(target), kind(kind) {}
 };

 // List of jumps for which the target is either unknown until finalization,
 // or cannot be known due to GC. Each entry here requires a unique entry
 // in the extended jump table, and is patched at finalization.
 js::Vector<RelativePatch, 8, SystemAllocPolicy> pendingJumps_;

 // Final output formatters.
 CompactBufferWriter jumpRelocations_;
 CompactBufferWriter dataRelocations_;
};

static const uint32_t NumIntArgRegs = 8;
static const uint32_t NumFloatArgRegs = 8;

class ABIArgGenerator : public ABIArgGeneratorShared {
public:
 explicit ABIArgGenerator(ABIKind kind)
     : ABIArgGeneratorShared(kind),
       intRegIndex_(0),
       floatRegIndex_(0),
       current_() {}

 ABIArg next(MIRType argType);
 ABIArg& current() { return current_; }

protected:
 unsigned intRegIndex_;
 unsigned floatRegIndex_;
 ABIArg current_;
};

// See "ABI special registers" in Assembler-shared.h for more information.
static constexpr Register ABINonArgReg0 = r8;
static constexpr Register ABINonArgReg1 = r9;
static constexpr Register ABINonArgReg2 = r10;
static constexpr Register ABINonArgReg3 = r11;

// See "ABI special registers" in Assembler-shared.h for more information.
// Avoid d31 which is the ScratchDoubleReg_.
static constexpr FloatRegister ABINonArgDoubleReg = {FloatRegisters::s16,
                                                    FloatRegisters::Single};

// See "ABI special registers" in Assembler-shared.h for more information.
static constexpr Register ABINonArgReturnReg0 = r8;
static constexpr Register ABINonArgReturnReg1 = r9;
static constexpr Register ABINonVolatileReg{Registers::x19};

// See "ABI special registers" in Assembler-shared.h for more information.
static constexpr Register ABINonArgReturnVolatileReg = r8;

// See "ABI special registers" in Assembler-shared.h, and "The WASM ABIs" in
// WasmFrame.h for more information.
static constexpr Register InstanceReg{Registers::x23};
static constexpr Register HeapReg{Registers::x21};

// Registers used for wasm table calls. These registers must be disjoint
// from the ABI argument registers, InstanceReg and each other.
static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;

// Registers used for ref calls.
static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
static constexpr Register WasmCallRefCallScratchReg2 = ABINonArgReg2;
static constexpr Register WasmCallRefReg = ABINonArgReg3;

// Registers used for wasm tail calls operations.
static constexpr Register WasmTailCallInstanceScratchReg = ABINonArgReg1;
static constexpr Register WasmTailCallRAScratchReg = lr;
static constexpr Register WasmTailCallFPScratchReg = ABINonArgReg3;

// Register used as a scratch along the return path in the fast js -> wasm stub
// code.  This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
// It must be a volatile register.
static constexpr Register WasmJitEntryReturnScratch = r9;

static inline bool GetIntArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
                               Register* out) {
 if (usedIntArgs >= NumIntArgRegs) {
   return false;
 }
 *out = Register::FromCode(usedIntArgs);
 return true;
}

static inline bool GetFloatArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
                                 FloatRegister* out) {
 if (usedFloatArgs >= NumFloatArgRegs) {
   return false;
 }
 *out = FloatRegister::FromCode(usedFloatArgs);
 return true;
}

// Get a register in which we plan to put a quantity that will be used as an
// integer argument.  This differs from GetIntArgReg in that if we have no more
// actual argument registers to use we will fall back on using whatever
// CallTempReg* don't overlap the argument registers, and only fail once those
// run out too.
static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
                                      uint32_t usedFloatArgs, Register* out) {
 if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
   return true;
 }
 // Unfortunately, we have to assume things about the point at which
 // GetIntArgReg returns false, because we need to know how many registers it
 // can allocate.
 usedIntArgs -= NumIntArgRegs;
 if (usedIntArgs >= NumCallTempNonArgRegs) {
   return false;
 }
 *out = CallTempNonArgRegs[usedIntArgs];
 return true;
}

// Forbids nop filling for testing purposes.  Nestable, but nested calls have
// no effect on the no-nops status; it is only the top level one that counts.
class AutoForbidNops {
protected:
 Assembler* asm_;

public:
 explicit AutoForbidNops(Assembler* asm_) : asm_(asm_) { asm_->enterNoNops(); }
 ~AutoForbidNops() { asm_->leaveNoNops(); }
};

// Forbids pool generation during a specified interval.  Nestable, but nested
// calls must imply a no-pool area of the assembler buffer that is completely
// contained within the area implied by the outermost level call.
class AutoForbidPoolsAndNops : public AutoForbidNops {
public:
 AutoForbidPoolsAndNops(Assembler* asm_, size_t maxInst)
     : AutoForbidNops(asm_) {
   asm_->enterNoPool(maxInst);
 }
 ~AutoForbidPoolsAndNops() { asm_->leaveNoPool(); }
};

}  // namespace jit
}  // namespace js

#endif  // A64_ASSEMBLER_A64_H_