tor-browser

Assembler-arm.h (79616B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_arm_Assembler_arm_h
#define jit_arm_Assembler_arm_h

#include "mozilla/Attributes.h"
#include "mozilla/MathAlgorithms.h"

#include <algorithm>
#include <type_traits>

#include "jit/arm/Architecture-arm.h"
#include "jit/arm/disasm/Disasm-arm.h"
#include "jit/CompactBuffer.h"
#include "jit/JitCode.h"
#include "jit/shared/Assembler-shared.h"
#include "jit/shared/Disassembler-shared.h"
#include "jit/shared/IonAssemblerBufferWithConstantPools.h"
#include "wasm/WasmTypeDecls.h"

union PoolHintPun;

namespace js {
namespace jit {

using LiteralDoc = DisassemblerSpew::LiteralDoc;
using LabelDoc = DisassemblerSpew::LabelDoc;

// NOTE: there are duplicates in this list! Sometimes we want to specifically
// refer to the link register as a link register (bl lr is much clearer than bl
// r14). HOWEVER, this register can easily be a gpr when it is not busy holding
// the return address.
static constexpr Register r0{Registers::r0};
static constexpr Register r1{Registers::r1};
static constexpr Register r2{Registers::r2};
static constexpr Register r3{Registers::r3};
static constexpr Register r4{Registers::r4};
static constexpr Register r5{Registers::r5};
static constexpr Register r6{Registers::r6};
static constexpr Register r7{Registers::r7};
static constexpr Register r8{Registers::r8};
static constexpr Register r9{Registers::r9};
static constexpr Register r10{Registers::r10};
static constexpr Register r11{Registers::r11};
static constexpr Register r12{Registers::ip};
static constexpr Register ip{Registers::ip};
static constexpr Register sp{Registers::sp};
static constexpr Register r14{Registers::lr};
static constexpr Register lr{Registers::lr};
static constexpr Register pc{Registers::pc};

static constexpr Register ScratchRegister{Registers::ip};

// Helper class for ScratchRegister usage. Asserts that only one piece
// of code thinks it has exclusive ownership of the scratch register.
struct ScratchRegisterScope : public AutoRegisterScope {
 explicit ScratchRegisterScope(MacroAssembler& masm)
     : AutoRegisterScope(masm, ScratchRegister) {}
};

struct SecondScratchRegisterScope : public AutoRegisterScope {
 explicit SecondScratchRegisterScope(MacroAssembler& masm);
};

class MOZ_RAII AutoNonDefaultSecondScratchRegister {
public:
 explicit AutoNonDefaultSecondScratchRegister(MacroAssembler& masm,
                                              Register reg);
 ~AutoNonDefaultSecondScratchRegister();

private:
 Register prevSecondScratch_;
 MacroAssembler& masm_;
};

static constexpr Register OsrFrameReg = r3;
static constexpr Register CallTempReg0 = r5;
static constexpr Register CallTempReg1 = r6;
static constexpr Register CallTempReg2 = r7;
static constexpr Register CallTempReg3 = r8;
static constexpr Register CallTempReg4 = r0;
static constexpr Register CallTempReg5 = r1;

static constexpr Register IntArgReg0 = r0;
static constexpr Register IntArgReg1 = r1;
static constexpr Register IntArgReg2 = r2;
static constexpr Register IntArgReg3 = r3;
static constexpr Register CallTempNonArgRegs[] = {r5, r6, r7, r8};
static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);

// These register assignments for the 64-bit atomic ops are frequently too
// constraining, but we have no way of expressing looser constraints to the
// register allocator.

// CompareExchange: Any two odd/even pairs would do for `new` and `out`, and any
// pair would do for `old`, so long as none of them overlap.

static constexpr Register CmpXchgOldLo = r4;
static constexpr Register CmpXchgOldHi = r5;
static constexpr Register64 CmpXchgOld64 =
   Register64(CmpXchgOldHi, CmpXchgOldLo);
static constexpr Register CmpXchgNewLo = IntArgReg2;
static constexpr Register CmpXchgNewHi = IntArgReg3;
static constexpr Register64 CmpXchgNew64 =
   Register64(CmpXchgNewHi, CmpXchgNewLo);
static constexpr Register CmpXchgOutLo = IntArgReg0;
static constexpr Register CmpXchgOutHi = IntArgReg1;
static constexpr Register64 CmpXchgOut64 =
   Register64(CmpXchgOutHi, CmpXchgOutLo);

// Exchange: Any two non-equal odd/even pairs would do for `new` and `out`.

static constexpr Register XchgNewLo = IntArgReg2;
static constexpr Register XchgNewHi = IntArgReg3;
static constexpr Register64 XchgNew64 = Register64(XchgNewHi, XchgNewLo);
static constexpr Register XchgOutLo = IntArgReg0;
static constexpr Register XchgOutHi = IntArgReg1;

// Atomic rmw operations: Any two odd/even pairs would do for `tmp` and `out`,
// and any pair would do for `val`, so long as none of them overlap.

static constexpr Register FetchOpValLo = r4;
static constexpr Register FetchOpValHi = r5;
static constexpr Register64 FetchOpVal64 =
   Register64(FetchOpValHi, FetchOpValLo);
static constexpr Register FetchOpTmpLo = IntArgReg2;
static constexpr Register FetchOpTmpHi = IntArgReg3;
static constexpr Register64 FetchOpTmp64 =
   Register64(FetchOpTmpHi, FetchOpTmpLo);
static constexpr Register FetchOpOutLo = IntArgReg0;
static constexpr Register FetchOpOutHi = IntArgReg1;
static constexpr Register64 FetchOpOut64 =
   Register64(FetchOpOutHi, FetchOpOutLo);

class ABIArgGenerator : public ABIArgGeneratorShared {
 unsigned intRegIndex_;
 unsigned floatRegIndex_;
 ABIArg current_;

 // ARM can either use HardFp (use float registers for float arguments), or
 // SoftFp (use general registers for float arguments) ABI.  We keep this
 // switch as a runtime switch because wasm always use the HardFp back-end
 // while the calls to native functions have to use the one provided by the
 // system.
 bool useHardFp_;

 ABIArg softNext(MIRType argType);
 ABIArg hardNext(MIRType argType);

public:
 explicit ABIArgGenerator(ABIKind kind);

 void setUseHardFp(bool useHardFp) {
   MOZ_ASSERT(intRegIndex_ == 0 && floatRegIndex_ == 0);
   MOZ_ASSERT_IF(kind_ == ABIKind::Wasm, useHardFp);
   useHardFp_ = useHardFp;
 }
 ABIArg next(MIRType argType);
 ABIArg& current() { return current_; }
};

bool IsUnaligned(const wasm::MemoryAccessDesc& access);

// See "ABI special registers" in Assembler-shared.h for more information.
static constexpr Register ABINonArgReg0 = r4;
static constexpr Register ABINonArgReg1 = r5;
static constexpr Register ABINonArgReg2 = r6;
static constexpr Register ABINonArgReg3 = r7;

// See "ABI special registers" in Assembler-shared.h for more information.
// Avoid d15 which is the ScratchDoubleReg_.
static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::d8,
                                                 VFPRegister::Double};

// See "ABI special registers" in Assembler-shared.h for more information.
static constexpr Register ABINonArgReturnReg0 = r4;
static constexpr Register ABINonArgReturnReg1 = r5;
static constexpr Register ABINonVolatileReg = r6;

// See "ABI special registers" in Assembler-shared.h for more information.
static constexpr Register ABINonArgReturnVolatileReg = lr;

// See "ABI special registers" in Assembler-shared.h, and "The WASM ABIs" in
// WasmFrame.h for more information.
static constexpr Register InstanceReg = r9;
static constexpr Register HeapReg = r10;

// Registers used for wasm table calls. These registers must be disjoint
// from the ABI argument registers, InstanceReg and each other.
static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;

// Registers used for ref calls.
static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
static constexpr Register WasmCallRefCallScratchReg2 = ABINonArgReg2;
static constexpr Register WasmCallRefReg = ABINonArgReg3;

// Registers used for wasm tail calls operations.
static constexpr Register WasmTailCallInstanceScratchReg = ABINonArgReg1;
static constexpr Register WasmTailCallRAScratchReg = lr;
static constexpr Register WasmTailCallFPScratchReg = ABINonArgReg3;

// Register used as a scratch along the return path in the fast js -> wasm stub
// code.  This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
// It must be a volatile register.
static constexpr Register WasmJitEntryReturnScratch = r5;

static constexpr Register PreBarrierReg = r1;

static constexpr Register InterpreterPCReg = r9;

static constexpr Register InvalidReg{Registers::invalid_reg};
static constexpr FloatRegister InvalidFloatReg;

static constexpr Register JSReturnReg_Type = r3;
static constexpr Register JSReturnReg_Data = r2;
static constexpr Register StackPointer = sp;
static constexpr Register FramePointer = r11;
static constexpr Register ReturnReg = r0;
static constexpr Register64 ReturnReg64(r1, r0);

// The attribute '__value_in_regs' alters the calling convention of a function
// so that a structure of up to four elements can be returned via the argument
// registers rather than being written to memory.
static constexpr Register ReturnRegVal0 = IntArgReg0;
static constexpr Register ReturnRegVal1 = IntArgReg1;
static constexpr Register ReturnRegVal2 = IntArgReg2;
static constexpr Register ReturnRegVal3 = IntArgReg3;

static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::d0,
                                                  VFPRegister::Single};
static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::d0,
                                                 VFPRegister::Double};
static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg;
static constexpr FloatRegister ScratchFloat32Reg_ = {FloatRegisters::s30,
                                                    VFPRegister::Single};
static constexpr FloatRegister ScratchDoubleReg_ = {FloatRegisters::d15,
                                                   VFPRegister::Double};
static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg;
static constexpr FloatRegister ScratchUIntReg = {FloatRegisters::d15,
                                                VFPRegister::UInt};
static constexpr FloatRegister ScratchIntReg = {FloatRegisters::d15,
                                               VFPRegister::Int};

// Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope
// instead.
struct ScratchFloat32Scope : public AutoFloatRegisterScope {
 explicit ScratchFloat32Scope(MacroAssembler& masm)
     : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
};

// Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead.
struct ScratchDoubleScope : public AutoFloatRegisterScope {
 explicit ScratchDoubleScope(MacroAssembler& masm)
     : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
};

// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use
// JSReturnOperand).
static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
static constexpr Register RegExpMatcherStringReg = CallTempReg1;
static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;

// Registers used by RegExpExecTest stub (do not use ReturnReg).
static constexpr Register RegExpExecTestRegExpReg = CallTempReg0;
static constexpr Register RegExpExecTestStringReg = CallTempReg1;

// Registers used by RegExpSearcher stub (do not use ReturnReg).
static constexpr Register RegExpSearcherRegExpReg = CallTempReg0;
static constexpr Register RegExpSearcherStringReg = CallTempReg1;
static constexpr Register RegExpSearcherLastIndexReg = CallTempReg2;

static constexpr FloatRegister d0 = {FloatRegisters::d0, VFPRegister::Double};
static constexpr FloatRegister d1 = {FloatRegisters::d1, VFPRegister::Double};
static constexpr FloatRegister d2 = {FloatRegisters::d2, VFPRegister::Double};
static constexpr FloatRegister d3 = {FloatRegisters::d3, VFPRegister::Double};
static constexpr FloatRegister d4 = {FloatRegisters::d4, VFPRegister::Double};
static constexpr FloatRegister d5 = {FloatRegisters::d5, VFPRegister::Double};
static constexpr FloatRegister d6 = {FloatRegisters::d6, VFPRegister::Double};
static constexpr FloatRegister d7 = {FloatRegisters::d7, VFPRegister::Double};
static constexpr FloatRegister d8 = {FloatRegisters::d8, VFPRegister::Double};
static constexpr FloatRegister d9 = {FloatRegisters::d9, VFPRegister::Double};
static constexpr FloatRegister d10 = {FloatRegisters::d10, VFPRegister::Double};
static constexpr FloatRegister d11 = {FloatRegisters::d11, VFPRegister::Double};
static constexpr FloatRegister d12 = {FloatRegisters::d12, VFPRegister::Double};
static constexpr FloatRegister d13 = {FloatRegisters::d13, VFPRegister::Double};
static constexpr FloatRegister d14 = {FloatRegisters::d14, VFPRegister::Double};
static constexpr FloatRegister d15 = {FloatRegisters::d15, VFPRegister::Double};

// For maximal awesomeness, 8 should be sufficent. ldrd/strd (dual-register
// load/store) operate in a single cycle when the address they are dealing with
// is 8 byte aligned. Also, the ARM abi wants the stack to be 8 byte aligned at
// function boundaries. I'm trying to make sure this is always true.
static constexpr uint32_t ABIStackAlignment = 8;
static constexpr uint32_t CodeAlignment = 8;
static constexpr uint32_t JitStackAlignment = 8;

static constexpr uint32_t JitStackValueAlignment =
   JitStackAlignment / sizeof(Value);
static_assert(JitStackAlignment % sizeof(Value) == 0 &&
                 JitStackValueAlignment >= 1,
             "Stack alignment should be a non-zero multiple of sizeof(Value)");

static constexpr uint32_t SimdMemoryAlignment = 8;

static_assert(CodeAlignment % SimdMemoryAlignment == 0,
             "Code alignment should be larger than any of the alignments "
             "which are used for "
             "the constant sections of the code buffer.  Thus it should be "
             "larger than the "
             "alignment for SIMD constants.");

static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
             "Stack alignment should be larger than any of the alignments "
             "which are used for "
             "spilled values.  Thus it should be larger than the alignment "
             "for SIMD accesses.");

static const uint32_t WasmStackAlignment = SimdMemoryAlignment;
static const uint32_t WasmTrapInstructionLength = 4;

// See comments in wasm::GenerateFunctionPrologue.  The difference between these
// is the size of the largest callable prologue on the platform.
static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;

static const Scale ScalePointer = TimesFour;

class Instruction;
class InstBranchImm;
uint32_t RM(Register r);
uint32_t RS(Register r);
uint32_t RD(Register r);
uint32_t RT(Register r);
uint32_t RN(Register r);

uint32_t maybeRD(Register r);
uint32_t maybeRT(Register r);
uint32_t maybeRN(Register r);

Register toRN(Instruction i);
Register toRM(Instruction i);
Register toRD(Instruction i);
Register toR(Instruction i);

class VFPRegister;
uint32_t VD(VFPRegister vr);
uint32_t VN(VFPRegister vr);
uint32_t VM(VFPRegister vr);

// For being passed into the generic vfp instruction generator when there is an
// instruction that only takes two registers.
static constexpr VFPRegister NoVFPRegister(VFPRegister::Double, 0, false, true);

struct ImmTag : public Imm32 {
 explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
};

struct ImmType : public ImmTag {
 explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {}
};

enum Index {
 Offset = 0 << 21 | 1 << 24,
 PreIndex = 1 << 21 | 1 << 24,
 PostIndex = 0 << 21 | 0 << 24
 // The docs were rather unclear on this. It sounds like
 // 1 << 21 | 0 << 24 encodes dtrt.
};

enum IsImmOp2_ { IsImmOp2 = 1 << 25, IsNotImmOp2 = 0 << 25 };
enum IsImmDTR_ { IsImmDTR = 0 << 25, IsNotImmDTR = 1 << 25 };
// For the extra memory operations, ldrd, ldrsb, ldrh.
enum IsImmEDTR_ { IsImmEDTR = 1 << 22, IsNotImmEDTR = 0 << 22 };

enum ShiftType {
 LSL = 0,   // << 5
 LSR = 1,   // << 5
 ASR = 2,   // << 5
 ROR = 3,   // << 5
 RRX = ROR  // RRX is encoded as ROR with a 0 offset.
};

// Modes for STM/LDM. Names are the suffixes applied to the instruction.
enum DTMMode {
 A = 0 << 24,  // empty / after
 B = 1 << 24,  // full / before
 D = 0 << 23,  // decrement
 I = 1 << 23,  // increment
 DA = D | A,
 DB = D | B,
 IA = I | A,
 IB = I | B
};

enum DTMWriteBack { WriteBack = 1 << 21, NoWriteBack = 0 << 21 };

// Condition code updating mode.
enum SBit {
 SetCC = 1 << 20,   // Set condition code.
 LeaveCC = 0 << 20  // Leave condition code unchanged.
};

enum LoadStore { IsLoad = 1 << 20, IsStore = 0 << 20 };

// You almost never want to use this directly. Instead, you wantto pass in a
// signed constant, and let this bit be implicitly set for you. This is however,
// necessary if we want a negative index.
enum IsUp_ { IsUp = 1 << 23, IsDown = 0 << 23 };
enum ALUOp {
 OpMov = 0xd << 21,
 OpMvn = 0xf << 21,
 OpAnd = 0x0 << 21,
 OpBic = 0xe << 21,
 OpEor = 0x1 << 21,
 OpOrr = 0xc << 21,
 OpAdc = 0x5 << 21,
 OpAdd = 0x4 << 21,
 OpSbc = 0x6 << 21,
 OpSub = 0x2 << 21,
 OpRsb = 0x3 << 21,
 OpRsc = 0x7 << 21,
 OpCmn = 0xb << 21,
 OpCmp = 0xa << 21,
 OpTeq = 0x9 << 21,
 OpTst = 0x8 << 21,
 OpInvalid = -1
};

enum MULOp {
 OpmMul = 0 << 21,
 OpmMla = 1 << 21,
 OpmUmaal = 2 << 21,
 OpmMls = 3 << 21,
 OpmUmull = 4 << 21,
 OpmUmlal = 5 << 21,
 OpmSmull = 6 << 21,
 OpmSmlal = 7 << 21
};
enum BranchTag {
 OpB = 0x0a000000,
 OpBMask = 0x0f000000,
 OpBDestMask = 0x00ffffff,
 OpBl = 0x0b000000,
 OpBlx = 0x012fff30,
 OpBx = 0x012fff10
};

// Just like ALUOp, but for the vfp instruction set.
enum VFPOp {
 OpvMul = 0x2 << 20,
 OpvAdd = 0x3 << 20,
 OpvSub = 0x3 << 20 | 0x1 << 6,
 OpvDiv = 0x8 << 20,
 OpvMov = 0xB << 20 | 0x1 << 6,
 OpvAbs = 0xB << 20 | 0x3 << 6,
 OpvNeg = 0xB << 20 | 0x1 << 6 | 0x1 << 16,
 OpvSqrt = 0xB << 20 | 0x3 << 6 | 0x1 << 16,
 OpvCmp = 0xB << 20 | 0x1 << 6 | 0x4 << 16,
 OpvCmpz = 0xB << 20 | 0x1 << 6 | 0x5 << 16
};

// Negate the operation, AND negate the immediate that we were passed in.
ALUOp ALUNeg(ALUOp op, Register dest, Register scratch, Imm32* imm,
            Register* negDest);
bool can_dbl(ALUOp op);
bool condsAreSafe(ALUOp op);

// If there is a variant of op that has a dest (think cmp/sub) return that
// variant of it.
ALUOp getDestVariant(ALUOp op);

static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type,
                                             JSReturnReg_Data};
static const ValueOperand softfpReturnOperand = ValueOperand(r1, r0);

// All of these classes exist solely to shuffle data into the various operands.
// For example Operand2 can be an imm8, a register-shifted-by-a-constant or a
// register-shifted-by-a-register. We represent this in C++ by having a base
// class Operand2, which just stores the 32 bits of data as they will be encoded
// in the instruction. You cannot directly create an Operand2 since it is
// tricky, and not entirely sane to do so. Instead, you create one of its child
// classes, e.g. Imm8. Imm8's constructor takes a single integer argument. Imm8
// will verify that its argument can be encoded as an ARM 12 bit imm8, encode it
// using an Imm8data, and finally call its parent's (Operand2) constructor with
// the Imm8data. The Operand2 constructor will then call the Imm8data's encode()
// function to extract the raw bits from it.
//
// In the future, we should be able to extract data from the Operand2 by asking
// it for its component Imm8data structures. The reason this is so horribly
// round-about is we wanted to have Imm8 and RegisterShiftedRegister inherit
// directly from Operand2 but have all of them take up only a single word of
// storage. We also wanted to avoid passing around raw integers at all since
// they are error prone.
class Op2Reg;
class O2RegImmShift;
class O2RegRegShift;

namespace datastore {

class Reg {
 // The "second register".
 uint32_t rm_ : 4;
 // Do we get another register for shifting.
 uint32_t rrs_ : 1;
 uint32_t type_ : 2;
 // We'd like this to be a more sensible encoding, but that would need to be
 // a struct and that would not pack :(
 uint32_t shiftAmount_ : 5;

protected:
 // Mark as a protected field to avoid unused private field warnings.
 uint32_t pad_ : 20;

public:
 Reg(uint32_t rm, ShiftType type, uint32_t rsr, uint32_t shiftAmount)
     : rm_(rm), rrs_(rsr), type_(type), shiftAmount_(shiftAmount), pad_(0) {}
 explicit Reg(const Op2Reg& op) { memcpy(this, &op, sizeof(*this)); }

 uint32_t shiftAmount() const { return shiftAmount_; }

 uint32_t encode() const {
   return rm_ | (rrs_ << 4) | (type_ << 5) | (shiftAmount_ << 7);
 }
};

// Op2 has a mode labelled "<imm8m>", which is arm's magical immediate encoding.
// Some instructions actually get 8 bits of data, which is called Imm8Data
// below. These should have edit distance > 1, but this is how it is for now.
class Imm8mData {
 uint32_t data_ : 8;
 uint32_t rot_ : 4;

protected:
 // Mark as a protected field to avoid unused private field warnings.
 uint32_t buff_ : 19;

private:
 // Throw in an extra bit that will be 1 if we can't encode this properly.
 // if we can encode it properly, a simple "|" will still suffice to meld it
 // into the instruction.
 uint32_t invalid_ : 1;

public:
 // Default constructor makes an invalid immediate.
 Imm8mData() : data_(0xff), rot_(0xf), buff_(0), invalid_(true) {}

 Imm8mData(uint32_t data, uint32_t rot)
     : data_(data), rot_(rot), buff_(0), invalid_(false) {
   MOZ_ASSERT(data == data_);
   MOZ_ASSERT(rot == rot_);
 }

 bool invalid() const { return invalid_; }

 uint32_t encode() const {
   MOZ_ASSERT(!invalid_);
   return data_ | (rot_ << 8);
 };
};

class Imm8Data {
 uint32_t imm4L_ : 4;

protected:
 // Mark as a protected field to avoid unused private field warnings.
 uint32_t pad_ : 4;

private:
 uint32_t imm4H_ : 4;

public:
 explicit Imm8Data(uint32_t imm) : imm4L_(imm & 0xf), imm4H_(imm >> 4) {
   MOZ_ASSERT(imm <= 0xff);
 }

 uint32_t encode() const { return imm4L_ | (imm4H_ << 8); };
};

// VLDR/VSTR take an 8 bit offset, which is implicitly left shifted by 2.
class Imm8VFPOffData {
 uint32_t data_;

public:
 explicit Imm8VFPOffData(uint32_t imm) : data_(imm) {
   MOZ_ASSERT((imm & ~(0xff)) == 0);
 }
 uint32_t encode() const { return data_; };
};

// ARM can magically encode 256 very special immediates to be moved into a
// register.
struct Imm8VFPImmData {
 // This structure's members are public and it has no constructor to
 // initialize them, for a very special reason. Were this structure to
 // have a constructor, the initialization for DoubleEncoder's internal
 // table (see below) would require a rather large static constructor on
 // some of our supported compilers. The known solution to this is to mark
 // the constructor constexpr, but, again, some of our supported
 // compilers don't support constexpr! So we are reduced to public
 // members and eschewing a constructor in hopes that the initialization
 // of DoubleEncoder's table is correct.
 uint32_t imm4L : 4;
 uint32_t imm4H : 4;
 int32_t isInvalid : 24;

 uint32_t encode() const {
   // This assert is an attempting at ensuring that we don't create random
   // instances of this structure and then asking to encode() it.
   MOZ_ASSERT(isInvalid == 0);
   return imm4L | (imm4H << 16);
 };
};

class Imm12Data {
 uint32_t data_ : 12;

public:
 explicit Imm12Data(uint32_t imm) : data_(imm) { MOZ_ASSERT(data_ == imm); }

 uint32_t encode() const { return data_; }
};

class RIS {
 uint32_t shiftAmount_ : 5;

public:
 explicit RIS(uint32_t imm) : shiftAmount_(imm) {
   MOZ_ASSERT(shiftAmount_ == imm);
 }

 explicit RIS(Reg r) : shiftAmount_(r.shiftAmount()) {}

 uint32_t encode() const { return shiftAmount_; }
};

class RRS {
protected:
 // Mark as a protected field to avoid unused private field warnings.
 uint32_t mustZero_ : 1;

private:
 // The register that holds the shift amount.
 uint32_t rs_ : 4;

public:
 explicit RRS(uint32_t rs) : rs_(rs) { MOZ_ASSERT(rs_ == rs); }

 uint32_t encode() const { return rs_ << 1; }
};

}  // namespace datastore

class MacroAssemblerARM;
class Operand;

class Operand2 {
 friend class Operand;
 friend class MacroAssemblerARM;
 friend class InstALU;

 uint32_t oper_ : 31;
 uint32_t invalid_ : 1;

protected:
 explicit Operand2(datastore::Imm8mData base)
     : oper_(base.invalid() ? -1 : (base.encode() | uint32_t(IsImmOp2))),
       invalid_(base.invalid()) {}

 explicit Operand2(datastore::Reg base)
     : oper_(base.encode() | uint32_t(IsNotImmOp2)), invalid_(false) {}

private:
 explicit Operand2(uint32_t blob) : oper_(blob), invalid_(false) {}

public:
 bool isO2Reg() const { return !(oper_ & IsImmOp2); }

 Op2Reg toOp2Reg() const;

 bool isImm8() const { return oper_ & IsImmOp2; }

 bool invalid() const { return invalid_; }

 uint32_t encode() const { return oper_; }
};

class Imm8 : public Operand2 {
public:
 explicit Imm8(uint32_t imm) : Operand2(EncodeImm(imm)) {}

 static datastore::Imm8mData EncodeImm(uint32_t imm) {
   // RotateLeft below may not be called with a shift of zero.
   if (imm <= 0xFF) {
     return datastore::Imm8mData(imm, 0);
   }

   // An encodable integer has a maximum of 8 contiguous set bits,
   // with an optional wrapped left rotation to even bit positions.
   for (int rot = 1; rot < 16; rot++) {
     uint32_t rotimm = mozilla::RotateLeft(imm, rot * 2);
     if (rotimm <= 0xFF) {
       return datastore::Imm8mData(rotimm, rot);
     }
   }
   return datastore::Imm8mData();
 }

 // Pair template?
 struct TwoImm8mData {
   datastore::Imm8mData fst_, snd_;

   TwoImm8mData() = default;

   TwoImm8mData(datastore::Imm8mData fst, datastore::Imm8mData snd)
       : fst_(fst), snd_(snd) {}

   datastore::Imm8mData fst() const { return fst_; }
   datastore::Imm8mData snd() const { return snd_; }
 };

 static TwoImm8mData EncodeTwoImms(uint32_t);
};

class Op2Reg : public Operand2 {
public:
 explicit Op2Reg(Register rm, ShiftType type, datastore::RIS shiftImm)
     : Operand2(datastore::Reg(rm.code(), type, 0, shiftImm.encode())) {}

 explicit Op2Reg(Register rm, ShiftType type, datastore::RRS shiftReg)
     : Operand2(datastore::Reg(rm.code(), type, 1, shiftReg.encode())) {}
};

static_assert(sizeof(Op2Reg) == sizeof(datastore::Reg),
             "datastore::Reg(const Op2Reg&) constructor relies on Reg/Op2Reg "
             "having same size");

class O2RegImmShift : public Op2Reg {
public:
 explicit O2RegImmShift(Register rn, ShiftType type, uint32_t shift)
     : Op2Reg(rn, type, datastore::RIS(shift)) {}
};

class O2RegRegShift : public Op2Reg {
public:
 explicit O2RegRegShift(Register rn, ShiftType type, Register rs)
     : Op2Reg(rn, type, datastore::RRS(rs.code())) {}
};

O2RegImmShift O2Reg(Register r);
O2RegImmShift lsl(Register r, int amt);
O2RegImmShift lsr(Register r, int amt);
O2RegImmShift asr(Register r, int amt);
O2RegImmShift rol(Register r, int amt);
O2RegImmShift ror(Register r, int amt);

O2RegRegShift lsl(Register r, Register amt);
O2RegRegShift lsr(Register r, Register amt);
O2RegRegShift asr(Register r, Register amt);
O2RegRegShift ror(Register r, Register amt);

// An offset from a register to be used for ldr/str. This should include the
// sign bit, since ARM has "signed-magnitude" offsets. That is it encodes an
// unsigned offset, then the instruction specifies if the offset is positive or
// negative. The +/- bit is necessary if the instruction set wants to be able to
// have a negative register offset e.g. ldr pc, [r1,-r2];
class DtrOff {
 uint32_t data_;

protected:
 explicit DtrOff(datastore::Imm12Data immdata, IsUp_ iu)
     : data_(immdata.encode() | uint32_t(IsImmDTR) | uint32_t(iu)) {}

 explicit DtrOff(datastore::Reg reg, IsUp_ iu = IsUp)
     : data_(reg.encode() | uint32_t(IsNotImmDTR) | iu) {}

public:
 uint32_t encode() const { return data_; }
};

class DtrOffImm : public DtrOff {
public:
 explicit DtrOffImm(int32_t imm)
     : DtrOff(datastore::Imm12Data(mozilla::Abs(imm)),
              imm >= 0 ? IsUp : IsDown) {
   MOZ_ASSERT(mozilla::Abs(imm) < 4096);
 }
};

class DtrOffReg : public DtrOff {
 // These are designed to be called by a constructor of a subclass.
 // Constructing the necessary RIS/RRS structures is annoying.

protected:
 explicit DtrOffReg(Register rn, ShiftType type, datastore::RIS shiftImm,
                    IsUp_ iu = IsUp)
     : DtrOff(datastore::Reg(rn.code(), type, 0, shiftImm.encode()), iu) {}

 explicit DtrOffReg(Register rn, ShiftType type, datastore::RRS shiftReg,
                    IsUp_ iu = IsUp)
     : DtrOff(datastore::Reg(rn.code(), type, 1, shiftReg.encode()), iu) {}
};

class DtrRegImmShift : public DtrOffReg {
public:
 explicit DtrRegImmShift(Register rn, ShiftType type, uint32_t shift,
                         IsUp_ iu = IsUp)
     : DtrOffReg(rn, type, datastore::RIS(shift), iu) {}
};

class DtrRegRegShift : public DtrOffReg {
public:
 explicit DtrRegRegShift(Register rn, ShiftType type, Register rs,
                         IsUp_ iu = IsUp)
     : DtrOffReg(rn, type, datastore::RRS(rs.code()), iu) {}
};

// We will frequently want to bundle a register with its offset so that we have
// an "operand" to a load instruction.
class DTRAddr {
 friend class Operand;

 uint32_t data_;

public:
 explicit DTRAddr(Register reg, DtrOff dtr)
     : data_(dtr.encode() | (reg.code() << 16)) {}

 uint32_t encode() const { return data_; }

 Register getBase() const { return Register::FromCode((data_ >> 16) & 0xf); }
};

// Offsets for the extended data transfer instructions:
// ldrsh, ldrd, ldrsb, etc.
class EDtrOff {
 uint32_t data_;

protected:
 explicit EDtrOff(datastore::Imm8Data imm8, IsUp_ iu = IsUp)
     : data_(imm8.encode() | IsImmEDTR | uint32_t(iu)) {}

 explicit EDtrOff(Register rm, IsUp_ iu = IsUp)
     : data_(rm.code() | IsNotImmEDTR | iu) {}

public:
 uint32_t encode() const { return data_; }
};

class EDtrOffImm : public EDtrOff {
public:
 explicit EDtrOffImm(int32_t imm)
     : EDtrOff(datastore::Imm8Data(mozilla::Abs(imm)),
               (imm >= 0) ? IsUp : IsDown) {
   MOZ_ASSERT(mozilla::Abs(imm) < 256);
 }
};

// This is the most-derived class, since the extended data transfer instructions
// don't support any sort of modifying the "index" operand.
class EDtrOffReg : public EDtrOff {
public:
 explicit EDtrOffReg(Register rm) : EDtrOff(rm) {}
};

class EDtrAddr {
 uint32_t data_;

public:
 explicit EDtrAddr(Register r, EDtrOff off) : data_(RN(r) | off.encode()) {}

 uint32_t encode() const { return data_; }
#ifdef DEBUG
 Register maybeOffsetRegister() const {
   if (data_ & IsImmEDTR) {
     return InvalidReg;
   }
   return Register::FromCode(data_ & 0xf);
 }
#endif
};

class VFPOff {
 uint32_t data_;

protected:
 explicit VFPOff(datastore::Imm8VFPOffData imm, IsUp_ isup)
     : data_(imm.encode() | uint32_t(isup)) {}

public:
 uint32_t encode() const { return data_; }
};

class VFPOffImm : public VFPOff {
public:
 explicit VFPOffImm(int32_t imm)
     : VFPOff(datastore::Imm8VFPOffData(mozilla::Abs(imm) / 4),
              imm < 0 ? IsDown : IsUp) {
   MOZ_ASSERT(mozilla::Abs(imm) <= 255 * 4);
 }
};

class VFPAddr {
 friend class Operand;

 uint32_t data_;

public:
 explicit VFPAddr(Register base, VFPOff off)
     : data_(RN(base) | off.encode()) {}

 uint32_t encode() const { return data_; }
};

class VFPImm {
 uint32_t data_;

public:
 explicit VFPImm(uint32_t topWordOfDouble);

 static const VFPImm One;

 uint32_t encode() const { return data_; }
 bool isValid() const { return data_ != (~0U); }
};

// A BOffImm is an immediate that is used for branches. Namely, it is the offset
// that will be encoded in the branch instruction. This is the only sane way of
// constructing a branch.
class BOffImm {
 friend class InstBranchImm;

 uint32_t data_;

public:
 explicit BOffImm(int offset) : data_((offset - 8) >> 2 & 0x00ffffff) {
   MOZ_ASSERT((offset & 0x3) == 0);
   if (!IsInRange(offset)) {
     MOZ_CRASH("BOffImm offset out of range");
   }
 }

 explicit BOffImm() : data_(INVALID) {}

private:
 explicit BOffImm(const Instruction& inst);

public:
 static const uint32_t INVALID = 0x00800000;

 uint32_t encode() const { return data_; }
 int32_t decode() const { return ((int32_t(data_) << 8) >> 6) + 8; }

 static bool IsInRange(int offset) {
   if ((offset - 8) < -33554432) {
     return false;
   }
   if ((offset - 8) > 33554428) {
     return false;
   }
   return true;
 }

 bool isInvalid() const { return data_ == INVALID; }
 Instruction* getDest(Instruction* src) const;
};

class Imm16 {
 uint32_t lower_ : 12;

protected:
 // Mark as a protected field to avoid unused private field warnings.
 uint32_t pad_ : 4;

private:
 uint32_t upper_ : 4;
 uint32_t invalid_ : 12;

public:
 explicit Imm16();
 explicit Imm16(uint32_t imm);
 explicit Imm16(Instruction& inst);

 uint32_t encode() const { return lower_ | (upper_ << 16); }
 uint32_t decode() const { return lower_ | (upper_ << 12); }

 bool isInvalid() const { return invalid_; }
};

// I would preffer that these do not exist, since there are essentially no
// instructions that would ever take more than one of these, however, the MIR
// wants to only have one type of arguments to functions, so bugger.
class Operand {
 // The encoding of registers is the same for OP2, DTR and EDTR yet the type
 // system doesn't let us express this, so choices must be made.
public:
 enum class Tag : uint8_t { OP2, MEM, FOP };

private:
 uint32_t tag_ : 8;
 uint32_t reg_ : 5;
 int32_t offset_;

protected:
 Operand(Tag tag, uint32_t regCode, int32_t offset)
     : tag_(static_cast<uint32_t>(tag)), reg_(regCode), offset_(offset) {}

public:
 explicit Operand(Register reg) : Operand(Tag::OP2, reg.code(), 0) {}

 explicit Operand(FloatRegister freg) : Operand(Tag::FOP, freg.code(), 0) {}

 explicit Operand(Register base, Imm32 off)
     : Operand(Tag::MEM, base.code(), off.value) {}

 explicit Operand(Register base, int32_t off)
     : Operand(Tag::MEM, base.code(), off) {}

 explicit Operand(const Address& addr)
     : Operand(Tag::MEM, addr.base.code(), addr.offset) {}

public:
 Tag tag() const { return static_cast<Tag>(tag_); }

 Operand2 toOp2() const {
   MOZ_ASSERT(tag() == Tag::OP2);
   return O2Reg(Register::FromCode(reg_));
 }

 Register toReg() const {
   MOZ_ASSERT(tag() == Tag::OP2);
   return Register::FromCode(reg_);
 }

 Address toAddress() const {
   MOZ_ASSERT(tag() == Tag::MEM);
   return Address(Register::FromCode(reg_), offset_);
 }
 int32_t disp() const {
   MOZ_ASSERT(tag() == Tag::MEM);
   return offset_;
 }

 int32_t base() const {
   MOZ_ASSERT(tag() == Tag::MEM);
   return reg_;
 }
 Register baseReg() const {
   MOZ_ASSERT(tag() == Tag::MEM);
   return Register::FromCode(reg_);
 }
 DTRAddr toDTRAddr() const {
   MOZ_ASSERT(tag() == Tag::MEM);
   return DTRAddr(baseReg(), DtrOffImm(offset_));
 }
 VFPAddr toVFPAddr() const {
   MOZ_ASSERT(tag() == Tag::MEM);
   return VFPAddr(baseReg(), VFPOffImm(offset_));
 }
};

class InstructionIterator {
private:
 Instruction* inst_;

public:
 explicit InstructionIterator(Instruction* inst) : inst_(inst) {
   maybeSkipAutomaticInstructions();
 }

 // Advances to the next intentionally-inserted instruction.
 Instruction* next();

 // Advances past any automatically-inserted instructions.
 Instruction* maybeSkipAutomaticInstructions();

 Instruction* cur() const { return inst_; }

protected:
 // Advances past the given number of instruction-length bytes.
 inline void advanceRaw(ptrdiff_t instructions = 1);
};

class Assembler;
using ARMBuffer =
   js::jit::AssemblerBufferWithConstantPools<1024, 4, Instruction, Assembler>;

class Assembler : public AssemblerShared {
public:
 // ARM conditional constants:
 enum ARMCondition : uint32_t {
   EQ = 0x00000000,  // Zero
   NE = 0x10000000,  // Non-zero
   CS = 0x20000000,
   CC = 0x30000000,
   MI = 0x40000000,
   PL = 0x50000000,
   VS = 0x60000000,
   VC = 0x70000000,
   HI = 0x80000000,
   LS = 0x90000000,
   GE = 0xa0000000,
   LT = 0xb0000000,
   GT = 0xc0000000,
   LE = 0xd0000000,
   AL = 0xe0000000
 };

 enum Condition : uint32_t {
   Equal = EQ,
   NotEqual = NE,
   Above = HI,
   AboveOrEqual = CS,
   Below = CC,
   BelowOrEqual = LS,
   GreaterThan = GT,
   GreaterThanOrEqual = GE,
   LessThan = LT,
   LessThanOrEqual = LE,
   Overflow = VS,
   CarrySet = CS,
   CarryClear = CC,
   Signed = MI,
   NotSigned = PL,
   Zero = EQ,
   NonZero = NE,
   Always = AL,

   VFP_NotEqualOrUnordered = NE,
   VFP_Equal = EQ,
   VFP_Unordered = VS,
   VFP_NotUnordered = VC,
   VFP_GreaterThanOrEqualOrUnordered = CS,
   VFP_GreaterThanOrEqual = GE,
   VFP_GreaterThanOrUnordered = HI,
   VFP_GreaterThan = GT,
   VFP_LessThanOrEqualOrUnordered = LE,
   VFP_LessThanOrEqual = LS,
   VFP_LessThanOrUnordered = LT,
   VFP_LessThan = CC  // MI is valid too.
 };

 // Bit set when a DoubleCondition does not map to a single ARM condition.
 // The macro assembler has to special-case these conditions, or else
 // ConditionFromDoubleCondition will complain.
 static const int DoubleConditionBitSpecial = 0x1;

 enum DoubleCondition : uint32_t {
   // These conditions will only evaluate to true if the comparison is
   // ordered - i.e. neither operand is NaN.
   DoubleOrdered = VFP_NotUnordered,
   DoubleEqual = VFP_Equal,
   DoubleNotEqual = VFP_NotEqualOrUnordered | DoubleConditionBitSpecial,
   DoubleGreaterThan = VFP_GreaterThan,
   DoubleGreaterThanOrEqual = VFP_GreaterThanOrEqual,
   DoubleLessThan = VFP_LessThan,
   DoubleLessThanOrEqual = VFP_LessThanOrEqual,
   // If either operand is NaN, these conditions always evaluate to true.
   DoubleUnordered = VFP_Unordered,
   DoubleEqualOrUnordered = VFP_Equal | DoubleConditionBitSpecial,
   DoubleNotEqualOrUnordered = VFP_NotEqualOrUnordered,
   DoubleGreaterThanOrUnordered = VFP_GreaterThanOrUnordered,
   DoubleGreaterThanOrEqualOrUnordered = VFP_GreaterThanOrEqualOrUnordered,
   DoubleLessThanOrUnordered = VFP_LessThanOrUnordered,
   DoubleLessThanOrEqualOrUnordered = VFP_LessThanOrEqualOrUnordered
 };

 Condition getCondition(uint32_t inst) {
   return (Condition)(0xf0000000 & inst);
 }
 static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
   MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));
   return static_cast<Condition>(cond);
 }

 enum BarrierOption {
   BarrierSY = 15,  // Full system barrier
   BarrierST = 14   // StoreStore barrier
 };

 // This should be protected, but since CodeGenerator wants to use it, it
 // needs to go out here :(

 BufferOffset nextOffset() { return m_buffer.nextOffset(); }

protected:
 // Shim around AssemblerBufferWithConstantPools::allocEntry.
 BufferOffset allocLiteralLoadEntry(size_t numInst, unsigned numPoolEntries,
                                    PoolHintPun& php, uint8_t* data,
                                    const LiteralDoc& doc = LiteralDoc(),
                                    ARMBuffer::PoolEntry* pe = nullptr,
                                    bool loadToPC = false);

 Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); }

#ifdef JS_DISASM_ARM
 using DisasmBuffer =
     disasm::EmbeddedVector<char, disasm::ReasonableBufferSize>;

 static void disassembleInstruction(const Instruction* i,
                                    DisasmBuffer& buffer);

 void initDisassembler();
 void finishDisassembler();
 void spew(Instruction* i);
 void spewBranch(Instruction* i, const LabelDoc& target);
 void spewLiteralLoad(PoolHintPun& php, bool loadToPC, const Instruction* offs,
                      const LiteralDoc& doc);
#endif

public:
 void resetCounter();
 static uint32_t NopFill;
 static uint32_t GetNopFill();
 static uint32_t AsmPoolMaxOffset;
 static uint32_t GetPoolMaxOffset();

protected:
 // Structure for fixing up pc-relative loads/jumps when a the machine code
 // gets moved (executable copy, gc, etc.).
 class RelativePatch {
   void* target_;
   RelocationKind kind_;

  public:
   RelativePatch(void* target, RelocationKind kind)
       : target_(target), kind_(kind) {}
   void* target() const { return target_; }
   RelocationKind kind() const { return kind_; }
 };

 // TODO: this should actually be a pool-like object. It is currently a big
 // hack, and probably shouldn't exist.
 js::Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;

 CompactBufferWriter jumpRelocations_;
 CompactBufferWriter dataRelocations_;

 ARMBuffer m_buffer;

#ifdef JS_DISASM_ARM
 DisassemblerSpew spew_;
#endif

public:
 // For the alignment fill use NOP: 0x0320f000 or (Always | InstNOP::NopInst).
 // For the nopFill use a branch to the next instruction: 0xeaffffff.
 Assembler()
     : m_buffer(1, 1, 8, GetPoolMaxOffset(), 8, 0xe320f000, 0xeaffffff,
                GetNopFill()),
       isFinished(false),
       dtmActive(false),
       dtmCond(Always) {
#ifdef JS_DISASM_ARM
   initDisassembler();
#endif
 }

 ~Assembler() {
#ifdef JS_DISASM_ARM
   finishDisassembler();
#endif
 }

 void setUnlimitedBuffer() { m_buffer.setUnlimited(); }

 static Condition InvertCondition(Condition cond);
 static Condition UnsignedCondition(Condition cond);
 static Condition ConditionWithoutEqual(Condition cond);

 static DoubleCondition InvertCondition(DoubleCondition cond);

 void writeDataRelocation(BufferOffset offset, ImmGCPtr ptr) {
   // Raw GC pointer relocations and Value relocations both end up in
   // Assembler::TraceDataRelocations.
   if (ptr.value) {
     if (gc::IsInsideNursery(ptr.value)) {
       embedsNurseryPointers_ = true;
     }
     dataRelocations_.writeUnsigned(offset.getOffset());
   }
 }

 enum RelocBranchStyle { B_MOVWT, B_LDR_BX, B_LDR, B_MOVW_ADD };

 enum RelocStyle { L_MOVWT, L_LDR };

public:
 // Given the start of a Control Flow sequence, grab the value that is
 // finally branched to given the start of a function that loads an address
 // into a register get the address that ends up in the register.
 template <class Iter>
 static const uint32_t* GetCF32Target(Iter* iter);

 static uintptr_t GetPointer(uint8_t*);
 template <class Iter>
 static const uint32_t* GetPtr32Target(Iter iter, Register* dest = nullptr,
                                       RelocStyle* rs = nullptr);

 bool oom() const;

 void setPrinter(Sprinter* sp) {
#ifdef JS_DISASM_ARM
   spew_.setPrinter(sp);
#endif
 }

 Register getStackPointer() const { return StackPointer; }

private:
 bool isFinished;

protected:
 LabelDoc refLabel(const Label* label) {
#ifdef JS_DISASM_ARM
   return spew_.refLabel(label);
#else
   return LabelDoc();
#endif
 }

public:
 void finish();
 bool appendRawCode(const uint8_t* code, size_t numBytes);
 bool reserve(size_t size);
 bool swapBuffer(wasm::Bytes& bytes);
 void copyJumpRelocationTable(uint8_t* dest);
 void copyDataRelocationTable(uint8_t* dest);

 // Size of the instruction stream, in bytes, after pools are flushed.
 size_t size() const;
 // Size of the jump relocation table, in bytes.
 size_t jumpRelocationTableBytes() const;
 size_t dataRelocationTableBytes() const;

 // Size of the data table, in bytes.
 size_t bytesNeeded() const;

 // Write a single instruction into the instruction stream.  Very hot,
 // inlined for performance
 MOZ_ALWAYS_INLINE BufferOffset writeInst(uint32_t x) {
   MOZ_ASSERT(hasCreator());
   BufferOffset offs = m_buffer.putInt(x);
#ifdef JS_DISASM_ARM
   spew(m_buffer.getInstOrNull(offs));
#endif
   return offs;
 }

 // As above, but also mark the instruction as a branch.  Very hot, inlined
 // for performance
 MOZ_ALWAYS_INLINE BufferOffset
 writeBranchInst(uint32_t x, const LabelDoc& documentation) {
   BufferOffset offs = m_buffer.putInt(x);
#ifdef JS_DISASM_ARM
   spewBranch(m_buffer.getInstOrNull(offs), documentation);
#endif
   return offs;
 }

 // Write a placeholder NOP for a branch into the instruction stream
 // (in order to adjust assembler addresses and mark it as a branch), it will
 // be overwritten subsequently.
 BufferOffset allocBranchInst();

 // A static variant for the cases where we don't want to have an assembler
 // object.
 static void WriteInstStatic(uint32_t x, uint32_t* dest);

public:
 void writeCodePointer(CodeLabel* label);

 void haltingAlign(int alignment);
 void nopAlign(int alignment);
 BufferOffset as_nop();
 BufferOffset as_alu(Register dest, Register src1, Operand2 op2, ALUOp op,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_mov(Register dest, Operand2 op2, SBit s = LeaveCC,
                     Condition c = Always);
 BufferOffset as_mvn(Register dest, Operand2 op2, SBit s = LeaveCC,
                     Condition c = Always);

 static void as_alu_patch(Register dest, Register src1, Operand2 op2, ALUOp op,
                          SBit s, Condition c, uint32_t* pos);
 static void as_mov_patch(Register dest, Operand2 op2, SBit s, Condition c,
                          uint32_t* pos);

 // Logical operations:
 BufferOffset as_and(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_bic(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_eor(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_orr(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 // Reverse byte operations:
 BufferOffset as_rev(Register dest, Register src, Condition c = Always);
 BufferOffset as_rev16(Register dest, Register src, Condition c = Always);
 BufferOffset as_revsh(Register dest, Register src, Condition c = Always);
 // Mathematical operations:
 BufferOffset as_adc(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_add(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_sbc(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_sub(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_rsb(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_rsc(Register dest, Register src1, Operand2 op2,
                     SBit s = LeaveCC, Condition c = Always);
 // Test operations:
 BufferOffset as_cmn(Register src1, Operand2 op2, Condition c = Always);
 BufferOffset as_cmp(Register src1, Operand2 op2, Condition c = Always);
 BufferOffset as_teq(Register src1, Operand2 op2, Condition c = Always);
 BufferOffset as_tst(Register src1, Operand2 op2, Condition c = Always);

 // Sign extension operations:
 BufferOffset as_sxtb(Register dest, Register src, int rotate,
                      Condition c = Always);
 BufferOffset as_sxth(Register dest, Register src, int rotate,
                      Condition c = Always);
 BufferOffset as_uxtb(Register dest, Register src, int rotate,
                      Condition c = Always);
 BufferOffset as_uxth(Register dest, Register src, int rotate,
                      Condition c = Always);

 // Not quite ALU worthy, but useful none the less: These also have the issue
 // of these being formatted completly differently from the standard ALU
 // operations.
 BufferOffset as_movw(Register dest, Imm16 imm, Condition c = Always);
 BufferOffset as_movt(Register dest, Imm16 imm, Condition c = Always);

 static void as_movw_patch(Register dest, Imm16 imm, Condition c,
                           Instruction* pos);
 static void as_movt_patch(Register dest, Imm16 imm, Condition c,
                           Instruction* pos);

 BufferOffset as_genmul(Register d1, Register d2, Register rm, Register rn,
                        MULOp op, SBit s, Condition c = Always);
 BufferOffset as_mul(Register dest, Register src1, Register src2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_mla(Register dest, Register acc, Register src1, Register src2,
                     SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_umaal(Register dest1, Register dest2, Register src1,
                       Register src2, Condition c = Always);
 BufferOffset as_mls(Register dest, Register acc, Register src1, Register src2,
                     Condition c = Always);
 BufferOffset as_umull(Register dest1, Register dest2, Register src1,
                       Register src2, SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_umlal(Register dest1, Register dest2, Register src1,
                       Register src2, SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_smull(Register dest1, Register dest2, Register src1,
                       Register src2, SBit s = LeaveCC, Condition c = Always);
 BufferOffset as_smlal(Register dest1, Register dest2, Register src1,
                       Register src2, SBit s = LeaveCC, Condition c = Always);

 BufferOffset as_sdiv(Register dest, Register num, Register div,
                      Condition c = Always);
 BufferOffset as_udiv(Register dest, Register num, Register div,
                      Condition c = Always);
 BufferOffset as_clz(Register dest, Register src, Condition c = Always);

 // Data transfer instructions: ldr, str, ldrb, strb.
 // Using an int to differentiate between 8 bits and 32 bits is overkill.
 BufferOffset as_dtr(LoadStore ls, int size, Index mode, Register rt,
                     DTRAddr addr, Condition c = Always);

 static void as_dtr_patch(LoadStore ls, int size, Index mode, Register rt,
                          DTRAddr addr, Condition c, uint32_t* dest);

 // Handles all of the other integral data transferring functions:
 // ldrsb, ldrsh, ldrd, etc. The size is given in bits.
 BufferOffset as_extdtr(LoadStore ls, int size, bool IsSigned, Index mode,
                        Register rt, EDtrAddr addr, Condition c = Always);

 BufferOffset as_dtm(LoadStore ls, Register rn, uint32_t mask, DTMMode mode,
                     DTMWriteBack wb, Condition c = Always);

 // Overwrite a pool entry with new data.
 static void WritePoolEntry(Instruction* addr, Condition c, uint32_t data);

 // Load a 32 bit immediate from a pool into a register.
 BufferOffset as_Imm32Pool(Register dest, uint32_t value,
                           Condition c = Always);

 // Load a 64 bit floating point immediate from a pool into a register.
 BufferOffset as_FImm64Pool(VFPRegister dest, double value,
                            Condition c = Always);
 // Load a 32 bit floating point immediate from a pool into a register.
 BufferOffset as_FImm32Pool(VFPRegister dest, float value,
                            Condition c = Always);

 // Atomic instructions: ldrexd, ldrex, ldrexh, ldrexb, strexd, strex, strexh,
 // strexb.
 //
 // The doubleword, halfword, and byte versions are available from ARMv6K
 // forward.
 //
 // The word versions are available from ARMv6 forward and can be used to
 // implement the halfword and byte versions on older systems.

 // LDREXD rt, rt2, [rn].  Constraint: rt even register, rt2=rt+1.
 BufferOffset as_ldrexd(Register rt, Register rt2, Register rn,
                        Condition c = Always);

 // LDREX rt, [rn]
 BufferOffset as_ldrex(Register rt, Register rn, Condition c = Always);
 BufferOffset as_ldrexh(Register rt, Register rn, Condition c = Always);
 BufferOffset as_ldrexb(Register rt, Register rn, Condition c = Always);

 // STREXD rd, rt, rt2, [rn].  Constraint: rt even register, rt2=rt+1.
 BufferOffset as_strexd(Register rd, Register rt, Register rt2, Register rn,
                        Condition c = Always);

 // STREX rd, rt, [rn].  Constraint: rd != rn, rd != rt.
 BufferOffset as_strex(Register rd, Register rt, Register rn,
                       Condition c = Always);
 BufferOffset as_strexh(Register rd, Register rt, Register rn,
                        Condition c = Always);
 BufferOffset as_strexb(Register rd, Register rt, Register rn,
                        Condition c = Always);

 // CLREX
 BufferOffset as_clrex();

 // Memory synchronization.
 // These are available from ARMv7 forward.
 BufferOffset as_dmb(BarrierOption option = BarrierSY);
 BufferOffset as_dsb(BarrierOption option = BarrierSY);
 BufferOffset as_isb();

 // Memory synchronization for architectures before ARMv7.
 BufferOffset as_dsb_trap();
 BufferOffset as_dmb_trap();
 BufferOffset as_isb_trap();

 // Speculation barrier
 BufferOffset as_csdb();

 // Move Special Register and Hints:

 // yield hint instruction.
 BufferOffset as_yield();

 // Control flow stuff:

 // bx can *only* branch to a register never to an immediate.
 BufferOffset as_bx(Register r, Condition c = Always);

 // Branch can branch to an immediate *or* to a register. Branches to
 // immediates are pc relative, branches to registers are absolute.
 BufferOffset as_b(BOffImm off, Condition c, Label* documentation = nullptr);

 BufferOffset as_b(Label* l, Condition c = Always);
 BufferOffset as_b(BOffImm off, Condition c, BufferOffset inst);

 // blx can go to either an immediate or a register. When blx'ing to a
 // register, we change processor mode depending on the low bit of the
 // register when blx'ing to an immediate, we *always* change processor
 // state.
 BufferOffset as_blx(Label* l);

 BufferOffset as_blx(Register r, Condition c = Always);
 BufferOffset as_bl(BOffImm off, Condition c, Label* documentation = nullptr);
 // bl can only branch+link to an immediate, never to a register it never
 // changes processor state.
 BufferOffset as_bl();
 // bl #imm can have a condition code, blx #imm cannot.
 // blx reg can be conditional.
 BufferOffset as_bl(Label* l, Condition c);
 BufferOffset as_bl(BOffImm off, Condition c, BufferOffset inst);

 BufferOffset as_mrs(Register r, Condition c = Always);
 BufferOffset as_msr(Register r, Condition c = Always);

 // VFP instructions!
private:
 enum vfp_size { IsDouble = 1 << 8, IsSingle = 0 << 8 };

 BufferOffset writeVFPInst(vfp_size sz, uint32_t blob);

 static void WriteVFPInstStatic(vfp_size sz, uint32_t blob, uint32_t* dest);

 // Unityped variants: all registers hold the same (ieee754 single/double)
 // notably not included are vcvt; vmov vd, #imm; vmov rt, vn.
 BufferOffset as_vfp_float(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                           VFPOp op, Condition c = Always);

public:
 BufferOffset as_vadd(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                      Condition c = Always);
 BufferOffset as_vdiv(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                      Condition c = Always);
 BufferOffset as_vmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                      Condition c = Always);
 BufferOffset as_vnmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                       Condition c = Always);
 BufferOffset as_vnmla(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                       Condition c = Always);
 BufferOffset as_vnmls(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                       Condition c = Always);
 BufferOffset as_vneg(VFPRegister vd, VFPRegister vm, Condition c = Always);
 BufferOffset as_vsqrt(VFPRegister vd, VFPRegister vm, Condition c = Always);
 BufferOffset as_vabs(VFPRegister vd, VFPRegister vm, Condition c = Always);
 BufferOffset as_vsub(VFPRegister vd, VFPRegister vn, VFPRegister vm,
                      Condition c = Always);
 BufferOffset as_vcmp(VFPRegister vd, VFPRegister vm, Condition c = Always);
 BufferOffset as_vcmpz(VFPRegister vd, Condition c = Always);

 // Specifically, a move between two same sized-registers.
 BufferOffset as_vmov(VFPRegister vd, VFPRegister vsrc, Condition c = Always);

 // Transfer between Core and VFP.
 enum FloatToCore_ { FloatToCore = 1 << 20, CoreToFloat = 0 << 20 };

private:
 enum VFPXferSize { WordTransfer = 0x02000010, DoubleTransfer = 0x00400010 };

public:
 // Unlike the next function, moving between the core registers and vfp
 // registers can't be *that* properly typed. Namely, since I don't want to
 // munge the type VFPRegister to also include core registers. Thus, the core
 // and vfp registers are passed in based on their type, and src/dest is
 // determined by the float2core.

 BufferOffset as_vxfer(Register vt1, Register vt2, VFPRegister vm,
                       FloatToCore_ f2c, Condition c = Always, int idx = 0);

 // Our encoding actually allows just the src and the dest (and their types)
 // to uniquely specify the encoding that we are going to use.
 BufferOffset as_vcvt(VFPRegister vd, VFPRegister vm, bool useFPSCR = false,
                      Condition c = Always);

 // Hard coded to a 32 bit fixed width result for now.
 BufferOffset as_vcvtFixed(VFPRegister vd, bool isSigned, uint32_t fixedPoint,
                           bool toFixed, Condition c = Always);

 // Convert between single- and half-precision. Both registers are single
 // precision.
 BufferOffset as_vcvtb_s2h(VFPRegister vd, VFPRegister vm,
                           Condition c = Always);
 BufferOffset as_vcvtb_h2s(VFPRegister vd, VFPRegister vm,
                           Condition c = Always);

 // Transfer between VFP and memory.
 BufferOffset as_vdtr(LoadStore ls, VFPRegister vd, VFPAddr addr,
                      Condition c = Always /* vfp doesn't have a wb option*/);

 static void as_vdtr_patch(LoadStore ls, VFPRegister vd, VFPAddr addr,
                           Condition c /* vfp doesn't have a wb option */,
                           uint32_t* dest);

 // VFP's ldm/stm work differently from the standard arm ones. You can only
 // transfer a range.

 BufferOffset as_vdtm(LoadStore st, Register rn, VFPRegister vd, int length,
                      /* also has update conditions */ Condition c = Always);

 // vldr/vstr variants that handle unaligned accesses.  These encode as NEON
 // single-element instructions and can only be used if NEON is available.
 // Here, vd must be tagged as a float or double register.
 BufferOffset as_vldr_unaligned(VFPRegister vd, Register rn);
 BufferOffset as_vstr_unaligned(VFPRegister vd, Register rn);

 BufferOffset as_vimm(VFPRegister vd, VFPImm imm, Condition c = Always);

 BufferOffset as_vmrs(Register r, Condition c = Always);
 BufferOffset as_vmsr(Register r, Condition c = Always);

 // Label operations.
 bool nextLink(BufferOffset b, BufferOffset* next);
 void bind(Label* label, BufferOffset boff = BufferOffset());
 void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }
 uint32_t currentOffset() { return nextOffset().getOffset(); }
 void retarget(Label* label, Label* target);
 // I'm going to pretend this doesn't exist for now.
 void retarget(Label* label, void* target, RelocationKind reloc);

 static void Bind(uint8_t* rawCode, const CodeLabel& label);
 static void PatchMovwt(Instruction* addr, uint32_t imm);

 void as_bkpt();
 BufferOffset as_illegal_trap();

public:
 static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
                                  CompactBufferReader& reader);
 static void TraceDataRelocations(JSTracer* trc, JitCode* code,
                                  CompactBufferReader& reader);

 void assertNoGCThings() const {
#ifdef DEBUG
   MOZ_ASSERT(dataRelocations_.length() == 0);
   for (auto& j : jumps_) {
     MOZ_ASSERT(j.kind() == RelocationKind::HARDCODED);
   }
#endif
 }

 static bool SupportsFloatingPoint() { return ARMFlags::HasVFP(); }
 static bool SupportsUnalignedAccesses() { return ARMFlags::HasARMv7(); }
 // Note, returning false here is technically wrong, but one has to go via the
 // as_vldr_unaligned and as_vstr_unaligned instructions to get proper behavior
 // and those are NEON-specific and have to be asked for specifically.
 static bool SupportsFastUnalignedFPAccesses() { return false; }
 static bool SupportsFloat64To16() { return false; }
 static bool SupportsFloat32To16() { return ARMFlags::HasFPHalfPrecision(); }

 static bool HasRoundInstruction(RoundingMode mode) { return false; }

protected:
 void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind) {
   enoughMemory_ &= jumps_.append(RelativePatch(target.value, kind));
   if (kind == RelocationKind::JITCODE) {
     jumpRelocations_.writeUnsigned(src.getOffset());
   }
 }

public:
 // The buffer is about to be linked, make sure any constant pools or excess
 // bookkeeping has been flushed to the instruction stream.
 void flush() {
   MOZ_ASSERT(!isFinished);
   m_buffer.flushPool();
   return;
 }

 void comment(const char* msg) {
#ifdef JS_DISASM_ARM
   spew_.spew("; %s", msg);
#endif
 }

 // Copy the assembly code to the given buffer, and perform any pending
 // relocations relying on the target address.
 void executableCopy(uint8_t* buffer);

 // Actual assembly emitting functions.

 // Since I can't think of a reasonable default for the mode, I'm going to
 // leave it as a required argument.
 void startDataTransferM(LoadStore ls, Register rm, DTMMode mode,
                         DTMWriteBack update = NoWriteBack,
                         Condition c = Always) {
   MOZ_ASSERT(!dtmActive);
   dtmUpdate = update;
   dtmBase = rm;
   dtmLoadStore = ls;
   dtmLastReg = -1;
   dtmRegBitField = 0;
   dtmActive = 1;
   dtmCond = c;
   dtmMode = mode;
 }

 void transferReg(Register rn) {
   MOZ_ASSERT(dtmActive);
   MOZ_ASSERT(rn.code() > dtmLastReg);
   dtmRegBitField |= 1 << rn.code();
   if (dtmLoadStore == IsLoad && rn.code() == 13 && dtmBase.code() == 13) {
     MOZ_CRASH("ARM Spec says this is invalid");
   }
 }
 void finishDataTransfer() {
   dtmActive = false;
   as_dtm(dtmLoadStore, dtmBase, dtmRegBitField, dtmMode, dtmUpdate, dtmCond);
 }

 void startFloatTransferM(LoadStore ls, Register rm, DTMMode mode,
                          DTMWriteBack update = NoWriteBack,
                          Condition c = Always) {
   MOZ_ASSERT(!dtmActive);
   dtmActive = true;
   dtmUpdate = update;
   dtmLoadStore = ls;
   dtmBase = rm;
   dtmCond = c;
   dtmLastReg = -1;
   dtmMode = mode;
   dtmDelta = 0;
 }
 void transferFloatReg(VFPRegister rn) {
   if (dtmLastReg == -1) {
     vdtmFirstReg = rn.code();
   } else {
     if (dtmDelta == 0) {
       dtmDelta = rn.code() - dtmLastReg;
       MOZ_ASSERT(dtmDelta == 1 || dtmDelta == -1);
     }
     MOZ_ASSERT(dtmLastReg >= 0);
     MOZ_ASSERT(rn.code() == unsigned(dtmLastReg) + dtmDelta);
   }

   dtmLastReg = rn.code();
 }
 void finishFloatTransfer() {
   MOZ_ASSERT(dtmActive);
   dtmActive = false;
   MOZ_ASSERT(dtmLastReg != -1);
   dtmDelta = dtmDelta ? dtmDelta : 1;
   // The operand for the vstr/vldr instruction is the lowest register in the
   // range.
   int low = std::min(dtmLastReg, vdtmFirstReg);
   int high = std::max(dtmLastReg, vdtmFirstReg);
   // Fencepost problem.
   int len = high - low + 1;
   // vdtm can only transfer 16 registers at once.  If we need to transfer
   // more, then either hoops are necessary, or we need to be updating the
   // register.
   MOZ_ASSERT_IF(len > 16, dtmUpdate == WriteBack);

   int adjustLow = dtmLoadStore == IsStore ? 0 : 1;
   int adjustHigh = dtmLoadStore == IsStore ? -1 : 0;
   while (len > 0) {
     // Limit the instruction to 16 registers.
     int curLen = std::min(len, 16);
     // If it is a store, we want to start at the high end and move down
     // (e.g. vpush d16-d31; vpush d0-d15).
     int curStart = (dtmLoadStore == IsStore) ? high - curLen + 1 : low;
     as_vdtm(dtmLoadStore, dtmBase,
             VFPRegister(FloatRegister::FromCode(curStart)), curLen, dtmCond);
     // Update the bounds.
     low += adjustLow * curLen;
     high += adjustHigh * curLen;
     // Update the length parameter.
     len -= curLen;
   }
 }

private:
 int dtmRegBitField;
 int vdtmFirstReg;
 int dtmLastReg;
 int dtmDelta;
 Register dtmBase;
 DTMWriteBack dtmUpdate;
 DTMMode dtmMode;
 LoadStore dtmLoadStore;
 bool dtmActive;
 Condition dtmCond;

public:
 enum {
   PadForAlign8 = (int)0x00,
   PadForAlign16 = (int)0x0000,
   PadForAlign32 = (int)0xe12fff7f  // 'bkpt 0xffff'
 };

 // API for speaking with the IonAssemblerBufferWithConstantPools generate an
 // initial placeholder instruction that we want to later fix up.
 static void InsertIndexIntoTag(uint8_t* load, uint32_t index);

 // Take the stub value that was written in before, and write in an actual
 // load using the index we'd computed previously as well as the address of
 // the pool start.
 static void PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr);

 // We're not tracking short-range branches for ARM for now.
 static void PatchShortRangeBranchToVeneer(ARMBuffer*, unsigned rangeIdx,
                                           BufferOffset deadline,
                                           BufferOffset veneer) {
   MOZ_CRASH();
 }
 // END API

 // Move our entire pool into the instruction stream. This is to force an
 // opportunistic dump of the pool, prefferably when it is more convenient to
 // do a dump.
 void flushBuffer();
 void enterNoPool(size_t maxInst);
 void leaveNoPool();
 void enterNoNops();
 void leaveNoNops();

 static void WritePoolHeader(uint8_t* start, Pool* p, bool isNatural);
 static void WritePoolGuard(BufferOffset branch, Instruction* inst,
                            BufferOffset dest);

 static uint32_t PatchWrite_NearCallSize();
 static uint32_t NopSize() { return 4; }
 static void PatchWrite_NearCall(CodeLocationLabel start,
                                 CodeLocationLabel toCall);
 static void PatchDataWithValueCheck(CodeLocationLabel label,
                                     PatchedImmPtr newValue,
                                     PatchedImmPtr expectedValue);
 static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
                                     ImmPtr expectedValue);
 static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm);

 static uint32_t AlignDoubleArg(uint32_t offset) { return (offset + 1) & ~1; }
 static uint8_t* NextInstruction(uint8_t* instruction,
                                 uint32_t* count = nullptr);

 // Toggle a jmp or cmp emitted by toggledJump().
 static void ToggleToJmp(CodeLocationLabel inst_);
 static void ToggleToCmp(CodeLocationLabel inst_);

 static size_t ToggledCallSize(uint8_t* code);
 static void ToggleCall(CodeLocationLabel inst_, bool enabled);

 void processCodeLabels(uint8_t* rawCode);

 void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
                                  const Disassembler::HeapAccess& heapAccess) {
   // Implement this if we implement a disassembler.
 }
};  // Assembler

// An Instruction is a structure for both encoding and decoding any and all ARM
// instructions. Many classes have not been implemented thus far.
class Instruction {
 uint32_t data;

protected:
 // This is not for defaulting to always, this is for instructions that
 // cannot be made conditional, and have the usually invalid 4b1111 cond
 // field.
 explicit Instruction(uint32_t data_, bool fake = false)
     : data(data_ | 0xf0000000) {
   MOZ_ASSERT(fake || ((data_ & 0xf0000000) == 0));
 }
 // Standard constructor.
 Instruction(uint32_t data_, Assembler::Condition c)
     : data(data_ | (uint32_t)c) {
   MOZ_ASSERT((data_ & 0xf0000000) == 0);
 }
 // You should never create an instruction directly. You should create a more
 // specific instruction which will eventually call one of these constructors
 // for you.
public:
 uint32_t encode() const { return data; }
 // Check if this instruction is really a particular case.
 template <class C>
 bool is() const {
   return C::IsTHIS(*this);
 }

 // Safely get a more specific variant of this pointer.
 template <class C>
 C* as() const {
   return C::AsTHIS(*this);
 }

 const Instruction& operator=(Instruction src) {
   data = src.data;
   return *this;
 }
 // Since almost all instructions have condition codes, the condition code
 // extractor resides in the base class.
 Assembler::Condition extractCond() const {
   MOZ_ASSERT(data >> 28 != 0xf,
              "The instruction does not have condition code");
   return (Assembler::Condition)(data & 0xf0000000);
 }

 // Sometimes, an api wants a uint32_t (or a pointer to it) rather than an
 // instruction. raw() just coerces this into a pointer to a uint32_t.
 const uint32_t* raw() const { return &data; }
 uint32_t size() const { return 4; }
};  // Instruction

// Make sure that it is the right size.
static_assert(sizeof(Instruction) == 4);

inline void InstructionIterator::advanceRaw(ptrdiff_t instructions) {
 inst_ = inst_ + instructions;
}

// Data Transfer Instructions.
class InstDTR : public Instruction {
public:
 enum IsByte_ { IsByte = 0x00400000, IsWord = 0x00000000 };
 static const int IsDTR = 0x04000000;
 static const int IsDTRMask = 0x0c000000;

 // TODO: Replace the initialization with something that is safer.
 InstDTR(LoadStore ls, IsByte_ ib, Index mode, Register rt, DTRAddr addr,
         Assembler::Condition c)
     : Instruction(std::underlying_type_t<LoadStore>(ls) |
                       std::underlying_type_t<IsByte_>(ib) |
                       std::underlying_type_t<Index>(mode) | RT(rt) |
                       addr.encode() | IsDTR,
                   c) {}

 static bool IsTHIS(const Instruction& i);
 static InstDTR* AsTHIS(const Instruction& i);
};
static_assert(sizeof(InstDTR) == sizeof(Instruction));

class InstLDR : public InstDTR {
public:
 InstLDR(Index mode, Register rt, DTRAddr addr, Assembler::Condition c)
     : InstDTR(IsLoad, IsWord, mode, rt, addr, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstLDR* AsTHIS(const Instruction& i);

 int32_t signedOffset() const {
   int32_t offset = encode() & 0xfff;
   if (IsUp_(encode() & IsUp) != IsUp) {
     return -offset;
   }
   return offset;
 }
 uint32_t* dest() const {
   int32_t offset = signedOffset();
   // When patching the load in PatchConstantPoolLoad, we ensure that the
   // offset is a multiple of 4, offset by 8 bytes from the actual
   // location.  Indeed, when the base register is PC, ARM's 3 stages
   // pipeline design makes it that PC is off by 8 bytes (= 2 *
   // sizeof(uint32*)) when we actually executed it.
   MOZ_ASSERT(offset % 4 == 0);
   offset >>= 2;
   return (uint32_t*)raw() + offset + 2;
 }
};
static_assert(sizeof(InstDTR) == sizeof(InstLDR));

class InstNOP : public Instruction {
public:
 static const uint32_t NopInst = 0x0320f000;

 InstNOP() : Instruction(NopInst, Assembler::Always) {}

 static bool IsTHIS(const Instruction& i);
 static InstNOP* AsTHIS(Instruction& i);
};

// Branching to a register, or calling a register
class InstBranchReg : public Instruction {
protected:
 // Don't use BranchTag yourself, use a derived instruction.
 enum BranchTag { IsBX = 0x012fff10, IsBLX = 0x012fff30 };

 static const uint32_t IsBRegMask = 0x0ffffff0;

 InstBranchReg(BranchTag tag, Register rm, Assembler::Condition c)
     : Instruction(tag | rm.code(), c) {}

public:
 static bool IsTHIS(const Instruction& i);
 static InstBranchReg* AsTHIS(const Instruction& i);

 // Get the register that is being branched to
 void extractDest(Register* dest);
 // Make sure we are branching to a pre-known register
 bool checkDest(Register dest);
};
static_assert(sizeof(InstBranchReg) == sizeof(Instruction));

// Branching to an immediate offset, or calling an immediate offset
class InstBranchImm : public Instruction {
protected:
 enum BranchTag { IsB = 0x0a000000, IsBL = 0x0b000000 };

 static const uint32_t IsBImmMask = 0x0f000000;

 InstBranchImm(BranchTag tag, BOffImm off, Assembler::Condition c)
     : Instruction(tag | off.encode(), c) {}

public:
 static bool IsTHIS(const Instruction& i);
 static InstBranchImm* AsTHIS(const Instruction& i);

 void extractImm(BOffImm* dest);
};
static_assert(sizeof(InstBranchImm) == sizeof(Instruction));

// Very specific branching instructions.
class InstBXReg : public InstBranchReg {
public:
 static bool IsTHIS(const Instruction& i);
 static InstBXReg* AsTHIS(const Instruction& i);
};

class InstBLXReg : public InstBranchReg {
public:
 InstBLXReg(Register reg, Assembler::Condition c)
     : InstBranchReg(IsBLX, reg, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstBLXReg* AsTHIS(const Instruction& i);
};

class InstBImm : public InstBranchImm {
public:
 InstBImm(BOffImm off, Assembler::Condition c) : InstBranchImm(IsB, off, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstBImm* AsTHIS(const Instruction& i);
};

class InstBLImm : public InstBranchImm {
public:
 InstBLImm(BOffImm off, Assembler::Condition c)
     : InstBranchImm(IsBL, off, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstBLImm* AsTHIS(const Instruction& i);
};

// Both movw and movt. The layout of both the immediate and the destination
// register is the same so the code is being shared.
class InstMovWT : public Instruction {
protected:
 enum WT { IsW = 0x03000000, IsT = 0x03400000 };
 static const uint32_t IsWTMask = 0x0ff00000;

 InstMovWT(Register rd, Imm16 imm, WT wt, Assembler::Condition c)
     : Instruction(RD(rd) | imm.encode() | wt, c) {}

public:
 void extractImm(Imm16* dest);
 void extractDest(Register* dest);
 bool checkImm(Imm16 dest);
 bool checkDest(Register dest);

 static bool IsTHIS(Instruction& i);
 static InstMovWT* AsTHIS(Instruction& i);
};
static_assert(sizeof(InstMovWT) == sizeof(Instruction));

class InstMovW : public InstMovWT {
public:
 InstMovW(Register rd, Imm16 imm, Assembler::Condition c)
     : InstMovWT(rd, imm, IsW, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstMovW* AsTHIS(const Instruction& i);
};

class InstMovT : public InstMovWT {
public:
 InstMovT(Register rd, Imm16 imm, Assembler::Condition c)
     : InstMovWT(rd, imm, IsT, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstMovT* AsTHIS(const Instruction& i);
};

class InstALU : public Instruction {
 static const int32_t ALUMask = 0xc << 24;

public:
 InstALU(Register rd, Register rn, Operand2 op2, ALUOp op, SBit s,
         Assembler::Condition c)
     : Instruction(maybeRD(rd) | maybeRN(rn) | op2.encode() | op | s, c) {}

 static bool IsTHIS(const Instruction& i);
 static InstALU* AsTHIS(const Instruction& i);

 void extractOp(ALUOp* ret);
 bool checkOp(ALUOp op);
 void extractDest(Register* ret);
 bool checkDest(Register rd);
 void extractOp1(Register* ret);
 bool checkOp1(Register rn);
 Operand2 extractOp2();
};

class InstCMP : public InstALU {
public:
 static bool IsTHIS(const Instruction& i);
 static InstCMP* AsTHIS(const Instruction& i);
};

class InstMOV : public InstALU {
public:
 static bool IsTHIS(const Instruction& i);
 static InstMOV* AsTHIS(const Instruction& i);
};

// Compile-time iterator over instructions, with a safe interface that
// references not-necessarily-linear Instructions by linear BufferOffset.
class BufferInstructionIterator
   : public ARMBuffer::AssemblerBufferInstIterator {
public:
 BufferInstructionIterator(BufferOffset bo, ARMBuffer* buffer)
     : ARMBuffer::AssemblerBufferInstIterator(bo, buffer) {}

 // Advances the buffer to the next intentionally-inserted instruction.
 Instruction* next() {
   advance(cur()->size());
   maybeSkipAutomaticInstructions();
   return cur();
 }

 // Advances the BufferOffset past any automatically-inserted instructions.
 Instruction* maybeSkipAutomaticInstructions();
};

static const uint32_t NumIntArgRegs = 4;

// There are 16 *float* registers available for arguments
// If doubles are used, only half the number of registers are available.
static const uint32_t NumFloatArgRegs = 16;

static inline bool GetIntArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
                               Register* out) {
 if (usedIntArgs >= NumIntArgRegs) {
   return false;
 }

 *out = Register::FromCode(usedIntArgs);
 return true;
}

// Get a register in which we plan to put a quantity that will be used as an
// integer argument. This differs from GetIntArgReg in that if we have no more
// actual argument registers to use we will fall back on using whatever
// CallTempReg* don't overlap the argument registers, and only fail once those
// run out too.
static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
                                      uint32_t usedFloatArgs, Register* out) {
 if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
   return true;
 }

 // Unfortunately, we have to assume things about the point at which
 // GetIntArgReg returns false, because we need to know how many registers it
 // can allocate.
 usedIntArgs -= NumIntArgRegs;
 if (usedIntArgs >= NumCallTempNonArgRegs) {
   return false;
 }

 *out = CallTempNonArgRegs[usedIntArgs];
 return true;
}

class DoubleEncoder {
 struct DoubleEntry {
   uint32_t dblTop;
   datastore::Imm8VFPImmData data;
 };

 static const DoubleEntry table[256];

public:
 bool lookup(uint32_t top, datastore::Imm8VFPImmData* ret) const {
   for (int i = 0; i < 256; i++) {
     if (table[i].dblTop == top) {
       *ret = table[i].data;
       return true;
     }
   }
   return false;
 }
};

// Forbids nop filling for testing purposes. Not nestable.
class AutoForbidNops {
protected:
 Assembler* masm_;

public:
 explicit AutoForbidNops(Assembler* masm) : masm_(masm) {
   masm_->enterNoNops();
 }
 ~AutoForbidNops() { masm_->leaveNoNops(); }
};

class AutoForbidPoolsAndNops : public AutoForbidNops {
public:
 // The maxInst argument is the maximum number of word sized instructions
 // that will be allocated within this context. It is used to determine if
 // the pool needs to be dumped before entering this content. The debug code
 // checks that no more than maxInst instructions are actually allocated.
 //
 // Allocation of pool entries is not supported within this content so the
 // code can not use large integers or float constants etc.
 AutoForbidPoolsAndNops(Assembler* masm, size_t maxInst)
     : AutoForbidNops(masm) {
   masm_->enterNoPool(maxInst);
 }

 ~AutoForbidPoolsAndNops() { masm_->leaveNoPool(); }
};

}  // namespace jit
}  // namespace js

#endif /* jit_arm_Assembler_arm_h */