tor-browser

LIR.h (72871B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_LIR_h
#define jit_LIR_h

// This file declares the core data structures for LIR: storage allocations for
// inputs and outputs, as well as the interface instructions must conform to.

#include "mozilla/Array.h"
#include "mozilla/Attributes.h"
#include "mozilla/Casting.h"

#include "jit/Bailouts.h"
#include "jit/FixedList.h"
#include "jit/InlineList.h"
#include "jit/JitAllocPolicy.h"
#include "jit/LIROpsGenerated.h"
#include "jit/MIR-wasm.h"
#include "jit/MIR.h"
#include "jit/MIRGraph.h"
#include "jit/Registers.h"
#include "jit/Safepoints.h"
#include "util/Memory.h"

namespace js {
namespace jit {

class LUse;
class LGeneralReg;
class LFloatReg;
class LStackSlot;
class LStackArea;
class LArgument;
class LConstantIndex;
class LInstruction;
class LNode;
class LDefinition;
class MBasicBlock;
class MIRGenerator;

static const uint32_t VREG_INCREMENT = 1;

static const uint32_t THIS_FRAME_ARGSLOT = 0;

#if defined(JS_NUNBOX32)
static const uint32_t BOX_PIECES = 2;
static const uint32_t VREG_TYPE_OFFSET = 0;
static const uint32_t VREG_DATA_OFFSET = 1;
static const uint32_t TYPE_INDEX = 0;
static const uint32_t PAYLOAD_INDEX = 1;
static const uint32_t INT64LOW_INDEX = 0;
static const uint32_t INT64HIGH_INDEX = 1;
#elif defined(JS_PUNBOX64)
static const uint32_t BOX_PIECES = 1;
#else
#  error "Unknown!"
#endif

static const uint32_t INT64_PIECES = sizeof(int64_t) / sizeof(uintptr_t);

// Represents storage for an operand. For constants, the pointer is tagged
// with a single bit, and the untagged pointer is a pointer to a Value.
class LAllocation {
 uintptr_t bits_;

 // 3 bits gives us enough for an interesting set of Kinds and also fits
 // within the alignment bits of pointers to Value, which are always
 // 8-byte aligned.
 static const uintptr_t KIND_BITS = 3;
 static const uintptr_t KIND_SHIFT = 0;
 static const uintptr_t KIND_MASK = (1 << KIND_BITS) - 1;

protected:
#ifdef JS_64BIT
 static const uintptr_t DATA_BITS = sizeof(uint32_t) * 8;
#else
 static const uintptr_t DATA_BITS = (sizeof(uint32_t) * 8) - KIND_BITS;
#endif
 static const uintptr_t DATA_SHIFT = KIND_SHIFT + KIND_BITS;

public:
 enum Kind {
   CONSTANT_VALUE,  // MConstant*.
   CONSTANT_INDEX,  // Constant arbitrary index.
   USE,         // Use of a virtual register, with physical allocation policy.
   GPR,         // General purpose register.
   FPU,         // Floating-point register.
   STACK_SLOT,  // Stack slot.
   STACK_AREA,  // Stack area.
   ARGUMENT_SLOT  // Argument slot.
 };

 static const uintptr_t DATA_MASK = (uintptr_t(1) << DATA_BITS) - 1;

protected:
 uint32_t data() const {
   MOZ_ASSERT(!hasIns());
   return mozilla::AssertedCast<uint32_t>(bits_ >> DATA_SHIFT);
 }
 void setData(uintptr_t data) {
   MOZ_ASSERT(!hasIns());
   MOZ_ASSERT(data <= DATA_MASK);
   bits_ &= ~(DATA_MASK << DATA_SHIFT);
   bits_ |= (data << DATA_SHIFT);
 }
 void setKindAndData(Kind kind, uintptr_t data) {
   MOZ_ASSERT(data <= DATA_MASK);
   bits_ = (uintptr_t(kind) << KIND_SHIFT) | data << DATA_SHIFT;
   MOZ_ASSERT(!hasIns());
 }

 bool hasIns() const { return isStackArea(); }
 const LInstruction* ins() const {
   MOZ_ASSERT(hasIns());
   return reinterpret_cast<const LInstruction*>(bits_ &
                                                ~(KIND_MASK << KIND_SHIFT));
 }
 LInstruction* ins() {
   MOZ_ASSERT(hasIns());
   return reinterpret_cast<LInstruction*>(bits_ & ~(KIND_MASK << KIND_SHIFT));
 }
 void setKindAndIns(Kind kind, LInstruction* ins) {
   uintptr_t data = reinterpret_cast<uintptr_t>(ins);
   MOZ_ASSERT((data & (KIND_MASK << KIND_SHIFT)) == 0);
   bits_ = data | (uintptr_t(kind) << KIND_SHIFT);
   MOZ_ASSERT(hasIns());
 }

 LAllocation(Kind kind, uintptr_t data) { setKindAndData(kind, data); }
 LAllocation(Kind kind, LInstruction* ins) { setKindAndIns(kind, ins); }
 explicit LAllocation(Kind kind) { setKindAndData(kind, 0); }

public:
 LAllocation() : bits_(0) { MOZ_ASSERT(isBogus()); }

 // The MConstant pointer must have its low bits cleared.
 explicit LAllocation(const MConstant* c) {
   MOZ_ASSERT(c);
   bits_ = uintptr_t(c);
   MOZ_ASSERT((bits_ & (KIND_MASK << KIND_SHIFT)) == 0);
   bits_ |= CONSTANT_VALUE << KIND_SHIFT;
 }
 inline explicit LAllocation(AnyRegister reg);

 Kind kind() const { return (Kind)((bits_ >> KIND_SHIFT) & KIND_MASK); }

 bool isBogus() const { return bits_ == 0; }
 bool isUse() const { return kind() == USE; }
 bool isConstant() const { return isConstantValue() || isConstantIndex(); }
 bool isConstantValue() const { return kind() == CONSTANT_VALUE; }
 bool isConstantIndex() const { return kind() == CONSTANT_INDEX; }
 bool isGeneralReg() const { return kind() == GPR; }
 bool isFloatReg() const { return kind() == FPU; }
 bool isStackSlot() const { return kind() == STACK_SLOT; }
 bool isStackArea() const { return kind() == STACK_AREA; }
 bool isArgument() const { return kind() == ARGUMENT_SLOT; }
 bool isAnyRegister() const { return isGeneralReg() || isFloatReg(); }
 bool isMemory() const { return isStackSlot() || isArgument(); }
 inline uint32_t memorySlot() const;
 inline LUse* toUse();
 inline const LUse* toUse() const;
 inline const LGeneralReg* toGeneralReg() const;
 inline const LFloatReg* toFloatReg() const;
 inline const LStackSlot* toStackSlot() const;
 inline LStackArea* toStackArea();
 inline const LStackArea* toStackArea() const;
 inline const LArgument* toArgument() const;
 inline const LConstantIndex* toConstantIndex() const;
 inline AnyRegister toAnyRegister() const;

 const MConstant* toConstant() const {
   MOZ_ASSERT(isConstantValue());
   return reinterpret_cast<const MConstant*>(bits_ &
                                             ~(KIND_MASK << KIND_SHIFT));
 }

 bool operator==(const LAllocation& other) const {
   return bits_ == other.bits_;
 }

 bool operator!=(const LAllocation& other) const {
   return bits_ != other.bits_;
 }

 HashNumber hash() const { return bits_; }

 uintptr_t asRawBits() const { return bits_; }

 bool aliases(const LAllocation& other) const;

#ifdef JS_JITSPEW
 UniqueChars toString() const;
 void dump() const;
#endif
};

class LUse : public LAllocation {
 static const uint32_t POLICY_BITS = 3;
 static const uint32_t POLICY_SHIFT = 0;
 static const uint32_t POLICY_MASK = (1 << POLICY_BITS) - 1;
#ifdef JS_CODEGEN_ARM64
 static const uint32_t REG_BITS = 7;
#else
 static const uint32_t REG_BITS = 6;
#endif
 static const uint32_t REG_SHIFT = POLICY_SHIFT + POLICY_BITS;
 static const uint32_t REG_MASK = (1 << REG_BITS) - 1;

 // Whether the physical register for this operand may be reused for a def.
 static const uint32_t USED_AT_START_BITS = 1;
 static const uint32_t USED_AT_START_SHIFT = REG_SHIFT + REG_BITS;
 static const uint32_t USED_AT_START_MASK = (1 << USED_AT_START_BITS) - 1;

 // The REG field will hold the register code for any Register or
 // FloatRegister, though not for an AnyRegister.
 static_assert(std::max(Registers::Total, FloatRegisters::Total) <=
                   REG_MASK + 1,
               "The field must be able to represent any register code");

public:
 // Virtual registers get the remaining bits.
 static const uint32_t VREG_BITS =
     DATA_BITS - (USED_AT_START_SHIFT + USED_AT_START_BITS);
 static const uint32_t VREG_SHIFT = USED_AT_START_SHIFT + USED_AT_START_BITS;
 static const uint32_t VREG_MASK = (1 << VREG_BITS) - 1;

 enum Policy {
   // Input should be in a read-only register or stack slot.
   ANY,

   // Input must be in a read-only register.
   REGISTER,

   // Input must be in a specific, read-only register.
   FIXED,

   // Keep the used virtual register alive, and use whatever allocation is
   // available. This is similar to ANY but hints to the register allocator
   // that it is never useful to optimize this site.
   KEEPALIVE,

   // Input must be allocated on the stack.  Only used when extracting stack
   // results from stack result areas.
   STACK,

   // For snapshot inputs, indicates that the associated instruction will
   // write this input to its output register before bailing out.
   // The register allocator may thus allocate that output register, and
   // does not need to keep the virtual register alive (alternatively,
   // this may be treated as KEEPALIVE).
   RECOVERED_INPUT
 };

 void set(Policy policy, uint32_t reg, bool usedAtStart) {
   MOZ_ASSERT(reg <= REG_MASK, "Register code must fit in field");
   setKindAndData(USE, (policy << POLICY_SHIFT) | (reg << REG_SHIFT) |
                           ((usedAtStart ? 1 : 0) << USED_AT_START_SHIFT));
 }

public:
 LUse(uint32_t vreg, Policy policy, bool usedAtStart = false) {
   set(policy, 0, usedAtStart);
   setVirtualRegister(vreg);
 }
 explicit LUse(Policy policy, bool usedAtStart = false) {
   set(policy, 0, usedAtStart);
 }
 explicit LUse(Register reg, bool usedAtStart = false) {
   set(FIXED, reg.code(), usedAtStart);
 }
 explicit LUse(FloatRegister reg, bool usedAtStart = false) {
   set(FIXED, reg.code(), usedAtStart);
 }
 LUse(Register reg, uint32_t virtualRegister, bool usedAtStart = false) {
   set(FIXED, reg.code(), usedAtStart);
   setVirtualRegister(virtualRegister);
 }
 LUse(FloatRegister reg, uint32_t virtualRegister, bool usedAtStart = false) {
   set(FIXED, reg.code(), usedAtStart);
   setVirtualRegister(virtualRegister);
 }

 void setVirtualRegister(uint32_t index) {
   MOZ_ASSERT(index < VREG_MASK);

   uint32_t old = data() & ~(VREG_MASK << VREG_SHIFT);
   setData(old | (index << VREG_SHIFT));
 }

 Policy policy() const {
   Policy policy = (Policy)((data() >> POLICY_SHIFT) & POLICY_MASK);
   return policy;
 }
 uint32_t virtualRegister() const {
   uint32_t index = (data() >> VREG_SHIFT) & VREG_MASK;
   MOZ_ASSERT(index != 0);
   return index;
 }
 uint32_t registerCode() const {
   MOZ_ASSERT(policy() == FIXED);
   return (data() >> REG_SHIFT) & REG_MASK;
 }
 bool isFixedRegister() const { return policy() == FIXED; }
 bool usedAtStart() const {
   return !!((data() >> USED_AT_START_SHIFT) & USED_AT_START_MASK);
 }
};

static const uint32_t MAX_VIRTUAL_REGISTERS = LUse::VREG_MASK;

class LBoxAllocation {
#ifdef JS_NUNBOX32
 LAllocation type_;
 LAllocation payload_;
#else
 LAllocation value_;
#endif

public:
#ifdef JS_NUNBOX32
 LBoxAllocation(LAllocation type, LAllocation payload)
     : type_(type), payload_(payload) {}

 LAllocation type() const { return type_; }
 LAllocation payload() const { return payload_; }
#else
 explicit LBoxAllocation(LAllocation value) : value_(value) {}

 LAllocation value() const { return value_; }
#endif
};

template <class ValT>
class LInt64Value {
#if JS_BITS_PER_WORD == 32
 ValT high_;
 ValT low_;
#else
 ValT value_;
#endif

public:
 LInt64Value() = default;

#if JS_BITS_PER_WORD == 32
 LInt64Value(ValT high, ValT low) : high_(high), low_(low) {}

 ValT high() const { return high_; }
 ValT low() const { return low_; }

 const ValT* pointerHigh() const { return &high_; }
 const ValT* pointerLow() const { return &low_; }
#else
 explicit LInt64Value(ValT value) : value_(value) {}

 ValT value() const { return value_; }
 const ValT* pointer() const { return &value_; }
#endif
};

using LInt64Allocation = LInt64Value<LAllocation>;

class LGeneralReg : public LAllocation {
public:
 explicit LGeneralReg(Register reg) : LAllocation(GPR, reg.code()) {}

 Register reg() const { return Register::FromCode(data()); }
};

class LFloatReg : public LAllocation {
public:
 explicit LFloatReg(FloatRegister reg) : LAllocation(FPU, reg.code()) {}

 FloatRegister reg() const { return FloatRegister::FromCode(data()); }
};

// Arbitrary constant index.
class LConstantIndex : public LAllocation {
 explicit LConstantIndex(uint32_t index)
     : LAllocation(CONSTANT_INDEX, index) {}

public:
 static LConstantIndex FromIndex(uint32_t index) {
   return LConstantIndex(index);
 }

 uint32_t index() const { return data(); }
};

// Stack slots are indices into the stack. The indices are byte indices.
class LStackSlot : public LAllocation {
 // Stack slots are aligned to 32-bit word boundaries.
 static constexpr uint32_t SLOT_ALIGNMENT = 4;

 // Stack slot width is stored in the two least significant bits.
 static constexpr uint32_t WIDTH_MASK = SLOT_ALIGNMENT - 1;

 // Remaining bits hold the stack slot offset.
 static constexpr uint32_t SLOT_MASK = ~WIDTH_MASK;

public:
 enum Width {
   Word,
   DoubleWord,
   QuadWord,
 };

 class SlotAndWidth {
   uint32_t data_;

   explicit SlotAndWidth(uint32_t data) : data_(data) {}

  public:
   static SlotAndWidth fromData(uint32_t data) { return SlotAndWidth(data); }

   explicit SlotAndWidth(uint32_t slot, Width width) {
     MOZ_ASSERT(slot % SLOT_ALIGNMENT == 0);
     MOZ_ASSERT(uint32_t(width) < SLOT_ALIGNMENT);
     data_ = slot | uint32_t(width);
   }
   uint32_t data() const { return data_; }
   uint32_t slot() const { return data_ & SLOT_MASK; }
   Width width() const { return Width(data_ & WIDTH_MASK); }
 };

 explicit LStackSlot(SlotAndWidth slotAndWidth)
     : LAllocation(STACK_SLOT, slotAndWidth.data()) {}

 LStackSlot(uint32_t slot, Width width)
     : LStackSlot(SlotAndWidth(slot, width)) {}

 uint32_t slot() const { return SlotAndWidth::fromData(data()).slot(); }
 Width width() const { return SlotAndWidth::fromData(data()).width(); }

 // |Type| is LDefinition::Type, but can't forward declare a nested definition.
 template <typename Type>
 static Width width(Type type);

 static uint32_t ByteWidth(Width width) {
   switch (width) {
     case Width::Word:
       return 4;
     case Width::DoubleWord:
       return 8;
     case Width::QuadWord:
       return 16;
   }
   MOZ_CRASH("invalid width");
 }
};

// Stack area indicates a contiguous stack allocation meant to receive call
// results that don't fit in registers.
class LStackArea : public LAllocation {
public:
 explicit LStackArea(LInstruction* stackArea)
     : LAllocation(STACK_AREA, stackArea) {}

 // Byte index of base of stack area, in the same coordinate space as
 // LStackSlot::slot().
 inline uint32_t base() const;
 inline void setBase(uint32_t base);

 // Size in bytes of the stack area.
 inline uint32_t size() const;
 inline uint32_t alignment() const { return 8; }

 class ResultIterator {
   const LStackArea& alloc_;
   uint32_t idx_;

  public:
   explicit ResultIterator(const LStackArea& alloc) : alloc_(alloc), idx_(0) {}

   inline bool done() const;
   inline void next();
   inline LAllocation alloc() const;
   inline bool isWasmAnyRef() const;

   explicit operator bool() const { return !done(); }
 };

 ResultIterator results() const { return ResultIterator(*this); }

 inline LStackSlot resultAlloc(LInstruction* lir, LDefinition* def) const;
};

// Arguments are reverse indices into the stack. The indices are byte indices.
class LArgument : public LAllocation {
public:
 explicit LArgument(uint32_t index) : LAllocation(ARGUMENT_SLOT, index) {}

 uint32_t index() const { return data(); }
};

inline uint32_t LAllocation::memorySlot() const {
 MOZ_ASSERT(isMemory());
 return isStackSlot() ? toStackSlot()->slot() : toArgument()->index();
}

// Represents storage for a definition.
class LDefinition {
 // Bits containing policy, type, and virtual register.
 uint32_t bits_;

 // Before register allocation, this optionally contains a fixed policy.
 // Register allocation assigns this field to a physical policy if none is
 // fixed.
 //
 // Right now, pre-allocated outputs are limited to the following:
 //   * Physical argument stack slots.
 //   * Physical registers.
 LAllocation output_;

 static const uint32_t TYPE_BITS = 4;
 static const uint32_t TYPE_SHIFT = 0;
 static const uint32_t TYPE_MASK = (1 << TYPE_BITS) - 1;
 static const uint32_t POLICY_BITS = 2;
 static const uint32_t POLICY_SHIFT = TYPE_SHIFT + TYPE_BITS;
 static const uint32_t POLICY_MASK = (1 << POLICY_BITS) - 1;

 static const uint32_t VREG_BITS =
     (sizeof(uint32_t) * 8) - (POLICY_BITS + TYPE_BITS);
 static const uint32_t VREG_SHIFT = POLICY_SHIFT + POLICY_BITS;
 static const uint32_t VREG_MASK = (1 << VREG_BITS) - 1;

public:
 // Note that definitions, by default, are always allocated a register,
 // unless the policy specifies that an input can be re-used and that input
 // is a stack slot.
 enum Policy {
   // The policy is predetermined by the LAllocation attached to this
   // definition. The allocation may be:
   //   * A register, which may not appear as any fixed temporary.
   //   * A stack slot or argument.
   //
   // Register allocation will not modify a fixed allocation.
   FIXED,

   // A random register of an appropriate class will be assigned.
   REGISTER,

   // An area on the stack must be assigned.  Used when defining stack results
   // and stack result areas.
   STACK,

   // One definition per instruction must re-use the first input
   // allocation, which (for now) must be a register.
   MUST_REUSE_INPUT
 };

 enum Type {
   GENERAL,  // Generic, integer or pointer-width data (GPR).
   INT32,    // int32 data (GPR).
   OBJECT,   // Pointer that may be collected as garbage (GPR).
   SLOTS,    // Slots/elements pointer that may be moved by minor GCs (GPR).
   WASM_ANYREF,       // Tagged pointer that may be collected as garbage (GPR).
   WASM_STRUCT_DATA,  // Pointer to wasm struct OOL storage that may be moved
                      // by minor GCs (GPR).
   WASM_ARRAY_DATA,   // Pointer to wasm array IL or OOL storage; in the OOL
                      // case it may be moved by minor GCs (GPR).
   FLOAT32,           // 32-bit floating-point value (FPU).
   DOUBLE,            // 64-bit floating-point value (FPU).
   SIMD128,           // 128-bit SIMD vector (FPU).
   STACKRESULTS,  // A variable-size stack allocation that may contain objects.
#ifdef JS_NUNBOX32
   // A type virtual register must be followed by a payload virtual
   // register, as both will be tracked as a single gcthing.
   TYPE,
   PAYLOAD
#else
   BOX  // Joined box, for punbox systems. (GPR, gcthing)
#endif
 };

 void set(uint32_t index, Type type, Policy policy) {
   static_assert(MAX_VIRTUAL_REGISTERS <= VREG_MASK);
   bits_ =
       (index << VREG_SHIFT) | (policy << POLICY_SHIFT) | (type << TYPE_SHIFT);
#ifndef ENABLE_WASM_SIMD
   MOZ_ASSERT(this->type() != SIMD128);
#endif
 }

public:
 LDefinition(uint32_t index, Type type, Policy policy = REGISTER) {
   set(index, type, policy);
 }

 explicit LDefinition(Type type, Policy policy = REGISTER) {
   set(0, type, policy);
 }

 LDefinition(Type type, const LAllocation& a) : output_(a) {
   set(0, type, FIXED);
 }

 LDefinition(uint32_t index, Type type, const LAllocation& a) : output_(a) {
   set(index, type, FIXED);
 }

 LDefinition() : bits_(0) { MOZ_ASSERT(isBogusTemp()); }

 static LDefinition BogusTemp() { return LDefinition(); }

 Policy policy() const {
   return (Policy)((bits_ >> POLICY_SHIFT) & POLICY_MASK);
 }
 Type type() const { return (Type)((bits_ >> TYPE_SHIFT) & TYPE_MASK); }

 static bool isFloatRegCompatible(Type type, FloatRegister reg) {
   if (type == FLOAT32) {
     return reg.isSingle();
   }
   if (type == DOUBLE) {
     return reg.isDouble();
   }
   MOZ_ASSERT(type == SIMD128);
   return reg.isSimd128();
 }

 bool isCompatibleReg(const AnyRegister& r) const {
   if (isFloatReg() && r.isFloat()) {
     return isFloatRegCompatible(type(), r.fpu());
   }
   return !isFloatReg() && !r.isFloat();
 }
 bool isCompatibleDef(const LDefinition& other) const {
#if defined(JS_CODEGEN_ARM)
   if (isFloatReg() && other.isFloatReg()) {
     return type() == other.type();
   }
   return !isFloatReg() && !other.isFloatReg();
#else
   return isFloatReg() == other.isFloatReg();
#endif
 }

 static bool isFloatReg(Type type) {
   return type == FLOAT32 || type == DOUBLE || type == SIMD128;
 }
 bool isFloatReg() const { return isFloatReg(type()); }

 uint32_t virtualRegister() const {
   uint32_t index = (bits_ >> VREG_SHIFT) & VREG_MASK;
   // MOZ_ASSERT(index != 0);
   return index;
 }
 LAllocation* output() { return &output_; }
 const LAllocation* output() const { return &output_; }
 bool isFixed() const { return policy() == FIXED; }
 bool isBogusTemp() const { return isFixed() && output()->isBogus(); }
 void setVirtualRegister(uint32_t index) {
   MOZ_ASSERT(index < VREG_MASK);
   bits_ &= ~(VREG_MASK << VREG_SHIFT);
   bits_ |= index << VREG_SHIFT;
 }
 void setOutput(const LAllocation& a) {
   output_ = a;
   if (!a.isUse()) {
     bits_ &= ~(POLICY_MASK << POLICY_SHIFT);
     bits_ |= FIXED << POLICY_SHIFT;
   }
 }
 void setReusedInput(uint32_t operand) {
   output_ = LConstantIndex::FromIndex(operand);
 }
 uint32_t getReusedInput() const {
   MOZ_ASSERT(policy() == LDefinition::MUST_REUSE_INPUT);
   return output_.toConstantIndex()->index();
 }

 // Returns true if this definition should be added to safepoints for GC
 // tracing. This includes Value type tags on 32-bit and slots/elements
 // pointers.
 inline bool isSafepointGCType(LNode* ins) const;

 static inline Type TypeFrom(MIRType type) {
   switch (type) {
     case MIRType::Boolean:
     case MIRType::Int32:
       // The stack slot allocator doesn't currently support allocating
       // 1-byte slots, so for now we lower MIRType::Boolean into INT32.
       static_assert(sizeof(bool) <= sizeof(int32_t),
                     "bool doesn't fit in an int32 slot");
       return LDefinition::INT32;
     case MIRType::String:
     case MIRType::Symbol:
     case MIRType::BigInt:
     case MIRType::Object:
       return LDefinition::OBJECT;
     case MIRType::Double:
       return LDefinition::DOUBLE;
     case MIRType::Float32:
       return LDefinition::FLOAT32;
#if defined(JS_PUNBOX64)
     case MIRType::Value:
       return LDefinition::BOX;
#endif
     case MIRType::Slots:
     case MIRType::Elements:
       return LDefinition::SLOTS;
     case MIRType::WasmAnyRef:
       return LDefinition::WASM_ANYREF;
     case MIRType::WasmStructData:
       return LDefinition::WASM_STRUCT_DATA;
     case MIRType::WasmArrayData:
       return LDefinition::WASM_ARRAY_DATA;
     case MIRType::Pointer:
     case MIRType::IntPtr:
       return LDefinition::GENERAL;
#if defined(JS_PUNBOX64)
     case MIRType::Int64:
       return LDefinition::GENERAL;
#endif
     case MIRType::StackResults:
       return LDefinition::STACKRESULTS;
     case MIRType::Simd128:
       return LDefinition::SIMD128;
     default:
       MOZ_CRASH("unexpected type");
   }
 }

 UniqueChars toString() const;

#ifdef JS_JITSPEW
 void dump() const;
#endif
};

class LInt64Definition : public LInt64Value<LDefinition> {
public:
 using LInt64Value<LDefinition>::LInt64Value;

 static LInt64Definition BogusTemp() { return LInt64Definition(); }

 bool isBogusTemp() const {
#if JS_BITS_PER_WORD == 32
   MOZ_ASSERT(high().isBogusTemp() == low().isBogusTemp());
   return high().isBogusTemp();
#else
   return value().isBogusTemp();
#endif
 }
};

template <>
inline LStackSlot::Width LStackSlot::width(LDefinition::Type type) {
 switch (type) {
#if JS_BITS_PER_WORD == 32
   case LDefinition::GENERAL:
   case LDefinition::OBJECT:
   case LDefinition::SLOTS:
   case LDefinition::WASM_ANYREF:
   case LDefinition::WASM_STRUCT_DATA:
   case LDefinition::WASM_ARRAY_DATA:
#endif
#ifdef JS_NUNBOX32
   case LDefinition::TYPE:
   case LDefinition::PAYLOAD:
#endif
   case LDefinition::INT32:
   case LDefinition::FLOAT32:
     return LStackSlot::Word;
#if JS_BITS_PER_WORD == 64
   case LDefinition::GENERAL:
   case LDefinition::OBJECT:
   case LDefinition::SLOTS:
   case LDefinition::WASM_ANYREF:
   case LDefinition::WASM_STRUCT_DATA:
   case LDefinition::WASM_ARRAY_DATA:
#endif
#ifdef JS_PUNBOX64
   case LDefinition::BOX:
#endif
   case LDefinition::DOUBLE:
     return LStackSlot::DoubleWord;
   case LDefinition::SIMD128:
     return LStackSlot::QuadWord;
   case LDefinition::STACKRESULTS:
     MOZ_CRASH("Stack results area must be allocated manually");
 }
 MOZ_CRASH("Unknown slot type");
}

// Forward declarations of LIR types.
#define LIROP(op) class L##op;
LIR_OPCODE_LIST(LIROP)
#undef LIROP

class LSnapshot;
class LSafepoint;
class LElementVisitor;

constexpr size_t MaxNumLInstructionOperands = 63;

// The common base class for LPhi and LInstruction.
class LNode {
protected:
 MDefinition* mir_;

private:
 LBlock* block_;
 uint32_t id_;

protected:
 // Bitfields below are all uint32_t to make sure MSVC packs them correctly.
 uint32_t op_ : 10;
 uint32_t isCall_ : 1;

 // LPhi::numOperands() may not fit in this bitfield, so we only use this
 // field for LInstruction.
 uint32_t nonPhiNumOperands_ : 6;
 static_assert((1 << 6) - 1 == MaxNumLInstructionOperands,
               "packing constraints");

 // For LInstruction, the first operand is stored at offset
 // sizeof(LInstruction) + nonPhiOperandsOffset_ * sizeof(uintptr_t).
 uint32_t nonPhiOperandsOffset_ : 5;
 uint32_t numDefs_ : 4;
 uint32_t numTemps_ : 4;

public:
 enum class Opcode {
#define LIROP(name) name,
   LIR_OPCODE_LIST(LIROP)
#undef LIROP
       Invalid
 };

 LNode(Opcode op, uint32_t nonPhiNumOperands, uint32_t numDefs,
       uint32_t numTemps)
     : mir_(nullptr),
       block_(nullptr),
       id_(0),
       op_(uint32_t(op)),
       isCall_(false),
       nonPhiNumOperands_(nonPhiNumOperands),
       nonPhiOperandsOffset_(0),
       numDefs_(numDefs),
       numTemps_(numTemps) {
   MOZ_ASSERT(op < Opcode::Invalid);
   MOZ_ASSERT(op_ == uint32_t(op), "opcode must fit in bitfield");
   MOZ_ASSERT(nonPhiNumOperands_ == nonPhiNumOperands,
              "nonPhiNumOperands must fit in bitfield");
   MOZ_ASSERT(numDefs_ == numDefs, "numDefs must fit in bitfield");
   MOZ_ASSERT(numTemps_ == numTemps, "numTemps must fit in bitfield");
 }

 const char* opName() {
   switch (op()) {
#define LIR_NAME_INS(name) \
 case Opcode::name:       \
   return #name;
     LIR_OPCODE_LIST(LIR_NAME_INS)
#undef LIR_NAME_INS
     default:
       MOZ_CRASH("Invalid op");
   }
 }

 // Hook for opcodes to add extra high level detail about what code will be
 // emitted for the op.
private:
 const char* extraName() const { return nullptr; }

public:
#ifdef JS_JITSPEW
 const char* getExtraName() const;
#endif

 Opcode op() const { return Opcode(op_); }

 bool isInstruction() const { return op() != Opcode::Phi; }
 inline LInstruction* toInstruction();
 inline const LInstruction* toInstruction() const;

 // Returns the number of outputs of this instruction. If an output is
 // unallocated, it is an LDefinition, defining a virtual register.
 size_t numDefs() const { return numDefs_; }

 bool isCall() const { return isCall_; }

 // Does this call preserve the given register?
 // By default, it is assumed that all registers are clobbered by a call.
 inline bool isCallPreserved(AnyRegister reg) const;

 uint32_t id() const { return id_; }
 void setId(uint32_t id) {
   MOZ_ASSERT(!id_);
   MOZ_ASSERT(id);
   id_ = id;
 }
 void setMir(MDefinition* mir) { mir_ = mir; }
 MDefinition* mirRaw() const {
   /* Untyped MIR for this op. Prefer mir() methods in subclasses. */
   return mir_;
 }
 LBlock* block() const { return block_; }
 void setBlock(LBlock* block) { block_ = block; }

 // For an instruction which has a MUST_REUSE_INPUT output, whether that
 // output register will be restored to its original value when bailing out.
 inline bool recoversInput() const;

#ifdef JS_JITSPEW
 void dump(GenericPrinter& out);
 void dump();
 static void printName(GenericPrinter& out, Opcode op);
 void printName(GenericPrinter& out);
 void printOperands(GenericPrinter& out);
#endif

public:
 // Opcode testing and casts.
#define LIROP(name)                                      \
 bool is##name() const { return op() == Opcode::name; } \
 inline L##name* to##name();                            \
 inline const L##name* to##name() const;
 LIR_OPCODE_LIST(LIROP)
#undef LIROP

// Note: GenerateOpcodeFiles.py generates LIROpsGenerated.h based on this
// macro.
#define LIR_HEADER(opcode) \
 static constexpr LNode::Opcode classOpcode = LNode::Opcode::opcode;
};

extern const char* const LIROpNames[];
inline const char* LIRCodeName(LNode::Opcode op) {
 return LIROpNames[static_cast<size_t>(op)];
}

class LInstruction : public LNode,
                    public TempObject,
                    public InlineListNode<LInstruction> {
 // This snapshot could be set after a ResumePoint.  It is used to restart
 // from the resume point pc.
 LSnapshot* snapshot_;

 // Structure capturing the set of stack slots and registers which are known
 // to hold either gcthings or Values.
 LSafepoint* safepoint_;

 LMoveGroup* inputMoves_;
 LMoveGroup* fixReuseMoves_;
 LMoveGroup* movesAfter_;

protected:
 LInstruction(Opcode opcode, uint32_t numOperands, uint32_t numDefs,
              uint32_t numTemps)
     : LNode(opcode, numOperands, numDefs, numTemps),
       snapshot_(nullptr),
       safepoint_(nullptr),
       inputMoves_(nullptr),
       fixReuseMoves_(nullptr),
       movesAfter_(nullptr) {}

 void setIsCall() { isCall_ = true; }

public:
 inline LDefinition* getDef(size_t index);

 void setDef(size_t index, const LDefinition& def) { *getDef(index) = def; }

 LAllocation* getOperand(size_t index) const {
   MOZ_ASSERT(index < numOperands());
   MOZ_ASSERT(nonPhiOperandsOffset_ > 0);
   uintptr_t p = reinterpret_cast<uintptr_t>(this + 1) +
                 nonPhiOperandsOffset_ * sizeof(uintptr_t);
   return reinterpret_cast<LAllocation*>(p) + index;
 }
 void setOperand(size_t index, const LAllocation& a) {
   *getOperand(index) = a;
 }

 LBoxAllocation getBoxOperand(size_t index) const {
#ifdef JS_NUNBOX32
   return LBoxAllocation(*getOperand(index + TYPE_INDEX),
                         *getOperand(index + PAYLOAD_INDEX));
#else
   return LBoxAllocation(*getOperand(index));
#endif
 }

 void initOperandsOffset(size_t offset) {
   MOZ_ASSERT(nonPhiOperandsOffset_ == 0);
   MOZ_ASSERT(offset >= sizeof(LInstruction));
   MOZ_ASSERT(((offset - sizeof(LInstruction)) % sizeof(uintptr_t)) == 0);
   offset = (offset - sizeof(LInstruction)) / sizeof(uintptr_t);
   nonPhiOperandsOffset_ = offset;
   MOZ_ASSERT(nonPhiOperandsOffset_ == offset, "offset must fit in bitfield");
 }

 void changePolicyOfReusedInputToAny(LDefinition* def) {
   // MUST_REUSE_INPUT is implemented by allocating an output register and
   // moving the input to it. Register hints are used to avoid unnecessary
   // moves. We give the input an LUse::ANY policy to avoid requiring a
   // register for the input.
   MOZ_ASSERT(def->policy() == LDefinition::MUST_REUSE_INPUT);
   LUse* inputUse = getOperand(def->getReusedInput())->toUse();
   MOZ_ASSERT(inputUse->policy() == LUse::REGISTER);
   MOZ_ASSERT(inputUse->usedAtStart());
   *inputUse = LUse(inputUse->virtualRegister(), LUse::ANY,
                    /* usedAtStart = */ true);
 }

 // Returns information about temporary registers needed. Each temporary
 // register is an LDefinition with a fixed or virtual register and
 // either GENERAL, FLOAT32, or DOUBLE type.
 size_t numTemps() const { return numTemps_; }
 inline LDefinition* getTemp(size_t index);

 LSnapshot* snapshot() const { return snapshot_; }
 LSafepoint* safepoint() const { return safepoint_; }
 LMoveGroup* inputMoves() const { return inputMoves_; }
 void setInputMoves(LMoveGroup* moves) { inputMoves_ = moves; }
 LMoveGroup* fixReuseMoves() const { return fixReuseMoves_; }
 void setFixReuseMoves(LMoveGroup* moves) { fixReuseMoves_ = moves; }
 LMoveGroup* movesAfter() const { return movesAfter_; }
 void setMovesAfter(LMoveGroup* moves) { movesAfter_ = moves; }
 uint32_t numOperands() const { return nonPhiNumOperands_; }
 void assignSnapshot(LSnapshot* snapshot);
 void initSafepoint(TempAllocator& alloc);

 // InputIter iterates over all operands including snapshot inputs.
 // NonSnapshotInputIter does not include snapshot inputs.
 //
 // There can be many snapshot inputs and these are always KEEPALIVE uses or
 // constants. NonSnapshotInputIter can be used in places where we're not
 // interested in those.
 template <bool WithSnapshotUses>
 class InputIterImpl;
 using InputIter = InputIterImpl<true>;
 using NonSnapshotInputIter = InputIterImpl<false>;

 // Iterators for an instruction's outputs and temps. These skip BogusTemp
 // definitions.
 template <bool Temps>
 class DefIterImpl;
 using TempIter = DefIterImpl<true>;
 using OutputIter = DefIterImpl<false>;
};

LInstruction* LNode::toInstruction() {
 MOZ_ASSERT(isInstruction());
 return static_cast<LInstruction*>(this);
}

const LInstruction* LNode::toInstruction() const {
 MOZ_ASSERT(isInstruction());
 return static_cast<const LInstruction*>(this);
}

class LElementVisitor {
#ifdef TRACK_SNAPSHOTS
 LInstruction* ins_ = nullptr;
#endif

protected:
#ifdef TRACK_SNAPSHOTS
 LInstruction* instruction() { return ins_; }

 void setElement(LInstruction* ins) { ins_ = ins; }
#else
 void setElement(LInstruction* ins) {}
#endif
};

using LInstructionIterator = InlineList<LInstruction>::iterator;
using LInstructionReverseIterator = InlineList<LInstruction>::reverse_iterator;

class MPhi;

// Phi is a pseudo-instruction that emits no code, and is an annotation for the
// register allocator. Like its equivalent in MIR, phis are collected at the
// top of blocks and are meant to be executed in parallel, choosing the input
// corresponding to the predecessor taken in the control flow graph.
class LPhi final : public LNode {
 LAllocation* const inputs_;
 LDefinition def_;

public:
 LIR_HEADER(Phi)

 LPhi(MPhi* ins, LAllocation* inputs)
     : LNode(classOpcode,
             /* nonPhiNumOperands = */ 0,
             /* numDefs = */ 1,
             /* numTemps = */ 0),
       inputs_(inputs) {
   setMir(ins);
 }

 LDefinition* getDef(size_t index) {
   MOZ_ASSERT(index == 0);
   return &def_;
 }
 void setDef(size_t index, const LDefinition& def) {
   MOZ_ASSERT(index == 0);
   def_ = def;
 }
 size_t numOperands() const { return mir_->toPhi()->numOperands(); }
 LAllocation* getOperand(size_t index) {
   MOZ_ASSERT(index < numOperands());
   return &inputs_[index];
 }
 void setOperand(size_t index, const LAllocation& a) {
   MOZ_ASSERT(index < numOperands());
   inputs_[index] = a;
 }

 // Phis don't have temps, so calling numTemps/getTemp is pointless.
 size_t numTemps() const = delete;
 LDefinition* getTemp(size_t index) = delete;
};

class LMoveGroup;
class LBlock {
 MBasicBlock* block_;
 FixedList<LPhi> phis_;
 InlineList<LInstruction> instructions_;
 LMoveGroup* entryMoveGroup_;
 LMoveGroup* exitMoveGroup_;
 Label label_;
 // If true, this block will be generated out of line.
 bool isOutOfLine_;

public:
 explicit LBlock(MBasicBlock* block);
 [[nodiscard]] bool init(TempAllocator& alloc);

 void add(LInstruction* ins) {
   ins->setBlock(this);
   instructions_.pushBack(ins);
 }
 size_t numPhis() const { return phis_.length(); }
 LPhi* getPhi(size_t index) { return &phis_[index]; }
 const LPhi* getPhi(size_t index) const { return &phis_[index]; }
 MBasicBlock* mir() const { return block_; }
 bool isOutOfLine() const { return isOutOfLine_; }
 LInstructionIterator begin() { return instructions_.begin(); }
 LInstructionIterator begin(LInstruction* at) {
   return instructions_.begin(at);
 }
 LInstructionIterator end() { return instructions_.end(); }
 LInstructionReverseIterator rbegin() { return instructions_.rbegin(); }
 LInstructionReverseIterator rbegin(LInstruction* at) {
   return instructions_.rbegin(at);
 }
 LInstructionReverseIterator rend() { return instructions_.rend(); }
 InlineList<LInstruction>& instructions() { return instructions_; }
 void insertAfter(LInstruction* at, LInstruction* ins) {
   instructions_.insertAfter(at, ins);
 }
 void insertBefore(LInstruction* at, LInstruction* ins) {
   instructions_.insertBefore(at, ins);
 }
 const LNode* firstElementWithId() const {
   return !phis_.empty() ? static_cast<const LNode*>(getPhi(0))
                         : firstInstructionWithId();
 }
 uint32_t firstId() const { return firstElementWithId()->id(); }
 uint32_t lastId() const { return lastInstructionWithId()->id(); }
 const LInstruction* firstInstructionWithId() const;
 const LInstruction* lastInstructionWithId() const {
   const LInstruction* last = *instructions_.rbegin();
   MOZ_ASSERT(last->id());
   // The last instruction is a control flow instruction which does not have
   // any output.
   MOZ_ASSERT(last->numDefs() == 0);
   return last;
 }

 // Return the label to branch to when branching to this block.
 Label* label() {
   MOZ_ASSERT(!isTrivial());
   return &label_;
 }

 LMoveGroup* getEntryMoveGroup(TempAllocator& alloc);
 LMoveGroup* getExitMoveGroup(TempAllocator& alloc);

 // Test whether this basic block is empty except for a simple goto, and
 // which is not forming a loop. No code will be emitted for such blocks.
 bool isTrivial() { return begin()->isGoto() && !mir()->isLoopHeader(); }

 // Test whether this basic block is a sequence of MoveGroups followed by a
 // simple goto, and is not a loop header.  If so return the target of the
 // jump, otherwise return nullptr.
 LBlock* isMoveGroupsThenGoto();

#ifdef JS_JITSPEW
 void dump(GenericPrinter& out);
 void dump();
#endif
};

namespace details {
template <size_t Defs, size_t Temps>
class LInstructionFixedDefsTempsHelper : public LInstruction {
 mozilla::Array<LDefinition, Defs + Temps> defsAndTemps_;

protected:
 LInstructionFixedDefsTempsHelper(Opcode opcode, uint32_t numOperands)
     : LInstruction(opcode, numOperands, Defs, Temps) {}

public:
 // Override the methods in LInstruction with more optimized versions
 // for when we know the exact instruction type.
 LDefinition* getDef(size_t index) {
   MOZ_ASSERT(index < Defs);
   return &defsAndTemps_[index];
 }
 LDefinition* getTemp(size_t index) {
   MOZ_ASSERT(index < Temps);
   return &defsAndTemps_[Defs + index];
 }
 LInt64Definition getInt64Temp(size_t index) {
   MOZ_ASSERT(index + INT64_PIECES <= Temps);
#if JS_BITS_PER_WORD == 32
   return LInt64Definition(defsAndTemps_[Defs + index + INT64HIGH_INDEX],
                           defsAndTemps_[Defs + index + INT64LOW_INDEX]);
#else
   return LInt64Definition(defsAndTemps_[Defs + index]);
#endif
 }

 void setDef(size_t index, const LDefinition& def) {
   MOZ_ASSERT(index < Defs);
   defsAndTemps_[index] = def;
 }
 void setTemp(size_t index, const LDefinition& a) {
   MOZ_ASSERT(index < Temps);
   defsAndTemps_[Defs + index] = a;
 }
 void setInt64Temp(size_t index, const LInt64Definition& a) {
#if JS_BITS_PER_WORD == 32
   setTemp(index, a.low());
   setTemp(index + 1, a.high());
#else
   setTemp(index, a.value());
#endif
 }

 // Default accessor, assuming a single output.
 const LDefinition* output() {
   MOZ_ASSERT(numDefs() == 1);
   return getDef(0);
 }
 static size_t offsetOfDef(size_t index) {
   using T = LInstructionFixedDefsTempsHelper<0, 0>;
   return offsetof(T, defsAndTemps_) + index * sizeof(LDefinition);
 }
 static size_t offsetOfTemp(uint32_t numDefs, uint32_t index) {
   using T = LInstructionFixedDefsTempsHelper<0, 0>;
   return offsetof(T, defsAndTemps_) + (numDefs + index) * sizeof(LDefinition);
 }
};
}  // namespace details

inline LDefinition* LInstruction::getDef(size_t index) {
 MOZ_ASSERT(index < numDefs());
 using T = details::LInstructionFixedDefsTempsHelper<0, 0>;
 uint8_t* p = reinterpret_cast<uint8_t*>(this) + T::offsetOfDef(index);
 return reinterpret_cast<LDefinition*>(p);
}

inline LDefinition* LInstruction::getTemp(size_t index) {
 MOZ_ASSERT(index < numTemps());
 using T = details::LInstructionFixedDefsTempsHelper<0, 0>;
 uint8_t* p =
     reinterpret_cast<uint8_t*>(this) + T::offsetOfTemp(numDefs(), index);
 return reinterpret_cast<LDefinition*>(p);
}

template <size_t Defs, size_t Operands, size_t Temps>
class LInstructionHelper
   : public details::LInstructionFixedDefsTempsHelper<Defs, Temps> {
 MOZ_NO_UNIQUE_ADDRESS mozilla::Array<LAllocation, Operands> operands_;

protected:
 explicit LInstructionHelper(LNode::Opcode opcode)
     : details::LInstructionFixedDefsTempsHelper<Defs, Temps>(opcode,
                                                              Operands) {
   static_assert(
       Operands == 0 || sizeof(operands_) == Operands * sizeof(LAllocation),
       "mozilla::Array should not contain other fields");
   if (Operands > 0) {
     using T = LInstructionHelper<Defs, Operands, Temps>;
     this->initOperandsOffset(offsetof(T, operands_));
   }
 }

public:
 // Override the methods in LInstruction with more optimized versions
 // for when we know the exact instruction type.
 LAllocation* getOperand(size_t index) { return &operands_[index]; }
 const LAllocation* getOperand(size_t index) const {
   return &operands_[index];
 }
 void setOperand(size_t index, const LAllocation& a) { operands_[index] = a; }
 LBoxAllocation getBoxOperand(size_t index) const {
#ifdef JS_NUNBOX32
   return LBoxAllocation(operands_[index + TYPE_INDEX],
                         operands_[index + PAYLOAD_INDEX]);
#else
   return LBoxAllocation(operands_[index]);
#endif
 }
 void setBoxOperand(size_t index, const LBoxAllocation& alloc) {
#ifdef JS_NUNBOX32
   operands_[index + TYPE_INDEX] = alloc.type();
   operands_[index + PAYLOAD_INDEX] = alloc.payload();
#else
   operands_[index] = alloc.value();
#endif
 }
 void setInt64Operand(size_t index, const LInt64Allocation& alloc) {
#if JS_BITS_PER_WORD == 32
   operands_[index + INT64LOW_INDEX] = alloc.low();
   operands_[index + INT64HIGH_INDEX] = alloc.high();
#else
   operands_[index] = alloc.value();
#endif
 }
 LInt64Allocation getInt64Operand(size_t offset) const {
#if JS_BITS_PER_WORD == 32
   return LInt64Allocation(operands_[offset + INT64HIGH_INDEX],
                           operands_[offset + INT64LOW_INDEX]);
#else
   return LInt64Allocation(operands_[offset]);
#endif
 }
};

template <size_t Defs, size_t Temps>
class LVariadicInstruction
   : public details::LInstructionFixedDefsTempsHelper<Defs, Temps> {
protected:
 LVariadicInstruction(LNode::Opcode opcode, size_t numOperands)
     : details::LInstructionFixedDefsTempsHelper<Defs, Temps>(opcode,
                                                              numOperands) {}

public:
 void setBoxOperand(size_t index, const LBoxAllocation& a) {
#ifdef JS_NUNBOX32
   this->setOperand(index + TYPE_INDEX, a.type());
   this->setOperand(index + PAYLOAD_INDEX, a.payload());
#else
   this->setOperand(index, a.value());
#endif
 }
};

template <size_t Defs, size_t Operands, size_t Temps>
class LCallInstructionHelper
   : public LInstructionHelper<Defs, Operands, Temps> {
protected:
 explicit LCallInstructionHelper(LNode::Opcode opcode)
     : LInstructionHelper<Defs, Operands, Temps>(opcode) {
   this->setIsCall();
 }
};

// Base class for control instructions (goto, branch, etc.)
template <size_t Succs, size_t Operands, size_t Temps>
class LControlInstructionHelper
   : public LInstructionHelper<0, Operands, Temps> {
 MOZ_NO_UNIQUE_ADDRESS mozilla::Array<MBasicBlock*, Succs> successors_;

protected:
 explicit LControlInstructionHelper(LNode::Opcode opcode)
     : LInstructionHelper<0, Operands, Temps>(opcode) {}

public:
 size_t numSuccessors() const { return Succs; }
 MBasicBlock* getSuccessor(size_t i) const { return successors_[i]; }

 void setSuccessor(size_t i, MBasicBlock* successor) {
   successors_[i] = successor;
 }
};

class LRecoverInfo : public TempObject {
public:
 using Instructions = Vector<MNode*, 2, JitAllocPolicy>;

private:
 // List of instructions needed to recover the stack frames.
 // Outer frames are stored before inner frames.
 Instructions instructions_;

 // Cached offset where this resume point is encoded.
 RecoverOffset recoverOffset_;

 // Whether this LRecoverInfo has any side-effect associated with it.
 bool hasSideEffects_ = false;

 explicit LRecoverInfo(TempAllocator& alloc);
 [[nodiscard]] bool init(MResumePoint* mir);

 // Fill the instruction vector such as all instructions needed for the
 // recovery are pushed before the current instruction.
 template <typename Node>
 [[nodiscard]] bool appendOperands(Node* ins);
 [[nodiscard]] bool appendDefinition(MDefinition* def);
 [[nodiscard]] bool appendResumePoint(MResumePoint* rp);

public:
 static LRecoverInfo* New(MIRGenerator* gen, MResumePoint* mir);

 // Resume point of the inner most function.
 MResumePoint* mir() const { return instructions_.back()->toResumePoint(); }
 RecoverOffset recoverOffset() const { return recoverOffset_; }
 void setRecoverOffset(RecoverOffset offset) {
   MOZ_ASSERT(recoverOffset_ == INVALID_RECOVER_OFFSET);
   recoverOffset_ = offset;
 }

 MNode** begin() { return instructions_.begin(); }
 MNode** end() { return instructions_.end(); }
 size_t numInstructions() const { return instructions_.length(); }
 bool hasSideEffects() { return hasSideEffects_; }

 class OperandIter {
  private:
   MNode** it_;
   MNode** end_;
   size_t op_;
   size_t opEnd_;
   MResumePoint* rp_;
   MNode* node_;

  public:
   explicit OperandIter(LRecoverInfo* recoverInfo)
       : it_(recoverInfo->begin()),
         end_(recoverInfo->end()),
         op_(0),
         opEnd_(0),
         rp_(nullptr),
         node_(nullptr) {
     settle();
   }

   void settle() {
     opEnd_ = (*it_)->numOperands();
     while (opEnd_ == 0) {
       ++it_;
       op_ = 0;
       opEnd_ = (*it_)->numOperands();
     }
     node_ = *it_;
     if (node_->isResumePoint()) {
       rp_ = node_->toResumePoint();
     }
   }

   MDefinition* operator*() {
     if (rp_) {  // de-virtualize MResumePoint::getOperand calls.
       return rp_->getOperand(op_);
     }
     return node_->getOperand(op_);
   }
   MDefinition* operator->() {
     if (rp_) {  // de-virtualize MResumePoint::getOperand calls.
       return rp_->getOperand(op_);
     }
     return node_->getOperand(op_);
   }

   OperandIter& operator++() {
     ++op_;
     if (op_ != opEnd_) {
       return *this;
     }
     op_ = 0;
     ++it_;
     node_ = rp_ = nullptr;
     if (!*this) {
       settle();
     }
     return *this;
   }

   explicit operator bool() const { return it_ == end_; }

#ifdef DEBUG
   bool canOptimizeOutIfUnused();
#endif
 };
};

// An LSnapshot is the reflection of an MResumePoint in LIR. Unlike
// MResumePoints, they cannot be shared, as they are filled in by the register
// allocator in order to capture the precise low-level stack state in between an
// instruction's input and output. During code generation, LSnapshots are
// compressed and saved in the compiled script.
class LSnapshot : public TempObject {
private:
 LAllocation* slots_;
 LRecoverInfo* recoverInfo_;
 SnapshotOffset snapshotOffset_;
 uint32_t numSlots_;
 BailoutKind bailoutKind_;

 LSnapshot(LRecoverInfo* recover, BailoutKind kind);
 [[nodiscard]] bool init(MIRGenerator* gen);

public:
 static LSnapshot* New(MIRGenerator* gen, LRecoverInfo* recover,
                       BailoutKind kind);

 size_t numEntries() const { return numSlots_; }
 size_t numSlots() const { return numSlots_ / BOX_PIECES; }
 LAllocation* payloadOfSlot(size_t i) {
   MOZ_ASSERT(i < numSlots());
   size_t entryIndex = (i * BOX_PIECES) + (BOX_PIECES - 1);
   return getEntry(entryIndex);
 }
#ifdef JS_NUNBOX32
 LAllocation* typeOfSlot(size_t i) {
   MOZ_ASSERT(i < numSlots());
   size_t entryIndex = (i * BOX_PIECES) + (BOX_PIECES - 2);
   return getEntry(entryIndex);
 }
#endif
 LAllocation* getEntry(size_t i) {
   MOZ_ASSERT(i < numSlots_);
   return &slots_[i];
 }
 void setEntry(size_t i, const LAllocation& alloc) {
   MOZ_ASSERT(i < numSlots_);
   slots_[i] = alloc;
 }
 LRecoverInfo* recoverInfo() const { return recoverInfo_; }
 MResumePoint* mir() const { return recoverInfo()->mir(); }
 SnapshotOffset snapshotOffset() const { return snapshotOffset_; }
 void setSnapshotOffset(SnapshotOffset offset) {
   MOZ_ASSERT(snapshotOffset_ == INVALID_SNAPSHOT_OFFSET);
   snapshotOffset_ = offset;
 }
 BailoutKind bailoutKind() const { return bailoutKind_; }
 void rewriteRecoveredInput(LUse input);
};

struct SafepointSlotEntry {
 // Flag indicating whether this is a slot in the stack or argument space.
 uint32_t stack : 1;

 // Byte offset of the slot, as in LStackSlot or LArgument.
 uint32_t slot : 31;

 SafepointSlotEntry() : stack(0), slot(0) {}
 SafepointSlotEntry(bool stack, uint32_t slot) : stack(stack), slot(slot) {}
 explicit SafepointSlotEntry(const LAllocation* a)
     : stack(a->isStackSlot()), slot(a->memorySlot()) {}
};

// Used for the type or payload half of a JS Value on 32-bit platforms.
class SafepointNunboxEntry {
 static constexpr size_t VregBits = 31;
 uint32_t isType_ : 1;
 uint32_t vreg_ : VregBits;
 LAllocation alloc_;

 static_assert(MAX_VIRTUAL_REGISTERS <= (uint32_t(1) << VregBits) - 1);

public:
 SafepointNunboxEntry(bool isType, uint32_t vreg, LAllocation alloc)
     : isType_(isType), vreg_(vreg), alloc_(alloc) {
   MOZ_ASSERT(alloc.isGeneralReg() || alloc.isMemory());
 }
 bool isType() const { return isType_; }
 uint32_t vreg() const { return vreg_; }
 LAllocation alloc() const { return alloc_; }
};

enum class WasmSafepointKind : uint8_t {
 // For wasm call instructions (isCall() == true) where registers are spilled
 // by register allocation.
 LirCall,
 // For wasm instructions (isCall() == false) which will spill/restore live
 // registers manually in codegen.
 CodegenCall,
 // For resumable wasm traps where registers will be spilled by the trap
 // handler.
 Trap,
 // For stack switch call.
 StackSwitch,
};

class LSafepoint : public TempObject {
 using SlotEntry = SafepointSlotEntry;
 using NunboxEntry = SafepointNunboxEntry;

public:
 using SlotList = Vector<SlotEntry, 0, JitAllocPolicy>;
 using NunboxList = Vector<NunboxEntry, 0, JitAllocPolicy>;

private:
 // The information in a safepoint describes the registers and gc related
 // values that are live at the start of the associated instruction.

 // The set of registers which are live at an OOL call made within the
 // instruction. This includes any registers for inputs which are not
 // use-at-start, any registers for temps, and any registers live after the
 // call except outputs of the instruction.
 //
 // For call instructions, the live regs are empty. Call instructions may
 // have register inputs or temporaries, which will *not* be in the live
 // registers: if passed to the call, the values passed will be marked via
 // TraceJitExitFrame, and no registers can be live after the instruction
 // except its outputs.
 LiveRegisterSet liveRegs_;

 // The subset of liveRegs which contains gcthing pointers.
 LiveGeneralRegisterSet gcRegs_;

#ifdef CHECK_OSIPOINT_REGISTERS
 // Clobbered regs of the current instruction. This set is never written to
 // the safepoint; it's only used by assertions during compilation.
 LiveRegisterSet clobberedRegs_;
#endif

 // Offset to a position in the safepoint stream, or
 // INVALID_SAFEPOINT_OFFSET.
 uint32_t safepointOffset_;

 // Assembler buffer displacement to OSI point's call location.
 uint32_t osiCallPointOffset_;

 // List of slots which have gcthing pointers.
 SlotList gcSlots_;

#ifdef JS_NUNBOX32
 // List of registers (in liveRegs) and slots which contain pieces of Values.
 NunboxList nunboxParts_;
#elif JS_PUNBOX64
 // List of slots which have Values.
 SlotList valueSlots_;

 // The subset of liveRegs which have Values.
 LiveGeneralRegisterSet valueRegs_;
#endif

 // The subset of liveRegs which contains pointers to slots/elements.
 LiveGeneralRegisterSet slotsOrElementsRegs_;
 // List of slots which have slots/elements pointers.
 SlotList slotsOrElementsSlots_;

 // The subset of liveRegs which contains wasm::AnyRef's.
 LiveGeneralRegisterSet wasmAnyRefRegs_;
 // List of slots which have wasm::AnyRef's.
 SlotList wasmAnyRefSlots_;

 // The subset of liveRegs which contains wasm struct data (OOL) pointers.
 LiveGeneralRegisterSet wasmStructDataRegs_;
 // List of slots which have have wasm struct data (OOL) pointers.
 SlotList wasmStructDataSlots_;

 // The subset of liveRegs which contains wasm array data (IL or OOL) pointers.
 LiveGeneralRegisterSet wasmArrayDataRegs_;
 // List of slots which have have wasm array data (IL or OOL) pointers.
 SlotList wasmArrayDataSlots_;

 // Wasm only: with what kind of instruction is this LSafepoint associated?
 WasmSafepointKind wasmSafepointKind_;

 // Wasm only: what is the value of masm.framePushed() that corresponds to
 // the lowest-addressed word covered by the StackMap that we will generate
 // from this LSafepoint?  This depends on the instruction:
 //
 // WasmSafepointKind::LirCall:
 //    masm.framePushed() - StackArgAreaSizeUnaligned(arg types for the call),
 //    because the map does not include the outgoing args themselves, but
 //    it does cover any and all alignment space above them.
 //
 // WasmSafepointKind::CodegenCall and WasmSafepointKind::Trap:
 //    masm.framePushed() unmodified. Note that when constructing the
 //    StackMap we will add entries below this point to take account of
 //    registers dumped on the stack.
 uint32_t framePushedAtStackMapBase_;

public:
 void assertInvariants() {
   // Every register in valueRegs, gcRegs, wasmAnyRefRegs, wasmStructDataRegs
   // and wasmArrayDataRegs should also be in liveRegs.
#ifndef JS_NUNBOX32
   MOZ_ASSERT((valueRegs().bits() & ~liveRegs().gprs().bits()) == 0);
#endif
   MOZ_ASSERT((gcRegs().bits() & ~liveRegs().gprs().bits()) == 0);
   MOZ_ASSERT((wasmAnyRefRegs().bits() & ~liveRegs().gprs().bits()) == 0);
   MOZ_ASSERT((wasmStructDataRegs().bits() & ~liveRegs().gprs().bits()) == 0);
   MOZ_ASSERT((wasmArrayDataRegs().bits() & ~liveRegs().gprs().bits()) == 0);
 }

 explicit LSafepoint(TempAllocator& alloc)
     : safepointOffset_(INVALID_SAFEPOINT_OFFSET),
       osiCallPointOffset_(0),
       gcSlots_(alloc),
#ifdef JS_NUNBOX32
       nunboxParts_(alloc),
#else
       valueSlots_(alloc),
#endif
       slotsOrElementsSlots_(alloc),
       wasmAnyRefSlots_(alloc),
       wasmStructDataSlots_(alloc),
       wasmArrayDataSlots_(alloc),
       wasmSafepointKind_(WasmSafepointKind::LirCall),
       framePushedAtStackMapBase_(0) {
   assertInvariants();
 }
 void addLiveRegister(AnyRegister reg) {
   liveRegs_.addUnchecked(reg);
   assertInvariants();
 }
 const LiveRegisterSet& liveRegs() const { return liveRegs_; }
#ifdef CHECK_OSIPOINT_REGISTERS
 void addClobberedRegister(AnyRegister reg) {
   clobberedRegs_.addUnchecked(reg);
   assertInvariants();
 }
 const LiveRegisterSet& clobberedRegs() const { return clobberedRegs_; }
#endif
 LiveGeneralRegisterSet gcRegs() const { return gcRegs_; }
 [[nodiscard]] bool addGcSlot(bool stack, uint32_t slot) {
   bool result = gcSlots_.append(SlotEntry(stack, slot));
   if (result) {
     assertInvariants();
   }
   return result;
 }
 SlotList& gcSlots() { return gcSlots_; }

 SlotList& slotsOrElementsSlots() { return slotsOrElementsSlots_; }
 LiveGeneralRegisterSet slotsOrElementsRegs() const {
   return slotsOrElementsRegs_;
 }
 [[nodiscard]] bool addSlotsOrElementsSlot(bool stack, uint32_t slot) {
   bool result = slotsOrElementsSlots_.append(SlotEntry(stack, slot));
   if (result) {
     assertInvariants();
   }
   return result;
 }
 [[nodiscard]] bool addSlotsOrElementsPointer(LAllocation alloc) {
   if (alloc.isMemory()) {
     return addSlotsOrElementsSlot(alloc.isStackSlot(), alloc.memorySlot());
   }
   MOZ_ASSERT(alloc.isGeneralReg());
   slotsOrElementsRegs_.addUnchecked(alloc.toGeneralReg()->reg());
   assertInvariants();
   return true;
 }

 SlotList& wasmStructDataSlots() { return wasmStructDataSlots_; }
 LiveGeneralRegisterSet wasmStructDataRegs() const {
   return wasmStructDataRegs_;
 }
 [[nodiscard]] bool addWasmStructDataSlot(bool stack, uint32_t slot) {
   bool result = wasmStructDataSlots_.append(SlotEntry(stack, slot));
   if (result) {
     assertInvariants();
   }
   return result;
 }
 [[nodiscard]] bool addWasmStructDataPointer(LAllocation alloc) {
   if (alloc.isMemory()) {
     return addWasmStructDataSlot(alloc.isStackSlot(), alloc.memorySlot());
   }
   MOZ_ASSERT(alloc.isGeneralReg());
   wasmStructDataRegs_.addUnchecked(alloc.toGeneralReg()->reg());
   assertInvariants();
   return true;
 }

 SlotList& wasmArrayDataSlots() { return wasmArrayDataSlots_; }
 LiveGeneralRegisterSet wasmArrayDataRegs() const {
   return wasmArrayDataRegs_;
 }
 [[nodiscard]] bool addWasmArrayDataSlot(bool stack, uint32_t slot) {
   bool result = wasmArrayDataSlots_.append(SlotEntry(stack, slot));
   if (result) {
     assertInvariants();
   }
   return result;
 }
 [[nodiscard]] bool addWasmArrayDataPointer(LAllocation alloc) {
   if (alloc.isMemory()) {
     return addWasmArrayDataSlot(alloc.isStackSlot(), alloc.memorySlot());
   }
   MOZ_ASSERT(alloc.isGeneralReg());
   wasmArrayDataRegs_.addUnchecked(alloc.toGeneralReg()->reg());
   assertInvariants();
   return true;
 }

 bool hasSlotsOrElementsPointer(LAllocation alloc) const {
   if (alloc.isGeneralReg()) {
     return slotsOrElementsRegs().has(alloc.toGeneralReg()->reg());
   }
   for (size_t i = 0; i < slotsOrElementsSlots_.length(); i++) {
     const SlotEntry& entry = slotsOrElementsSlots_[i];
     if (entry.stack == alloc.isStackSlot() &&
         entry.slot == alloc.memorySlot()) {
       return true;
     }
   }
   return false;
 }

 [[nodiscard]] bool addGcPointer(LAllocation alloc) {
   if (alloc.isMemory()) {
     return addGcSlot(alloc.isStackSlot(), alloc.memorySlot());
   }
   MOZ_ASSERT(alloc.isGeneralReg());
   gcRegs_.addUnchecked(alloc.toGeneralReg()->reg());
   assertInvariants();
   return true;
 }

 bool hasGcPointer(LAllocation alloc) const {
   if (alloc.isGeneralReg()) {
     return gcRegs().has(alloc.toGeneralReg()->reg());
   }
   MOZ_ASSERT(alloc.isMemory());
   for (size_t i = 0; i < gcSlots_.length(); i++) {
     if (gcSlots_[i].stack == alloc.isStackSlot() &&
         gcSlots_[i].slot == alloc.memorySlot()) {
       return true;
     }
   }
   return false;
 }

 LiveGeneralRegisterSet wasmAnyRefRegs() const { return wasmAnyRefRegs_; }

 [[nodiscard]] bool addWasmAnyRefSlot(bool stack, uint32_t slot) {
   bool result = wasmAnyRefSlots_.append(SlotEntry(stack, slot));
   if (result) {
     assertInvariants();
   }
   return result;
 }
 SlotList& wasmAnyRefSlots() { return wasmAnyRefSlots_; }

 [[nodiscard]] bool addWasmAnyRef(LAllocation alloc) {
   if (alloc.isMemory()) {
     return addWasmAnyRefSlot(alloc.isStackSlot(), alloc.memorySlot());
   }
   MOZ_ASSERT(alloc.isGeneralReg());
   wasmAnyRefRegs_.addUnchecked(alloc.toGeneralReg()->reg());
   assertInvariants();
   return true;
 }
 bool hasWasmAnyRef(LAllocation alloc) const {
   if (alloc.isGeneralReg()) {
     return wasmAnyRefRegs().has(alloc.toGeneralReg()->reg());
   }
   MOZ_ASSERT(alloc.isMemory());
   for (size_t i = 0; i < wasmAnyRefSlots_.length(); i++) {
     if (wasmAnyRefSlots_[i].stack == alloc.isStackSlot() &&
         wasmAnyRefSlots_[i].slot == alloc.memorySlot()) {
       return true;
     }
   }
   return false;
 }

 // Return true if all GC-managed pointers from `alloc` are recorded in this
 // safepoint.
 bool hasAllWasmAnyRefsFromStackArea(LAllocation alloc) const {
   for (LStackArea::ResultIterator iter = alloc.toStackArea()->results(); iter;
        iter.next()) {
     if (iter.isWasmAnyRef() && !hasWasmAnyRef(iter.alloc())) {
       return false;
     }
   }
   return true;
 }

#ifdef JS_NUNBOX32
 [[nodiscard]] bool addNunboxPart(bool isType, uint32_t vreg,
                                  LAllocation alloc) {
   bool result = nunboxParts_.emplaceBack(isType, vreg, alloc);
   if (result) {
     assertInvariants();
   }
   return result;
 }

#  ifdef DEBUG
 bool hasNunboxPart(bool isType, LAllocation alloc) const {
   for (auto entry : nunboxParts_) {
     if (entry.alloc() == alloc && entry.isType() == isType) {
       return true;
     }
   }
   return false;
 }
#  endif

 NunboxList& nunboxParts() { return nunboxParts_; }

#elif JS_PUNBOX64
 [[nodiscard]] bool addValueSlot(bool stack, uint32_t slot) {
   bool result = valueSlots_.append(SlotEntry(stack, slot));
   if (result) {
     assertInvariants();
   }
   return result;
 }
 SlotList& valueSlots() { return valueSlots_; }

 bool hasValueSlot(bool stack, uint32_t slot) const {
   for (size_t i = 0; i < valueSlots_.length(); i++) {
     if (valueSlots_[i].stack == stack && valueSlots_[i].slot == slot) {
       return true;
     }
   }
   return false;
 }

 LiveGeneralRegisterSet valueRegs() const { return valueRegs_; }

 [[nodiscard]] bool addBoxedValue(LAllocation alloc) {
   if (alloc.isMemory()) {
     return addValueSlot(alloc.isStackSlot(), alloc.memorySlot());
   }
   MOZ_ASSERT(alloc.isGeneralReg());
   valueRegs_.addUnchecked(alloc.toGeneralReg()->reg());
   assertInvariants();
   return true;
 }

 bool hasBoxedValue(LAllocation alloc) const {
   if (alloc.isGeneralReg()) {
     return valueRegs().has(alloc.toGeneralReg()->reg());
   }
   return hasValueSlot(alloc.isStackSlot(), alloc.memorySlot());
 }

#endif  // JS_PUNBOX64

 [[nodiscard]] bool addGCAllocation(uint32_t vregId, LDefinition* def,
                                    LAllocation a);

 bool encoded() const { return safepointOffset_ != INVALID_SAFEPOINT_OFFSET; }
 uint32_t offset() const {
   MOZ_ASSERT(encoded());
   return safepointOffset_;
 }
 void setOffset(uint32_t offset) { safepointOffset_ = offset; }
 uint32_t osiReturnPointOffset() const {
   // In general, pointer arithmetic on code is bad, but in this case,
   // getting the return address from a call instruction, stepping over pools
   // would be wrong.
   return osiCallPointOffset_ + Assembler::PatchWrite_NearCallSize();
 }
 uint32_t osiCallPointOffset() const { return osiCallPointOffset_; }
 void setOsiCallPointOffset(uint32_t osiCallPointOffset) {
   MOZ_ASSERT(!osiCallPointOffset_);
   osiCallPointOffset_ = osiCallPointOffset;
 }

 WasmSafepointKind wasmSafepointKind() const { return wasmSafepointKind_; }
 void setWasmSafepointKind(WasmSafepointKind kind) {
   wasmSafepointKind_ = kind;
 }

 // See comment on framePushedAtStackMapBase_.
 uint32_t framePushedAtStackMapBase() const {
   return framePushedAtStackMapBase_;
 }
 void setFramePushedAtStackMapBase(uint32_t n) {
   MOZ_ASSERT(framePushedAtStackMapBase_ == 0);
   framePushedAtStackMapBase_ = n;
 }
};

struct WasmRefIsSubtypeDefs {
 LAllocation superSTV;
 LDefinition scratch1;
 LDefinition scratch2;
};

template <bool WithSnapshotUses>
class LInstruction::InputIterImpl {
private:
 LInstruction& ins_;
 size_t idx_ = 0;
 bool snapshot_ = false;

 void handleOperandsEnd() {
   // Iterate on the snapshot when iteration over all operands is done.
   if constexpr (WithSnapshotUses) {
     if (!snapshot_ && idx_ == ins_.numOperands() && ins_.snapshot()) {
       idx_ = 0;
       snapshot_ = true;
     }
   }
 }

public:
 explicit InputIterImpl(LInstruction& ins) : ins_(ins) { handleOperandsEnd(); }

 bool more() const {
   if (WithSnapshotUses && snapshot_) {
     return idx_ < ins_.snapshot()->numEntries();
   }
   if (idx_ < ins_.numOperands()) {
     return true;
   }
   if (WithSnapshotUses && ins_.snapshot() && ins_.snapshot()->numEntries()) {
     return true;
   }
   return false;
 }

 bool isSnapshotInput() const { return snapshot_; }

 void next() {
   MOZ_ASSERT(more());
   idx_++;
   handleOperandsEnd();
 }

 void replace(const LAllocation& alloc) {
   if (WithSnapshotUses && snapshot_) {
     ins_.snapshot()->setEntry(idx_, alloc);
   } else {
     ins_.setOperand(idx_, alloc);
   }
 }

 LAllocation* operator*() const {
   if (WithSnapshotUses && snapshot_) {
     return ins_.snapshot()->getEntry(idx_);
   }
   return ins_.getOperand(idx_);
 }

 LAllocation* operator->() const { return **this; }
};

// Iterator for instruction outputs or temps. Skips BogusTemp definitions.
template <bool Temps>
class LInstruction::DefIterImpl {
private:
 LInstruction* ins_;
 size_t idx_ = 0;
 const size_t len_;

public:
 explicit DefIterImpl(LInstruction* ins)
     : ins_(ins), len_(Temps ? ins->numTemps() : ins->numDefs()) {
   settle();
 }
 bool done() const {
   MOZ_ASSERT(idx_ <= len_);
   return idx_ == len_;
 }
 void operator++(int) {
   MOZ_ASSERT(!done());
   idx_++;
   settle();
 }
 void settle() {
   while (!done() && get()->isBogusTemp()) {
     idx_++;
   }
 }
 size_t index() const {
   MOZ_ASSERT(!done());
   return idx_;
 }
 LDefinition* get() const {
   MOZ_ASSERT(!done());
   if constexpr (Temps) {
     return ins_->getTemp(idx_);
   }
   return ins_->getDef(idx_);
 }
 LDefinition* operator*() const { return get(); }
 LDefinition* operator->() const { return **this; }
};

bool LDefinition::isSafepointGCType(LNode* ins) const {
 switch (type()) {
   case LDefinition::OBJECT:
   case LDefinition::SLOTS:
   case LDefinition::WASM_ANYREF:
   case LDefinition::WASM_STRUCT_DATA:
   case LDefinition::WASM_ARRAY_DATA:
#ifdef JS_NUNBOX32
   case LDefinition::TYPE:
   case LDefinition::PAYLOAD:
#else
   case LDefinition::BOX:
#endif
     return true;
   case LDefinition::STACKRESULTS: {
     LStackArea alloc(ins->toInstruction());
     for (auto iter = alloc.results(); iter; iter.next()) {
       if (iter.isWasmAnyRef()) {
         return true;
       }
     }
     return false;
   }
   case LDefinition::GENERAL:
   case LDefinition::INT32:
   case LDefinition::FLOAT32:
   case LDefinition::DOUBLE:
   case LDefinition::SIMD128:
     return false;
 }
 MOZ_CRASH("invalid type");
}

class LIRGraph {
 struct ValueHasher {
   using Lookup = Value;
   static HashNumber hash(const Value& v) { return HashNumber(v.asRawBits()); }
   static bool match(const Value& lhs, const Value& rhs) { return lhs == rhs; }
 };

 FixedList<LBlock> blocks_;

 // constantPool_ is a mozilla::Vector, not a js::Vector, because
 // js::Vector<Value> is prohibited as unsafe. This particular Vector of
 // Values is safe because it is only used within the scope of an
 // AutoSuppressGC (in IonCompile), which inhibits GC.
 mozilla::Vector<Value, 0, JitAllocPolicy> constantPool_;
 using ConstantPoolMap = HashMap<Value, uint32_t, ValueHasher, JitAllocPolicy>;
 ConstantPoolMap constantPoolMap_;
 uint32_t numVirtualRegisters_;
 uint32_t numInstructions_;

 // Number of instructions with a safepoint.
 uint32_t numSafepoints_ = 0;
 // Number of non-call instructions with a safepoint.
 uint32_t numNonCallSafepoints_ = 0;

 // Number of call-instructions in this LIR graph.
 uint32_t numCallInstructions_ = 0;

 // Size of stack slots needed for local spills.
 uint32_t localSlotsSize_;
 // Number of JS::Value stack slots needed for argument construction for calls.
 uint32_t argumentSlotCount_;
 // Count the number of extra times a single safepoint would be encoded.
 uint32_t extraSafepointUses_;

 MIRGraph& mir_;

public:
 explicit LIRGraph(MIRGraph* mir);

 [[nodiscard]] bool init() {
   return blocks_.init(mir_.alloc(), mir_.numBlocks());
 }
 MIRGraph& mir() const { return mir_; }
 size_t numBlocks() const { return blocks_.length(); }
 LBlock* getBlock(size_t i) { return &blocks_[i]; }
 uint32_t numBlockIds() const { return mir_.numBlockIds(); }
 [[nodiscard]] bool initBlock(MBasicBlock* mir) {
   auto* block = &blocks_[mir->id()];
   auto* lir = new (block) LBlock(mir);
   return lir->init(mir_.alloc());
 }
 uint32_t getVirtualRegister() {
   numVirtualRegisters_ += VREG_INCREMENT;
   return numVirtualRegisters_;
 }
 uint32_t numVirtualRegisters() const {
   // Virtual registers are 1-based, not 0-based, so add one as a
   // convenience for 0-based arrays.
   return numVirtualRegisters_ + 1;
 }
 uint32_t getInstructionId() { return numInstructions_++; }
 uint32_t numInstructions() const { return numInstructions_; }

 void incNumCallInstructions() { numCallInstructions_++; }
 uint32_t numCallInstructions() const { return numCallInstructions_; }

 void setLocalSlotsSize(uint32_t localSlotsSize) {
   localSlotsSize_ = localSlotsSize;
 }
 uint32_t localSlotsSize() const { return localSlotsSize_; }
 void setArgumentSlotCount(uint32_t argumentSlotCount) {
   argumentSlotCount_ = argumentSlotCount;
 }
 uint32_t argumentSlotCount() const { return argumentSlotCount_; }
 void addExtraSafepointUses(uint32_t extra) { extraSafepointUses_ += extra; }
 uint32_t extraSafepointUses() const { return extraSafepointUses_; }
 [[nodiscard]] bool addConstantToPool(const Value& v, uint32_t* index);
 size_t numConstants() const { return constantPool_.length(); }
 Value* constantPool() { return &constantPool_[0]; }

 void noteNeedsSafepoint(LInstruction* ins) {
   numSafepoints_++;
   if (!ins->isCall()) {
     numNonCallSafepoints_++;
   }
 }

 size_t numNonCallSafepoints() const { return numNonCallSafepoints_; }
 size_t numSafepoints() const { return numSafepoints_; }

#ifdef JS_JITSPEW
 void dump(GenericPrinter& out);
 void dump();
#endif
};

LAllocation::LAllocation(AnyRegister reg) {
 if (reg.isFloat()) {
   *this = LFloatReg(reg.fpu());
 } else {
   *this = LGeneralReg(reg.gpr());
 }
}

AnyRegister LAllocation::toAnyRegister() const {
 MOZ_ASSERT(isAnyRegister());
 if (isFloatReg()) {
   return AnyRegister(toFloatReg()->reg());
 }
 return AnyRegister(toGeneralReg()->reg());
}

}  // namespace jit
}  // namespace js

#include "jit/shared/LIR-shared.h"
#if defined(JS_CODEGEN_X86)
#  include "jit/x86/LIR-x86.h"
#elif defined(JS_CODEGEN_X64)
#  include "jit/x64/LIR-x64.h"
#elif defined(JS_CODEGEN_ARM)
#  include "jit/arm/LIR-arm.h"
#elif defined(JS_CODEGEN_ARM64)
#  include "jit/arm64/LIR-arm64.h"
#elif defined(JS_CODEGEN_LOONG64)
#  include "jit/loong64/LIR-loong64.h"
#elif defined(JS_CODEGEN_RISCV64)
#  include "jit/riscv64/LIR-riscv64.h"
#elif defined(JS_CODEGEN_MIPS64)
#  include "jit/mips-shared/LIR-mips-shared.h"
#  include "jit/mips64/LIR-mips64.h"
#elif defined(JS_CODEGEN_WASM32)
#  include "jit/wasm32/LIR-wasm32.h"
#elif defined(JS_CODEGEN_NONE)
#  include "jit/none/LIR-none.h"
#else
#  error "Unknown architecture!"
#endif

#undef LIR_HEADER

namespace js {
namespace jit {

#define LIROP(name)                           \
 L##name* LNode::to##name() {                \
   MOZ_ASSERT(is##name());                   \
   return static_cast<L##name*>(this);       \
 }                                           \
 const L##name* LNode::to##name() const {    \
   MOZ_ASSERT(is##name());                   \
   return static_cast<const L##name*>(this); \
 }
LIR_OPCODE_LIST(LIROP)
#undef LIROP

#define LALLOC_CAST(type)               \
 L##type* LAllocation::to##type() {    \
   MOZ_ASSERT(is##type());             \
   return static_cast<L##type*>(this); \
 }
#define LALLOC_CONST_CAST(type)                  \
 const L##type* LAllocation::to##type() const { \
   MOZ_ASSERT(is##type());                      \
   return static_cast<const L##type*>(this);    \
 }

LALLOC_CAST(Use)
LALLOC_CONST_CAST(Use)
LALLOC_CONST_CAST(GeneralReg)
LALLOC_CONST_CAST(FloatReg)
LALLOC_CONST_CAST(StackSlot)
LALLOC_CAST(StackArea)
LALLOC_CONST_CAST(StackArea)
LALLOC_CONST_CAST(Argument)
LALLOC_CONST_CAST(ConstantIndex)

#undef LALLOC_CAST

}  // namespace jit
}  // namespace js

#endif /* jit_LIR_h */