tor-browser

CodeGenerator-arm.cpp (89030B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "jit/arm/CodeGenerator-arm.h"

#include "mozilla/DebugOnly.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/Maybe.h"

#include "jsnum.h"

#include "jit/CodeGenerator.h"
#include "jit/InlineScriptTree.h"
#include "jit/JitRuntime.h"
#include "jit/MIR-wasm.h"
#include "jit/MIR.h"
#include "jit/MIRGraph.h"
#include "js/Conversions.h"
#include "js/ScalarType.h"  // js::Scalar::Type
#include "vm/JSContext.h"
#include "vm/Realm.h"
#include "vm/Shape.h"

#include "jit/MacroAssembler-inl.h"
#include "jit/shared/CodeGenerator-shared-inl.h"
#include "vm/JSScript-inl.h"

using namespace js;
using namespace js::jit;

using JS::GenericNaN;
using JS::ToInt32;
using mozilla::DebugOnly;
using mozilla::FloorLog2;
using mozilla::NegativeInfinity;

// shared
CodeGeneratorARM::CodeGeneratorARM(MIRGenerator* gen, LIRGraph* graph,
                                  MacroAssembler* masm,
                                  const wasm::CodeMetadata* wasmCodeMeta)
   : CodeGeneratorShared(gen, graph, masm, wasmCodeMeta) {}

void CodeGeneratorARM::emitBranch(Assembler::Condition cond,
                                 MBasicBlock* mirTrue, MBasicBlock* mirFalse) {
 if (isNextBlock(mirFalse->lir())) {
   jumpToBlock(mirTrue, cond);
 } else {
   jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
   jumpToBlock(mirTrue);
 }
}

bool CodeGeneratorARM::generateOutOfLineCode() {
 if (!CodeGeneratorShared::generateOutOfLineCode()) {
   return false;
 }

 if (deoptLabel_.used()) {
   // All non-table-based bailouts will go here.
   masm.bind(&deoptLabel_);

   // Push the frame size, so the handler can recover the IonScript.
   masm.push(Imm32(frameSize()));

   TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
   masm.jump(handler);
 }

 return !masm.oom();
}

void CodeGeneratorARM::emitBailoutOOL(LSnapshot* snapshot) {
 masm.push(Imm32(snapshot->snapshotOffset()));
 masm.ma_b(&deoptLabel_);
}

void CodeGeneratorARM::bailoutIf(Assembler::Condition condition,
                                LSnapshot* snapshot) {
 encode(snapshot);

 InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
 auto* ool = new (alloc()) LambdaOutOfLineCode(
     [=, this](OutOfLineCode& ool) { emitBailoutOOL(snapshot); });

 // All bailout code is associated with the bytecodeSite of the block we are
 // bailing out from.
 addOutOfLineCode(ool,
                  new (alloc()) BytecodeSite(tree, tree->script()->code()));

 masm.ma_b(ool->entry(), condition);
}

void CodeGeneratorARM::bailoutFrom(Label* label, LSnapshot* snapshot) {
 MOZ_ASSERT_IF(!masm.oom(), label->used());
 MOZ_ASSERT_IF(!masm.oom(), !label->bound());

 encode(snapshot);

 InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
 auto* ool = new (alloc()) LambdaOutOfLineCode(
     [=, this](OutOfLineCode& ool) { emitBailoutOOL(snapshot); });

 // All bailout code is associated with the bytecodeSite of the block we are
 // bailing out from.
 addOutOfLineCode(ool,
                  new (alloc()) BytecodeSite(tree, tree->script()->code()));

 masm.retarget(label, ool->entry());
}

void CodeGeneratorARM::bailout(LSnapshot* snapshot) {
 Label label;
 masm.ma_b(&label);
 bailoutFrom(&label, snapshot);
}

void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
 FloatRegister first = ToFloatRegister(ins->first());
 FloatRegister second = ToFloatRegister(ins->second());

 MOZ_ASSERT(first == ToFloatRegister(ins->output()));

 if (ins->mir()->isMax()) {
   masm.maxDouble(second, first, true);
 } else {
   masm.minDouble(second, first, true);
 }
}

void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
 FloatRegister first = ToFloatRegister(ins->first());
 FloatRegister second = ToFloatRegister(ins->second());

 MOZ_ASSERT(first == ToFloatRegister(ins->output()));

 if (ins->mir()->isMax()) {
   masm.maxFloat32(second, first, true);
 } else {
   masm.minFloat32(second, first, true);
 }
}

void CodeGenerator::visitAddI(LAddI* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();

 ScratchRegisterScope scratch(masm);

 if (rhs->isConstant()) {
   masm.ma_add(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
               SetCC);
 } else if (rhs->isGeneralReg()) {
   masm.ma_add(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), SetCC);
 } else {
   masm.ma_add(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
               SetCC);
 }

 if (ins->snapshot()) {
   bailoutIf(Assembler::Overflow, ins->snapshot());
 }
}

void CodeGenerator::visitAddIntPtr(LAddIntPtr* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();

 ScratchRegisterScope scratch(masm);

 if (rhs->isConstant()) {
   masm.ma_add(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
               LeaveCC);
 } else if (rhs->isGeneralReg()) {
   masm.ma_add(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), LeaveCC);
 } else {
   masm.ma_add(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
               LeaveCC);
 }
}

void CodeGenerator::visitAddI64(LAddI64* lir) {
 LInt64Allocation lhs = lir->lhs();
 LInt64Allocation rhs = lir->rhs();

 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));

 if (IsConstant(rhs)) {
   masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
   return;
 }

 masm.add64(ToRegister64(rhs), ToRegister64(lhs));
}

void CodeGenerator::visitSubI(LSubI* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();

 ScratchRegisterScope scratch(masm);

 if (rhs->isConstant()) {
   masm.ma_sub(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
               SetCC);
 } else if (rhs->isGeneralReg()) {
   masm.ma_sub(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), SetCC);
 } else {
   masm.ma_sub(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
               SetCC);
 }

 if (ins->snapshot()) {
   bailoutIf(Assembler::Overflow, ins->snapshot());
 }
}

void CodeGenerator::visitSubIntPtr(LSubIntPtr* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();

 ScratchRegisterScope scratch(masm);

 if (rhs->isConstant()) {
   masm.ma_sub(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
               LeaveCC);
 } else if (rhs->isGeneralReg()) {
   masm.ma_sub(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), LeaveCC);
 } else {
   masm.ma_sub(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
               LeaveCC);
 }
}

void CodeGenerator::visitSubI64(LSubI64* lir) {
 LInt64Allocation lhs = lir->lhs();
 LInt64Allocation rhs = lir->rhs();

 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));

 if (IsConstant(rhs)) {
   masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
   return;
 }

 masm.sub64(ToRegister64(rhs), ToRegister64(lhs));
}

void CodeGenerator::visitMulI(LMulI* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();
 MMul* mul = ins->mir();
 MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
               !mul->canBeNegativeZero() && !mul->canOverflow());

 if (rhs->isConstant()) {
   // Bailout when this condition is met.
   Assembler::Condition c = Assembler::Overflow;
   // Bailout on -0.0
   int32_t constant = ToInt32(rhs);
   if (mul->canBeNegativeZero() && constant <= 0) {
     Assembler::Condition bailoutCond =
         (constant == 0) ? Assembler::LessThan : Assembler::Equal;
     masm.as_cmp(ToRegister(lhs), Imm8(0));
     bailoutIf(bailoutCond, ins->snapshot());
   }
   // TODO: move these to ma_mul.
   switch (constant) {
     case -1:
       masm.as_rsb(ToRegister(dest), ToRegister(lhs), Imm8(0), SetCC);
       break;
     case 0:
       masm.ma_mov(Imm32(0), ToRegister(dest));
       return;  // Escape overflow check;
     case 1:
       // Nop
       masm.ma_mov(ToRegister(lhs), ToRegister(dest));
       return;  // Escape overflow check;
     case 2:
       masm.ma_add(ToRegister(lhs), ToRegister(lhs), ToRegister(dest), SetCC);
       // Overflow is handled later.
       break;
     default: {
       bool handled = false;
       if (constant > 0) {
         // Try shift and add sequences for a positive constant.
         if (!mul->canOverflow()) {
           // If it cannot overflow, we can do lots of optimizations.
           Register src = ToRegister(lhs);
           uint32_t shift = FloorLog2(constant);
           uint32_t rest = constant - (1 << shift);
           // See if the constant has one bit set, meaning it can be
           // encoded as a bitshift.
           if ((1 << shift) == constant) {
             masm.ma_lsl(Imm32(shift), src, ToRegister(dest));
             handled = true;
           } else {
             // If the constant cannot be encoded as (1 << C1), see
             // if it can be encoded as (1 << C1) | (1 << C2), which
             // can be computed using an add and a shift.
             uint32_t shift_rest = FloorLog2(rest);
             if ((1u << shift_rest) == rest) {
               masm.as_add(ToRegister(dest), src,
                           lsl(src, shift - shift_rest));
               if (shift_rest != 0) {
                 masm.ma_lsl(Imm32(shift_rest), ToRegister(dest),
                             ToRegister(dest));
               }
               handled = true;
             }
           }
         } else if (ToRegister(lhs) != ToRegister(dest)) {
           // To stay on the safe side, only optimize things that are a
           // power of 2.

           uint32_t shift = FloorLog2(constant);
           if ((1 << shift) == constant) {
             // dest = lhs * pow(2,shift)
             masm.ma_lsl(Imm32(shift), ToRegister(lhs), ToRegister(dest));
             // At runtime, check (lhs == dest >> shift), if this
             // does not hold, some bits were lost due to overflow,
             // and the computation should be resumed as a double.
             masm.as_cmp(ToRegister(lhs), asr(ToRegister(dest), shift));
             c = Assembler::NotEqual;
             handled = true;
           }
         }
       }

       if (!handled) {
         ScratchRegisterScope scratch(masm);
         if (mul->canOverflow()) {
           c = masm.ma_check_mul(ToRegister(lhs), Imm32(ToInt32(rhs)),
                                 ToRegister(dest), scratch, c);
         } else {
           masm.ma_mul(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest),
                       scratch);
         }
       }
     }
   }
   // Bailout on overflow.
   if (mul->canOverflow()) {
     bailoutIf(c, ins->snapshot());
   }
 } else {
   Assembler::Condition c = Assembler::Overflow;

   if (mul->canOverflow()) {
     ScratchRegisterScope scratch(masm);
     c = masm.ma_check_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest),
                           scratch, c);
   } else {
     masm.ma_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest));
   }

   // Bailout on overflow.
   if (mul->canOverflow()) {
     bailoutIf(c, ins->snapshot());
   }

   if (mul->canBeNegativeZero()) {
     Label done;
     masm.as_cmp(ToRegister(dest), Imm8(0));
     masm.ma_b(&done, Assembler::NotEqual);

     // Result is -0 if lhs or rhs is negative.
     masm.ma_cmn(ToRegister(lhs), ToRegister(rhs));
     bailoutIf(Assembler::Signed, ins->snapshot());

     masm.bind(&done);
   }
 }
}

void CodeGenerator::visitMulIntPtr(LMulIntPtr* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();

 if (rhs->isConstant()) {
   intptr_t constant = ToIntPtr(rhs);

   switch (constant) {
     case -1:
       masm.as_rsb(ToRegister(dest), ToRegister(lhs), Imm8(0), LeaveCC);
       return;
     case 0:
       masm.ma_mov(Imm32(0), ToRegister(dest));
       return;
     case 1:
       masm.ma_mov(ToRegister(lhs), ToRegister(dest));
       return;
     case 2:
       masm.ma_add(ToRegister(lhs), ToRegister(lhs), ToRegister(dest),
                   LeaveCC);
       return;
   }

   // Use shift if constant is a power of 2.
   if (constant > 0 && mozilla::IsPowerOfTwo(uintptr_t(constant))) {
     uint32_t shift = mozilla::FloorLog2(constant);
     masm.ma_lsl(Imm32(shift), ToRegister(lhs), ToRegister(dest));
     return;
   }

   ScratchRegisterScope scratch(masm);
   masm.ma_mul(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest),
               scratch);
 } else {
   masm.ma_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest));
 }
}

void CodeGenerator::visitMulI64(LMulI64* lir) {
 LInt64Allocation lhs = lir->lhs();
 LInt64Allocation rhs = lir->rhs();

 MOZ_ASSERT(ToRegister64(lhs) == ToOutRegister64(lir));

 if (IsConstant(rhs)) {
   int64_t constant = ToInt64(rhs);
   switch (constant) {
     case -1:
       masm.neg64(ToRegister64(lhs));
       return;
     case 0:
       masm.move64(Imm64(0), ToRegister64(lhs));
       return;
     case 1:
       // nop
       return;
     case 2:
       masm.add64(ToRegister64(lhs), ToRegister64(lhs));
       return;
     default:
       if (constant > 0) {
         // Use shift if constant is power of 2.
         int32_t shift = mozilla::FloorLog2(constant);
         if (int64_t(1) << shift == constant) {
           masm.lshift64(Imm32(shift), ToRegister64(lhs));
           return;
         }
       }
       Register temp = ToTempRegisterOrInvalid(lir->temp0());
       masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
   }
 } else {
   Register temp = ToTempRegisterOrInvalid(lir->temp0());
   masm.mul64(ToRegister64(rhs), ToRegister64(lhs), temp);
 }
}

void CodeGeneratorARM::divICommon(MDiv* mir, Register lhs, Register rhs,
                                 Register output, LSnapshot* snapshot,
                                 Label& done) {
 ScratchRegisterScope scratch(masm);

 if (mir->canBeNegativeOverflow()) {
   // Handle INT32_MIN / -1;
   // The integer division will give INT32_MIN, but we want -(double)INT32_MIN.

   // Sets EQ if lhs == INT32_MIN.
   masm.ma_cmp(lhs, Imm32(INT32_MIN), scratch);
   // If EQ (LHS == INT32_MIN), sets EQ if rhs == -1.
   masm.ma_cmp(rhs, Imm32(-1), scratch, Assembler::Equal);
   if (mir->canTruncateOverflow()) {
     if (mir->trapOnError()) {
       Label ok;
       masm.ma_b(&ok, Assembler::NotEqual);
       masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->trapSiteDesc());
       masm.bind(&ok);
     } else {
       // (-INT32_MIN)|0 = INT32_MIN
       Label skip;
       masm.ma_b(&skip, Assembler::NotEqual);
       masm.ma_mov(Imm32(INT32_MIN), output);
       masm.ma_b(&done);
       masm.bind(&skip);
     }
   } else {
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Equal, snapshot);
   }
 }

 // Handle divide by zero.
 if (mir->canBeDivideByZero()) {
   masm.as_cmp(rhs, Imm8(0));
   if (mir->canTruncateInfinities()) {
     if (mir->trapOnError()) {
       Label nonZero;
       masm.ma_b(&nonZero, Assembler::NotEqual);
       masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc());
       masm.bind(&nonZero);
     } else {
       // Infinity|0 == 0
       Label skip;
       masm.ma_b(&skip, Assembler::NotEqual);
       masm.ma_mov(Imm32(0), output);
       masm.ma_b(&done);
       masm.bind(&skip);
     }
   } else {
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Equal, snapshot);
   }
 }

 // Handle negative 0.
 if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
   Label nonzero;
   masm.as_cmp(lhs, Imm8(0));
   masm.ma_b(&nonzero, Assembler::NotEqual);
   masm.as_cmp(rhs, Imm8(0));
   MOZ_ASSERT(mir->fallible());
   bailoutIf(Assembler::LessThan, snapshot);
   masm.bind(&nonzero);
 }
}

void CodeGenerator::visitDivI(LDivI* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register temp = ToRegister(ins->temp0());
 Register output = ToRegister(ins->output());
 MDiv* mir = ins->mir();

 Label done;
 divICommon(mir, lhs, rhs, output, ins->snapshot(), done);

 if (mir->canTruncateRemainder()) {
   masm.ma_sdiv(lhs, rhs, output);
 } else {
   {
     ScratchRegisterScope scratch(masm);
     masm.ma_sdiv(lhs, rhs, temp);
     masm.ma_mul(temp, rhs, scratch);
     masm.ma_cmp(lhs, scratch);
   }
   bailoutIf(Assembler::NotEqual, ins->snapshot());
   masm.ma_mov(temp, output);
 }

 masm.bind(&done);
}

extern "C" {
extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int);
extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int);
}

void CodeGenerator::visitSoftDivI(LSoftDivI* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register output = ToRegister(ins->output());
 MDiv* mir = ins->mir();

 Label done;
 divICommon(mir, lhs, rhs, output, ins->snapshot(), done);

 if (gen->compilingWasm()) {
   masm.Push(InstanceReg);
   int32_t framePushedAfterInstance = masm.framePushed();
   masm.setupWasmABICall(wasm::SymbolicAddress::aeabi_idivmod);
   masm.passABIArg(lhs);
   masm.passABIArg(rhs);
   int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
   masm.callWithABI(mir->trapSiteDesc().bytecodeOffset,
                    wasm::SymbolicAddress::aeabi_idivmod,
                    mozilla::Some(instanceOffset));
   masm.Pop(InstanceReg);
 } else {
   using Fn = int64_t (*)(int, int);
   masm.setupAlignedABICall();
   masm.passABIArg(lhs);
   masm.passABIArg(rhs);
   masm.callWithABI<Fn, __aeabi_idivmod>(
       ABIType::Int64, CheckUnsafeCallWithABI::DontCheckOther);
 }

 // idivmod returns the quotient in r0, and the remainder in r1.
 if (!mir->canTruncateRemainder()) {
   MOZ_ASSERT(mir->fallible());
   masm.as_cmp(r1, Imm8(0));
   bailoutIf(Assembler::NonZero, ins->snapshot());
 }

 masm.bind(&done);
}

void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
 MDiv* mir = ins->mir();
 Register lhs = ToRegister(ins->numerator());
 Register output = ToRegister(ins->output());
 int32_t shift = ins->shift();

 if (shift == 0) {
   masm.ma_mov(lhs, output);
   return;
 }

 if (!mir->isTruncated()) {
   // If the remainder is != 0, bailout since this must be a double.
   {
     // The bailout code also needs the scratch register.
     // Here it is only used as a dummy target to set CC flags.
     ScratchRegisterScope scratch(masm);
     masm.as_mov(scratch, lsl(lhs, 32 - shift), SetCC);
   }
   bailoutIf(Assembler::NonZero, ins->snapshot());
 }

 if (!mir->canBeNegativeDividend()) {
   // Numerator is unsigned, so needs no adjusting. Do the shift.
   masm.as_mov(output, asr(lhs, shift));
   return;
 }

 // Adjust the value so that shifting produces a correctly rounded result
 // when the numerator is negative. See 10-1 "Signed Division by a Known
 // Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight.
 ScratchRegisterScope scratch(masm);

 if (shift > 1) {
   masm.as_mov(scratch, asr(lhs, 31));
   masm.as_add(scratch, lhs, lsr(scratch, 32 - shift));
 } else {
   masm.as_add(scratch, lhs, lsr(lhs, 32 - shift));
 }

 // Do the shift.
 masm.as_mov(output, asr(scratch, shift));
}

void CodeGeneratorARM::modICommon(MMod* mir, Register lhs, Register rhs,
                                 Register output, LSnapshot* snapshot,
                                 Label& done) {
 // X % 0 is bad because it will give garbage (or abort), when it should give
 // NaN.

 if (mir->canBeDivideByZero()) {
   masm.as_cmp(rhs, Imm8(0));
   if (mir->isTruncated()) {
     Label nonZero;
     masm.ma_b(&nonZero, Assembler::NotEqual);
     if (mir->trapOnError()) {
       masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc());
     } else {
       // NaN|0 == 0
       masm.ma_mov(Imm32(0), output);
       masm.ma_b(&done);
     }
     masm.bind(&nonZero);
   } else {
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Equal, snapshot);
   }
 }
}

void CodeGenerator::visitModI(LModI* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register output = ToRegister(ins->output());
 MMod* mir = ins->mir();

 // Contrary to other architectures (notably x86) INT_MIN % -1 doesn't need to
 // be handled separately. |ma_smod| computes the remainder using the |SDIV|
 // and the |MLS| instructions. On overflow, |SDIV| truncates the result to
 // 32-bit and returns INT_MIN, see ARM Architecture Reference Manual, SDIV
 // instruction.
 //
 //   mls(INT_MIN, sdiv(INT_MIN, -1), -1)
 // = INT_MIN - (sdiv(INT_MIN, -1) * -1)
 // = INT_MIN - (INT_MIN * -1)
 // = INT_MIN - INT_MIN
 // = 0
 //
 // And a zero remainder with a negative dividend is already handled below.

 Label done;
 modICommon(mir, lhs, rhs, output, ins->snapshot(), done);

 {
   ScratchRegisterScope scratch(masm);
   masm.ma_smod(lhs, rhs, output, scratch);
 }

 // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0.
 if (mir->canBeNegativeDividend()) {
   if (mir->isTruncated()) {
     // -0.0|0 == 0
   } else {
     MOZ_ASSERT(mir->fallible());
     // See if X < 0
     masm.as_cmp(output, Imm8(0));
     masm.ma_b(&done, Assembler::NotEqual);
     masm.as_cmp(lhs, Imm8(0));
     bailoutIf(Assembler::Signed, ins->snapshot());
   }
 }

 masm.bind(&done);
}

void CodeGenerator::visitSoftModI(LSoftModI* ins) {
 // Extract the registers from this instruction.
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register output = ToRegister(ins->output());
 Register callTemp = ToRegister(ins->temp0());
 MMod* mir = ins->mir();
 Label done;

 // Save the lhs in case we end up with a 0 that should be a -0.0 because lhs <
 // 0.
 MOZ_ASSERT(callTemp != lhs);
 MOZ_ASSERT(callTemp != rhs);
 masm.ma_mov(lhs, callTemp);

 // Prevent INT_MIN % -1.
 //
 // |aeabi_idivmod| is allowed to return any arbitrary value when called with
 // |(INT_MIN, -1)|, see "Run-time ABI for the ARM architecture manual". Most
 // implementations perform a non-trapping signed integer division and
 // return the expected result, i.e. INT_MIN. But since we can't rely on this
 // behavior, handle this case separately here.
 if (mir->canBeNegativeDividend()) {
   {
     ScratchRegisterScope scratch(masm);
     // Sets EQ if lhs == INT_MIN
     masm.ma_cmp(lhs, Imm32(INT_MIN), scratch);
     // If EQ (LHS == INT_MIN), sets EQ if rhs == -1
     masm.ma_cmp(rhs, Imm32(-1), scratch, Assembler::Equal);
   }
   if (mir->isTruncated()) {
     // (INT_MIN % -1)|0 == 0
     Label skip;
     masm.ma_b(&skip, Assembler::NotEqual);
     masm.ma_mov(Imm32(0), output);
     masm.ma_b(&done);
     masm.bind(&skip);
   } else {
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Equal, ins->snapshot());
   }
 }

 modICommon(mir, lhs, rhs, output, ins->snapshot(), done);

 if (gen->compilingWasm()) {
   masm.Push(InstanceReg);
   int32_t framePushedAfterInstance = masm.framePushed();
   masm.setupWasmABICall(wasm::SymbolicAddress::aeabi_idivmod);
   masm.passABIArg(lhs);
   masm.passABIArg(rhs);
   int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
   masm.callWithABI(mir->trapSiteDesc().bytecodeOffset,
                    wasm::SymbolicAddress::aeabi_idivmod,
                    mozilla::Some(instanceOffset));
   masm.Pop(InstanceReg);
 } else {
   using Fn = int64_t (*)(int, int);
   masm.setupAlignedABICall();
   masm.passABIArg(lhs);
   masm.passABIArg(rhs);
   masm.callWithABI<Fn, __aeabi_idivmod>(
       ABIType::Int64, CheckUnsafeCallWithABI::DontCheckOther);
 }

 MOZ_ASSERT(r1 != output);
 masm.move32(r1, output);

 // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0
 if (mir->canBeNegativeDividend()) {
   if (mir->isTruncated()) {
     // -0.0|0 == 0
   } else {
     MOZ_ASSERT(mir->fallible());
     // See if X < 0
     masm.as_cmp(output, Imm8(0));
     masm.ma_b(&done, Assembler::NotEqual);
     masm.as_cmp(callTemp, Imm8(0));
     bailoutIf(Assembler::Signed, ins->snapshot());
   }
 }

 masm.bind(&done);
}

void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
 Register in = ToRegister(ins->input());
 Register out = ToRegister(ins->output());
 MMod* mir = ins->mir();
 Label fin;
 // bug 739870, jbramley has a different sequence that may help with speed
 // here.

 masm.ma_mov(in, out, SetCC);
 masm.ma_b(&fin, Assembler::Zero);
 masm.as_rsb(out, out, Imm8(0), LeaveCC, Assembler::Signed);
 {
   ScratchRegisterScope scratch(masm);
   masm.ma_and(Imm32((1 << ins->shift()) - 1), out, scratch);
 }
 masm.as_rsb(out, out, Imm8(0), SetCC, Assembler::Signed);
 if (mir->canBeNegativeDividend()) {
   if (!mir->isTruncated()) {
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Zero, ins->snapshot());
   } else {
     // -0|0 == 0
   }
 }
 masm.bind(&fin);
}

void CodeGenerator::visitModMaskI(LModMaskI* ins) {
 Register src = ToRegister(ins->input());
 Register dest = ToRegister(ins->output());
 Register tmp1 = ToRegister(ins->temp0());
 Register tmp2 = ToRegister(ins->temp1());
 MMod* mir = ins->mir();

 ScratchRegisterScope scratch(masm);
 SecondScratchRegisterScope scratch2(masm);

 masm.ma_mod_mask(src, dest, tmp1, tmp2, scratch, scratch2, ins->shift());

 if (mir->canBeNegativeDividend()) {
   if (!mir->isTruncated()) {
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Zero, ins->snapshot());
   } else {
     // -0|0 == 0
   }
 }
}

void CodeGeneratorARM::emitBigIntPtrDiv(LBigIntPtrDiv* ins, Register dividend,
                                       Register divisor, Register output) {
 // Callers handle division by zero and integer overflow.

 if (ARMFlags::HasIDIV()) {
   masm.ma_sdiv(dividend, divisor, /* result= */ output);
   return;
 }

 // idivmod returns the quotient in r0, and the remainder in r1.
 MOZ_ASSERT(ToRegister(ins->temp0()) == r0);
 MOZ_ASSERT(ToRegister(ins->temp1()) == r1);

 LiveRegisterSet volatileRegs = liveVolatileRegs(ins);
 volatileRegs.takeUnchecked(output);

 masm.PushRegsInMask(volatileRegs);

 using Fn = int64_t (*)(int, int);
 masm.setupUnalignedABICall(output);
 masm.passABIArg(dividend);
 masm.passABIArg(divisor);
 masm.callWithABI<Fn, __aeabi_idivmod>(ABIType::Int64,
                                       CheckUnsafeCallWithABI::DontCheckOther);
 masm.move32(r0, output);

 masm.PopRegsInMask(volatileRegs);
}

void CodeGeneratorARM::emitBigIntPtrMod(LBigIntPtrMod* ins, Register dividend,
                                       Register divisor, Register output) {
 // Callers handle division by zero and integer overflow.

 if (ARMFlags::HasIDIV()) {
   ScratchRegisterScope scratch(masm);
   masm.ma_smod(dividend, divisor, /* result= */ output, scratch);
   return;
 }

 // idivmod returns the quotient in r0, and the remainder in r1.
 MOZ_ASSERT(ToRegister(ins->temp0()) == r0);
 MOZ_ASSERT(ToRegister(ins->temp1()) == r1);

 LiveRegisterSet volatileRegs = liveVolatileRegs(ins);
 volatileRegs.takeUnchecked(output);

 masm.PushRegsInMask(volatileRegs);

 using Fn = int64_t (*)(int, int);
 masm.setupUnalignedABICall(output);
 masm.passABIArg(dividend);
 masm.passABIArg(divisor);
 masm.callWithABI<Fn, __aeabi_idivmod>(ABIType::Int64,
                                       CheckUnsafeCallWithABI::DontCheckOther);
 masm.move32(r1, output);

 masm.PopRegsInMask(volatileRegs);
}

void CodeGenerator::visitBitNotI(LBitNotI* ins) {
 const LAllocation* input = ins->input();
 const LDefinition* dest = ins->output();
 // This will not actually be true on arm. We can not an imm8m in order to
 // get a wider range of numbers
 MOZ_ASSERT(!input->isConstant());

 masm.ma_mvn(ToRegister(input), ToRegister(dest));
}

void CodeGenerator::visitBitOpI(LBitOpI* ins) {
 const LAllocation* lhs = ins->lhs();
 const LAllocation* rhs = ins->rhs();
 const LDefinition* dest = ins->output();

 ScratchRegisterScope scratch(masm);

 // All of these bitops should be either imm32's, or integer registers.
 switch (ins->bitop()) {
   case JSOp::BitOr:
     if (rhs->isConstant()) {
       masm.ma_orr(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
                   scratch);
     } else {
       masm.ma_orr(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
     }
     break;
   case JSOp::BitXor:
     if (rhs->isConstant()) {
       masm.ma_eor(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
                   scratch);
     } else {
       masm.ma_eor(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
     }
     break;
   case JSOp::BitAnd:
     if (rhs->isConstant()) {
       masm.ma_and(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
                   scratch);
     } else {
       masm.ma_and(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
     }
     break;
   default:
     MOZ_CRASH("unexpected binary opcode");
 }
}

void CodeGenerator::visitShiftI(LShiftI* ins) {
 Register lhs = ToRegister(ins->lhs());
 const LAllocation* rhs = ins->rhs();
 Register dest = ToRegister(ins->output());

 if (rhs->isConstant()) {
   int32_t shift = ToInt32(rhs) & 0x1F;
   switch (ins->bitop()) {
     case JSOp::Lsh:
       if (shift) {
         masm.ma_lsl(Imm32(shift), lhs, dest);
       } else {
         masm.ma_mov(lhs, dest);
       }
       break;
     case JSOp::Rsh:
       if (shift) {
         masm.ma_asr(Imm32(shift), lhs, dest);
       } else {
         masm.ma_mov(lhs, dest);
       }
       break;
     case JSOp::Ursh:
       if (shift) {
         masm.ma_lsr(Imm32(shift), lhs, dest);
       } else {
         // x >>> 0 can overflow.
         masm.ma_mov(lhs, dest);
         if (ins->mir()->toUrsh()->fallible()) {
           masm.as_cmp(dest, Imm8(0));
           bailoutIf(Assembler::LessThan, ins->snapshot());
         }
       }
       break;
     default:
       MOZ_CRASH("Unexpected shift op");
   }
 } else {
   // The shift amounts should be AND'ed into the 0-31 range since arm
   // shifts by the lower byte of the register (it will attempt to shift by
   // 250 if you ask it to).
   ScratchRegisterScope scratch(masm);
   masm.as_and(scratch, ToRegister(rhs), Imm8(0x1F));

   switch (ins->bitop()) {
     case JSOp::Lsh:
       masm.ma_lsl(scratch, lhs, dest);
       break;
     case JSOp::Rsh:
       masm.ma_asr(scratch, lhs, dest);
       break;
     case JSOp::Ursh:
       masm.ma_lsr(scratch, lhs, dest);
       if (ins->mir()->toUrsh()->fallible()) {
         // x >>> 0 can overflow.
         masm.as_cmp(dest, Imm8(0));
         bailoutIf(Assembler::LessThan, ins->snapshot());
       }
       break;
     default:
       MOZ_CRASH("Unexpected shift op");
   }
 }
}

void CodeGenerator::visitShiftIntPtr(LShiftIntPtr* ins) {
 Register lhs = ToRegister(ins->lhs());
 const LAllocation* rhs = ins->rhs();
 Register dest = ToRegister(ins->output());

 if (rhs->isConstant()) {
   int32_t shift = ToIntPtr(rhs) & 0x1F;
   switch (ins->bitop()) {
     case JSOp::Lsh:
       if (shift) {
         masm.ma_lsl(Imm32(shift), lhs, dest);
       } else {
         masm.ma_mov(lhs, dest);
       }
       break;
     case JSOp::Rsh:
       if (shift) {
         masm.ma_asr(Imm32(shift), lhs, dest);
       } else {
         masm.ma_mov(lhs, dest);
       }
       break;
     case JSOp::Ursh:
       if (shift) {
         masm.ma_lsr(Imm32(shift), lhs, dest);
       } else {
         masm.ma_mov(lhs, dest);
       }
       break;
     default:
       MOZ_CRASH("Unexpected shift op");
   }
 } else {
   // The shift amounts should be AND'ed into the 0-31 range since arm
   // shifts by the lower byte of the register (it will attempt to shift by
   // 250 if you ask it to).
   masm.as_and(dest, ToRegister(rhs), Imm8(0x1F));

   switch (ins->bitop()) {
     case JSOp::Lsh:
       masm.ma_lsl(dest, lhs, dest);
       break;
     case JSOp::Rsh:
       masm.ma_asr(dest, lhs, dest);
       break;
     case JSOp::Ursh:
       masm.ma_lsr(dest, lhs, dest);
       break;
     default:
       MOZ_CRASH("Unexpected shift op");
   }
 }
}

void CodeGenerator::visitUrshD(LUrshD* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register temp = ToRegister(ins->temp0());

 const LAllocation* rhs = ins->rhs();
 FloatRegister out = ToFloatRegister(ins->output());

 if (rhs->isConstant()) {
   int32_t shift = ToInt32(rhs) & 0x1F;
   if (shift) {
     masm.ma_lsr(Imm32(shift), lhs, temp);
   } else {
     masm.ma_mov(lhs, temp);
   }
 } else {
   masm.as_and(temp, ToRegister(rhs), Imm8(0x1F));
   masm.ma_lsr(temp, lhs, temp);
 }

 masm.convertUInt32ToDouble(temp, out);
}

void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
 FloatRegister input = ToFloatRegister(ins->input());
 FloatRegister output = ToFloatRegister(ins->output());
 ScratchDoubleScope scratch(masm);

 Label done;

 // Masm.pow(-Infinity, 0.5) == Infinity.
 masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
 masm.compareDouble(input, scratch);
 masm.ma_vneg(scratch, output, Assembler::Equal);
 masm.ma_b(&done, Assembler::Equal);

 // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
 // Adding 0 converts any -0 to 0.
 masm.loadConstantDouble(0.0, scratch);
 masm.ma_vadd(scratch, input, output);
 masm.ma_vsqrt(output, output);

 masm.bind(&done);
}

MoveOperand CodeGeneratorARM::toMoveOperand(LAllocation a) const {
 if (a.isGeneralReg()) {
   return MoveOperand(ToRegister(a));
 }
 if (a.isFloatReg()) {
   return MoveOperand(ToFloatRegister(a));
 }
 MoveOperand::Kind kind = a.isStackArea() ? MoveOperand::Kind::EffectiveAddress
                                          : MoveOperand::Kind::Memory;
 Address addr = ToAddress(a);
 MOZ_ASSERT((addr.offset & 3) == 0);
 return MoveOperand(addr, kind);
}

class js::jit::OutOfLineTableSwitch
   : public OutOfLineCodeBase<CodeGeneratorARM> {
 MTableSwitch* mir_;
 Vector<CodeLabel, 8, JitAllocPolicy> codeLabels_;

 void accept(CodeGeneratorARM* codegen) override {
   codegen->visitOutOfLineTableSwitch(this);
 }

public:
 OutOfLineTableSwitch(TempAllocator& alloc, MTableSwitch* mir)
     : mir_(mir), codeLabels_(alloc) {}

 MTableSwitch* mir() const { return mir_; }

 bool addCodeLabel(CodeLabel label) { return codeLabels_.append(label); }
 CodeLabel codeLabel(unsigned i) { return codeLabels_[i]; }
};

void CodeGeneratorARM::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool) {
 MTableSwitch* mir = ool->mir();

 size_t numCases = mir->numCases();
 for (size_t i = 0; i < numCases; i++) {
   LBlock* caseblock =
       skipTrivialBlocks(mir->getCase(numCases - 1 - i))->lir();
   Label* caseheader = caseblock->label();
   uint32_t caseoffset = caseheader->offset();

   // The entries of the jump table need to be absolute addresses and thus
   // must be patched after codegen is finished.
   CodeLabel cl = ool->codeLabel(i);
   cl.target()->bind(caseoffset);
   masm.addCodeLabel(cl);
 }
}

void CodeGeneratorARM::emitTableSwitchDispatch(MTableSwitch* mir,
                                              Register index, Register base) {
 // The code generated by this is utter hax.
 // The end result looks something like:
 // SUBS index, input, #base
 // RSBSPL index, index, #max
 // LDRPL pc, pc, index lsl 2
 // B default

 // If the range of targets in N through M, we first subtract off the lowest
 // case (N), which both shifts the arguments into the range 0 to (M - N)
 // with and sets the MInus flag if the argument was out of range on the low
 // end.

 // Then we a reverse subtract with the size of the jump table, which will
 // reverse the order of range (It is size through 0, rather than 0 through
 // size). The main purpose of this is that we set the same flag as the lower
 // bound check for the upper bound check. Lastly, we do this conditionally
 // on the previous check succeeding.

 // Then we conditionally load the pc offset by the (reversed) index (times
 // the address size) into the pc, which branches to the correct case. NOTE:
 // when we go to read the pc, the value that we get back is the pc of the
 // current instruction *PLUS 8*. This means that ldr foo, [pc, +0] reads
 // $pc+8. In other words, there is an empty word after the branch into the
 // switch table before the table actually starts. Since the only other
 // unhandled case is the default case (both out of range high and out of
 // range low) I then insert a branch to default case into the extra slot,
 // which ensures we don't attempt to execute the address table.
 Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();

 ScratchRegisterScope scratch(masm);

 int32_t cases = mir->numCases();
 // Lower value with low value.
 masm.ma_sub(index, Imm32(mir->low()), index, scratch, SetCC);
 masm.ma_rsb(index, Imm32(cases - 1), index, scratch, SetCC,
             Assembler::NotSigned);
 // Inhibit pools within the following sequence because we are indexing into
 // a pc relative table. The region will have one instruction for ma_ldr, one
 // for ma_b, and each table case takes one word.
 AutoForbidPoolsAndNops afp(&masm, 1 + 1 + cases);
 masm.ma_ldr(DTRAddr(pc, DtrRegImmShift(index, LSL, 2)), pc, Offset,
             Assembler::NotSigned);
 masm.ma_b(defaultcase);

 // To fill in the CodeLabels for the case entries, we need to first generate
 // the case entries (we don't yet know their offsets in the instruction
 // stream).
 OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(alloc(), mir);
 for (int32_t i = 0; i < cases; i++) {
   CodeLabel cl;
   masm.writeCodePointer(&cl);
   masm.propagateOOM(ool->addCodeLabel(cl));
 }
 addOutOfLineCode(ool, mir);
}

void CodeGenerator::visitMathD(LMathD* math) {
 FloatRegister src1 = ToFloatRegister(math->lhs());
 FloatRegister src2 = ToFloatRegister(math->rhs());
 FloatRegister output = ToFloatRegister(math->output());

 switch (math->jsop()) {
   case JSOp::Add:
     masm.ma_vadd(src1, src2, output);
     break;
   case JSOp::Sub:
     masm.ma_vsub(src1, src2, output);
     break;
   case JSOp::Mul:
     masm.ma_vmul(src1, src2, output);
     break;
   case JSOp::Div:
     masm.ma_vdiv(src1, src2, output);
     break;
   default:
     MOZ_CRASH("unexpected opcode");
 }
}

void CodeGenerator::visitMathF(LMathF* math) {
 FloatRegister src1 = ToFloatRegister(math->lhs());
 FloatRegister src2 = ToFloatRegister(math->rhs());
 FloatRegister output = ToFloatRegister(math->output());

 switch (math->jsop()) {
   case JSOp::Add:
     masm.ma_vadd_f32(src1, src2, output);
     break;
   case JSOp::Sub:
     masm.ma_vsub_f32(src1, src2, output);
     break;
   case JSOp::Mul:
     masm.ma_vmul_f32(src1, src2, output);
     break;
   case JSOp::Div:
     masm.ma_vdiv_f32(src1, src2, output);
     break;
   default:
     MOZ_CRASH("unexpected opcode");
 }
}

void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
 emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
                    ins->mir());
}

void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
   LWasmBuiltinTruncateDToInt32* ins) {
 emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
                    ins->mir());
}

void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
 emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
                     ins->mir());
}

void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
   LWasmBuiltinTruncateFToInt32* ins) {
 emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
                     ins->mir());
}

void CodeGenerator::visitBox(LBox* box) {
 const LDefinition* type = box->getDef(TYPE_INDEX);

 MOZ_ASSERT(!box->payload()->isConstant());

 // On arm, the input operand and the output payload have the same virtual
 // register. All that needs to be written is the type tag for the type
 // definition.
 masm.ma_mov(Imm32(MIRTypeToTag(box->type())), ToRegister(type));
}

void CodeGenerator::visitBoxFloatingPoint(LBoxFloatingPoint* box) {
 const AnyRegister in = ToAnyRegister(box->input());
 const ValueOperand out = ToOutValue(box);

 masm.moveValue(TypedOrValueRegister(box->type(), in), out);
}

void CodeGenerator::visitUnbox(LUnbox* unbox) {
 // Note that for unbox, the type and payload indexes are switched on the
 // inputs.
 MUnbox* mir = unbox->mir();
 Register type = ToRegister(unbox->type());
 Register payload = ToRegister(unbox->payload());
 Register output = ToRegister(unbox->output());

 mozilla::Maybe<ScratchRegisterScope> scratch;
 scratch.emplace(masm);

 JSValueTag tag = MIRTypeToTag(mir->type());
 if (mir->fallible()) {
   masm.ma_cmp(type, Imm32(tag), *scratch);
   bailoutIf(Assembler::NotEqual, unbox->snapshot());
 } else {
#ifdef DEBUG
   Label ok;
   masm.ma_cmp(type, Imm32(tag), *scratch);
   masm.ma_b(&ok, Assembler::Equal);
   scratch.reset();
   masm.assumeUnreachable("Infallible unbox type mismatch");
   masm.bind(&ok);
#endif
 }

 // Note: If spectreValueMasking is disabled, then this instruction will
 // default to a no-op as long as the lowering allocate the same register for
 // the output and the payload.
 masm.unboxNonDouble(ValueOperand(type, payload), output,
                     ValueTypeFromMIRType(mir->type()));
}

void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
 const LAllocation* opd = test->input();
 masm.ma_vcmpz(ToFloatRegister(opd));
 masm.as_vmrs(pc);

 MBasicBlock* ifTrue = test->ifTrue();
 MBasicBlock* ifFalse = test->ifFalse();
 // If the compare set the 0 bit, then the result is definitely false.
 jumpToBlock(ifFalse, Assembler::Zero);
 // It is also false if one of the operands is NAN, which is shown as
 // Overflow.
 jumpToBlock(ifFalse, Assembler::Overflow);
 jumpToBlock(ifTrue);
}

void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
 const LAllocation* opd = test->input();
 masm.ma_vcmpz_f32(ToFloatRegister(opd));
 masm.as_vmrs(pc);

 MBasicBlock* ifTrue = test->ifTrue();
 MBasicBlock* ifFalse = test->ifFalse();
 // If the compare set the 0 bit, then the result is definitely false.
 jumpToBlock(ifFalse, Assembler::Zero);
 // It is also false if one of the operands is NAN, which is shown as
 // Overflow.
 jumpToBlock(ifFalse, Assembler::Overflow);
 jumpToBlock(ifTrue);
}

void CodeGenerator::visitCompareD(LCompareD* comp) {
 FloatRegister lhs = ToFloatRegister(comp->left());
 FloatRegister rhs = ToFloatRegister(comp->right());

 Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
 masm.compareDouble(lhs, rhs);
 masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
              ToRegister(comp->output()));
}

void CodeGenerator::visitCompareF(LCompareF* comp) {
 FloatRegister lhs = ToFloatRegister(comp->left());
 FloatRegister rhs = ToFloatRegister(comp->right());

 Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
 masm.compareFloat(lhs, rhs);
 masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
              ToRegister(comp->output()));
}

void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
 FloatRegister lhs = ToFloatRegister(comp->left());
 FloatRegister rhs = ToFloatRegister(comp->right());

 Assembler::DoubleCondition cond =
     JSOpToDoubleCondition(comp->cmpMir()->jsop());
 masm.compareDouble(lhs, rhs);
 emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
            comp->ifFalse());
}

void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
 FloatRegister lhs = ToFloatRegister(comp->left());
 FloatRegister rhs = ToFloatRegister(comp->right());

 Assembler::DoubleCondition cond =
     JSOpToDoubleCondition(comp->cmpMir()->jsop());
 masm.compareFloat(lhs, rhs);
 emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
            comp->ifFalse());
}

void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
 masm.convertUInt32ToDouble(ToRegister(lir->input()),
                            ToFloatRegister(lir->output()));
}

void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
 masm.convertUInt32ToFloat32(ToRegister(lir->input()),
                             ToFloatRegister(lir->output()));
}

void CodeGenerator::visitNotD(LNotD* ins) {
 // Since this operation is not, we want to set a bit if the double is
 // falsey, which means 0.0, -0.0 or NaN. When comparing with 0, an input of
 // 0 will set the Z bit (30) and NaN will set the V bit (28) of the APSR.
 FloatRegister opd = ToFloatRegister(ins->input());
 Register dest = ToRegister(ins->output());

 // Do the compare.
 masm.ma_vcmpz(opd);
 // TODO There are three variations here to compare performance-wise.
 bool nocond = true;
 if (nocond) {
   // Load the value into the dest register.
   masm.as_vmrs(dest);
   masm.ma_lsr(Imm32(28), dest, dest);
   // 28 + 2 = 30
   masm.ma_alu(dest, lsr(dest, 2), dest, OpOrr);
   masm.as_and(dest, dest, Imm8(1));
 } else {
   masm.as_vmrs(pc);
   masm.ma_mov(Imm32(0), dest);
   masm.ma_mov(Imm32(1), dest, Assembler::Equal);
   masm.ma_mov(Imm32(1), dest, Assembler::Overflow);
 }
}

void CodeGenerator::visitNotF(LNotF* ins) {
 // Since this operation is not, we want to set a bit if the double is
 // falsey, which means 0.0, -0.0 or NaN. When comparing with 0, an input of
 // 0 will set the Z bit (30) and NaN will set the V bit (28) of the APSR.
 FloatRegister opd = ToFloatRegister(ins->input());
 Register dest = ToRegister(ins->output());

 // Do the compare.
 masm.ma_vcmpz_f32(opd);
 // TODO There are three variations here to compare performance-wise.
 bool nocond = true;
 if (nocond) {
   // Load the value into the dest register.
   masm.as_vmrs(dest);
   masm.ma_lsr(Imm32(28), dest, dest);
   // 28 + 2 = 30
   masm.ma_alu(dest, lsr(dest, 2), dest, OpOrr);
   masm.as_and(dest, dest, Imm8(1));
 } else {
   masm.as_vmrs(pc);
   masm.ma_mov(Imm32(0), dest);
   masm.ma_mov(Imm32(1), dest, Assembler::Equal);
   masm.ma_mov(Imm32(1), dest, Assembler::Overflow);
 }
}

void CodeGeneratorARM::generateInvalidateEpilogue() {
 // Ensure that there is enough space in the buffer for the OsiPoint patching
 // to occur. Otherwise, we could overwrite the invalidation epilogue.
 for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
   masm.nop();
 }

 masm.bind(&invalidate_);

 // Push the return address of the point that we bailed out at onto the stack.
 masm.Push(lr);

 // Push the Ion script onto the stack (when we determine what that pointer
 // is).
 invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));

 // Jump to the invalidator which will replace the current frame.
 TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
 masm.jump(thunk);
}

void CodeGenerator::visitCompareExchangeTypedArrayElement(
   LCompareExchangeTypedArrayElement* lir) {
 Register elements = ToRegister(lir->elements());
 AnyRegister output = ToAnyRegister(lir->output());
 Register temp = ToTempRegisterOrInvalid(lir->temp0());

 Register oldval = ToRegister(lir->oldval());
 Register newval = ToRegister(lir->newval());

 Scalar::Type arrayType = lir->mir()->arrayType();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 dest.match([&](const auto& dest) {
   masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
                          newval, temp, output);
 });
}

void CodeGenerator::visitAtomicExchangeTypedArrayElement(
   LAtomicExchangeTypedArrayElement* lir) {
 Register elements = ToRegister(lir->elements());
 AnyRegister output = ToAnyRegister(lir->output());
 Register temp = ToTempRegisterOrInvalid(lir->temp0());

 Register value = ToRegister(lir->value());

 Scalar::Type arrayType = lir->mir()->arrayType();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 dest.match([&](const auto& dest) {
   masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
                         output);
 });
}

void CodeGenerator::visitAtomicTypedArrayElementBinop(
   LAtomicTypedArrayElementBinop* lir) {
 MOZ_ASSERT(!lir->mir()->isForEffect());

 AnyRegister output = ToAnyRegister(lir->output());
 Register elements = ToRegister(lir->elements());
 Register flagTemp = ToRegister(lir->temp0());
 Register outTemp = ToTempRegisterOrInvalid(lir->temp1());
 Register value = ToRegister(lir->value());

 Scalar::Type arrayType = lir->mir()->arrayType();

 auto mem = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 mem.match([&](const auto& mem) {
   masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
                        lir->mir()->operation(), value, mem, flagTemp, outTemp,
                        output);
 });
}

void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
   LAtomicTypedArrayElementBinopForEffect* lir) {
 MOZ_ASSERT(lir->mir()->isForEffect());

 Register elements = ToRegister(lir->elements());
 Register flagTemp = ToRegister(lir->temp0());
 Register value = ToRegister(lir->value());
 Scalar::Type arrayType = lir->mir()->arrayType();

 auto mem = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 mem.match([&](const auto& mem) {
   masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
                         lir->mir()->operation(), value, mem, flagTemp);
 });
}

void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
 Register elements = ToRegister(lir->elements());
 Register64 out = ToOutRegister64(lir);

 Scalar::Type storageType = lir->mir()->storageType();

 auto source = ToAddressOrBaseIndex(elements, lir->index(), storageType);

 source.match([&](const auto& source) {
   masm.atomicLoad64(Synchronization::Load(), source, out);
 });
}

void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
 Register elements = ToRegister(lir->elements());
 Register64 value = ToRegister64(lir->value());
 Register64 temp = ToRegister64(lir->temp0());

 Scalar::Type writeType = lir->mir()->writeType();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), writeType);

 dest.match([&](const auto& dest) {
   masm.atomicStore64(Synchronization::Store(), dest, value, temp);
 });
}

void CodeGenerator::visitCompareExchangeTypedArrayElement64(
   LCompareExchangeTypedArrayElement64* lir) {
 Register elements = ToRegister(lir->elements());
 Register64 oldval = ToRegister64(lir->oldval());
 Register64 newval = ToRegister64(lir->newval());
 Register64 out = ToOutRegister64(lir);

 Scalar::Type arrayType = lir->mir()->arrayType();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 dest.match([&](const auto& dest) {
   masm.compareExchange64(Synchronization::Full(), dest, oldval, newval, out);
 });
}

void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
   LAtomicExchangeTypedArrayElement64* lir) {
 Register elements = ToRegister(lir->elements());
 Register64 value = ToRegister64(lir->value());
 Register64 out = ToOutRegister64(lir);

 Scalar::Type arrayType = lir->mir()->arrayType();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 dest.match([&](const auto& dest) {
   masm.atomicExchange64(Synchronization::Full(), dest, value, out);
 });
}

void CodeGenerator::visitAtomicTypedArrayElementBinop64(
   LAtomicTypedArrayElementBinop64* lir) {
 MOZ_ASSERT(!lir->mir()->isForEffect());

 Register elements = ToRegister(lir->elements());
 Register64 value = ToRegister64(lir->value());
 Register64 temp = ToRegister64(lir->temp0());
 Register64 out = ToOutRegister64(lir);

 Scalar::Type arrayType = lir->mir()->arrayType();
 AtomicOp atomicOp = lir->mir()->operation();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 dest.match([&](const auto& dest) {
   masm.atomicFetchOp64(Synchronization::Full(), atomicOp, value, dest, temp,
                        out);
 });
}

void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
   LAtomicTypedArrayElementBinopForEffect64* lir) {
 MOZ_ASSERT(lir->mir()->isForEffect());

 Register elements = ToRegister(lir->elements());
 Register64 value = ToRegister64(lir->value());
 Register64 temp = ToRegister64(lir->temp0());

 Scalar::Type arrayType = lir->mir()->arrayType();
 AtomicOp atomicOp = lir->mir()->operation();

 auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);

 dest.match([&](const auto& dest) {
   masm.atomicEffectOp64(Synchronization::Full(), atomicOp, value, dest, temp);
 });
}

void CodeGenerator::visitWasmSelect(LWasmSelect* ins) {
 MIRType mirType = ins->mir()->type();

 Register cond = ToRegister(ins->condExpr());
 masm.as_cmp(cond, Imm8(0));

 if (mirType == MIRType::Int32 || mirType == MIRType::WasmAnyRef) {
   Register falseExpr = ToRegister(ins->falseExpr());
   Register out = ToRegister(ins->output());
   MOZ_ASSERT(ToRegister(ins->trueExpr()) == out,
              "true expr input is reused for output");
   masm.ma_mov(falseExpr, out, LeaveCC, Assembler::Zero);
   return;
 }

 FloatRegister out = ToFloatRegister(ins->output());
 MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out,
            "true expr input is reused for output");

 FloatRegister falseExpr = ToFloatRegister(ins->falseExpr());

 if (mirType == MIRType::Double) {
   masm.moveDouble(falseExpr, out, Assembler::Zero);
 } else if (mirType == MIRType::Float32) {
   masm.moveFloat32(falseExpr, out, Assembler::Zero);
 } else {
   MOZ_CRASH("unhandled type in visitWasmSelect!");
 }
}

// We expect to handle only the case where compare is {U,}Int32 and select is
// {U,}Int32, and the "true" input is reused for the output.
void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
 bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
                   ins->compareType() == MCompare::Compare_UInt32;
 bool selIs32bit = ins->mir()->type() == MIRType::Int32;

 MOZ_RELEASE_ASSERT(
     cmpIs32bit && selIs32bit,
     "CodeGenerator::visitWasmCompareAndSelect: unexpected types");

 Register trueExprAndDest = ToRegister(ins->output());
 MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
            "true expr input is reused for output");

 Assembler::Condition cond = Assembler::InvertCondition(
     JSOpToCondition(ins->compareType(), ins->jsop()));
 const LAllocation* rhs = ins->rightExpr();
 const LAllocation* falseExpr = ins->ifFalseExpr();
 Register lhs = ToRegister(ins->leftExpr());

 masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
                  trueExprAndDest);
}

void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
 const MAsmJSLoadHeap* mir = ins->mir();

 const LAllocation* ptr = ins->ptr();
 const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();

 Scalar::Type accessType = mir->access().type();
 bool isSigned = Scalar::isSignedIntType(accessType);
 int size = Scalar::byteSize(accessType) * 8;
 bool isFloat = Scalar::isFloatingType(accessType);

 if (ptr->isConstant()) {
   MOZ_ASSERT(!mir->needsBoundsCheck());
   int32_t ptrImm = ptr->toConstant()->toInt32();
   MOZ_ASSERT(ptrImm >= 0);
   if (isFloat) {
     ScratchRegisterScope scratch(masm);
     VFPRegister vd(ToFloatRegister(ins->output()));
     if (size == 32) {
       masm.ma_vldr(Address(HeapReg, ptrImm), vd.singleOverlay(), scratch,
                    Assembler::Always);
     } else {
       masm.ma_vldr(Address(HeapReg, ptrImm), vd, scratch, Assembler::Always);
     }
   } else {
     ScratchRegisterScope scratch(masm);
     masm.ma_dataTransferN(IsLoad, size, isSigned, HeapReg, Imm32(ptrImm),
                           ToRegister(ins->output()), scratch, Offset,
                           Assembler::Always);
   }
 } else {
   Register ptrReg = ToRegister(ptr);
   if (isFloat) {
     FloatRegister output = ToFloatRegister(ins->output());
     if (size == 32) {
       output = output.singleOverlay();
     }

     Assembler::Condition cond = Assembler::Always;
     if (mir->needsBoundsCheck()) {
       Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
       masm.as_cmp(ptrReg, O2Reg(boundsCheckLimitReg));
       if (size == 32) {
         masm.ma_vimm_f32(GenericNaN(), output, Assembler::AboveOrEqual);
       } else {
         masm.ma_vimm(GenericNaN(), output, Assembler::AboveOrEqual);
       }
       cond = Assembler::Below;
     }

     ScratchRegisterScope scratch(masm);
     masm.ma_vldr(output, HeapReg, ptrReg, scratch, 0, cond);
   } else {
     Register output = ToRegister(ins->output());

     Assembler::Condition cond = Assembler::Always;
     if (mir->needsBoundsCheck()) {
       Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
       masm.as_cmp(ptrReg, O2Reg(boundsCheckLimitReg));
       masm.ma_mov(Imm32(0), output, Assembler::AboveOrEqual);
       cond = Assembler::Below;
     }

     ScratchRegisterScope scratch(masm);
     masm.ma_dataTransferN(IsLoad, size, isSigned, HeapReg, ptrReg, output,
                           scratch, Offset, cond);
   }
 }
}

template <typename T>
void CodeGeneratorARM::emitWasmLoad(T* lir) {
 const MWasmLoad* mir = lir->mir();
 MIRType resultType = mir->type();
 Register ptr;
 Register memoryBase = ToRegister(lir->memoryBase());

 if (mir->access().offset32() || mir->access().type() == Scalar::Int64) {
   ptr = ToRegister(lir->temp0());
 } else {
   MOZ_ASSERT(lir->temp0()->isBogusTemp());
   ptr = ToRegister(lir->ptr());
 }

 if (resultType == MIRType::Int64) {
   masm.wasmLoadI64(mir->access(), memoryBase, ptr, ptr, ToOutRegister64(lir));
 } else {
   masm.wasmLoad(mir->access(), memoryBase, ptr, ptr,
                 ToAnyRegister(lir->output()));
 }
}

void CodeGenerator::visitWasmLoad(LWasmLoad* lir) { emitWasmLoad(lir); }

void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) { emitWasmLoad(lir); }

void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
 MWasmAddOffset* mir = lir->mir();
 Register base = ToRegister(lir->base());
 Register out = ToRegister(lir->output());

 ScratchRegisterScope scratch(masm);
 masm.ma_add(base, Imm32(mir->offset()), out, scratch, SetCC);
 auto* ool = new (alloc()) LambdaOutOfLineCode([=, this](OutOfLineCode& ool) {
   masm.wasmTrap(wasm::Trap::OutOfBounds, mir->trapSiteDesc());
 });
 addOutOfLineCode(ool, mir);
 masm.ma_b(ool->entry(), Assembler::CarrySet);
}

void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
 MWasmAddOffset* mir = lir->mir();
 Register64 base = ToRegister64(lir->base());
 Register64 out = ToOutRegister64(lir);
 MOZ_ASSERT(base.low != out.high && base.high != out.low);

 ScratchRegisterScope scratch(masm);
 masm.ma_add(base.low, Imm32(mir->offset()), out.low, scratch, SetCC);
 masm.ma_adc(base.high, Imm32(mir->offset() >> 32), out.high, scratch, SetCC);
 auto* ool = new (alloc()) LambdaOutOfLineCode([=, this](OutOfLineCode& ool) {
   masm.wasmTrap(wasm::Trap::OutOfBounds, mir->trapSiteDesc());
 });
 addOutOfLineCode(ool, mir);
 masm.ma_b(ool->entry(), Assembler::CarrySet);
}

template <typename T>
void CodeGeneratorARM::emitWasmStore(T* lir) {
 const MWasmStore* mir = lir->mir();
 Register memoryBase = ToRegister(lir->memoryBase());

 if constexpr (std::is_same_v<T, LWasmStoreI64>) {
   Register64 value = ToRegister64(lir->value());
   Register ptr = ToRegister(lir->temp0());
   masm.wasmStoreI64(mir->access(), value, memoryBase, ptr, ptr);
 } else {
   // Maybe add the offset.
   Register ptr;
   if (mir->access().offset32()) {
     ptr = ToRegister(lir->temp0());
   } else {
     MOZ_ASSERT(lir->temp0()->isBogusTemp());
     ptr = ToRegister(lir->ptr());
   }

   masm.wasmStore(mir->access(), ToAnyRegister(lir->value()), memoryBase, ptr,
                  ptr);
 }
}

void CodeGenerator::visitWasmStore(LWasmStore* lir) { emitWasmStore(lir); }

void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) {
 emitWasmStore(lir);
}

void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
 const MAsmJSStoreHeap* mir = ins->mir();

 const LAllocation* ptr = ins->ptr();
 const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();

 Scalar::Type accessType = mir->access().type();
 bool isSigned = accessType == Scalar::Int32 || accessType == Scalar::Uint32;
 int size = Scalar::byteSize(accessType) * 8;
 bool isFloat = Scalar::isFloatingType(accessType);

 if (ptr->isConstant()) {
   MOZ_ASSERT(!mir->needsBoundsCheck());
   int32_t ptrImm = ptr->toConstant()->toInt32();
   MOZ_ASSERT(ptrImm >= 0);
   if (isFloat) {
     VFPRegister vd(ToFloatRegister(ins->value()));
     Address addr(HeapReg, ptrImm);
     if (size == 32) {
       masm.storeFloat32(vd, addr);
     } else {
       masm.storeDouble(vd, addr);
     }
   } else {
     ScratchRegisterScope scratch(masm);
     masm.ma_dataTransferN(IsStore, size, isSigned, HeapReg, Imm32(ptrImm),
                           ToRegister(ins->value()), scratch, Offset,
                           Assembler::Always);
   }
 } else {
   Register ptrReg = ToRegister(ptr);

   Assembler::Condition cond = Assembler::Always;
   if (mir->needsBoundsCheck()) {
     Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
     masm.as_cmp(ptrReg, O2Reg(boundsCheckLimitReg));
     cond = Assembler::Below;
   }

   if (isFloat) {
     ScratchRegisterScope scratch(masm);
     FloatRegister value = ToFloatRegister(ins->value());
     if (size == 32) {
       value = value.singleOverlay();
     }

     masm.ma_vstr(value, HeapReg, ptrReg, scratch, 0, Assembler::Below);
   } else {
     ScratchRegisterScope scratch(masm);
     Register value = ToRegister(ins->value());
     masm.ma_dataTransferN(IsStore, size, isSigned, HeapReg, ptrReg, value,
                           scratch, Offset, cond);
   }
 }
}

void CodeGenerator::visitWasmCompareExchangeHeap(
   LWasmCompareExchangeHeap* ins) {
 MWasmCompareExchangeHeap* mir = ins->mir();

 const LAllocation* ptr = ins->ptr();
 Register ptrReg = ToRegister(ptr);
 Register memoryBase = ToRegister(ins->memoryBase());
 BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32());

 Register oldval = ToRegister(ins->oldValue());
 Register newval = ToRegister(ins->newValue());
 Register out = ToRegister(ins->output());

 masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, out);
}

void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
 MWasmAtomicExchangeHeap* mir = ins->mir();

 Register ptrReg = ToRegister(ins->ptr());
 Register value = ToRegister(ins->value());
 Register memoryBase = ToRegister(ins->memoryBase());
 Register output = ToRegister(ins->output());
 BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32());

 masm.wasmAtomicExchange(mir->access(), srcAddr, value, output);
}

void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
 MWasmAtomicBinopHeap* mir = ins->mir();
 MOZ_ASSERT(mir->hasUses());

 Register ptrReg = ToRegister(ins->ptr());
 Register memoryBase = ToRegister(ins->memoryBase());
 Register flagTemp = ToRegister(ins->temp0());
 Register output = ToRegister(ins->output());
 const LAllocation* value = ins->value();
 AtomicOp op = mir->operation();

 BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32());
 masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), srcAddr,
                        flagTemp, output);
}

void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
   LWasmAtomicBinopHeapForEffect* ins) {
 MWasmAtomicBinopHeap* mir = ins->mir();
 MOZ_ASSERT(!mir->hasUses());

 Register ptrReg = ToRegister(ins->ptr());
 Register memoryBase = ToRegister(ins->memoryBase());
 Register flagTemp = ToRegister(ins->temp0());
 const LAllocation* value = ins->value();
 AtomicOp op = mir->operation();

 BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32());
 masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), srcAddr,
                         flagTemp);
}

void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
 const MWasmStackArg* mir = ins->mir();
 Address dst(StackPointer, mir->spOffset());
 ScratchRegisterScope scratch(masm);
 SecondScratchRegisterScope scratch2(masm);

 if (ins->arg()->isConstant()) {
   masm.ma_mov(Imm32(ToInt32(ins->arg())), scratch);
   masm.ma_str(scratch, dst, scratch2);
 } else {
   if (ins->arg()->isGeneralReg()) {
     masm.ma_str(ToRegister(ins->arg()), dst, scratch);
   } else {
     masm.ma_vstr(ToFloatRegister(ins->arg()), dst, scratch);
   }
 }
}

void CodeGenerator::visitUDiv(LUDiv* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register output = ToRegister(ins->output());

 Label done;
 generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), ins->mir());

 masm.ma_udiv(lhs, rhs, output);

 // Check for large unsigned result - represent as double.
 if (!ins->mir()->isTruncated()) {
   MOZ_ASSERT(ins->mir()->fallible());
   masm.as_cmp(output, Imm8(0));
   bailoutIf(Assembler::LessThan, ins->snapshot());
 }

 // Check for non-zero remainder if not truncating to int.
 if (!ins->mir()->canTruncateRemainder()) {
   MOZ_ASSERT(ins->mir()->fallible());
   {
     ScratchRegisterScope scratch(masm);
     masm.ma_mul(rhs, output, scratch);
     masm.ma_cmp(scratch, lhs);
   }
   bailoutIf(Assembler::NotEqual, ins->snapshot());
 }

 if (done.used()) {
   masm.bind(&done);
 }
}

void CodeGenerator::visitUMod(LUMod* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register output = ToRegister(ins->output());

 Label done;
 generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), ins->mir());

 {
   ScratchRegisterScope scratch(masm);
   masm.ma_umod(lhs, rhs, output, scratch);
 }

 // Check for large unsigned result - represent as double.
 if (!ins->mir()->isTruncated()) {
   MOZ_ASSERT(ins->mir()->fallible());
   masm.as_cmp(output, Imm8(0));
   bailoutIf(Assembler::LessThan, ins->snapshot());
 }

 if (done.used()) {
   masm.bind(&done);
 }
}

template <class T>
void CodeGeneratorARM::generateUDivModZeroCheck(Register rhs, Register output,
                                               Label* done,
                                               LSnapshot* snapshot, T* mir) {
 if (!mir) {
   return;
 }
 if (mir->canBeDivideByZero()) {
   masm.as_cmp(rhs, Imm8(0));
   if (mir->isTruncated()) {
     if (mir->trapOnError()) {
       Label nonZero;
       masm.ma_b(&nonZero, Assembler::NotEqual);
       masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc());
       masm.bind(&nonZero);
     } else {
       Label skip;
       masm.ma_b(&skip, Assembler::NotEqual);
       // Infinity|0 == 0
       masm.ma_mov(Imm32(0), output);
       masm.ma_b(done);
       masm.bind(&skip);
     }
   } else {
     // Bailout for divide by zero
     MOZ_ASSERT(mir->fallible());
     bailoutIf(Assembler::Equal, snapshot);
   }
 }
}

void CodeGenerator::visitSoftUDivOrMod(LSoftUDivOrMod* ins) {
 Register lhs = ToRegister(ins->lhs());
 Register rhs = ToRegister(ins->rhs());
 Register output = ToRegister(ins->output());

 MOZ_ASSERT(lhs == r0);
 MOZ_ASSERT(rhs == r1);
 MOZ_ASSERT(output == r0);

 Label done;
 MDiv* div = ins->mir()->isDiv() ? ins->mir()->toDiv() : nullptr;
 MMod* mod = !div ? ins->mir()->toMod() : nullptr;

 generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), div);
 generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), mod);

 if (gen->compilingWasm()) {
   masm.Push(InstanceReg);
   int32_t framePushedAfterInstance = masm.framePushed();
   masm.setupWasmABICall(wasm::SymbolicAddress::aeabi_uidivmod);
   masm.passABIArg(lhs);
   masm.passABIArg(rhs);
   wasm::BytecodeOffset bytecodeOffset =
       (div ? div->trapSiteDesc().bytecodeOffset
            : mod->trapSiteDesc().bytecodeOffset);
   int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
   masm.callWithABI(bytecodeOffset, wasm::SymbolicAddress::aeabi_uidivmod,
                    mozilla::Some(instanceOffset));
   masm.Pop(InstanceReg);
 } else {
   using Fn = int64_t (*)(int, int);
   masm.setupAlignedABICall();
   masm.passABIArg(lhs);
   masm.passABIArg(rhs);
   masm.callWithABI<Fn, __aeabi_uidivmod>(
       ABIType::Int64, CheckUnsafeCallWithABI::DontCheckOther);
 }

 if (mod) {
   MOZ_ASSERT(output == r0, "output should not be r1 for mod");
   masm.move32(r1, output);
 }

 // uidivmod returns the quotient in r0, and the remainder in r1.
 if (div && !div->canTruncateRemainder()) {
   MOZ_ASSERT(div->fallible());
   masm.as_cmp(r1, Imm8(0));
   bailoutIf(Assembler::NonZero, ins->snapshot());
 }

 // Bailout for big unsigned results
 if ((div && !div->isTruncated()) || (mod && !mod->isTruncated())) {
   DebugOnly<bool> isFallible =
       (div && div->fallible()) || (mod && mod->fallible());
   MOZ_ASSERT(isFallible);
   masm.as_cmp(output, Imm8(0));
   bailoutIf(Assembler::LessThan, ins->snapshot());
 }

 masm.bind(&done);
}

void CodeGenerator::visitEffectiveAddress3(LEffectiveAddress3* ins) {
 const MEffectiveAddress3* mir = ins->mir();
 Register base = ToRegister(ins->base());
 Register index = ToRegister(ins->index());
 Register output = ToRegister(ins->output());

 masm.as_add(output, base, lsl(index, mir->scale()));
 if (mir->displacement() != 0) {
   ScratchRegisterScope scratch(masm);
   masm.ma_add(Imm32(mir->displacement()), output, scratch);
 }
}

void CodeGenerator::visitEffectiveAddress2(LEffectiveAddress2* ins) {
 const MEffectiveAddress2* mir = ins->mir();
 Register index = ToRegister(ins->index());
 Register output = ToRegister(ins->output());

 masm.ma_lsl(Imm32(mir->scale()), index, output);
 ScratchRegisterScope scratch(masm);
 masm.ma_add(Imm32(mir->displacement()), output, scratch);
}

void CodeGenerator::visitNegI(LNegI* ins) {
 Register input = ToRegister(ins->input());
 masm.ma_neg(input, ToRegister(ins->output()));
}

void CodeGenerator::visitNegI64(LNegI64* ins) {
 Register64 input = ToRegister64(ins->input());
 MOZ_ASSERT(input == ToOutRegister64(ins));
 masm.neg64(input);
}

void CodeGenerator::visitNegD(LNegD* ins) {
 FloatRegister input = ToFloatRegister(ins->input());
 masm.ma_vneg(input, ToFloatRegister(ins->output()));
}

void CodeGenerator::visitNegF(LNegF* ins) {
 FloatRegister input = ToFloatRegister(ins->input());
 masm.ma_vneg_f32(input, ToFloatRegister(ins->output()));
}

void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
 auto input = ToFloatRegister(lir->input());
 auto output = ToRegister(lir->output());

 MWasmTruncateToInt32* mir = lir->mir();
 MIRType fromType = mir->input()->type();

 OutOfLineWasmTruncateCheck* ool = nullptr;
 Label* oolEntry = nullptr;
 if (!lir->mir()->isSaturating()) {
   ool = new (alloc())
       OutOfLineWasmTruncateCheck(mir, input, Register::Invalid());
   addOutOfLineCode(ool, mir);
   oolEntry = ool->entry();
 }

 masm.wasmTruncateToInt32(input, output, fromType, mir->isUnsigned(),
                          mir->isSaturating(), oolEntry);

 if (!lir->mir()->isSaturating()) {
   masm.bind(ool->rejoin());
 }
}

void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
 MOZ_ASSERT(gen->compilingWasm());
 MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg);
 masm.Push(InstanceReg);
 int32_t framePushedAfterInstance = masm.framePushed();

 FloatRegister input = ToFloatRegister(lir->input());
 FloatRegister inputDouble = input;
 Register64 output = ToOutRegister64(lir);

 MWasmBuiltinTruncateToInt64* mir = lir->mir();
 MIRType fromType = mir->input()->type();

 OutOfLineWasmTruncateCheck* ool = nullptr;
 if (!lir->mir()->isSaturating()) {
   ool = new (alloc())
       OutOfLineWasmTruncateCheck(mir, input, Register64::Invalid());
   addOutOfLineCode(ool, mir);
 }

 ScratchDoubleScope fpscratch(masm);
 if (fromType == MIRType::Float32) {
   inputDouble = fpscratch;
   masm.convertFloat32ToDouble(input, inputDouble);
 }

 masm.Push(input);

 wasm::SymbolicAddress callee;
 if (lir->mir()->isSaturating()) {
   if (lir->mir()->isUnsigned()) {
     callee = wasm::SymbolicAddress::SaturatingTruncateDoubleToUint64;
   } else {
     callee = wasm::SymbolicAddress::SaturatingTruncateDoubleToInt64;
   }
 } else {
   if (lir->mir()->isUnsigned()) {
     callee = wasm::SymbolicAddress::TruncateDoubleToUint64;
   } else {
     callee = wasm::SymbolicAddress::TruncateDoubleToInt64;
   }
 }

 masm.setupWasmABICall(callee);
 masm.passABIArg(inputDouble, ABIType::Float64);

 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
 masm.callWithABI(mir->trapSiteDesc().bytecodeOffset, callee,
                  mozilla::Some(instanceOffset));

 masm.Pop(input);
 masm.Pop(InstanceReg);

 // TruncateDoubleTo{UI,I}nt64 returns 0x8000000000000000 to indicate
 // exceptional results, so check for that and produce the appropriate
 // traps. The Saturating form always returns a normal value and never
 // needs traps.
 if (!lir->mir()->isSaturating()) {
   ScratchRegisterScope scratch(masm);
   masm.ma_cmp(output.high, Imm32(0x80000000), scratch);
   masm.as_cmp(output.low, Imm8(0x00000000), Assembler::Equal);
   masm.ma_b(ool->entry(), Assembler::Equal);

   masm.bind(ool->rejoin());
 }

 MOZ_ASSERT(ReturnReg64 == output);
}

void CodeGeneratorARM::visitOutOfLineWasmTruncateCheck(
   OutOfLineWasmTruncateCheck* ool) {
 // On ARM, saturating truncation codegen handles saturating itself rather than
 // relying on out-of-line fixup code.
 if (ool->isSaturating()) {
   return;
 }

 masm.outOfLineWasmTruncateToIntCheck(ool->input(), ool->fromType(),
                                      ool->toType(), ool->isUnsigned(),
                                      ool->rejoin(), ool->trapSiteDesc());
}

void CodeGenerator::visitInt64ToFloatingPointCall(
   LInt64ToFloatingPointCall* lir) {
 MOZ_ASSERT(gen->compilingWasm());
 MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg);
 masm.Push(InstanceReg);
 int32_t framePushedAfterInstance = masm.framePushed();

 Register64 input = ToRegister64(lir->input());

 MBuiltinInt64ToFloatingPoint* mir = lir->mir();
 MIRType toType = mir->type();

 bool isUnsigned = mir->isUnsigned();
 wasm::SymbolicAddress callee =
     toType == MIRType::Float32
         ? (isUnsigned ? wasm::SymbolicAddress::Uint64ToFloat32
                       : wasm::SymbolicAddress::Int64ToFloat32)
         : (isUnsigned ? wasm::SymbolicAddress::Uint64ToDouble
                       : wasm::SymbolicAddress::Int64ToDouble);
 masm.setupWasmABICall(callee);
 masm.passABIArg(input.high);
 masm.passABIArg(input.low);

 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
 ABIType result =
     toType == MIRType::Float32 ? ABIType::Float32 : ABIType::Float64;
 masm.callWithABI(mir->bytecodeOffset(), callee, mozilla::Some(instanceOffset),
                  result);

 DebugOnly<FloatRegister> output(ToFloatRegister(lir->output()));
 MOZ_ASSERT_IF(toType == MIRType::Double, output.value == ReturnDoubleReg);
 MOZ_ASSERT_IF(toType == MIRType::Float32, output.value == ReturnFloat32Reg);

 masm.Pop(InstanceReg);
}

void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
 LInt64Allocation input = lir->input();
 Register output = ToRegister(lir->output());

 if (lir->mir()->bottomHalf()) {
   masm.move32(ToRegister(input.low()), output);
 } else {
   masm.move32(ToRegister(input.high()), output);
 }
}

void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
 Register64 output = ToOutRegister64(lir);
 MOZ_ASSERT(ToRegister(lir->input()) == output.low);

 if (lir->mir()->isUnsigned()) {
   masm.ma_mov(Imm32(0), output.high);
 } else {
   masm.ma_asr(Imm32(31), output.low, output.high);
 }
}

void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) {
 Register64 input = ToRegister64(lir->input());
 Register64 output = ToOutRegister64(lir);
 switch (lir->mir()->mode()) {
   case MSignExtendInt64::Byte:
     masm.move8SignExtend(input.low, output.low);
     break;
   case MSignExtendInt64::Half:
     masm.move16SignExtend(input.low, output.low);
     break;
   case MSignExtendInt64::Word:
     masm.move32(input.low, output.low);
     break;
 }
 masm.ma_asr(Imm32(31), output.low, output.high);
}

void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index*) {
 MOZ_CRASH("64-bit only");
}

void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
 // Generates no code on this platform because we just return the low part of
 // the input register pair.
 MOZ_ASSERT(ToRegister(lir->input()) == ToRegister(lir->output()));
}

void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
 MOZ_ASSERT(gen->compilingWasm());
 MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg);
 masm.Push(InstanceReg);
 int32_t framePushedAfterInstance = masm.framePushed();

 Register64 lhs = ToRegister64(lir->lhs());
 Register64 rhs = ToRegister64(lir->rhs());
 Register64 output = ToOutRegister64(lir);

 MOZ_ASSERT(output == ReturnReg64);

 Label done;

 // Handle divide by zero.
 if (lir->canBeDivideByZero()) {
   Label nonZero;
   // We can use InstanceReg as temp register because we preserved it
   // before.
   masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero);
   masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->trapSiteDesc());
   masm.bind(&nonZero);
 }

 auto* mir = lir->mir();

 // Handle an integer overflow exception from INT64_MIN / -1.
 if (lir->canBeNegativeOverflow()) {
   Label notmin;
   masm.branch64(Assembler::NotEqual, lhs, Imm64(INT64_MIN), &notmin);
   masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), &notmin);
   if (mir->isWasmBuiltinModI64()) {
     masm.xor64(output, output);
   } else {
     masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->trapSiteDesc());
   }
   masm.jump(&done);
   masm.bind(&notmin);
 }

 wasm::SymbolicAddress callee = mir->isWasmBuiltinModI64()
                                    ? wasm::SymbolicAddress::ModI64
                                    : wasm::SymbolicAddress::DivI64;
 masm.setupWasmABICall(callee);
 masm.passABIArg(lhs.high);
 masm.passABIArg(lhs.low);
 masm.passABIArg(rhs.high);
 masm.passABIArg(rhs.low);

 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
 masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, callee,
                  mozilla::Some(instanceOffset));

 MOZ_ASSERT(ReturnReg64 == output);

 masm.bind(&done);
 masm.Pop(InstanceReg);
}

void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
 MOZ_ASSERT(gen->compilingWasm());
 MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg);
 masm.Push(InstanceReg);
 int32_t framePushedAfterInstance = masm.framePushed();

 Register64 lhs = ToRegister64(lir->lhs());
 Register64 rhs = ToRegister64(lir->rhs());

 MOZ_ASSERT(ToOutRegister64(lir) == ReturnReg64);

 // Prevent divide by zero.
 if (lir->canBeDivideByZero()) {
   Label nonZero;
   // We can use InstanceReg as temp register because we preserved it
   // before.
   masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero);
   masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->trapSiteDesc());
   masm.bind(&nonZero);
 }

 MDefinition* mir = lir->mir();
 wasm::SymbolicAddress callee = mir->isWasmBuiltinModI64()
                                    ? wasm::SymbolicAddress::UModI64
                                    : wasm::SymbolicAddress::UDivI64;
 masm.setupWasmABICall(callee);
 masm.passABIArg(lhs.high);
 masm.passABIArg(lhs.low);
 masm.passABIArg(rhs.high);
 masm.passABIArg(rhs.low);

 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
 masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, callee,
                  mozilla::Some(instanceOffset));
 masm.Pop(InstanceReg);
}

void CodeGenerator::visitShiftI64(LShiftI64* lir) {
 LInt64Allocation lhs = lir->lhs();
 const LAllocation* rhs = lir->rhs();

 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));

 if (rhs->isConstant()) {
   int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
   switch (lir->bitop()) {
     case JSOp::Lsh:
       if (shift) {
         masm.lshift64(Imm32(shift), ToRegister64(lhs));
       }
       break;
     case JSOp::Rsh:
       if (shift) {
         masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
       }
       break;
     case JSOp::Ursh:
       if (shift) {
         masm.rshift64(Imm32(shift), ToRegister64(lhs));
       }
       break;
     default:
       MOZ_CRASH("Unexpected shift op");
   }
   return;
 }

 switch (lir->bitop()) {
   case JSOp::Lsh:
     masm.lshift64(ToRegister(rhs), ToRegister64(lhs));
     break;
   case JSOp::Rsh:
     masm.rshift64Arithmetic(ToRegister(rhs), ToRegister64(lhs));
     break;
   case JSOp::Ursh:
     masm.rshift64(ToRegister(rhs), ToRegister64(lhs));
     break;
   default:
     MOZ_CRASH("Unexpected shift op");
 }
}

void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
 LInt64Allocation lhs = lir->lhs();
 LInt64Allocation rhs = lir->rhs();

 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));

 switch (lir->bitop()) {
   case JSOp::BitOr:
     if (IsConstant(rhs)) {
       masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
     } else {
       masm.or64(ToRegister64(rhs), ToRegister64(lhs));
     }
     break;
   case JSOp::BitXor:
     if (IsConstant(rhs)) {
       masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
     } else {
       masm.xor64(ToRegister64(rhs), ToRegister64(lhs));
     }
     break;
   case JSOp::BitAnd:
     if (IsConstant(rhs)) {
       masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
     } else {
       masm.and64(ToRegister64(rhs), ToRegister64(lhs));
     }
     break;
   default:
     MOZ_CRASH("unexpected binary opcode");
 }
}

void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
 const MWasmStackArg* mir = ins->mir();
 Address dst(StackPointer, mir->spOffset());
 if (IsConstant(ins->arg())) {
   masm.store64(Imm64(ToInt64(ins->arg())), dst);
 } else {
   masm.store64(ToRegister64(ins->arg()), dst);
 }
}

void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
 Register cond = ToRegister(lir->condExpr());
 LInt64Allocation falseExpr = lir->falseExpr();

 Register64 out = ToOutRegister64(lir);
 MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
            "true expr is reused for input");

 masm.as_cmp(cond, Imm8(0));
 if (falseExpr.low().isGeneralReg()) {
   masm.ma_mov(ToRegister(falseExpr.low()), out.low, LeaveCC,
               Assembler::Equal);
   masm.ma_mov(ToRegister(falseExpr.high()), out.high, LeaveCC,
               Assembler::Equal);
 } else {
   ScratchRegisterScope scratch(masm);
   masm.ma_ldr(ToAddress(falseExpr.low()), out.low, scratch, Offset,
               Assembler::Equal);
   masm.ma_ldr(ToAddress(falseExpr.high()), out.high, scratch, Offset,
               Assembler::Equal);
 }
}

void CodeGenerator::visitBitNotI64(LBitNotI64* lir) {
 Register64 input = ToRegister64(lir->input());
 MOZ_ASSERT(input == ToOutRegister64(lir));
 masm.ma_mvn(input.high, input.high);
 masm.ma_mvn(input.low, input.low);
}

void CodeGenerator::visitWasmAtomicLoadI64(LWasmAtomicLoadI64* lir) {
 Register ptr = ToRegister(lir->ptr());
 Register memoryBase = ToRegister(lir->memoryBase());
 Register64 output = ToOutRegister64(lir);
 Register64 tmp(InvalidReg, InvalidReg);

 BaseIndex addr(memoryBase, ptr, TimesOne, lir->mir()->access().offset32());
 masm.wasmAtomicLoad64(lir->mir()->access(), addr, tmp, output);
}

void CodeGenerator::visitWasmAtomicStoreI64(LWasmAtomicStoreI64* lir) {
 Register ptr = ToRegister(lir->ptr());
 Register memoryBase = ToRegister(lir->memoryBase());
 Register64 value = ToRegister64(lir->value());
 Register64 tmp = ToRegister64(lir->temp0());

 BaseIndex addr(memoryBase, ptr, TimesOne, lir->mir()->access().offset32());
 masm.wasmAtomicExchange64(lir->mir()->access(), addr, value, tmp);
}

void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir) {
 Register ptr = ToRegister(lir->ptr());
 Register64 expected = ToRegister64(lir->expected());
 Register64 replacement = ToRegister64(lir->replacement());
 Register memoryBase = ToRegister(lir->memoryBase());
 Register64 out = ToOutRegister64(lir);

 BaseIndex addr(memoryBase, ptr, TimesOne, lir->mir()->access().offset32());
 masm.wasmCompareExchange64(lir->mir()->access(), addr, expected, replacement,
                            out);
}

void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir) {
 Register ptr = ToRegister(lir->ptr());
 Register64 value = ToRegister64(lir->value());
 Register memoryBase = ToRegister(lir->memoryBase());
 Register64 out = ToOutRegister64(lir);

 BaseIndex addr(memoryBase, ptr, TimesOne, lir->access().offset32());
 Register64 tmp = ToRegister64(lir->temp0());
 masm.wasmAtomicFetchOp64(lir->access(), lir->operation(), value, addr, tmp,
                          out);
}

void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir) {
 Register ptr = ToRegister(lir->ptr());
 Register64 value = ToRegister64(lir->value());
 Register memoryBase = ToRegister(lir->memoryBase());
 Register64 out = ToOutRegister64(lir);

 BaseIndex addr(memoryBase, ptr, TimesOne, lir->access().offset32());
 masm.wasmAtomicExchange64(lir->access(), addr, value, out);
}

void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); }

void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmBinarySimd128WithConstant(
   LWasmBinarySimd128WithConstant* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmVariableShiftSimd128(
   LWasmVariableShiftSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmConstantShiftSimd128(
   LWasmConstantShiftSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmSignReplicationSimd128(
   LWasmSignReplicationSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
   LWasmReplaceInt64LaneSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmReduceAndBranchSimd128(
   LWasmReduceAndBranchSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmReduceSimd128ToInt64(
   LWasmReduceSimd128ToInt64* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
 MOZ_CRASH("No SIMD");
}

void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
 MOZ_CRASH("No SIMD");
}