commit dd6cf89b0202235f5b4752584ef37d482df6c754 parent 0dc775dede89fe0be5c399744f58d36e98ccf8b0 Author: Sandor Molnar <smolnar@mozilla.com> Date: Wed, 5 Nov 2025 21:16:26 +0200 Revert "Bug 1997975 - Part 10: Add test case for fallible unsigned div/mod instructions. r=spidermonkey-reviewers,iain" for causing spider-monkey perma failures @ Lowering.cpp This reverts commit 5603d97b64819e5f13d0035b0b94b748351b4ecb. Revert "Bug 1997975 - Part 9: Split (U)DivOrModI64 for arm64. r=spidermonkey-reviewers,iain" This reverts commit 7efbe4081709985c01983eb546c02a0d9cb4ab97. Revert "Bug 1997975 - Part 8: Use compare-branch-zero instructions for bailouts on arm64. r=spidermonkey-reviewers,iain" This reverts commit 5c0692df6f560540ce176a222429f6e4c8dfda07. Revert "Bug 1997975 - Part 7: Remove unused function bailoutTestPtr. r=spidermonkey-reviewers,iain" This reverts commit 6fb6191f439410b3fcf76bb187a386c6626c7081. Revert "Bug 1997975 - Part 6: Remove non-working LModMask for arm64. r=spidermonkey-reviewers,iain" This reverts commit 0b470379848d5f74b6621dd0a2e282fc2bc6b9a6. Revert "Bug 1997975 - Part 5: Support modulus with constant in arm64. r=spidermonkey-reviewers,iain" This reverts commit 5806379c0c6697756c4be1fee05c866c2d613c3d. Revert "Bug 1997975 - Part 4: Remove no longer used cmp32 with vixl::Operand. r=spidermonkey-reviewers,iain" This reverts commit 01981e567d954015eea7b0d67b376ee2afc81eb2. Revert "Bug 1997975 - Part 3: Align LModI/LUMod with LDivI/LUDiv on arm64. r=spidermonkey-reviewers,iain" This reverts commit 59bc21f9fe2af280e2175e5b49e6ea6dcdffbc9a. Revert "Bug 1997975 - Part 2: Improve register allocation and codegen for LDivI on arm64. r=spidermonkey-reviewers,iain" This reverts commit 42c0c51869eb33be68252045d5cd7379035c1805. Revert "Bug 1997975 - Part 1: Move unsigned-check to shared lowering. r=spidermonkey-reviewers,iain" This reverts commit 0ac764d6a155e0336f8e19aebc569d3e4ddc3f80. Diffstat:
24 files changed, 565 insertions(+), 603 deletions(-)
diff --git a/js/src/jit-test/tests/ion/udivormod-fallible.js b/js/src/jit-test/tests/ion/udivormod-fallible.js @@ -1,105 +0,0 @@ -// Test case for fallible unsigned division and modulus instructions. -// -// Doesn't include test that bailouts are correctly implemented. - -// |MBinaryInstruction::unsignedOperands()| calls MustBeUInt32, which only -// treats MUrsh as unsigned when bailouts are disabled. -// -// |MUrsh::collectRangeInfoPreTrunc()| sets |MUrsh::bailoutsDisabled_| if -// the lower bound of the lhs operand is >= 0 and the lower bound of the rhs -// operand is >= 1. -// -// Use |Math.max(v, 0)| below to ensure the operand is non-negative, so that -// the |MUrsh| is marked as bailouts disabled. - -// MMod::computeRange sets |MMod::unsigned_| flag. -// -// Range Analysis uses IsUint32Type to check for Uint32 types. IsUint32Type -// doesn't require that |Ursh| is marked as bailouts disabled. -// -// These don't need an explicit bit-or operation to mark as truncated. -function umod_by_constant(x) { - return (x >>> 0) % 3; -} -function umod_by_constant_pow_two(x) { - return (x >>> 0) % 4; -} -function umod(dividend, divisor) { - // Ensure range of |divisor| doesn't include zero. - divisor = Math.min(Math.max(divisor, 1), 10); - - // Make sure |IsUint32Type| returns true for both operands. - return (dividend >>> 0) % (divisor >>> 0); -} - -// |MMod::computeRange| also marks the operation as unsigned when the lower -// bound of the dividend is >= 0 (and the divisor's range doesn't include zero.) -function umod_by_constant_no_ursh(dividend, divisor) { - // Ensure lower bound of |dividend| range is >= 0. - dividend = Math.max(dividend, 0); - - return dividend % 5; -} -function umod_no_ursh(dividend, divisor) { - // Ensure lower bound of |dividend| range is >= 0. - dividend = Math.max(dividend, 0); - - // Ensure range of |divisor| doesn't include zero. - divisor = Math.min(Math.max(divisor, 1), 10); - - return dividend % divisor; -} - -// MMod::truncate sets |MMod::unsigned_| flag. Bit-or is needed to mark as truncated. -function umod_truncate(dividend, divisor) { - dividend = Math.max(dividend, 0); - divisor = Math.max(divisor, 0); - return ((dividend >>> 0) % (divisor >>> 0))|0; -} - -// MDiv::truncate sets |MDiv::unsigned_| flag. Bit-or is needed to mark as truncated. -// -// Note: MDiv::computeRange never sets |MDiv::unsigned_|. -function udiv_by_constant(dividend) { - dividend = Math.max(dividend, 0); - return ((dividend >>> 0) / 3)|0; -} -function udiv_by_constant_pow_two(dividend) { - dividend = Math.max(dividend, 0); - return ((dividend >>> 0) / 8)|0; -} -function udiv(dividend, divisor) { - dividend = Math.max(dividend, 0); - divisor = Math.max(divisor, 0); - return ((dividend >>> 0) / (divisor >>> 0))|0; -} - -// Don't Ion compile the top-level script. -with ({}); - -for (let i = 0; i < 100; ++i) { - assertEq(umod_by_constant(i), i % 3); - assertEq(umod_by_constant_pow_two(i), i % 4); - - assertEq(umod(i, 1), i % 1); - assertEq(umod(i, 3), i % 3); - assertEq(umod(i, 5), i % 5); - - assertEq(umod_by_constant_no_ursh(i), i % 5); - - assertEq(umod_no_ursh(i, 1), i % 1); - assertEq(umod_no_ursh(i, 7), i % 7); - assertEq(umod_no_ursh(i, 9), i % 9); - - assertEq(umod_truncate(i, 1), i % 1); - assertEq(umod_truncate(i, 6), i % 6); - assertEq(umod_truncate(i, 11), i % 11); - - // Use multiples of the divisor to ensure CacheIR doesn't emit a Double division. (bug 1554721) - assertEq(udiv_by_constant(i * 3), i); - assertEq(udiv_by_constant_pow_two(i * 8), i); - - assertEq(udiv(i * 1, 1), i); - assertEq(udiv(i * 3, 3), i); - assertEq(udiv(i * 5, 5), i); -} diff --git a/js/src/jit/LIROps.yaml b/js/src/jit/LIROps.yaml @@ -4403,33 +4403,11 @@ #endif #ifdef JS_CODEGEN_ARM64 -- name: DivI64 - result_type: Int64 - operands: - lhs: WordSized - rhs: WordSized - mir_op: Div - -- name: ModI64 - result_type: Int64 - operands: - lhs: WordSized - rhs: WordSized - mir_op: Mod - -- name: UDivI64 - result_type: Int64 - operands: - lhs: WordSized - rhs: WordSized - mir_op: Div +- name: DivOrModI64 + gen_boilerplate: false -- name: UModI64 - result_type: Int64 - operands: - lhs: WordSized - rhs: WordSized - mir_op: Mod +- name: UDivOrModI64 + gen_boilerplate: false - name: DivConstantI result_type: WordSized @@ -4437,30 +4415,25 @@ numerator: WordSized arguments: denominator: int32_t + num_temps: 1 mir_op: Div -- name: UDivConstant - result_type: WordSized - operands: - numerator: WordSized - arguments: - denominator: uint32_t - mir_op: Div - -- name: ModConstantI +- name: UDivConstantI result_type: WordSized operands: numerator: WordSized arguments: denominator: int32_t - mir_op: Mod + num_temps: 1 + mir_op: Div -- name: UModConstant +- name: ModMaskI result_type: WordSized operands: - numerator: WordSized + input: WordSized arguments: - denominator: uint32_t + shift: int32_t + num_temps: 2 mir_op: Mod - name: UDiv @@ -4468,6 +4441,7 @@ operands: lhs: WordSized rhs: WordSized + num_temps: 1 mir_op: Div - name: UMod diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp @@ -2454,21 +2454,13 @@ void LIRGenerator::visitDiv(MDiv* ins) { if (ins->type() == MIRType::Int32) { MOZ_ASSERT(lhs->type() == MIRType::Int32); - if (ins->isUnsigned()) { - lowerUDiv(ins); - } else { - lowerDivI(ins); - } + lowerDivI(ins); return; } if (ins->type() == MIRType::Int64) { MOZ_ASSERT(lhs->type() == MIRType::Int64); - if (ins->isUnsigned()) { - lowerUDivI64(ins); - } else { - lowerDivI64(ins); - } + lowerDivI64(ins); return; } @@ -2520,22 +2512,14 @@ void LIRGenerator::visitMod(MMod* ins) { if (ins->type() == MIRType::Int32) { MOZ_ASSERT(ins->type() == MIRType::Int32); MOZ_ASSERT(ins->lhs()->type() == MIRType::Int32); - if (ins->isUnsigned()) { - lowerUMod(ins); - } else { - lowerModI(ins); - } + lowerModI(ins); return; } if (ins->type() == MIRType::Int64) { MOZ_ASSERT(ins->type() == MIRType::Int64); MOZ_ASSERT(ins->lhs()->type() == MIRType::Int64); - if (ins->isUnsigned()) { - lowerUModI64(ins); - } else { - lowerModI64(ins); - } + lowerModI64(ins); return; } diff --git a/js/src/jit/ReciprocalMulConstants.h b/js/src/jit/ReciprocalMulConstants.h @@ -7,8 +7,6 @@ #ifndef jit_ReciprocalMulConstants_h #define jit_ReciprocalMulConstants_h -#include "mozilla/MathAlgorithms.h" - #include <stdint.h> namespace js::jit { @@ -17,8 +15,8 @@ struct ReciprocalMulConstants { int64_t multiplier; int32_t shiftAmount; - static ReciprocalMulConstants computeSignedDivisionConstants(int32_t d) { - return computeDivisionConstants(mozilla::Abs(d), 31); + static ReciprocalMulConstants computeSignedDivisionConstants(uint32_t d) { + return computeDivisionConstants(d, 31); } static ReciprocalMulConstants computeUnsignedDivisionConstants(uint32_t d) { diff --git a/js/src/jit/arm/CodeGenerator-arm.h b/js/src/jit/arm/CodeGenerator-arm.h @@ -41,6 +41,11 @@ class CodeGeneratorARM : public CodeGeneratorShared { masm.cmpPtr(lhs, rhs); bailoutIf(c, snapshot); } + void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs, + LSnapshot* snapshot) { + masm.testPtr(lhs, rhs); + bailoutIf(c, snapshot); + } template <typename T1, typename T2> void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) { diff --git a/js/src/jit/arm/Lowering-arm.cpp b/js/src/jit/arm/Lowering-arm.cpp @@ -309,6 +309,11 @@ template void LIRGeneratorARM::lowerForShiftInt64(LRotateI64* ins, MDefinition* rhs); void LIRGeneratorARM::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + // Division instructions are slow. Division by constant denominators can be // rewritten to use other instructions. if (div->rhs()->isConstant()) { @@ -369,6 +374,11 @@ void LIRGeneratorARM::lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs) { } void LIRGeneratorARM::lowerModI(MMod* mod) { + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } + if (mod->rhs()->isConstant()) { int32_t rhs = mod->rhs()->toConstant()->toInt32(); int32_t shift = FloorLog2(rhs); diff --git a/js/src/jit/arm64/CodeGenerator-arm64.cpp b/js/src/jit/arm64/CodeGenerator-arm64.cpp @@ -91,25 +91,6 @@ void CodeGeneratorARM64::bailoutIf(Assembler::Condition condition, masm.B(ool->entry(), condition); } -void CodeGeneratorARM64::bailoutIfZero(Assembler::Condition condition, - ARMRegister rt, LSnapshot* snapshot) { - MOZ_ASSERT(condition == Assembler::Zero || condition == Assembler::NonZero); - - encode(snapshot); - - InlineScriptTree* tree = snapshot->mir()->block()->trackedTree(); - auto* ool = new (alloc()) LambdaOutOfLineCode( - [=](OutOfLineCode& ool) { emitBailoutOOL(snapshot); }); - addOutOfLineCode(ool, - new (alloc()) BytecodeSite(tree, tree->script()->code())); - - if (condition == Assembler::Zero) { - masm.Cbz(rt, ool->entry()); - } else { - masm.Cbnz(rt, ool->entry()); - } -} - void CodeGeneratorARM64::bailoutFrom(Label* label, LSnapshot* snapshot) { MOZ_ASSERT_IF(!masm.oom(), label->used()); MOZ_ASSERT_IF(!masm.oom(), !label->bound()); @@ -239,7 +220,8 @@ void CodeGenerator::visitMulI(LMulI* ins) { if (mul->canBeNegativeZero() && constant <= 0) { Assembler::Condition bailoutCond = (constant == 0) ? Assembler::LessThan : Assembler::Equal; - bailoutCmp32(bailoutCond, lhsreg, Imm32(0), ins->snapshot()); + masm.Cmp(toWRegister(lhs), Operand(0)); + bailoutIf(bailoutCond, ins->snapshot()); } switch (constant) { @@ -337,65 +319,60 @@ void CodeGenerator::visitMulI(LMulI* ins) { } } -template <class LIR> -static void TrapIfDivideByZero(MacroAssembler& masm, LIR* lir, - ARMRegister rhs) { - auto* mir = lir->mir(); - MOZ_ASSERT(mir->trapOnError()); - - if (mir->canBeDivideByZero()) { - Label nonZero; - masm.Cbnz(rhs, &nonZero); - masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); - masm.bind(&nonZero); - } -} - void CodeGenerator::visitDivI(LDivI* ins) { - Register lhs = ToRegister(ins->lhs()); - Register rhs = ToRegister(ins->rhs()); + const Register lhs = ToRegister(ins->lhs()); + const Register rhs = ToRegister(ins->rhs()); + const Register output = ToRegister(ins->output()); - ARMRegister lhs32 = toWRegister(ins->lhs()); - ARMRegister rhs32 = toWRegister(ins->rhs()); - ARMRegister output32 = toWRegister(ins->output()); + const ARMRegister lhs32 = toWRegister(ins->lhs()); + const ARMRegister rhs32 = toWRegister(ins->rhs()); + const ARMRegister temp32 = toWRegister(ins->temp0()); + const ARMRegister output32 = toWRegister(ins->output()); MDiv* mir = ins->mir(); + Label done; + // Handle division by zero. if (mir->canBeDivideByZero()) { + masm.test32(rhs, rhs); if (mir->trapOnError()) { - TrapIfDivideByZero(masm, ins, rhs32); + Label nonZero; + masm.j(Assembler::NonZero, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); + masm.bind(&nonZero); } else if (mir->canTruncateInfinities()) { - // SDIV returns zero for division by zero, exactly what we want for - // truncated division. Remainder computation expects a non-zero divisor, - // so we must also be allowed to truncate the remainder. - MOZ_ASSERT(mir->canTruncateRemainder(), - "remainder computation expects a non-zero divisor"); + // Truncated division by zero is zero: (Infinity|0 = 0). + Label nonZero; + masm.j(Assembler::NonZero, &nonZero); + masm.Mov(output32, wzr); + masm.jump(&done); + masm.bind(&nonZero); } else { MOZ_ASSERT(mir->fallible()); - bailoutTest32(Assembler::Zero, rhs, rhs, ins->snapshot()); + bailoutIf(Assembler::Zero, ins->snapshot()); } } // Handle an integer overflow from (INT32_MIN / -1). // The integer division gives INT32_MIN, but should be -(double)INT32_MIN. - // - // SDIV returns INT32_MIN for (INT32_MIN / -1), so no extra code needed when - // truncation is allowed. - if (mir->canBeNegativeOverflow() && - (mir->trapOnError() || !mir->canTruncateOverflow())) { + if (mir->canBeNegativeOverflow()) { Label notOverflow; // Branch to handle the non-overflow cases. masm.branch32(Assembler::NotEqual, lhs, Imm32(INT32_MIN), ¬Overflow); + masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), ¬Overflow); // Handle overflow. if (mir->trapOnError()) { - masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), ¬Overflow); masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->trapSiteDesc()); + } else if (mir->canTruncateOverflow()) { + // (-INT32_MIN)|0 == INT32_MIN, which is already in lhs. + masm.move32(lhs, output); + masm.jump(&done); } else { MOZ_ASSERT(mir->fallible()); - bailoutCmp32(Assembler::Equal, rhs, Imm32(-1), ins->snapshot()); + bailout(ins->snapshot()); } masm.bind(¬Overflow); } @@ -404,23 +381,28 @@ void CodeGenerator::visitDivI(LDivI* ins) { if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) { Label nonZero; masm.branch32(Assembler::NotEqual, lhs, Imm32(0), &nonZero); - bailoutCmp32(Assembler::LessThan, rhs, Imm32(0), ins->snapshot()); + masm.cmp32(rhs, Imm32(0)); + bailoutIf(Assembler::LessThan, ins->snapshot()); masm.bind(&nonZero); } // Perform integer division. - masm.Sdiv(output32, lhs32, rhs32); - - if (!mir->canTruncateRemainder()) { + if (mir->canTruncateRemainder()) { + masm.Sdiv(output32, lhs32, rhs32); + } else { vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister remainder32 = temps.AcquireW(); - Register remainder = remainder32.asUnsized(); + ARMRegister scratch32 = temps.AcquireW(); - // Compute the remainder: remainder = lhs - (output * rhs). - masm.Msub(remainder32, output32, rhs32, lhs32); - - bailoutTest32(Assembler::NonZero, remainder, remainder, ins->snapshot()); + // ARM does not automatically calculate the remainder. + // The ISR suggests multiplication to determine whether a remainder exists. + masm.Sdiv(scratch32, lhs32, rhs32); + masm.Mul(temp32, scratch32, rhs32); + masm.Cmp(lhs32, temp32); + bailoutIf(Assembler::NotEqual, ins->snapshot()); + masm.Mov(output32, scratch32); } + + masm.bind(&done); } void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) { @@ -506,25 +488,21 @@ void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) { } } -template <class LDivOrMod> -static void DivideWithConstant(MacroAssembler& masm, LDivOrMod* ins) { - ARMRegister lhs32 = toWRegister(ins->numerator()); - ARMRegister lhs64 = toXRegister(ins->numerator()); - ARMRegister output32 = toWRegister(ins->output()); - ARMRegister output64 = toXRegister(ins->output()); +void CodeGenerator::visitDivConstantI(LDivConstantI* ins) { + const ARMRegister lhs32 = toWRegister(ins->numerator()); + const ARMRegister lhs64 = toXRegister(ins->numerator()); + const ARMRegister const32 = toWRegister(ins->temp0()); + const ARMRegister output32 = toWRegister(ins->output()); + const ARMRegister output64 = toXRegister(ins->output()); int32_t d = ins->denominator(); - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister const32 = temps.AcquireW(); - // The absolute value of the denominator isn't a power of 2. - MOZ_ASSERT(!mozilla::IsPowerOfTwo(mozilla::Abs(d))); - - auto* mir = ins->mir(); + using mozilla::Abs; + MOZ_ASSERT((Abs(d) & (Abs(d) - 1)) != 0); // We will first divide by Abs(d), and negate the answer if d is negative. // If desired, this can be avoided by generalizing computeDivisionConstants. - auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(d); + auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(Abs(d)); // We first compute (M * n) >> 32, where M = rmc.multiplier. masm.Mov(const32, int32_t(rmc.multiplier)); @@ -552,7 +530,7 @@ static void DivideWithConstant(MacroAssembler& masm, LDivOrMod* ins) { // We'll subtract -1 instead of adding 1, because (n < 0 ? -1 : 0) can be // computed with just a sign-extending shift of 31 bits. - if (mir->canBeNegativeDividend()) { + if (ins->mir()->canBeNegativeDividend()) { masm.Asr(const32, lhs32, 31); masm.Sub(output32, output32, const32); } @@ -561,76 +539,56 @@ static void DivideWithConstant(MacroAssembler& masm, LDivOrMod* ins) { if (d < 0) { masm.Neg(output32, output32); } -} - -void CodeGenerator::visitDivConstantI(LDivConstantI* ins) { - ARMRegister lhs32 = toWRegister(ins->numerator()); - ARMRegister output32 = toWRegister(ins->output()); - int32_t d = ins->denominator(); - - MDiv* mir = ins->mir(); - - if (d == 0) { - if (mir->trapOnError()) { - masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); - } else if (mir->canTruncateInfinities()) { - masm.Mov(output32, wzr); - } else { - MOZ_ASSERT(mir->fallible()); - bailout(ins->snapshot()); - } - return; - } - - // Compute the truncated division result in output32. - DivideWithConstant(masm, ins); - - if (!mir->isTruncated()) { - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister temp32 = temps.AcquireW(); - Register temp = temp32.asUnsized(); + if (!ins->mir()->isTruncated()) { // This is a division op. Multiply the obtained value by d to check if // the correct answer is an integer. This cannot overflow, since |d| > 1. - masm.Mov(temp32, d); - masm.Msub(temp32, output32, temp32, lhs32); - - if (d > 0) { - // bailout if (lhs - output * d != 0) - bailoutTest32(Assembler::NonZero, temp, temp, ins->snapshot()); - } else { - MOZ_ASSERT(d < 0); - - // bailout if (lhs - output * d != 0) - masm.Cmp(temp32, wzr); + masm.Mov(const32, d); + masm.Msub(const32, output32, const32, lhs32); + // bailout if (lhs - output * d != 0) + masm.Cmp(const32, wzr); + auto bailoutCond = Assembler::NonZero; - // If lhs is zero and the divisor is negative, the answer should have - // been -0. - // + // If lhs is zero and the divisor is negative, the answer should have + // been -0. + if (d < 0) { // or bailout if (lhs == 0). // ^ ^ // | '-- masm.Ccmp(lhs32, lhs32, .., ..) - // '-- masm.Ccmp(.., .., vixl::ZFlag, Assembler::Zero) + // '-- masm.Ccmp(.., .., vixl::ZFlag, ! bailoutCond) masm.Ccmp(lhs32, wzr, vixl::ZFlag, Assembler::Zero); - - // bailout if (lhs - output * d != 0) or (lhs == 0) - bailoutIf(Assembler::Zero, ins->snapshot()); + bailoutCond = Assembler::Zero; } + + // bailout if (lhs - output * d != 0) or (d < 0 && lhs == 0) + bailoutIf(bailoutCond, ins->snapshot()); } } -template <class LUDivOrUMod> -static void UnsignedDivideWithConstant(MacroAssembler& masm, LUDivOrUMod* ins) { - ARMRegister lhs32 = toWRegister(ins->numerator()); - ARMRegister lhs64 = toXRegister(ins->numerator()); - ARMRegister output64 = toXRegister(ins->output()); +void CodeGenerator::visitUDivConstantI(LUDivConstantI* ins) { + const ARMRegister lhs32 = toWRegister(ins->numerator()); + const ARMRegister lhs64 = toXRegister(ins->numerator()); + const ARMRegister const32 = toWRegister(ins->temp0()); + const ARMRegister output32 = toWRegister(ins->output()); + const ARMRegister output64 = toXRegister(ins->output()); uint32_t d = ins->denominator(); - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister const32 = temps.AcquireW(); + if (d == 0) { + if (ins->mir()->isTruncated()) { + if (ins->mir()->trapOnError()) { + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, + ins->mir()->trapSiteDesc()); + } else { + masm.Mov(output32, wzr); + } + } else { + bailout(ins->snapshot()); + } + return; + } // The denominator isn't a power of 2 (see LDivPowTwoI). - MOZ_ASSERT(!mozilla::IsPowerOfTwo(d)); + MOZ_ASSERT((d & (d - 1)) != 0); auto rmc = ReciprocalMulConstants::computeUnsignedDivisionConstants(d); @@ -660,77 +618,53 @@ static void UnsignedDivideWithConstant(MacroAssembler& masm, LUDivOrUMod* ins) { // (M * n) >> (32 + shift) is the truncated division answer. masm.Lsr(output64, output64, 32 + rmc.shiftAmount); } -} - -void CodeGenerator::visitUDivConstant(LUDivConstant* ins) { - ARMRegister lhs32 = toWRegister(ins->numerator()); - ARMRegister output32 = toWRegister(ins->output()); - uint32_t d = ins->denominator(); - - MDiv* mir = ins->mir(); - - if (d == 0) { - if (ins->mir()->trapOnError()) { - masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); - } else if (mir->canTruncateInfinities()) { - masm.Mov(output32, wzr); - } else { - MOZ_ASSERT(mir->fallible()); - bailout(ins->snapshot()); - } - return; - } - - // Compute the truncated division result in output32. - UnsignedDivideWithConstant(masm, ins); // We now have the truncated division value. We are checking whether the // division resulted in an integer, we multiply the obtained value by d and // check the remainder of the division. - if (!mir->isTruncated()) { - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister temp32 = temps.AcquireW(); - Register temp = temp32.asUnsized(); - - masm.Mov(temp32, d); - masm.Msub(temp32, output32, temp32, lhs32); - + if (!ins->mir()->isTruncated()) { + masm.Mov(const32, d); + masm.Msub(const32, output32, const32, lhs32); // bailout if (lhs - output * d != 0) - bailoutTest32(Assembler::NonZero, temp, temp, ins->snapshot()); + masm.Cmp(const32, const32); + bailoutIf(Assembler::NonZero, ins->snapshot()); } } void CodeGenerator::visitModI(LModI* ins) { - Register lhs = ToRegister(ins->lhs()); - Register rhs = ToRegister(ins->rhs()); - - ARMRegister lhs32 = toWRegister(ins->lhs()); - ARMRegister rhs32 = toWRegister(ins->rhs()); - ARMRegister output32 = toWRegister(ins->output()); + ARMRegister lhs = toWRegister(ins->lhs()); + ARMRegister rhs = toWRegister(ins->rhs()); + ARMRegister output = toWRegister(ins->output()); Label done; MMod* mir = ins->mir(); // Prevent divide by zero. if (mir->canBeDivideByZero()) { - if (mir->trapOnError()) { - TrapIfDivideByZero(masm, ins, rhs32); - } else if (mir->isTruncated()) { - // Truncated division by zero yields integer zero. - masm.Mov(output32, wzr); - masm.Cbz(rhs32, &done); + if (mir->isTruncated()) { + if (mir->trapOnError()) { + Label nonZero; + masm.Cbnz(rhs, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); + masm.bind(&nonZero); + } else { + // Truncated division by zero yields integer zero. + masm.Mov(output, rhs); + masm.Cbz(rhs, &done); + } } else { // Non-truncated division by zero produces a non-integer. - MOZ_ASSERT(mir->fallible()); - bailoutTest32(Assembler::Zero, rhs, rhs, ins->snapshot()); + MOZ_ASSERT(!gen->compilingWasm()); + masm.Cmp(rhs, Operand(0)); + bailoutIf(Assembler::Equal, ins->snapshot()); } } // Signed division. - masm.Sdiv(output32, lhs32, rhs32); + masm.Sdiv(output, lhs, rhs); // Compute the remainder: output = lhs - (output * rhs). - masm.Msub(output32, output32, rhs32, lhs32); + masm.Msub(output, output, rhs, lhs); if (mir->canBeNegativeDividend() && !mir->isTruncated()) { // If output == 0 and lhs < 0, then the result should be double -0.0. @@ -738,7 +672,7 @@ void CodeGenerator::visitModI(LModI* ins) { // output = INT_MIN - (INT_MIN / -1) * -1 // = INT_MIN - INT_MIN // = 0 - masm.Cbnz(output32, &done); + masm.Cbnz(output, &done); bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot()); } @@ -787,82 +721,97 @@ void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) { } } -void CodeGenerator::visitModConstantI(LModConstantI* ins) { - Register lhs = ToRegister(ins->numerator()); - ARMRegister lhs32 = toWRegister(ins->numerator()); - ARMRegister output32 = toWRegister(ins->output()); - +void CodeGenerator::visitModMaskI(LModMaskI* ins) { MMod* mir = ins->mir(); + int32_t shift = ins->shift(); - int32_t d = ins->denominator(); - if (d == 0) { - if (mir->trapOnError()) { - masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); - } else if (mir->isTruncated()) { - masm.Mov(output32, wzr); - } else { - MOZ_ASSERT(mir->fallible()); - bailout(ins->snapshot()); - } - return; - } + const Register src = ToRegister(ins->input()); + const Register dest = ToRegister(ins->output()); + const Register hold = ToRegister(ins->temp0()); + const Register remain = ToRegister(ins->temp1()); - // Compute the truncated division result in output32. - DivideWithConstant(masm, ins); + const ARMRegister src32 = ARMRegister(src, 32); + const ARMRegister dest32 = ARMRegister(dest, 32); + const ARMRegister remain32 = ARMRegister(remain, 32); - // Compute the remainder: output = lhs - (output * rhs). + vixl::UseScratchRegisterScope temps(&masm.asVIXL()); + const ARMRegister scratch32 = temps.AcquireW(); + const Register scratch = scratch32.asUnsized(); + + // We wish to compute x % (1<<y) - 1 for a known constant, y. + // + // 1. Let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit dividend as + // a number in base b, namely c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n + // + // 2. Since both addition and multiplication commute with modulus: + // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C == + // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)... + // + // 3. Since b == C + 1, b % C == 1, and b^n % C == 1 the whole thing + // simplifies to: c_0 + c_1 + c_2 ... c_n % C + // + // Each c_n can easily be computed by a shift/bitextract, and the modulus + // can be maintained by simply subtracting by C whenever the number gets + // over C. + int32_t mask = (1 << shift) - 1; + Label loop; + + // Register 'hold' holds -1 if the value was negative, 1 otherwise. + // The remain reg holds the remaining bits that have not been processed. + // The scratch reg serves as a temporary location to store extracted bits. + // The dest reg is the accumulator, becoming final result. + // + // Move the whole value into the remain. + masm.Mov(remain32, src32); + // Zero out the dest. + masm.Mov(dest32, wzr); + // Set the hold appropriately. { - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister rhs32 = temps.AcquireW(); + Label negative; + masm.branch32(Assembler::Signed, remain, Imm32(0), &negative); + masm.move32(Imm32(1), hold); + masm.jump(&loop); - masm.Mov(rhs32, d); - masm.Msub(output32, output32, rhs32, lhs32); + masm.bind(&negative); + masm.move32(Imm32(-1), hold); + masm.neg32(remain); } - if (mir->canBeNegativeDividend() && !mir->isTruncated()) { - // If output == 0 and lhs < 0, then the result should be double -0.0. + // Begin the main loop. + masm.bind(&loop); + { + // Extract the bottom bits into scratch. + masm.And(scratch32, remain32, Operand(mask)); + // Add those bits to the accumulator. + masm.Add(dest32, dest32, scratch32); + // Do a trial subtraction. This functions as a cmp but remembers the result. + masm.Subs(scratch32, dest32, Operand(mask)); + // If (sum - C) > 0, store sum - C back into sum, thus performing a modulus. + { + Label sumSigned; + masm.branch32(Assembler::Signed, scratch, scratch, &sumSigned); + masm.Mov(dest32, scratch32); + masm.bind(&sumSigned); + } + // Get rid of the bits that we extracted before. + masm.Lsr(remain32, remain32, shift); + // If the shift produced zero, finish, otherwise, continue in the loop. + masm.branchTest32(Assembler::NonZero, remain, remain, &loop); + } + + // Check the hold to see if we need to negate the result. + { Label done; - masm.Cbnz(output32, &done); - bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot()); - masm.bind(&done); - } -} - -void CodeGenerator::visitUModConstant(LUModConstant* ins) { - Register output = ToRegister(ins->output()); - ARMRegister lhs32 = toWRegister(ins->numerator()); - ARMRegister output32 = toWRegister(ins->output()); - - MMod* mir = ins->mir(); - uint32_t d = ins->denominator(); - if (d == 0) { - if (ins->mir()->trapOnError()) { - masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); - } else if (mir->isTruncated()) { - masm.Mov(output32, wzr); - } else { - MOZ_ASSERT(mir->fallible()); - bailout(ins->snapshot()); + // If the hold was non-zero, negate the result to match JS expectations. + masm.branchTest32(Assembler::NotSigned, hold, hold, &done); + if (mir->canBeNegativeDividend() && !mir->isTruncated()) { + // Bail in case of negative zero hold. + bailoutTest32(Assembler::Zero, hold, hold, ins->snapshot()); } - return; - } - - // Compute the truncated division result in output32. - UnsignedDivideWithConstant(masm, ins); - - // Compute the remainder: output = lhs - (output * rhs). - { - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister rhs32 = temps.AcquireW(); - masm.Mov(rhs32, d); - masm.Msub(output32, output32, rhs32, lhs32); - } - - // Bail if not truncated and the remainder is in the range [2^31, 2^32). - if (!ins->mir()->isTruncated()) { - bailoutTest32(Assembler::Signed, output, output, ins->snapshot()); + masm.neg32(dest); + masm.bind(&done); } } @@ -1713,16 +1662,17 @@ void CodeGenerator::visitUDiv(LUDiv* ins) { // Prevent divide by zero. if (mir->canBeDivideByZero()) { - if (mir->trapOnError()) { - TrapIfDivideByZero(masm, ins, rhs32); - } else if (mir->canTruncateInfinities()) { - // Udiv returns zero for division by zero, exactly what we want for - // truncated division. Remainder computation expects a non-zero divisor, - // so we must also be allowed to truncate the remainder. - MOZ_ASSERT(mir->canTruncateRemainder(), - "remainder computation expects a non-zero divisor"); + if (mir->isTruncated()) { + if (mir->trapOnError()) { + Label nonZero; + masm.branchTest32(Assembler::NonZero, rhs, rhs, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); + masm.bind(&nonZero); + } else { + // ARM64 UDIV instruction will return 0 when divided by 0. + // No need for extra tests. + } } else { - MOZ_ASSERT(mir->fallible()); bailoutTest32(Assembler::Zero, rhs, rhs, ins->snapshot()); } } @@ -1732,9 +1682,8 @@ void CodeGenerator::visitUDiv(LUDiv* ins) { // If the remainder is > 0, bailout since this must be a double. if (!mir->canTruncateRemainder()) { - vixl::UseScratchRegisterScope temps(&masm.asVIXL()); - ARMRegister remainder32 = temps.AcquireW(); - Register remainder = remainder32.asUnsized(); + Register remainder = ToRegister(ins->temp0()); + ARMRegister remainder32 = ARMRegister(remainder, 32); // Compute the remainder: remainder = lhs - (output * rhs). masm.Msub(remainder32, output32, rhs32, lhs32); @@ -1750,34 +1699,36 @@ void CodeGenerator::visitUDiv(LUDiv* ins) { } void CodeGenerator::visitUMod(LUMod* ins) { - Register rhs = ToRegister(ins->rhs()); - Register output = ToRegister(ins->output()); - - ARMRegister lhs32 = toWRegister(ins->lhs()); - ARMRegister rhs32 = toWRegister(ins->rhs()); - ARMRegister output32 = toWRegister(ins->output()); - Label done; - MMod* mir = ins->mir(); + ARMRegister lhs = toWRegister(ins->lhs()); + ARMRegister rhs = toWRegister(ins->rhs()); + ARMRegister output = toWRegister(ins->output()); + Label done; if (mir->canBeDivideByZero()) { - if (mir->trapOnError()) { - TrapIfDivideByZero(masm, ins, rhs32); - } else if (mir->isTruncated()) { - // Truncated division by zero yields integer zero. - masm.Mov(output32, wzr); - masm.Cbz(rhs32, &done); + if (mir->isTruncated()) { + if (mir->trapOnError()) { + Label nonZero; + masm.Cbnz(rhs, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); + masm.bind(&nonZero); + } else { + // Truncated division by zero yields integer zero. + masm.Mov(output, rhs); + masm.Cbz(rhs, &done); + } } else { // Non-truncated division by zero produces a non-integer. - bailoutTest32(Assembler::Zero, rhs, rhs, ins->snapshot()); + masm.Cmp(rhs, Operand(0)); + bailoutIf(Assembler::Equal, ins->snapshot()); } } // Unsigned division. - masm.Udiv(output32, lhs32, rhs32); + masm.Udiv(output, lhs, rhs); // Compute the remainder: output = lhs - (output * rhs). - masm.Msub(output32, output32, rhs32, lhs32); + masm.Msub(output, output, rhs, lhs); if (!mir->isTruncated()) { // Bail if the output would be negative. @@ -2653,68 +2604,66 @@ void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) { } } -void CodeGenerator::visitDivI64(LDivI64* lir) { +void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) { Register lhs = ToRegister(lir->lhs()); Register rhs = ToRegister(lir->rhs()); + Register output = ToRegister(lir->output()); - ARMRegister lhs64 = toXRegister(lir->lhs()); - ARMRegister rhs64 = toXRegister(lir->rhs()); - ARMRegister output64 = toXRegister(lir->output()); - - MDiv* mir = lir->mir(); + Label done; // Handle divide by zero. - TrapIfDivideByZero(masm, lir, rhs64); + if (lir->canBeDivideByZero()) { + Label isNotDivByZero; + masm.Cbnz(ARMRegister(rhs, 64), &isNotDivByZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->trapSiteDesc()); + masm.bind(&isNotDivByZero); + } // Handle an integer overflow exception from INT64_MIN / -1. - if (mir->canBeNegativeOverflow()) { + if (lir->canBeNegativeOverflow()) { Label noOverflow; masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &noOverflow); masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &noOverflow); - masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->trapSiteDesc()); + if (lir->mir()->isMod()) { + masm.movePtr(ImmWord(0), output); + } else { + masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->trapSiteDesc()); + } + masm.jump(&done); masm.bind(&noOverflow); } - masm.Sdiv(output64, lhs64, rhs64); -} - -void CodeGenerator::visitModI64(LModI64* lir) { - ARMRegister lhs64 = toXRegister(lir->lhs()); - ARMRegister rhs64 = toXRegister(lir->rhs()); - ARMRegister output64 = toXRegister(lir->output()); - - // Handle divide by zero. - TrapIfDivideByZero(masm, lir, rhs64); - - masm.Sdiv(output64, lhs64, rhs64); - - // Compute the remainder: output = lhs - (output * rhs). - masm.Msub(output64, output64, rhs64, lhs64); + masm.Sdiv(ARMRegister(output, 64), ARMRegister(lhs, 64), + ARMRegister(rhs, 64)); + if (lir->mir()->isMod()) { + masm.Msub(ARMRegister(output, 64), ARMRegister(output, 64), + ARMRegister(rhs, 64), ARMRegister(lhs, 64)); + } + masm.bind(&done); } -void CodeGenerator::visitUDivI64(LUDivI64* lir) { - ARMRegister lhs64 = toXRegister(lir->lhs()); - ARMRegister rhs64 = toXRegister(lir->rhs()); - ARMRegister output64 = toXRegister(lir->output()); - - // Handle divide by zero. - TrapIfDivideByZero(masm, lir, rhs64); - - masm.Udiv(output64, lhs64, rhs64); -} +void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) { + Register lhs = ToRegister(lir->lhs()); + Register rhs = ToRegister(lir->rhs()); + Register output = ToRegister(lir->output()); -void CodeGenerator::visitUModI64(LUModI64* lir) { - ARMRegister lhs64 = toXRegister(lir->lhs()); - ARMRegister rhs64 = toXRegister(lir->rhs()); - ARMRegister output64 = toXRegister(lir->output()); + Label done; // Handle divide by zero. - TrapIfDivideByZero(masm, lir, rhs64); - - masm.Udiv(output64, lhs64, rhs64); + if (lir->canBeDivideByZero()) { + Label isNotDivByZero; + masm.Cbnz(ARMRegister(rhs, 64), &isNotDivByZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->trapSiteDesc()); + masm.bind(&isNotDivByZero); + } - // Compute the remainder: output = lhs - (output * rhs). - masm.Msub(output64, output64, rhs64, lhs64); + masm.Udiv(ARMRegister(output, 64), ARMRegister(lhs, 64), + ARMRegister(rhs, 64)); + if (lir->mir()->isMod()) { + masm.Msub(ARMRegister(output, 64), ARMRegister(output, 64), + ARMRegister(rhs, 64), ARMRegister(lhs, 64)); + } + masm.bind(&done); } void CodeGenerator::visitSimd128(LSimd128* ins) { diff --git a/js/src/jit/arm64/CodeGenerator-arm64.h b/js/src/jit/arm64/CodeGenerator-arm64.h @@ -31,59 +31,29 @@ class CodeGeneratorARM64 : public CodeGeneratorShared { MoveOperand toMoveOperand(const LAllocation a) const; void bailoutIf(Assembler::Condition condition, LSnapshot* snapshot); - void bailoutIfZero(Assembler::Condition condition, ARMRegister rt, - LSnapshot* snapshot); void bailoutFrom(Label* label, LSnapshot* snapshot); void bailout(LSnapshot* snapshot); template <typename T1, typename T2> void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) { - if constexpr (std::is_same_v<T1, Register> && - (std::is_same_v<T2, Imm32> || std::is_same_v<T2, Imm64> || - std::is_same_v<T2, ImmWord> || std::is_same_v<T2, ImmPtr>)) { - if (rhs.value == 0) { - if (c == Assembler::Equal) { - bailoutIfZero(Assembler::Zero, ARMRegister(lhs, 64), snapshot); - return; - } - if (c == Assembler::NotEqual) { - bailoutIfZero(Assembler::NonZero, ARMRegister(lhs, 64), snapshot); - return; - } - } - } masm.cmpPtr(lhs, rhs); return bailoutIf(c, snapshot); } + void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs, + LSnapshot* snapshot) { + masm.testPtr(lhs, rhs); + return bailoutIf(c, snapshot); + } template <typename T1, typename T2> void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) { - if constexpr (std::is_same_v<T1, Register> && std::is_same_v<T2, Imm32>) { - if (rhs.value == 0) { - if (c == Assembler::Equal) { - bailoutIfZero(Assembler::Zero, ARMRegister(lhs, 32), snapshot); - return; - } - if (c == Assembler::NotEqual) { - bailoutIfZero(Assembler::NonZero, ARMRegister(lhs, 32), snapshot); - return; - } - } - } masm.cmp32(lhs, rhs); return bailoutIf(c, snapshot); } template <typename T1, typename T2> void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) { - if constexpr (std::is_same_v<T1, Register> && - std::is_same_v<T2, Register>) { - if (lhs == rhs && (c == Assembler::Zero || c == Assembler::NonZero)) { - bailoutIfZero(c, ARMRegister(lhs, 32), snapshot); - return; - } - } masm.test32(lhs, rhs); return bailoutIf(c, snapshot); } diff --git a/js/src/jit/arm64/LIR-arm64.h b/js/src/jit/arm64/LIR-arm64.h @@ -26,6 +26,75 @@ class LUnbox : public LInstructionHelper<1, BOX_PIECES, 0> { const char* extraName() const { return StringFromMIRType(mir()->type()); } }; +class LDivOrModI64 : public LBinaryMath<0> { + public: + LIR_HEADER(DivOrModI64) + + LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + } + + MBinaryArithInstruction* mir() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + return static_cast<MBinaryArithInstruction*>(mir_); + } + + bool canBeDivideByZero() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeDivideByZero(); + } + return mir_->toDiv()->canBeDivideByZero(); + } + bool canBeNegativeOverflow() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeNegativeDividend(); + } + return mir_->toDiv()->canBeNegativeOverflow(); + } + wasm::TrapSiteDesc trapSiteDesc() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + if (mir_->isMod()) { + return mir_->toMod()->trapSiteDesc(); + } + return mir_->toDiv()->trapSiteDesc(); + } +}; + +class LUDivOrModI64 : public LBinaryMath<0> { + public: + LIR_HEADER(UDivOrModI64); + + LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + } + + const char* extraName() const { + return mir()->isTruncated() ? "Truncated" : nullptr; + } + + MBinaryArithInstruction* mir() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + return static_cast<MBinaryArithInstruction*>(mir_); + } + bool canBeDivideByZero() const { + if (mir_->isMod()) { + return mir_->toMod()->canBeDivideByZero(); + } + return mir_->toDiv()->canBeDivideByZero(); + } + wasm::TrapSiteDesc trapSiteDesc() const { + MOZ_ASSERT(mir_->isDiv() || mir_->isMod()); + if (mir_->isMod()) { + return mir_->toMod()->trapSiteDesc(); + } + return mir_->toDiv()->trapSiteDesc(); + } +}; + } // namespace jit } // namespace js diff --git a/js/src/jit/arm64/Lowering-arm64.cpp b/js/src/jit/arm64/Lowering-arm64.cpp @@ -19,6 +19,8 @@ using namespace js; using namespace js::jit; +using mozilla::FloorLog2; + LBoxAllocation LIRGeneratorARM64::useBoxFixed(MDefinition* mir, Register reg1, Register, bool useAtStart) { MOZ_ASSERT(mir->type() == MIRType::Value); @@ -202,41 +204,36 @@ void LIRGeneratorARM64::lowerForShift(LInstructionHelper<1, 2, 0>* ins, } void LIRGeneratorARM64::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + if (div->rhs()->isConstant()) { LAllocation lhs = useRegister(div->lhs()); - int32_t rhs = div->rhs()->toConstant()->toInt32(); int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs)); if (rhs != 0 && uint32_t(1) << shift == mozilla::Abs(rhs)) { - auto* lir = new (alloc()) LDivPowTwoI(lhs, shift, rhs < 0); + LDivPowTwoI* lir = new (alloc()) LDivPowTwoI(lhs, shift, rhs < 0); if (div->fallible()) { assignSnapshot(lir, div->bailoutKind()); } define(lir, div); return; } - - auto* lir = new (alloc()) LDivConstantI(lhs, rhs); - if (div->fallible()) { - assignSnapshot(lir, div->bailoutKind()); + if (rhs != 0) { + LDivConstantI* lir = new (alloc()) LDivConstantI(lhs, temp(), rhs); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + define(lir, div); + return; } - define(lir, div); - return; - } - - LAllocation lhs, rhs; - if (div->canTruncateRemainder()) { - lhs = useRegisterAtStart(div->lhs()); - rhs = useRegisterAtStart(div->rhs()); - } else { - lhs = useRegister(div->lhs()); - rhs = useRegister(div->rhs()); } - // ARM64 has plenty of scratch registers, so we don't need to request an - // additonal temp register from the register allocator. - auto* lir = new (alloc()) LDivI(lhs, rhs, LDefinition::BogusTemp()); + LDivI* lir = new (alloc()) + LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp()); if (div->fallible()) { assignSnapshot(lir, div->bailoutKind()); } @@ -263,30 +260,34 @@ void LIRGeneratorARM64::lowerMulI(MMul* mul, MDefinition* lhs, } void LIRGeneratorARM64::lowerModI(MMod* mod) { - LAllocation lhs = useRegister(mod->lhs()); + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } if (mod->rhs()->isConstant()) { int32_t rhs = mod->rhs()->toConstant()->toInt32(); - int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs)); - - if (rhs != 0 && uint32_t(1) << shift == mozilla::Abs(rhs)) { - auto* lir = new (alloc()) LModPowTwoI(lhs, shift); + int32_t shift = FloorLog2(rhs); + if (rhs > 0 && 1 << shift == rhs) { + LModPowTwoI* lir = + new (alloc()) LModPowTwoI(useRegister(mod->lhs()), shift); if (mod->fallible()) { assignSnapshot(lir, mod->bailoutKind()); } define(lir, mod); return; + } else if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) { + LModMaskI* lir = new (alloc()) + LModMaskI(useRegister(mod->lhs()), temp(), temp(), shift + 1); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); } - - auto* lir = new (alloc()) LModConstantI(lhs, rhs); - if (mod->fallible()) { - assignSnapshot(lir, mod->bailoutKind()); - } - define(lir, mod); - return; } - auto* lir = new (alloc()) LModI(lhs, useRegister(mod->rhs())); + LModI* lir = + new (alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs())); if (mod->fallible()) { assignSnapshot(lir, mod->bailoutKind()); } @@ -294,26 +295,25 @@ void LIRGeneratorARM64::lowerModI(MMod* mod) { } void LIRGeneratorARM64::lowerDivI64(MDiv* div) { - auto* lir = new (alloc()) - LDivI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs())); - defineInt64(lir, div); -} + if (div->isUnsigned()) { + lowerUDivI64(div); + return; + } -void LIRGeneratorARM64::lowerModI64(MMod* mod) { - auto* lir = - new (alloc()) LModI64(useRegister(mod->lhs()), useRegister(mod->rhs())); - defineInt64(lir, mod); + LDivOrModI64* lir = new (alloc()) + LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs())); + defineInt64(lir, div); } void LIRGeneratorARM64::lowerUDivI64(MDiv* div) { - auto* lir = new (alloc()) - LUDivI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs())); + LUDivOrModI64* lir = new (alloc()) + LUDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs())); defineInt64(lir, div); } void LIRGeneratorARM64::lowerUModI64(MMod* mod) { - auto* lir = - new (alloc()) LUModI64(useRegister(mod->lhs()), useRegister(mod->rhs())); + LUDivOrModI64* lir = new (alloc()) + LUDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs())); defineInt64(lir, mod); } @@ -321,6 +321,17 @@ void LIRGeneratorARM64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { MOZ_CRASH("We don't use runtime div for this architecture"); } +void LIRGeneratorARM64::lowerModI64(MMod* mod) { + if (mod->isUnsigned()) { + lowerUModI64(mod); + return; + } + + LDivOrModI64* lir = new (alloc()) + LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs())); + defineInt64(lir, mod); +} + void LIRGeneratorARM64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) { MOZ_CRASH("We don't use runtime mod for this architecture"); } @@ -493,15 +504,14 @@ bool LIRGeneratorARM64::canEmitWasmReduceSimd128AtUses( #endif void LIRGeneratorARM64::lowerUDiv(MDiv* div) { + LAllocation lhs = useRegister(div->lhs()); if (div->rhs()->isConstant()) { - LAllocation lhs = useRegister(div->lhs()); - // NOTE: the result of toInt32 is coerced to uint32_t. uint32_t rhs = div->rhs()->toConstant()->toInt32(); int32_t shift = mozilla::FloorLog2(rhs); if (rhs != 0 && uint32_t(1) << shift == rhs) { - auto* lir = new (alloc()) LDivPowTwoI(lhs, shift, false); + LDivPowTwoI* lir = new (alloc()) LDivPowTwoI(lhs, shift, false); if (div->fallible()) { assignSnapshot(lir, div->bailoutKind()); } @@ -509,7 +519,7 @@ void LIRGeneratorARM64::lowerUDiv(MDiv* div) { return; } - auto* lir = new (alloc()) LUDivConstant(lhs, rhs); + LUDivConstantI* lir = new (alloc()) LUDivConstantI(lhs, temp(), rhs); if (div->fallible()) { assignSnapshot(lir, div->bailoutKind()); } @@ -518,16 +528,13 @@ void LIRGeneratorARM64::lowerUDiv(MDiv* div) { } // Generate UDiv - LAllocation lhs, rhs; - if (div->canTruncateRemainder()) { - lhs = useRegisterAtStart(div->lhs()); - rhs = useRegisterAtStart(div->rhs()); - } else { - lhs = useRegister(div->lhs()); - rhs = useRegister(div->rhs()); + LAllocation rhs = useRegister(div->rhs()); + LDefinition remainder = LDefinition::BogusTemp(); + if (!div->canTruncateRemainder()) { + remainder = temp(); } - auto* lir = new (alloc()) LUDiv(lhs, rhs); + LUDiv* lir = new (alloc()) LUDiv(lhs, rhs, remainder); if (div->fallible()) { assignSnapshot(lir, div->bailoutKind()); } @@ -535,31 +542,8 @@ void LIRGeneratorARM64::lowerUDiv(MDiv* div) { } void LIRGeneratorARM64::lowerUMod(MMod* mod) { - LAllocation lhs = useRegister(mod->lhs()); - - if (mod->rhs()->isConstant()) { - // NOTE: the result of toInt32 is coerced to uint32_t. - uint32_t rhs = mod->rhs()->toConstant()->toInt32(); - int32_t shift = mozilla::FloorLog2(rhs); - - if (rhs != 0 && uint32_t(1) << shift == rhs) { - auto* lir = new (alloc()) LModPowTwoI(lhs, shift); - if (mod->fallible()) { - assignSnapshot(lir, mod->bailoutKind()); - } - define(lir, mod); - return; - } - - auto* lir = new (alloc()) LUModConstant(lhs, rhs); - if (mod->fallible()) { - assignSnapshot(lir, mod->bailoutKind()); - } - define(lir, mod); - return; - } - - auto* lir = new (alloc()) LUMod(lhs, useRegister(mod->rhs())); + LUMod* lir = new (alloc()) + LUMod(useRegister(mod->getOperand(0)), useRegister(mod->getOperand(1))); if (mod->fallible()) { assignSnapshot(lir, mod->bailoutKind()); } diff --git a/js/src/jit/arm64/MacroAssembler-arm64.h b/js/src/jit/arm64/MacroAssembler-arm64.h @@ -977,6 +977,18 @@ class MacroAssemblerCompat : public vixl::MacroAssembler { Ldr(scratch32, toMemOperand(lhs)); Cmp(scratch32, Operand(ARMRegister(rhs, 32))); } + void cmp32(const vixl::Operand& lhs, Imm32 rhs) { + vixl::UseScratchRegisterScope temps(this); + const ARMRegister scratch32 = temps.AcquireW(); + Mov(scratch32, lhs); + Cmp(scratch32, Operand(rhs.value)); + } + void cmp32(const vixl::Operand& lhs, Register rhs) { + vixl::UseScratchRegisterScope temps(this); + const ARMRegister scratch32 = temps.AcquireW(); + Mov(scratch32, lhs); + Cmp(scratch32, Operand(ARMRegister(rhs, 32))); + } void cmn32(Register lhs, Imm32 rhs) { Cmn(ARMRegister(lhs, 32), Operand(rhs.value)); diff --git a/js/src/jit/loong64/CodeGenerator-loong64.h b/js/src/jit/loong64/CodeGenerator-loong64.h @@ -55,6 +55,13 @@ class CodeGeneratorLOONG64 : public CodeGeneratorShared { masm.branchPtr(c, lhs, rhs, &bail); bailoutFrom(&bail, snapshot); } + template <typename T1, typename T2> + void bailoutTestPtr(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchTestPtr(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) { Label bail; UseScratchRegisterScope temps(masm); diff --git a/js/src/jit/loong64/Lowering-loong64.cpp b/js/src/jit/loong64/Lowering-loong64.cpp @@ -170,6 +170,11 @@ void LIRGeneratorLOONG64::lowerMulI(MMul* mul, MDefinition* lhs, } void LIRGeneratorLOONG64::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + // Division instructions are slow. Division by constant denominators can be // rewritten to use other instructions. if (div->rhs()->isConstant()) { @@ -200,12 +205,22 @@ void LIRGeneratorLOONG64::lowerDivI(MDiv* div) { } void LIRGeneratorLOONG64::lowerDivI64(MDiv* div) { + if (div->isUnsigned()) { + lowerUDivI64(div); + return; + } + auto* lir = new (alloc()) LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs())); defineInt64(lir, div); } void LIRGeneratorLOONG64::lowerModI(MMod* mod) { + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } + if (mod->rhs()->isConstant()) { int32_t rhs = mod->rhs()->toConstant()->toInt32(); int32_t shift = FloorLog2(rhs); @@ -237,6 +252,11 @@ void LIRGeneratorLOONG64::lowerModI(MMod* mod) { } void LIRGeneratorLOONG64::lowerModI64(MMod* mod) { + if (mod->isUnsigned()) { + lowerUModI64(mod); + return; + } + auto* lir = new (alloc()) LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs())); defineInt64(lir, mod); diff --git a/js/src/jit/mips-shared/CodeGenerator-mips-shared.h b/js/src/jit/mips-shared/CodeGenerator-mips-shared.h @@ -55,6 +55,13 @@ class CodeGeneratorMIPSShared : public CodeGeneratorShared { masm.branchPtr(c, lhs, rhs, &bail); bailoutFrom(&bail, snapshot); } + template <typename T1, typename T2> + void bailoutTestPtr(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchTestPtr(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } template <typename T> void bailoutIfFalseBool(T reg, LSnapshot* snapshot) { Label bail; diff --git a/js/src/jit/mips-shared/Lowering-mips-shared.cpp b/js/src/jit/mips-shared/Lowering-mips-shared.cpp @@ -138,6 +138,11 @@ void LIRGeneratorMIPSShared::lowerForShift(LInstructionHelper<1, 2, 0>* ins, } void LIRGeneratorMIPSShared::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + // Division instructions are slow. Division by constant denominators can be // rewritten to use other instructions. if (div->rhs()->isConstant()) { @@ -187,6 +192,11 @@ void LIRGeneratorMIPSShared::lowerMulI(MMul* mul, MDefinition* lhs, } void LIRGeneratorMIPSShared::lowerModI(MMod* mod) { + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } + if (mod->rhs()->isConstant()) { int32_t rhs = mod->rhs()->toConstant()->toInt32(); int32_t shift = FloorLog2(rhs); diff --git a/js/src/jit/mips64/Lowering-mips64.cpp b/js/src/jit/mips64/Lowering-mips64.cpp @@ -36,6 +36,11 @@ LBoxAllocation LIRGeneratorMIPS64::useBoxFixed(MDefinition* mir, Register reg1, LDefinition LIRGeneratorMIPS64::tempToUnbox() { return temp(); } void LIRGeneratorMIPS64::lowerDivI64(MDiv* div) { + if (div->isUnsigned()) { + lowerUDivI64(div); + return; + } + auto* lir = new (alloc()) LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs())); defineInt64(lir, div); @@ -46,6 +51,11 @@ void LIRGeneratorMIPS64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { } void LIRGeneratorMIPS64::lowerModI64(MMod* mod) { + if (mod->isUnsigned()) { + lowerUModI64(mod); + return; + } + auto* lir = new (alloc()) LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs())); defineInt64(lir, mod); diff --git a/js/src/jit/none/CodeGenerator-none.h b/js/src/jit/none/CodeGenerator-none.h @@ -33,6 +33,9 @@ class CodeGeneratorNone : public CodeGeneratorShared { void bailoutCmpPtr(Assembler::Condition, T1, T2, LSnapshot*) { MOZ_CRASH(); } + void bailoutTestPtr(Assembler::Condition, Register, Register, LSnapshot*) { + MOZ_CRASH(); + } void bailoutIfFalseBool(Register, LSnapshot*) { MOZ_CRASH(); } void bailoutFrom(Label*, LSnapshot*) { MOZ_CRASH(); } void bailout(LSnapshot*) { MOZ_CRASH(); } diff --git a/js/src/jit/riscv64/CodeGenerator-riscv64.h b/js/src/jit/riscv64/CodeGenerator-riscv64.h @@ -56,6 +56,13 @@ class CodeGeneratorRiscv64 : public CodeGeneratorShared { masm.branchPtr(c, lhs, rhs, &bail); bailoutFrom(&bail, snapshot); } + template <typename T1, typename T2> + void bailoutTestPtr(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchTestPtr(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) { Label bail; UseScratchRegisterScope temps(&masm); diff --git a/js/src/jit/riscv64/Lowering-riscv64.cpp b/js/src/jit/riscv64/Lowering-riscv64.cpp @@ -170,6 +170,11 @@ void LIRGeneratorRiscv64::lowerMulI(MMul* mul, MDefinition* lhs, } void LIRGeneratorRiscv64::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + // Division instructions are slow. Division by constant denominators can be // rewritten to use other instructions. if (div->rhs()->isConstant()) { @@ -208,12 +213,22 @@ void LIRGeneratorRiscv64::lowerDivI(MDiv* div) { } void LIRGeneratorRiscv64::lowerDivI64(MDiv* div) { + if (div->isUnsigned()) { + lowerUDivI64(div); + return; + } + auto* lir = new (alloc()) LDivOrModI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs())); defineInt64(lir, div); } void LIRGeneratorRiscv64::lowerModI(MMod* mod) { + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } + if (mod->rhs()->isConstant()) { int32_t rhs = mod->rhs()->toConstant()->toInt32(); int32_t shift = FloorLog2(rhs); @@ -253,6 +268,11 @@ void LIRGeneratorRiscv64::lowerModI(MMod* mod) { } void LIRGeneratorRiscv64::lowerModI64(MMod* mod) { + if (mod->isUnsigned()) { + lowerUModI64(mod); + return; + } + auto* lir = new (alloc()) LDivOrModI64(useRegisterAtStart(mod->lhs()), useRegisterAtStart(mod->rhs())); defineInt64(lir, mod); diff --git a/js/src/jit/wasm32/CodeGenerator-wasm32.h b/js/src/jit/wasm32/CodeGenerator-wasm32.h @@ -32,6 +32,9 @@ class CodeGeneratorWasm32 : public CodeGeneratorShared { void bailoutCmpPtr(Assembler::Condition, T1, T2, LSnapshot*) { MOZ_CRASH(); } + void bailoutTestPtr(Assembler::Condition, Register, Register, LSnapshot*) { + MOZ_CRASH(); + } void bailoutIfFalseBool(Register, LSnapshot*) { MOZ_CRASH(); } void bailoutFrom(Label*, LSnapshot*) { MOZ_CRASH(); } void bailout(LSnapshot*) { MOZ_CRASH(); } diff --git a/js/src/jit/x64/Lowering-x64.cpp b/js/src/jit/x64/Lowering-x64.cpp @@ -526,6 +526,11 @@ void LIRGenerator::visitSubstr(MSubstr* ins) { } void LIRGeneratorX64::lowerDivI64(MDiv* div) { + if (div->isUnsigned()) { + lowerUDivI64(div); + return; + } + LDivOrModI64* lir = new (alloc()) LDivOrModI64( useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx)); defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax)))); @@ -536,6 +541,11 @@ void LIRGeneratorX64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { } void LIRGeneratorX64::lowerModI64(MMod* mod) { + if (mod->isUnsigned()) { + lowerUModI64(mod); + return; + } + LDivOrModI64* lir = new (alloc()) LDivOrModI64( useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax)); defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx)))); diff --git a/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp b/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp @@ -1103,11 +1103,11 @@ void CodeGenerator::visitDivOrModConstantI(LDivOrModConstantI* ins) { // The absolute value of the denominator isn't a power of 2 (see LDivPowTwoI // and LModPowTwoI). - MOZ_ASSERT(!mozilla::IsPowerOfTwo(mozilla::Abs(d))); + MOZ_ASSERT((Abs(d) & (Abs(d) - 1)) != 0); // We will first divide by Abs(d), and negate the answer if d is negative. // If desired, this can be avoided by generalizing computeDivisionConstants. - auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(d); + auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(Abs(d)); // We first compute (M * n) >> 32, where M = rmc.multiplier. masm.movl(Imm32(rmc.multiplier), eax); diff --git a/js/src/jit/x86-shared/CodeGenerator-x86-shared.h b/js/src/jit/x86-shared/CodeGenerator-x86-shared.h @@ -55,6 +55,11 @@ class CodeGeneratorX86Shared : public CodeGeneratorShared { masm.cmpPtr(lhs, rhs); bailoutIf(c, snapshot); } + void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs, + LSnapshot* snapshot) { + masm.testPtr(lhs, rhs); + bailoutIf(c, snapshot); + } template <typename T1, typename T2> void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) { diff --git a/js/src/jit/x86-shared/Lowering-x86-shared.cpp b/js/src/jit/x86-shared/Lowering-x86-shared.cpp @@ -169,6 +169,11 @@ void LIRGeneratorX86Shared::lowerMulI(MMul* mul, MDefinition* lhs, } void LIRGeneratorX86Shared::lowerDivI(MDiv* div) { + if (div->isUnsigned()) { + lowerUDiv(div); + return; + } + // Division instructions are slow. Division by constant denominators can be // rewritten to use other instructions. if (div->rhs()->isConstant()) { @@ -220,6 +225,11 @@ void LIRGeneratorX86Shared::lowerDivI(MDiv* div) { } void LIRGeneratorX86Shared::lowerModI(MMod* mod) { + if (mod->isUnsigned()) { + lowerUMod(mod); + return; + } + if (mod->rhs()->isConstant()) { int32_t rhs = mod->rhs()->toConstant()->toInt32(); int32_t shift = FloorLog2(Abs(rhs));