commit 527c3ea6fc525402a16460a8d8e7e547e92bd05d parent 1ab1eb6ce51b15cc04140d297bfb664985a28683 Author: Ryan Hunt <rhunt@eqrion.net> Date: Thu, 18 Dec 2025 16:45:28 +0000 Bug 2002625 - wasm: Use Wasm ABI for builtin thunks. r=yury Builtin thunks are going to need more scratch registers for the prologue/epilogue in order to do stack switches. Right now we are using the sytem ABI for builtin thunks and this leaves us with only 2 registers for the prologue and 1 register for the epilogue. It is not feasible to expand this. We could save/restore non-volatile registers, but this gets really complicated when the stack is switching to figure out which stack has the saved register on it. An easier solution is to move builtin thunks back to using the wasm ABI, which has no non-volatile registers and therefore has a good amount of scratch registers that don't alias the arg/result registers. We still keep the code to call a builtin using the system ABI and use that for builtins that don't have a thunk (such as intgemm). A future commit will expand the number of builtins which go without a thunk to reduce the overhead of the stack switch. A necessary change here is that we must now use saveLive/ restoreLive in some code that is around calling builtins because we no longer have non-volatile registers. Differential Revision: https://phabricator.services.mozilla.com/D275544 Diffstat:
26 files changed, 242 insertions(+), 261 deletions(-)
diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp @@ -10247,8 +10247,8 @@ void CodeGenerator::visitWasmRegisterResult(LWasmRegisterResult* lir) { #endif } -void CodeGenerator::visitWasmBuiltinFloatRegisterResult( - LWasmBuiltinFloatRegisterResult* lir) { +void CodeGenerator::visitWasmSystemFloatRegisterResult( + LWasmSystemFloatRegisterResult* lir) { MOZ_ASSERT(lir->mir()->type() == MIRType::Float32 || lir->mir()->type() == MIRType::Double); MOZ_ASSERT_IF(lir->mir()->type() == MIRType::Float32, @@ -10257,7 +10257,7 @@ void CodeGenerator::visitWasmBuiltinFloatRegisterResult( ToFloatRegister(lir->output()) == ReturnDoubleReg); #ifdef JS_CODEGEN_ARM - MWasmBuiltinFloatRegisterResult* mir = lir->mir(); + MWasmSystemFloatRegisterResult* mir = lir->mir(); if (!mir->hardFP()) { if (mir->type() == MIRType::Float32) { // Move float32 from r0 to ReturnFloatReg. @@ -10270,7 +10270,7 @@ void CodeGenerator::visitWasmBuiltinFloatRegisterResult( } } #elif JS_CODEGEN_X86 - MWasmBuiltinFloatRegisterResult* mir = lir->mir(); + MWasmSystemFloatRegisterResult* mir = lir->mir(); if (mir->type() == MIRType::Double) { masm.reserveStack(sizeof(double)); masm.fstp(Operand(esp, 0)); @@ -10512,7 +10512,7 @@ void CodeGenerator::callWasmUpdateSuspenderState( masm.move32(Imm32(uint32_t(kind)), temp); - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::UpdateSuspenderState); masm.passABIArg(InstanceReg); masm.passABIArg(suspender); masm.passABIArg(temp); @@ -11368,12 +11368,12 @@ void CodeGenerator::visitWasmPostWriteBarrierWholeCell( wasm::CheckWholeCellLastElementCache(masm, InstanceReg, object, temp, ool.rejoin()); - saveLiveVolatile(lir); + saveLive(lir); masm.Push(InstanceReg); int32_t framePushedAfterInstance = masm.framePushed(); // Call Instance::postBarrierWholeCell - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::PostBarrierWholeCell); masm.passABIArg(InstanceReg); masm.passABIArg(object); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; @@ -11382,7 +11382,7 @@ void CodeGenerator::visitWasmPostWriteBarrierWholeCell( mozilla::Some(instanceOffset), ABIType::General); masm.Pop(InstanceReg); - restoreLiveVolatile(lir); + restoreLive(lir); masm.jump(ool.rejoin()); }); @@ -11403,7 +11403,7 @@ void CodeGenerator::visitWasmPostWriteBarrierEdgeAtIndex( Register temp = ToRegister(lir->temp0()); MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg); auto* ool = new (alloc()) LambdaOutOfLineCode([=, this](OutOfLineCode& ool) { - saveLiveVolatile(lir); + saveLive(lir); masm.Push(InstanceReg); int32_t framePushedAfterInstance = masm.framePushed(); @@ -11418,7 +11418,7 @@ void CodeGenerator::visitWasmPostWriteBarrierEdgeAtIndex( } // Call Instance::postBarrier - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::PostBarrierEdge); masm.passABIArg(InstanceReg); masm.passABIArg(temp); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; @@ -11427,7 +11427,7 @@ void CodeGenerator::visitWasmPostWriteBarrierEdgeAtIndex( mozilla::Some(instanceOffset), ABIType::General); masm.Pop(InstanceReg); - restoreLiveVolatile(lir); + restoreLive(lir); masm.jump(ool.rejoin()); }); @@ -12171,7 +12171,7 @@ void CodeGenerator::visitWasmBuiltinModD(LWasmBuiltinModD* ins) { MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg); - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::ModD); masm.passABIArg(lhs, ABIType::Float64); masm.passABIArg(rhs, ABIType::Float64); @@ -21276,7 +21276,7 @@ void CodeGenerator::callWasmStructAllocFun( int32_t framePushedAfterInstance = masm.framePushed(); saveLive(lir); - masm.setupWasmABICall(); + masm.setupWasmABICall(fun); masm.passABIArg(InstanceReg); masm.passABIArg(typeDefIndex); masm.passABIArg(allocSite); @@ -21360,7 +21360,7 @@ void CodeGenerator::callWasmArrayAllocFun( int32_t framePushedAfterInstance = masm.framePushed(); saveLive(lir); - masm.setupWasmABICall(); + masm.setupWasmABICall(fun); masm.passABIArg(InstanceReg); masm.passABIArg(numElements); masm.passABIArg(typeDefIndex); diff --git a/js/src/jit/LIROps.yaml b/js/src/jit/LIROps.yaml @@ -3434,7 +3434,7 @@ - name: WasmRegisterPairResult gen_boilerplate: false -- name: WasmBuiltinFloatRegisterResult +- name: WasmSystemFloatRegisterResult gen_boilerplate: false - name: WasmStackResultArea diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp @@ -7256,9 +7256,9 @@ void LIRGenerator::visitWasmFloatRegisterResult(MWasmFloatRegisterResult* ins) { addUnchecked(lir, ins); } -void LIRGenerator::visitWasmBuiltinFloatRegisterResult( - MWasmBuiltinFloatRegisterResult* ins) { - auto* lir = new (alloc()) LWasmBuiltinFloatRegisterResult(); +void LIRGenerator::visitWasmSystemFloatRegisterResult( + MWasmSystemFloatRegisterResult* ins) { + auto* lir = new (alloc()) LWasmSystemFloatRegisterResult(); uint32_t vreg = getVirtualRegister(); auto type = LDefinition::TypeFrom(ins->type()); lir->setDef(0, LDefinition(vreg, type, LFloatReg(ins->loc()))); diff --git a/js/src/jit/MIR-wasm.h b/js/src/jit/MIR-wasm.h @@ -1730,14 +1730,14 @@ class MWasmFloatRegisterResult : public MWasmResultBase<FloatRegister> { AliasSet getAliasSet() const override { return AliasSet::None(); } }; -class MWasmBuiltinFloatRegisterResult : public MWasmResultBase<FloatRegister> { - MWasmBuiltinFloatRegisterResult(MIRType type, FloatRegister reg, bool hardFP) +class MWasmSystemFloatRegisterResult : public MWasmResultBase<FloatRegister> { + MWasmSystemFloatRegisterResult(MIRType type, FloatRegister reg, bool hardFP) : MWasmResultBase(classOpcode, type, reg), hardFP_(hardFP) {} bool hardFP_; public: - INSTRUCTION_HEADER(WasmBuiltinFloatRegisterResult) + INSTRUCTION_HEADER(WasmSystemFloatRegisterResult) TRIVIAL_NEW_WRAPPERS AliasSet getAliasSet() const override { return AliasSet::None(); } diff --git a/js/src/jit/MIROps.yaml b/js/src/jit/MIROps.yaml @@ -4350,7 +4350,7 @@ - name: WasmFloatRegisterResult gen_boilerplate: false -- name: WasmBuiltinFloatRegisterResult +- name: WasmSystemFloatRegisterResult gen_boilerplate: false - name: WasmRegister64Result diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp @@ -4249,7 +4249,7 @@ void MacroAssembler::outOfLineTruncateSlow(FloatRegister src, Register dest, Push(InstanceReg); int32_t framePushedAfterInstance = framePushed(); - setupWasmABICall(); + setupWasmABICall(wasm::SymbolicAddress::ToInt32); passABIArg(src, ABIType::Float64); int32_t instanceOffset = framePushed() - framePushedAfterInstance; @@ -4916,9 +4916,9 @@ void MacroAssembler::setupNativeABICall() { setupABICallHelper(ABIKind::System); } -void MacroAssembler::setupWasmABICall() { +void MacroAssembler::setupWasmABICall(wasm::SymbolicAddress builtin) { MOZ_ASSERT(IsCompilingWasm(), "non-wasm should use setupAlignedABICall"); - setupABICallHelper(ABIKind::System); + setupABICallHelper(wasm::ABIForBuiltin(builtin)); dynamicAlignment_ = false; } @@ -5103,7 +5103,7 @@ CodeOffset MacroAssembler::callWithABI(wasm::BytecodeOffset bytecode, CodeOffset raOffset = call( wasm::CallSiteDesc(bytecode.offset(), wasm::CallSiteKind::Symbolic), imm); - callWithABIPost(stackAdjust, result, /* callFromWasm = */ true); + callWithABIPost(stackAdjust, result); #ifdef JS_CHECK_UNSAFE_CALL_WITH_ABI if (checkUnsafeCallWithABI) { @@ -5119,7 +5119,7 @@ void MacroAssembler::callDebugWithABI(wasm::SymbolicAddress imm, uint32_t stackAdjust; callWithABIPre(&stackAdjust, /* callFromWasm = */ false); call(imm); - callWithABIPost(stackAdjust, result, /* callFromWasm = */ false); + callWithABIPost(stackAdjust, result); } // =============================================================== diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h @@ -772,7 +772,7 @@ class MacroAssembler : public MacroAssemblerSpecific { // As setupAlignedABICall, but for WebAssembly native ABI calls, which pass // through a builtin thunk that uses the wasm ABI. All the wasm ABI calls // can be native, since we always know the stack alignment a priori. - void setupWasmABICall(); + void setupWasmABICall(wasm::SymbolicAddress builtin); // Setup an ABI call for when the alignment is not known. This may need a // scratch register. @@ -831,8 +831,7 @@ class MacroAssembler : public MacroAssemblerSpecific { void callWithABINoProfiler(const Address& fun, ABIType result) PER_ARCH; // Restore the stack to its state before the setup function call. - void callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm = false) PER_ARCH; + void callWithABIPost(uint32_t stackAdjust, ABIType result) PER_ARCH; #ifdef JS_CHECK_UNSAFE_CALL_WITH_ABI // Set the JSContext::inUnsafeCallWithABI flag using InstanceReg. diff --git a/js/src/jit/arm/CodeGenerator-arm.cpp b/js/src/jit/arm/CodeGenerator-arm.cpp @@ -568,7 +568,7 @@ void CodeGenerator::visitSoftDivI(LSoftDivI* ins) { if (gen->compilingWasm()) { masm.Push(InstanceReg); int32_t framePushedAfterInstance = masm.framePushed(); - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::aeabi_idivmod); masm.passABIArg(lhs); masm.passABIArg(rhs); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; @@ -758,7 +758,7 @@ void CodeGenerator::visitSoftModI(LSoftModI* ins) { if (gen->compilingWasm()) { masm.Push(InstanceReg); int32_t framePushedAfterInstance = masm.framePushed(); - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::aeabi_idivmod); masm.passABIArg(lhs); masm.passABIArg(rhs); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; @@ -2121,7 +2121,7 @@ void CodeGenerator::visitSoftUDivOrMod(LSoftUDivOrMod* ins) { if (gen->compilingWasm()) { masm.Push(InstanceReg); int32_t framePushedAfterInstance = masm.framePushed(); - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::aeabi_uidivmod); masm.passABIArg(lhs); masm.passABIArg(rhs); wasm::BytecodeOffset bytecodeOffset = @@ -2260,32 +2260,28 @@ void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) { masm.Push(input); - masm.setupWasmABICall(); - masm.passABIArg(inputDouble, ABIType::Float64); - - int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; + wasm::SymbolicAddress callee; if (lir->mir()->isSaturating()) { if (lir->mir()->isUnsigned()) { - masm.callWithABI(mir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::SaturatingTruncateDoubleToUint64, - mozilla::Some(instanceOffset)); + callee = wasm::SymbolicAddress::SaturatingTruncateDoubleToUint64; } else { - masm.callWithABI(mir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::SaturatingTruncateDoubleToInt64, - mozilla::Some(instanceOffset)); + callee = wasm::SymbolicAddress::SaturatingTruncateDoubleToInt64; } } else { if (lir->mir()->isUnsigned()) { - masm.callWithABI(mir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::TruncateDoubleToUint64, - mozilla::Some(instanceOffset)); + callee = wasm::SymbolicAddress::TruncateDoubleToUint64; } else { - masm.callWithABI(mir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::TruncateDoubleToInt64, - mozilla::Some(instanceOffset)); + callee = wasm::SymbolicAddress::TruncateDoubleToInt64; } } + masm.setupWasmABICall(callee); + masm.passABIArg(inputDouble, ABIType::Float64); + + int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; + masm.callWithABI(mir->trapSiteDesc().bytecodeOffset, callee, + mozilla::Some(instanceOffset)); + masm.Pop(input); masm.Pop(InstanceReg); @@ -2330,10 +2326,6 @@ void CodeGenerator::visitInt64ToFloatingPointCall( MBuiltinInt64ToFloatingPoint* mir = lir->mir(); MIRType toType = mir->type(); - masm.setupWasmABICall(); - masm.passABIArg(input.high); - masm.passABIArg(input.low); - bool isUnsigned = mir->isUnsigned(); wasm::SymbolicAddress callee = toType == MIRType::Float32 @@ -2341,6 +2333,9 @@ void CodeGenerator::visitInt64ToFloatingPointCall( : wasm::SymbolicAddress::Int64ToFloat32) : (isUnsigned ? wasm::SymbolicAddress::Uint64ToDouble : wasm::SymbolicAddress::Int64ToDouble); + masm.setupWasmABICall(callee); + masm.passABIArg(input.high); + masm.passABIArg(input.low); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; ABIType result = @@ -2444,22 +2439,18 @@ void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) { masm.bind(¬min); } - masm.setupWasmABICall(); + wasm::SymbolicAddress callee = mir->isWasmBuiltinModI64() + ? wasm::SymbolicAddress::ModI64 + : wasm::SymbolicAddress::DivI64; + masm.setupWasmABICall(callee); masm.passABIArg(lhs.high); masm.passABIArg(lhs.low); masm.passABIArg(rhs.high); masm.passABIArg(rhs.low); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; - if (mir->isWasmBuiltinModI64()) { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::ModI64, - mozilla::Some(instanceOffset)); - } else { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::DivI64, - mozilla::Some(instanceOffset)); - } + masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, callee, + mozilla::Some(instanceOffset)); MOZ_ASSERT(ReturnReg64 == output); @@ -2488,23 +2479,19 @@ void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) { masm.bind(&nonZero); } - masm.setupWasmABICall(); + MDefinition* mir = lir->mir(); + wasm::SymbolicAddress callee = mir->isWasmBuiltinModI64() + ? wasm::SymbolicAddress::UModI64 + : wasm::SymbolicAddress::UDivI64; + masm.setupWasmABICall(callee); masm.passABIArg(lhs.high); masm.passABIArg(lhs.low); masm.passABIArg(rhs.high); masm.passABIArg(rhs.low); - MDefinition* mir = lir->mir(); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; - if (mir->isWasmBuiltinModI64()) { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::UModI64, - mozilla::Some(instanceOffset)); - } else { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::UDivI64, - mozilla::Some(instanceOffset)); - } + masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, callee, + mozilla::Some(instanceOffset)); masm.Pop(InstanceReg); } diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -4611,13 +4611,12 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { } } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { if (secondScratchReg_ != lr) { ma_mov(secondScratchReg_, lr); } - if (!ARMFlags::UseHardFpABI()) { + if (abiArgs_.abi() == ABIKind::System && !ARMFlags::UseHardFpABI()) { switch (result) { case ABIType::Float64: // Move double from r0/r1 to ReturnFloatReg. diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp @@ -1861,11 +1861,7 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { - // wasm operates without the need for dynamic alignment of SP. - MOZ_ASSERT(!(dynamicAlignment_ && callFromWasm)); - +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { // Call boundaries communicate stack via SP, so we must resync PSP now. initPseudoStackPtr(); diff --git a/js/src/jit/loong64/MacroAssembler-loong64.cpp b/js/src/jit/loong64/MacroAssembler-loong64.cpp @@ -3024,8 +3024,7 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { // Restore ra value (as stored in callWithABIPre()). loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); diff --git a/js/src/jit/mips64/MacroAssembler-mips64.cpp b/js/src/jit/mips64/MacroAssembler-mips64.cpp @@ -2662,8 +2662,7 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { // Restore ra value (as stored in callWithABIPre()). loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); diff --git a/js/src/jit/riscv64/MacroAssembler-riscv64.cpp b/js/src/jit/riscv64/MacroAssembler-riscv64.cpp @@ -3338,8 +3338,7 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { // Restore ra value (as stored in callWithABIPre()). loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h @@ -793,6 +793,7 @@ class ABIArgGeneratorShared { explicit ABIArgGeneratorShared(ABIKind kind); public: + ABIKind abi() const { return kind_; } uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } }; diff --git a/js/src/jit/shared/LIR-shared.h b/js/src/jit/shared/LIR-shared.h @@ -909,14 +909,14 @@ class LWasmRegisterPairResult : public LInstructionHelper<2, 0, 0> { MDefinition* mir() const { return mirRaw(); } }; -class LWasmBuiltinFloatRegisterResult : public LInstructionHelper<1, 0, 0> { +class LWasmSystemFloatRegisterResult : public LInstructionHelper<1, 0, 0> { public: - LIR_HEADER(WasmBuiltinFloatRegisterResult); + LIR_HEADER(WasmSystemFloatRegisterResult); - LWasmBuiltinFloatRegisterResult() : LInstructionHelper(classOpcode) {} + LWasmSystemFloatRegisterResult() : LInstructionHelper(classOpcode) {} - MWasmBuiltinFloatRegisterResult* mir() const { - return mir_->toWasmBuiltinFloatRegisterResult(); + MWasmSystemFloatRegisterResult* mir() const { + return mir_->toWasmSystemFloatRegisterResult(); } }; diff --git a/js/src/jit/wasm32/MacroAssembler-wasm32.cpp b/js/src/jit/wasm32/MacroAssembler-wasm32.cpp @@ -439,8 +439,7 @@ void MacroAssembler::call(ImmPtr imm) { MOZ_CRASH(); } void MacroAssembler::call(JitCode* c) { MOZ_CRASH(); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { MOZ_CRASH(); } diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp @@ -984,8 +984,7 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { freeStack(stackAdjust); if (dynamicAlignment_) { pop(rsp); diff --git a/js/src/jit/x86/CodeGenerator-x86.cpp b/js/src/jit/x86/CodeGenerator-x86.cpp @@ -674,7 +674,7 @@ void CodeGeneratorX86::visitOutOfLineTruncate(OutOfLineTruncate* ool) { saveVolatile(output); if (gen->compilingWasm()) { - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::ToInt32); masm.passABIArg(input, ABIType::Float64); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; @@ -779,7 +779,7 @@ void CodeGeneratorX86::visitOutOfLineTruncateFloat32( masm.Push(input.asDouble()); if (gen->compilingWasm()) { - masm.setupWasmABICall(); + masm.setupWasmABICall(wasm::SymbolicAddress::ToInt32); } else { masm.setupUnalignedABICall(output); } @@ -891,22 +891,18 @@ void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) { masm.bind(¬Overflow); } - masm.setupWasmABICall(); + wasm::SymbolicAddress callee = mir->isWasmBuiltinModI64() + ? wasm::SymbolicAddress::ModI64 + : wasm::SymbolicAddress::DivI64; + masm.setupWasmABICall(callee); masm.passABIArg(lhs.high); masm.passABIArg(lhs.low); masm.passABIArg(rhs.high); masm.passABIArg(rhs.low); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; - if (mir->isWasmBuiltinModI64()) { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::ModI64, - mozilla::Some(instanceOffset)); - } else { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::DivI64, - mozilla::Some(instanceOffset)); - } + masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, callee, + mozilla::Some(instanceOffset)); // output in edx:eax, move to output register. masm.movl(edx, output.high); @@ -939,23 +935,19 @@ void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) { masm.bind(&nonZero); } - masm.setupWasmABICall(); + MDefinition* mir = lir->mir(); + wasm::SymbolicAddress callee = mir->isWasmBuiltinModI64() + ? wasm::SymbolicAddress::UModI64 + : wasm::SymbolicAddress::UDivI64; + masm.setupWasmABICall(callee); masm.passABIArg(lhs.high); masm.passABIArg(lhs.low); masm.passABIArg(rhs.high); masm.passABIArg(rhs.low); - MDefinition* mir = lir->mir(); int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance; - if (mir->isWasmBuiltinModI64()) { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::UModI64, - mozilla::Some(instanceOffset)); - } else { - masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, - wasm::SymbolicAddress::UDivI64, - mozilla::Some(instanceOffset)); - } + masm.callWithABI(lir->trapSiteDesc().bytecodeOffset, callee, + mozilla::Some(instanceOffset)); // output in edx:eax, move to output register. masm.movl(edx, output.high); diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -888,22 +888,23 @@ void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { assertStackAlignment(ABIStackAlignment); } -void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, - bool callFromWasm) { +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { freeStack(stackAdjust); - // Calls to native functions in wasm pass through a thunk which already - // fixes up the return value for us. - if (result == ABIType::Float64) { - reserveStack(sizeof(double)); - fstp(Operand(esp, 0)); - loadDouble(Operand(esp, 0), ReturnDoubleReg); - freeStack(sizeof(double)); - } else if (result == ABIType::Float32) { - reserveStack(sizeof(float)); - fstp32(Operand(esp, 0)); - loadFloat32(Operand(esp, 0), ReturnFloat32Reg); - freeStack(sizeof(float)); + // If this was a call to a system ABI function, we need to adapt the FP + // results to the expected return registers for JIT code. + if (abiArgs_.abi() == ABIKind::System) { + if (result == ABIType::Float64) { + reserveStack(sizeof(double)); + fstp(Operand(esp, 0)); + loadDouble(Operand(esp, 0), ReturnDoubleReg); + freeStack(sizeof(double)); + } else if (result == ABIType::Float32) { + reserveStack(sizeof(float)); + fstp32(Operand(esp, 0)); + loadFloat32(Operand(esp, 0), ReturnFloat32Reg); + freeStack(sizeof(float)); + } } if (dynamicAlignment_) { diff --git a/js/src/wasm/WasmBCClass.h b/js/src/wasm/WasmBCClass.h @@ -156,10 +156,6 @@ struct FunctionCall { MOZ_ASSERT_IF(abiKind == ABIKind::System, restoreState == RestoreState::None || restoreState == RestoreState::PinnedRegs); - // Our uses of the wasm ABI either preserves everything or nothing. - MOZ_ASSERT_IF(abiKind == ABIKind::Wasm, - restoreState == RestoreState::None || - restoreState == RestoreState::All); if (abiKind == ABIKind::System) { // Builtin calls use the system hardFP setting on ARM32. #if defined(JS_CODEGEN_ARM) diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp @@ -1999,7 +1999,6 @@ void BaseCompiler::popStackResultsAfterWasmCall(const StackResultsLoc& results, void BaseCompiler::pushBuiltinCallResult(const FunctionCall& call, MIRType type) { - MOZ_ASSERT(call.abiKind == ABIKind::System); switch (type) { case MIRType::Int32: { RegI32 rv = captureReturnedI32(); @@ -5783,7 +5782,7 @@ bool BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, uint32_t numArgs = signature.length(); size_t stackSpace = stackConsumed(numArgs); - FunctionCall baselineCall(ABIKind::System, RestoreState::None); + FunctionCall baselineCall(ABIForBuiltin(callee), RestoreState::None); beginCall(baselineCall); if (!emitCallArgs(signature, NoCallResults(), &baselineCall, @@ -5828,7 +5827,7 @@ bool BaseCompiler::emitDivOrModI64BuiltinCall(SymbolicAddress callee, checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(true)); } - masm.setupWasmABICall(); + masm.setupWasmABICall(callee); masm.passABIArg(srcDest.high); masm.passABIArg(srcDest.low); masm.passABIArg(rhs.high); @@ -5857,7 +5856,7 @@ bool BaseCompiler::emitConvertInt64ToFloatingCallout(SymbolicAddress callee, FunctionCall call(ABIKind::Wasm, RestoreState::None); - masm.setupWasmABICall(); + masm.setupWasmABICall(callee); # ifdef JS_PUNBOX64 MOZ_CRASH("BaseCompiler platform hook: emitConvertInt64ToFloatingCallout"); # else @@ -5907,7 +5906,7 @@ bool BaseCompiler::emitConvertFloatingToInt64Callout(SymbolicAddress callee, FunctionCall call(ABIKind::Wasm, RestoreState::None); - masm.setupWasmABICall(); + masm.setupWasmABICall(callee); masm.passABIArg(doubleInput, ABIType::Float64); CodeOffset raOffset = masm.callWithABI( bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset())); @@ -6534,7 +6533,8 @@ bool BaseCompiler::emitInstanceCall(const SymbolicAddressSignature& builtin) { uint32_t numNonInstanceArgs = builtin.numArgs - 1 /* instance */; size_t stackSpace = stackConsumed(numNonInstanceArgs); - FunctionCall baselineCall(ABIKind::System, RestoreState::PinnedRegs); + FunctionCall baselineCall(ABIForBuiltin(builtin.identity), + RestoreState::PinnedRegs); beginCall(baselineCall); ABIArg instanceArg = reservePointerArgument(&baselineCall); diff --git a/js/src/wasm/WasmBuiltins.cpp b/js/src/wasm/WasmBuiltins.cpp @@ -2143,9 +2143,11 @@ bool wasm::EnsureBuiltinThunksInitialized( ExitReason exitReason(sym); + // All thunked builtins must use the wasm ABI. + MOZ_ASSERT(ABIForBuiltin(sym) == ABIKind::Wasm); + CallableOffsets offsets; - if (!GenerateBuiltinThunk(masm, ABIKind::System, abiType, exitReason, - funcPtr, &offsets)) { + if (!GenerateBuiltinThunk(masm, abiType, exitReason, funcPtr, &offsets)) { return false; } if (!thunks->codeRanges.emplaceBack(CodeRange::BuiltinThunk, offsets)) { @@ -2173,8 +2175,7 @@ bool wasm::EnsureBuiltinThunksInitialized( ExitReason exitReason = ExitReason::Fixed::BuiltinNative; CallableOffsets offsets; - if (!GenerateBuiltinThunk(masm, ABIKind::Wasm, abiType, exitReason, funcPtr, - &offsets)) { + if (!GenerateBuiltinThunk(masm, abiType, exitReason, funcPtr, &offsets)) { return false; } if (!thunks->codeRanges.emplaceBack(CodeRange::BuiltinThunk, offsets)) { diff --git a/js/src/wasm/WasmBuiltins.h b/js/src/wasm/WasmBuiltins.h @@ -308,6 +308,18 @@ bool IsRoundingFunction(SymbolicAddress callee, jit::RoundingMode* mode); bool NeedsBuiltinThunk(SymbolicAddress sym); +// Returns the ABI that needs to be used to call a builtin. +inline jit::ABIKind ABIForBuiltin(SymbolicAddress sym) { + // Builtin thunks use the WebAssembly ABI. See GenerateBuiltinThunk for more + // information. + if (NeedsBuiltinThunk(sym)) { + return jit::ABIKind::Wasm; + } + + // Otherwise non-thunked builtins use the System ABI directly. + return jit::ABIKind::System; +} + // This function queries whether pc is in one of the process's builtin thunks // and, if so, returns the CodeRange and pointer to the code segment that the // CodeRange is relative to. diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp @@ -2502,10 +2502,6 @@ class FunctionCompiler { [[nodiscard]] bool collectBuiltinCallResult(MIRType type, MDefinition** result, CallCompileState* callState) { - // This function is for collecting results of builtin calls. Use - // collectWasmCallResults for wasm calls. - MOZ_ASSERT(callState->abiKind == ABIKind::System); - MInstruction* def; switch (type) { case MIRType::Int32: @@ -2514,14 +2510,26 @@ class FunctionCompiler { case MIRType::Int64: def = MWasmRegister64Result::New(alloc(), ReturnReg64); break; - case MIRType::Float32: - def = MWasmBuiltinFloatRegisterResult::New( - alloc(), type, ReturnFloat32Reg, callState->hardFP); + case MIRType::Float32: { + if (callState->abiKind == ABIKind::System) { + def = MWasmSystemFloatRegisterResult::New( + alloc(), type, ReturnFloat32Reg, callState->hardFP); + } else { + def = MWasmFloatRegisterResult::New(alloc(), MIRType::Float32, + ReturnFloat32Reg); + } break; - case MIRType::Double: - def = MWasmBuiltinFloatRegisterResult::New( - alloc(), type, ReturnDoubleReg, callState->hardFP); + } + case MIRType::Double: { + if (callState->abiKind == ABIKind::System) { + def = MWasmSystemFloatRegisterResult::New( + alloc(), type, ReturnDoubleReg, callState->hardFP); + } else { + def = MWasmFloatRegisterResult::New(alloc(), MIRType::Double, + ReturnDoubleReg); + } break; + } #ifdef ENABLE_WASM_SIMD case MIRType::Simd128: MOZ_CRASH("SIMD128 not supported in builtin ABI"); @@ -2549,7 +2557,7 @@ class FunctionCompiler { [[nodiscard]] bool collectWasmCallResults(const ResultType& type, CallCompileState* callState, DefVector* results) { - // This function uses wasm::ABIResultIter which does not handle the builtin + // This function uses wasm::ABIResultIter which does not handle the system // ABI. Use collectBuiltinCallResult instead for builtin calls. MOZ_ASSERT(callState->abiKind == ABIKind::Wasm); MOZ_ASSERT(callState->hardFP); @@ -3064,7 +3072,7 @@ class FunctionCompiler { bool builtinCall1(const SymbolicAddressSignature& builtin, uint32_t lineOrBytecode, MDefinition* arg, MDefinition** result) { - CallCompileState callState(ABIKind::System); + CallCompileState callState(ABIForBuiltin(builtin.identity)); return passCallArg(arg, builtin.argTypes[0], &callState) && finishCallArgs(&callState) && builtinCall(&callState, builtin, lineOrBytecode, result); @@ -3074,7 +3082,7 @@ class FunctionCompiler { bool builtinCall2(const SymbolicAddressSignature& builtin, uint32_t lineOrBytecode, MDefinition* arg1, MDefinition* arg2, MDefinition** result) { - CallCompileState callState(ABIKind::System); + CallCompileState callState(ABIForBuiltin(builtin.identity)); return passCallArg(arg1, builtin.argTypes[0], &callState) && passCallArg(arg2, builtin.argTypes[1], &callState) && finishCallArgs(&callState) && @@ -3086,7 +3094,7 @@ class FunctionCompiler { uint32_t lineOrBytecode, MDefinition* arg1, MDefinition* arg2, MDefinition* arg3, MDefinition* arg4, MDefinition* arg5, MDefinition** result) { - CallCompileState callState(ABIKind::System); + CallCompileState callState(ABIForBuiltin(builtin.identity)); return passCallArg(arg1, builtin.argTypes[0], &callState) && passCallArg(arg2, builtin.argTypes[1], &callState) && passCallArg(arg3, builtin.argTypes[2], &callState) && @@ -3102,7 +3110,7 @@ class FunctionCompiler { MDefinition* arg2, MDefinition* arg3, MDefinition* arg4, MDefinition* arg5, MDefinition* arg6, MDefinition** result) { - CallCompileState callState(ABIKind::System); + CallCompileState callState(ABIForBuiltin(builtin.identity)); return passCallArg(arg1, builtin.argTypes[0], &callState) && passCallArg(arg2, builtin.argTypes[1], &callState) && passCallArg(arg3, builtin.argTypes[2], &callState) && @@ -3209,7 +3217,7 @@ class FunctionCompiler { } // Finally, construct the call. - CallCompileState callState(ABIKind::System); + CallCompileState callState(ABIForBuiltin(callee.identity)); if (!passInstanceCallArg(callee.argTypes[0], &callState)) { return false; } @@ -3446,7 +3454,7 @@ class FunctionCompiler { // `memoryBase`, and make the call. const SymbolicAddressSignature& callee = *builtinModuleFunc.sig(); - CallCompileState callState(ABIKind::System); + CallCompileState callState(ABIForBuiltin(callee.identity)); if (!passInstanceCallArg(callee.argTypes[0], &callState) || !passCallArgs(params, builtinModuleFunc.funcType()->args(), &callState)) { diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp @@ -166,7 +166,8 @@ void ABIResultIter::settlePrev() { #ifdef WASM_CODEGEN_DEBUG template <class Closure> static void GenPrint(DebugChannel channel, MacroAssembler& masm, - const Maybe<Register>& taken, Closure passArgAndCall) { + const Maybe<Register>& taken, SymbolicAddress builtin, + Closure passArgAndCall) { if (!IsCodegenDebugEnabled(channel)) { return; } @@ -185,7 +186,7 @@ static void GenPrint(DebugChannel channel, MacroAssembler& masm, "codegen debug checks require a jit context"); # ifdef JS_CODEGEN_ARM64 if (IsCompilingWasm()) { - masm.setupWasmABICall(); + masm.setupWasmABICall(builtin); } else { // JS ARM64 has an extra stack pointer which is not managed in WASM. masm.setupUnalignedABICall(temp); @@ -206,52 +207,55 @@ static void GenPrintf(DebugChannel channel, MacroAssembler& masm, UniqueChars str = JS_vsmprintf(fmt, ap); va_end(ap); - GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) { - // If we've gone this far, it means we're actually using the debugging - // strings. In this case, we leak them! This is only for debugging, and - // doing the right thing is cumbersome (in Ion, it'd mean add a vec of - // strings to the IonScript; in wasm, it'd mean add it to the current - // Module and serialize it properly). - const char* text = str.release(); - - masm.movePtr(ImmPtr((void*)text, ImmPtr::NoCheckToken()), temp); - masm.passABIArg(temp); - if (inWasm) { - masm.callDebugWithABI(SymbolicAddress::PrintText); - } else { - using Fn = void (*)(const char* output); - masm.callWithABI<Fn, PrintText>(ABIType::General, - CheckUnsafeCallWithABI::DontCheckOther); - } - }); + GenPrint(channel, masm, Nothing(), SymbolicAddress::PrintText, + [&](bool inWasm, Register temp) { + // If we've gone this far, it means we're actually using the + // debugging strings. In this case, we leak them! This is only for + // debugging, and doing the right thing is cumbersome (in Ion, it'd + // mean add a vec of strings to the IonScript; in wasm, it'd mean + // add it to the current Module and serialize it properly). + const char* text = str.release(); + + masm.movePtr(ImmPtr((void*)text, ImmPtr::NoCheckToken()), temp); + masm.passABIArg(temp); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintText); + } else { + using Fn = void (*)(const char* output); + masm.callWithABI<Fn, PrintText>( + ABIType::General, CheckUnsafeCallWithABI::DontCheckOther); + } + }); } static void GenPrintIsize(DebugChannel channel, MacroAssembler& masm, const Register& src) { - GenPrint(channel, masm, Some(src), [&](bool inWasm, Register _temp) { - masm.passABIArg(src); - if (inWasm) { - masm.callDebugWithABI(SymbolicAddress::PrintI32); - } else { - using Fn = void (*)(int32_t val); - masm.callWithABI<Fn, PrintI32>(ABIType::General, - CheckUnsafeCallWithABI::DontCheckOther); - } - }); + GenPrint(channel, masm, Some(src), SymbolicAddress::PrintI32, + [&](bool inWasm, Register _temp) { + masm.passABIArg(src); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintI32); + } else { + using Fn = void (*)(int32_t val); + masm.callWithABI<Fn, PrintI32>( + ABIType::General, CheckUnsafeCallWithABI::DontCheckOther); + } + }); } static void GenPrintPtr(DebugChannel channel, MacroAssembler& masm, const Register& src) { - GenPrint(channel, masm, Some(src), [&](bool inWasm, Register _temp) { - masm.passABIArg(src); - if (inWasm) { - masm.callDebugWithABI(SymbolicAddress::PrintPtr); - } else { - using Fn = void (*)(uint8_t* val); - masm.callWithABI<Fn, PrintPtr>(ABIType::General, - CheckUnsafeCallWithABI::DontCheckOther); - } - }); + GenPrint(channel, masm, Some(src), SymbolicAddress::PrintPtr, + [&](bool inWasm, Register _temp) { + masm.passABIArg(src); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintPtr); + } else { + using Fn = void (*)(uint8_t* val); + masm.callWithABI<Fn, PrintPtr>( + ABIType::General, CheckUnsafeCallWithABI::DontCheckOther); + } + }); } static void GenPrintI64(DebugChannel channel, MacroAssembler& masm, @@ -269,30 +273,32 @@ static void GenPrintI64(DebugChannel channel, MacroAssembler& masm, static void GenPrintF32(DebugChannel channel, MacroAssembler& masm, const FloatRegister& src) { - GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) { - masm.passABIArg(src, ABIType::Float32); - if (inWasm) { - masm.callDebugWithABI(SymbolicAddress::PrintF32); - } else { - using Fn = void (*)(float val); - masm.callWithABI<Fn, PrintF32>(ABIType::General, - CheckUnsafeCallWithABI::DontCheckOther); - } - }); + GenPrint(channel, masm, Nothing(), SymbolicAddress::PrintF32, + [&](bool inWasm, Register temp) { + masm.passABIArg(src, ABIType::Float32); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintF32); + } else { + using Fn = void (*)(float val); + masm.callWithABI<Fn, PrintF32>( + ABIType::General, CheckUnsafeCallWithABI::DontCheckOther); + } + }); } static void GenPrintF64(DebugChannel channel, MacroAssembler& masm, const FloatRegister& src) { - GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) { - masm.passABIArg(src, ABIType::Float64); - if (inWasm) { - masm.callDebugWithABI(SymbolicAddress::PrintF64); - } else { - using Fn = void (*)(double val); - masm.callWithABI<Fn, PrintF64>(ABIType::General, - CheckUnsafeCallWithABI::DontCheckOther); - } - }); + GenPrint(channel, masm, Nothing(), SymbolicAddress::PrintF64, + [&](bool inWasm, Register temp) { + masm.passABIArg(src, ABIType::Float64); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintF64); + } else { + using Fn = void (*)(double val); + masm.callWithABI<Fn, PrintF64>( + ABIType::General, CheckUnsafeCallWithABI::DontCheckOther); + } + }); } # ifdef ENABLE_WASM_SIMD @@ -1302,7 +1308,8 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex, // Baseline and Ion call C++ runtime via BuiltinThunk with wasm abi, so to // unify the BuiltinThunk's interface we call it here with wasm abi. - jit::ABIArgIter<MIRTypeVector> argsIter(coerceArgTypes, ABIKind::System); + jit::ABIArgIter<MIRTypeVector> argsIter( + coerceArgTypes, ABIForBuiltin(SymbolicAddress::CoerceInPlace_JitEntry)); // argument 0: function index. if (argsIter->kind() == ABIArg::GPR) { @@ -2505,16 +2512,9 @@ struct ABIFunctionArgs { } }; -bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIKind abiKind, - ABIFunctionType abiType, ExitReason exitReason, - void* funcPtr, CallableOffsets* offsets) { - // This is used to generate 'typed natives' (see "JS Fast Wasm Imports") in - // WasmBuiltins.cpp, and 'builtin thunks' (see "Process-wide builtin thunk - // set") in WasmBuiltins.cpp. Typed natives use the wasm ABI, as they are - // directly imported into wasm and act as normal functions. Builtin thunks use - // the system ABI and are called from JIT code. - MOZ_ASSERT(abiKind == ABIKind::System || abiKind == ABIKind::Wasm); - +bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIFunctionType abiType, + ExitReason exitReason, void* funcPtr, + CallableOffsets* offsets) { AssertExpectedSP(masm); masm.setFramePushed(0); @@ -2527,12 +2527,12 @@ bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIKind abiKind, GenerateExitPrologue(masm, framePushed, exitReason, offsets); // Copy out and convert caller arguments, if needed. We are translating from - // our 'self' ABI which is either 'wasm' or 'system' to a system ABI builtin. + // the wasm ABI to the system ABI. Register scratch = ABINonArgReturnReg0; // Use two arg iterators to track the different offsets that arguments must - // go. - ABIArgIter selfArgs(args, abiKind); + // go. We are translating from the wasm ABI to the system ABI. + ABIArgIter selfArgs(args, ABIKind::Wasm); ABIArgIter callArgs(args, ABIKind::System); // `selfArgs` gives us offsets from 'arg base' which is the SP immediately @@ -2549,9 +2549,9 @@ bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIKind abiKind, if (selfArgs->argInRegister()) { #ifdef JS_CODEGEN_ARM - // If our ABI is wasm, we must adapt FP args when using the soft-float - // ABI to go into GPRs. - if (abiKind == ABIKind::Wasm && !ARMFlags::UseHardFpABI() && + // The system ABI may use soft-FP, while the wasm ABI will always use + // hard-FP. We must adapt FP args in this case. + if (!ARMFlags::UseHardFpABI() && IsFloatingPointType(selfArgs.mirType())) { FloatRegister input = selfArgs->fpu(); if (selfArgs.mirType() == MIRType::Float32) { @@ -2582,28 +2582,23 @@ bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIKind abiKind, #if defined(JS_CODEGEN_X64) // No widening is required, as the caller will widen. #elif defined(JS_CODEGEN_X86) - // If our ABI is wasm, we must adapt the system x86 return value from a - // to x87 FP stack to a FPR that wasm expects. - if (abiKind == ABIKind::Wasm) { - // x86 passes the return value on the x87 FP stack. - Operand op(esp, 0); - MIRType retType = ToMIRType(ABIType( - std::underlying_type_t<ABIFunctionType>(abiType) & ABITypeArgMask)); - if (retType == MIRType::Float32) { - masm.fstp32(op); - masm.loadFloat32(op, ReturnFloat32Reg); - } else if (retType == MIRType::Double) { - masm.fstp(op); - masm.loadDouble(op, ReturnDoubleReg); - } + // The wasm ABI always uses SSE for floating point returns, and so we must + // convert the x87 FP stack result over. + Operand op(esp, 0); + MIRType retType = ToMIRType(ABIType( + std::underlying_type_t<ABIFunctionType>(abiType) & ABITypeArgMask)); + if (retType == MIRType::Float32) { + masm.fstp32(op); + masm.loadFloat32(op, ReturnFloat32Reg); + } else if (retType == MIRType::Double) { + masm.fstp(op); + masm.loadDouble(op, ReturnDoubleReg); } #elif defined(JS_CODEGEN_ARM) - // If our ABI is wasm, we must adapt the system soft-fp return value from a - // GPR to a FPR. + // We must adapt the system soft-fp return value from a GPR to a FPR. MIRType retType = ToMIRType(ABIType( std::underlying_type_t<ABIFunctionType>(abiType) & ABITypeArgMask)); - if (abiKind == ABIKind::Wasm && !ARMFlags::UseHardFpABI() && - IsFloatingPointType(retType)) { + if (!ARMFlags::UseHardFpABI() && IsFloatingPointType(retType)) { masm.ma_vxfer(r0, r1, d0); } #endif diff --git a/js/src/wasm/WasmStubs.h b/js/src/wasm/WasmStubs.h @@ -243,7 +243,6 @@ class ABIResultIter { }; extern bool GenerateBuiltinThunk(jit::MacroAssembler& masm, - jit::ABIKind abiKind, jit::ABIFunctionType abiType, ExitReason exitReason, void* funcPtr, CallableOffsets* offsets);