commit 2a4ecd4c3acc79cf571c40f7894171b7e704a630
parent 36888c1cec9770fbaef277b52a8a01075c20f733
Author: Rong "Mantle" Bao <webmaster@csmantle.top>
Date: Thu, 27 Nov 2025 11:57:05 +0000
Bug 1996840 - [riscv64] Part 2: Pad branches to match veneer size. r=nbp
IonAssemblerBufferWithConstantPools assumes that all short branches are
as long as the veneers, so that no "decompression" happens during veneer
insertion. For RISCV64, we use an `auipc; j` sequence to implement
veneers, so we need to insert a `nop` after each short jump to
compensate for the difference in size.
Differential Revision: https://phabricator.services.mozilla.com/D270941
Diffstat:
15 files changed, 81 insertions(+), 54 deletions(-)
diff --git a/js/src/jit/BaselineCodeGen.cpp b/js/src/jit/BaselineCodeGen.cpp
@@ -771,8 +771,8 @@ bool BaselineInterpreterCodeGen::emitNextIC() {
saveInterpreterPCReg();
masm.loadPtr(frame.addressOfInterpreterICEntry(), ICStubReg);
masm.loadPtr(Address(ICStubReg, ICEntry::offsetOfFirstStub()), ICStubReg);
- masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
- uint32_t returnOffset = masm.currentOffset();
+ uint32_t returnOffset =
+ masm.call(Address(ICStubReg, ICStub::offsetOfStubCode())).offset();
restoreInterpreterPCReg();
// If this is an IC for a bytecode op where Ion may inline scripts, we need to
@@ -6422,14 +6422,14 @@ bool BaselineCodeGen<Handler>::emit_Resume() {
// generator returns.
Label genStart, returnTarget;
#ifdef JS_USE_LINK_REGISTER
- masm.call(&genStart);
+ const CodeOffset retAddr = masm.call(&genStart);
#else
masm.callAndPushReturnAddress(&genStart);
+ const CodeOffset retAddr = CodeOffset(masm.currentOffset());
#endif
// Record the return address so the return offset -> pc mapping works.
- if (!handler.recordCallRetAddr(RetAddrEntry::Kind::IC,
- masm.currentOffset())) {
+ if (!handler.recordCallRetAddr(RetAddrEntry::Kind::IC, retAddr.offset())) {
return false;
}
diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h
@@ -596,8 +596,8 @@ class MacroAssembler : public MacroAssemblerSpecific {
// immediately following the call; that is, for the return point.
CodeOffset call(Register reg) PER_SHARED_ARCH;
CodeOffset call(Label* label) PER_SHARED_ARCH;
+ CodeOffset call(const Address& addr) PER_SHARED_ARCH;
- void call(const Address& addr) PER_SHARED_ARCH;
void call(ImmWord imm) PER_SHARED_ARCH;
// Call a target native function, which is neither traceable nor movable.
void call(ImmPtr imm) PER_SHARED_ARCH;
diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp
@@ -4426,9 +4426,9 @@ CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
return call(CallReg);
}
-void MacroAssembler::call(const Address& addr) {
+CodeOffset MacroAssembler::call(const Address& addr) {
loadPtr(addr, CallReg);
- call(CallReg);
+ return call(CallReg);
}
void MacroAssembler::call(JitCode* c) {
diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp
@@ -1639,7 +1639,7 @@ CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
return CodeOffset(currentOffset());
}
-void MacroAssembler::call(const Address& addr) {
+CodeOffset MacroAssembler::call(const Address& addr) {
vixl::UseScratchRegisterScope temps(this);
const Register scratch = temps.AcquireX().asUnsized();
// This sync has been observed (and is expected) to be necessary.
@@ -1647,6 +1647,7 @@ void MacroAssembler::call(const Address& addr) {
syncStackPtr();
loadPtr(addr, scratch);
Blr(ARMRegister(scratch, 64));
+ return CodeOffset(currentOffset());
}
void MacroAssembler::call(JitCode* c) {
diff --git a/js/src/jit/loong64/MacroAssembler-loong64.cpp b/js/src/jit/loong64/MacroAssembler-loong64.cpp
@@ -2910,9 +2910,9 @@ CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
return call(CallReg);
}
-void MacroAssembler::call(const Address& addr) {
+CodeOffset MacroAssembler::call(const Address& addr) {
loadPtr(addr, CallReg);
- call(CallReg);
+ return call(CallReg);
}
void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); }
diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp
@@ -1845,9 +1845,9 @@ CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
return call(CallReg);
}
-void MacroAssembler::call(const Address& addr) {
+CodeOffset MacroAssembler::call(const Address& addr) {
loadPtr(addr, CallReg);
- call(CallReg);
+ return call(CallReg);
}
void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); }
diff --git a/js/src/jit/riscv64/MacroAssembler-riscv64.cpp b/js/src/jit/riscv64/MacroAssembler-riscv64.cpp
@@ -2476,10 +2476,7 @@ void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
// ===============================================================
// MacroAssembler high-level usage.
bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
-CodeOffset MacroAssembler::call(Label* label) {
- BranchAndLink(label);
- return CodeOffset(currentOffset());
-}
+CodeOffset MacroAssembler::call(Label* label) { return BranchAndLink(label); }
CodeOffset MacroAssembler::call(Register reg) {
jalr(reg, 0);
return CodeOffset(currentOffset());
@@ -3271,11 +3268,11 @@ void MacroAssembler::branchValueIsNurseryCell(Condition cond,
Label* label) {
branchValueIsNurseryCellImpl(cond, value, temp, label);
}
-void MacroAssembler::call(const Address& addr) {
+CodeOffset MacroAssembler::call(const Address& addr) {
UseScratchRegisterScope temps(this);
temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
loadPtr(addr, CallReg);
- call(CallReg);
+ return call(CallReg);
}
void MacroAssembler::call(ImmPtr target) {
BufferOffset bo = m_buffer.nextOffset();
@@ -4824,6 +4821,7 @@ bool MacroAssemblerRiscv64::CalculateOffset(Label* L, int32_t* offset,
void MacroAssemblerRiscv64::BranchShortHelper(int32_t offset, Label* L) {
MOZ_ASSERT(L == nullptr || offset == 0);
+ BlockTrampolinePoolScope block_trampoline_pool(this, 2);
offset = GetOffset(offset, L, OffsetSize::kOffset21);
Assembler::j(offset);
}
@@ -4998,12 +4996,13 @@ void MacroAssemblerRiscv64::BranchLong(Label* L) {
GenPCRelativeJump(scratch, imm);
}
-void MacroAssemblerRiscv64::BranchAndLinkLong(Label* L) {
+CodeOffset MacroAssemblerRiscv64::BranchAndLinkLong(Label* L) {
// Generate position independent long branch and link.
int32_t imm = branch_long_offset(L);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
GenPCRelativeJumpAndLink(scratch, imm);
+ return CodeOffset(currentOffset());
}
void MacroAssemblerRiscv64::ma_branch(Label* L, Condition cond, Register rs,
@@ -6911,30 +6910,32 @@ void MacroAssemblerRiscv64::GenPCRelativeJumpAndLink(Register rd,
jalr(rd, Lo12); // jump PC + Hi20 + Lo12
}
-void MacroAssemblerRiscv64::BranchAndLinkShortHelper(int32_t offset, Label* L) {
+CodeOffset MacroAssemblerRiscv64::BranchAndLinkShortHelper(int32_t offset,
+ Label* L) {
MOZ_ASSERT(L == nullptr || offset == 0);
+ BlockTrampolinePoolScope block_trampoline_pool(this, 2);
offset = GetOffset(offset, L, OffsetSize::kOffset21);
- jal(offset);
+ return jal(offset);
}
-void MacroAssemblerRiscv64::BranchAndLinkShort(int32_t offset) {
+CodeOffset MacroAssemblerRiscv64::BranchAndLinkShort(int32_t offset) {
MOZ_ASSERT(is_int21(offset));
- BranchAndLinkShortHelper(offset, nullptr);
+ return BranchAndLinkShortHelper(offset, nullptr);
}
-void MacroAssemblerRiscv64::BranchAndLinkShort(Label* L) {
- BranchAndLinkShortHelper(0, L);
+CodeOffset MacroAssemblerRiscv64::BranchAndLinkShort(Label* L) {
+ return BranchAndLinkShortHelper(0, L);
}
-void MacroAssemblerRiscv64::BranchAndLink(Label* L) {
+CodeOffset MacroAssemblerRiscv64::BranchAndLink(Label* L) {
if (L->bound()) {
if (is_near(L)) {
- BranchAndLinkShort(L);
+ return BranchAndLinkShort(L);
} else {
- BranchAndLinkLong(L);
+ return BranchAndLinkLong(L);
}
} else {
- BranchAndLinkShort(L);
+ return BranchAndLinkShort(L);
}
}
diff --git a/js/src/jit/riscv64/MacroAssembler-riscv64.h b/js/src/jit/riscv64/MacroAssembler-riscv64.h
@@ -168,11 +168,11 @@ class MacroAssemblerRiscv64 : public Assembler {
JumpKind jumpKind = LongJump);
void ma_b(Register lhs, Imm32 imm, Label* l, Condition c,
JumpKind jumpKind = LongJump);
- void BranchAndLinkShort(Label* L);
- void BranchAndLink(Label* label);
- void BranchAndLinkShort(int32_t offset);
- void BranchAndLinkShortHelper(int32_t offset, Label* L);
- void BranchAndLinkLong(Label* L);
+ CodeOffset BranchAndLinkShort(Label* L);
+ CodeOffset BranchAndLink(Label* label);
+ CodeOffset BranchAndLinkShort(int32_t offset);
+ CodeOffset BranchAndLinkShortHelper(int32_t offset, Label* L);
+ CodeOffset BranchAndLinkLong(Label* L);
void GenPCRelativeJumpAndLink(Register rd, int32_t imm32);
#define DEFINE_INSTRUCTION(instr) \
diff --git a/js/src/jit/riscv64/SharedICHelpers-riscv64.h b/js/src/jit/riscv64/SharedICHelpers-riscv64.h
@@ -29,8 +29,7 @@ inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg());
// Call the stubcode via a direct jump-and-link
- masm.call(R2.scratchReg());
- *callOffset = CodeOffset(masm.currentOffset());
+ *callOffset = masm.call(R2.scratchReg());
}
inline void EmitReturnFromIC(MacroAssembler& masm) { masm.branch(ra); }
inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
diff --git a/js/src/jit/riscv64/extension/base-riscv-i.cc b/js/src/jit/riscv64/extension/base-riscv-i.cc
@@ -18,8 +18,13 @@ void AssemblerRISCVI::auipc(Register rd, int32_t imm20) {
// Jumps
-void AssemblerRISCVI::jal(Register rd, int32_t imm21) {
+CodeOffset AssemblerRISCVI::jal(Register rd, int32_t imm21) {
GenInstrJ(JAL, rd, imm21);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ CodeOffset retAddr = CodeOffset(currentOffset());
+ addi(zero, zero, 0);
+ return retAddr;
}
BufferOffset AssemblerRISCVI::jalr(Register rd, Register rs1, int16_t imm12) {
@@ -30,26 +35,44 @@ BufferOffset AssemblerRISCVI::jalr(Register rd, Register rs1, int16_t imm12) {
void AssemblerRISCVI::beq(Register rs1, Register rs2, int16_t imm13) {
GenInstrBranchCC_rri(0b000, rs1, rs2, imm13);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ addi(zero, zero, 0);
}
void AssemblerRISCVI::bne(Register rs1, Register rs2, int16_t imm13) {
GenInstrBranchCC_rri(0b001, rs1, rs2, imm13);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ addi(zero, zero, 0);
}
void AssemblerRISCVI::blt(Register rs1, Register rs2, int16_t imm13) {
GenInstrBranchCC_rri(0b100, rs1, rs2, imm13);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ addi(zero, zero, 0);
}
void AssemblerRISCVI::bge(Register rs1, Register rs2, int16_t imm13) {
GenInstrBranchCC_rri(0b101, rs1, rs2, imm13);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ addi(zero, zero, 0);
}
void AssemblerRISCVI::bltu(Register rs1, Register rs2, int16_t imm13) {
GenInstrBranchCC_rri(0b110, rs1, rs2, imm13);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ addi(zero, zero, 0);
}
void AssemblerRISCVI::bgeu(Register rs1, Register rs2, int16_t imm13) {
GenInstrBranchCC_rri(0b111, rs1, rs2, imm13);
+ // FIXME: Pad short branches so that we don't need to care about decompression
+ // when inserting veneers
+ addi(zero, zero, 0);
}
// Loads
diff --git a/js/src/jit/riscv64/extension/base-riscv-i.h b/js/src/jit/riscv64/extension/base-riscv-i.h
@@ -18,7 +18,7 @@ class AssemblerRISCVI : public AssemblerRiscvBase {
void auipc(Register rd, int32_t imm20);
// Jumps
- void jal(Register rd, int32_t imm20);
+ CodeOffset jal(Register rd, int32_t imm20);
BufferOffset jalr(Register rd, Register rs1, int16_t imm12);
// Branches
@@ -224,11 +224,11 @@ class AssemblerRISCVI : public AssemblerRiscvBase {
bleu(rs1, rs2, branch_offset(L));
}
- void j(int32_t imm21) { jal(zero_reg, imm21); }
- void j(Label* L) { j(jump_offset(L)); }
- void b(Label* L) { j(L); }
- void jal(int32_t imm21) { jal(ra, imm21); }
- void jal(Label* L) { jal(jump_offset(L)); }
+ CodeOffset j(int32_t imm21) { return jal(zero_reg, imm21); }
+ CodeOffset j(Label* L) { return j(jump_offset(L)); }
+ CodeOffset b(Label* L) { return j(L); }
+ CodeOffset jal(int32_t imm21) { return jal(ra, imm21); }
+ CodeOffset jal(Label* L) { return jal(jump_offset(L)); }
void jr(Register rs) { jalr(zero_reg, rs, 0); }
void jr(Register rs, int32_t imm12) { jalr(zero_reg, rs, imm12); }
void jalr(Register rs, int32_t imm12) { jalr(ra, rs, imm12); }
diff --git a/js/src/jit/wasm32/MacroAssembler-wasm32.cpp b/js/src/jit/wasm32/MacroAssembler-wasm32.cpp
@@ -431,7 +431,7 @@ void MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result) {
MOZ_CRASH();
}
-void MacroAssembler::call(const Address& addr) { MOZ_CRASH(); }
+CodeOffset MacroAssembler::call(const Address& addr) { MOZ_CRASH(); }
void MacroAssembler::call(ImmWord imm) { MOZ_CRASH(); }
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp
@@ -704,8 +704,9 @@ CodeOffset MacroAssembler::call(Register reg) { return Assembler::call(reg); }
CodeOffset MacroAssembler::call(Label* label) { return Assembler::call(label); }
-void MacroAssembler::call(const Address& addr) {
+CodeOffset MacroAssembler::call(const Address& addr) {
Assembler::call(Operand(addr.base, addr.offset));
+ return CodeOffset(currentOffset());
}
CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp
@@ -916,9 +916,8 @@ void BaseCompiler::insertBreakablePoint(CallSiteKind kind) {
Label L;
masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugStub()), scratch);
masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), &L);
- masm.call(&perFunctionDebugStub_);
- masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind),
- CodeOffset(masm.currentOffset()));
+ const CodeOffset retAddr = masm.call(&perFunctionDebugStub_);
+ masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), retAddr);
masm.bind(&L);
#else
MOZ_CRASH("BaseCompiler platform hook: insertBreakablePoint");
@@ -1153,9 +1152,10 @@ class OutOfLineRequestTierUp : public OutOfLineCode {
}
#endif
// Call the stub
- masm->call(Address(InstanceReg, Instance::offsetOfRequestTierUpStub()));
+ const CodeOffset retAddr =
+ masm->call(Address(InstanceReg, Instance::offsetOfRequestTierUpStub()));
masm->append(CallSiteDesc(lastOpcodeOffset_, CallSiteKind::RequestTierUp),
- CodeOffset(masm->currentOffset()));
+ retAddr);
// And swap again, if we swapped above.
#ifndef RABALDR_PIN_INSTANCE
if (Register(instance_) != InstanceReg) {
diff --git a/js/src/wasm/WasmGC.cpp b/js/src/wasm/WasmGC.cpp
@@ -312,11 +312,13 @@ bool wasm::IsPlausibleStackMapKey(const uint8_t* nextPC) {
// TODO(loong64): Implement IsValidStackMapKey.
return true;
# elif defined(JS_CODEGEN_RISCV64)
- const uint32_t* insn = (const uint32_t*)nextPC;
+ const uint32_t* insn = reinterpret_cast<const uint32_t*>(nextPC);
return (((uintptr_t(insn) & 3) == 0) &&
((insn[-1] == 0x00006037 && insn[-2] == 0x00100073) || // break;
- ((insn[-1] & kBaseOpcodeMask) == JALR) ||
- ((insn[-1] & kBaseOpcodeMask) == JAL) ||
+ ((insn[-1] & kBaseOpcodeMask) == JALR) || // jalr
+ ((insn[-1] & kBaseOpcodeMask) == JAL) || // jal
+ ((insn[-2] & kBaseOpcodeMask) == JAL &&
+ insn[-1] == 0x00000013 /* addi zero, zero, 0 */) || // jal; nop
(insn[-1] == 0x00100073 &&
(insn[-2] & kITypeMask) == RO_CSRRWI))); // wasm trap
# else
