commit 8705f290a1a4e82a0bd18c8966af6f1183b5eebe
parent 5b63e963ea7196f6a8c46c375154e2b84f1564ba
Author: Rong "Mantle" Bao <webmaster@csmantle.top>
Date: Fri, 12 Dec 2025 14:17:57 +0000
Bug 2004879 - [riscv64] Part 3: Find a good slot to insert pending uses when the list head is out of reach. r=jandem
The current implementation is largely learned from ARM[64], since these
have similar approaches to label linking, refilling and veneers.
Differential Revision: https://phabricator.services.mozilla.com/D275569
Diffstat:
3 files changed, 148 insertions(+), 99 deletions(-)
diff --git a/js/src/jit/riscv64/Assembler-riscv64.cpp b/js/src/jit/riscv64/Assembler-riscv64.cpp
@@ -958,45 +958,41 @@ int Assembler::jumpChainTargetAt(Instruction* instruction, BufferOffset pos,
if (imm13 == kEndOfJumpChain) {
// EndOfChain sentinel is returned directly, not relative to pc or pos.
return kEndOfChain;
- } else {
- DEBUG_PRINTF("\t jumpChainTargetAt: %d %d\n", imm13,
- pos.getOffset() + imm13);
- return pos.getOffset() + imm13;
}
+ DEBUG_PRINTF("\t jumpChainTargetAt: %d %d\n", imm13,
+ pos.getOffset() + imm13);
+ return pos.getOffset() + imm13;
}
case JAL: {
int32_t imm21 = JumpOffset(instr);
if (imm21 == kEndOfJumpChain) {
// EndOfChain sentinel is returned directly, not relative to pc or pos.
return kEndOfChain;
- } else {
- DEBUG_PRINTF("\t jumpChainTargetAt: %d %d\n", imm21,
- pos.getOffset() + imm21);
- return pos.getOffset() + imm21;
}
+ DEBUG_PRINTF("\t jumpChainTargetAt: %d %d\n", imm21,
+ pos.getOffset() + imm21);
+ return pos.getOffset() + imm21;
}
case JALR: {
int32_t imm12 = instr >> 20;
if (imm12 == kEndOfJumpChain) {
// EndOfChain sentinel is returned directly, not relative to pc or pos.
return kEndOfChain;
- } else {
- DEBUG_PRINTF("\t jumpChainTargetAt: %d %d\n", imm12,
- pos.getOffset() + imm12);
- return pos.getOffset() + imm12;
}
+ DEBUG_PRINTF("\t jumpChainTargetAt: %d %d\n", imm12,
+ pos.getOffset() + imm12);
+ return pos.getOffset() + imm12;
}
case LUI: {
uintptr_t imm = jumpChainTargetAddressAt(instruction);
uintptr_t instr_address = reinterpret_cast<uintptr_t>(instruction);
if (imm == kEndOfJumpChain) {
return kEndOfChain;
- } else {
- MOZ_ASSERT(instr_address - imm < INT_MAX);
- int32_t delta = static_cast<int32_t>(instr_address - imm);
- MOZ_ASSERT(pos.getOffset() > delta);
- return pos.getOffset() - delta;
}
+ MOZ_ASSERT(instr_address - imm < INT_MAX);
+ int32_t delta = static_cast<int32_t>(instr_address - imm);
+ MOZ_ASSERT(pos.getOffset() > delta);
+ return pos.getOffset() - delta;
}
case AUIPC: {
MOZ_ASSERT(instruction2 != nullptr);
@@ -1022,12 +1018,11 @@ uint32_t Assembler::jumpChainNextLink(Label* L, bool is_internal) {
if (link == kEndOfChain) {
L->reset();
return LabelBase::INVALID_OFFSET;
- } else {
- MOZ_ASSERT(link >= 0);
- DEBUG_PRINTF("next: %p to offset %d\n", L, link);
- L->use(link);
- return link;
}
+ MOZ_ASSERT(link >= 0);
+ DEBUG_PRINTF("next: %p to offset %d\n", L, link);
+ L->use(link);
+ return link;
}
void Assembler::bind(Label* label, BufferOffset boff) {
@@ -1128,101 +1123,115 @@ bool Assembler::is_near_branch(Label* L) {
return is_intn((currentOffset() - L->offset()), kBranchOffsetBits);
}
-int32_t Assembler::branchLongOffset(Label* L) {
+int32_t Assembler::branchLongOffsetHelper(Label* L) {
if (oom()) {
return kEndOfJumpChain;
}
- intptr_t target_pos;
+
BufferOffset next_instr_offset = nextInstrOffset(2);
- DEBUG_PRINTF("\branchLongOffset: %p to (%d)\n", L,
+ DEBUG_PRINTF("\tbranchLongOffsetHelper: %p to (%d)\n", L,
next_instr_offset.getOffset());
+
if (L->bound()) {
+ // The label is bound: all uses are already linked.
JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
next_instr_offset.getOffset());
- target_pos = L->offset();
- } else {
- if (L->used()) {
- LabelCache::Ptr p = label_cache_.lookup(L->offset());
- MOZ_ASSERT(p);
- MOZ_ASSERT(p->key() == L->offset());
- target_pos = p->value().getOffset();
- if (!jumpChainPutTargetAt(BufferOffset(target_pos), next_instr_offset)) {
- DEBUG_PRINTF("\tLabel %p can't be added to link: %d -> %d\n", L,
- BufferOffset(target_pos).getOffset(),
- next_instr_offset.getOffset());
- return kEndOfJumpChain;
- }
- DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
- next_instr_offset.getOffset());
- bool ok = label_cache_.put(L->offset(), next_instr_offset);
- if (!ok) {
- NoEnoughLabelCache();
- }
- return kEndOfJumpChain;
- } else {
- JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
- next_instr_offset.getOffset());
- L->use(next_instr_offset.getOffset());
- DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
- next_instr_offset.getOffset());
- bool ok = label_cache_.putNew(L->offset(), next_instr_offset);
- if (!ok) {
- NoEnoughLabelCache();
- }
- return kEndOfJumpChain;
+ intptr_t offset = L->offset() - next_instr_offset.getOffset();
+ MOZ_ASSERT((offset & 3) == 0);
+ MOZ_ASSERT(is_int32(offset));
+ return static_cast<int32_t>(offset);
+ }
+
+ // The label is unbound and previously unused: Store the offset in the label
+ // itself for patching by bind().
+ if (!L->used()) {
+ JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
+ next_instr_offset.getOffset());
+ L->use(next_instr_offset.getOffset());
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ if (!label_cache_.putNew(L->offset(), next_instr_offset)) {
+ NoEnoughLabelCache();
}
+ return kEndOfJumpChain;
}
- intptr_t offset = target_pos - next_instr_offset.getOffset();
- MOZ_ASSERT((offset & 3) == 0);
- MOZ_ASSERT(is_int32(offset));
- return static_cast<int32_t>(offset);
+
+ LabelCache::Ptr p = label_cache_.lookup(L->offset());
+ MOZ_ASSERT(p);
+ MOZ_ASSERT(p->key() == L->offset());
+ const int32_t target_pos = p->value().getOffset();
+
+ // If the existing instruction at the head of the list is within reach of the
+ // new branch, we can simply insert the new branch at the front of the list.
+ if (jumpChainPutTargetAt(BufferOffset(target_pos), next_instr_offset)) {
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ if (!label_cache_.put(L->offset(), next_instr_offset)) {
+ NoEnoughLabelCache();
+ }
+ } else {
+ DEBUG_PRINTF("\tLabel %p can't be added to link: %d -> %d\n", L,
+ BufferOffset(target_pos).getOffset(),
+ next_instr_offset.getOffset());
+
+ // The label already has a linked list of uses, but we can't reach the head
+ // of the list with the allowed branch range. Insert this branch at a
+ // different position in the list. We need to find an existing branch
+ // `exbr`.
+ //
+ // In particular, the end of the list is always a viable candidate, so we'll
+ // just get that.
+ //
+ // See also vixl::MozBaseAssembler::LinkAndGetOffsetTo.
+
+ BufferOffset next(L);
+ BufferOffset exbr;
+ do {
+ exbr = next;
+ const int link = jumpChainTargetAt(next, false);
+ next = link == kEndOfChain ? BufferOffset() : BufferOffset(link);
+ } while (next.assigned());
+ mozilla::DebugOnly<bool> ok = jumpChainPutTargetAt(exbr, next_instr_offset);
+ MOZ_ASSERT(ok, "Still can't reach list head");
+ }
+
+ return kEndOfJumpChain;
}
int32_t Assembler::branchOffsetHelper(Label* L, OffsetSize bits) {
if (oom()) {
return kEndOfJumpChain;
}
- int32_t target_pos;
+
BufferOffset next_instr_offset = nextInstrOffset();
DEBUG_PRINTF("\tbranchOffsetHelper: %p to %d\n", L,
next_instr_offset.getOffset());
- // This is the last possible branch target.
+
if (L->bound()) {
+ // The label is bound: all uses are already linked.
JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
next_instr_offset.getOffset());
- target_pos = L->offset();
- } else {
- BufferOffset deadline(next_instr_offset.getOffset() +
- ImmBranchMaxForwardOffset(bits));
- DEBUG_PRINTF("\tregisterBranchDeadline %d type %d\n", deadline.getOffset(),
- OffsetSizeToImmBranchRangeType(bits));
- m_buffer.registerBranchDeadline(OffsetSizeToImmBranchRangeType(bits),
- deadline);
- if (L->used()) {
- LabelCache::Ptr p = label_cache_.lookup(L->offset());
- MOZ_ASSERT(p);
- MOZ_ASSERT(p->key() == L->offset());
- target_pos = p->value().getOffset();
- if (!jumpChainPutTargetAt(BufferOffset(target_pos), next_instr_offset)) {
- DEBUG_PRINTF("\tLabel %p can't be added to link: %d -> %d\n", L,
- BufferOffset(target_pos).getOffset(),
- next_instr_offset.getOffset());
- return kEndOfJumpChain;
- }
- DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
- next_instr_offset.getOffset());
- bool ok = label_cache_.put(L->offset(), next_instr_offset);
- if (!ok) {
- NoEnoughLabelCache();
- }
- return kEndOfJumpChain;
- }
+ int32_t offset = L->offset() - next_instr_offset.getOffset();
+ DEBUG_PRINTF("\toffset = %d\n", offset);
+ MOZ_ASSERT(is_intn(offset, bits));
+ MOZ_ASSERT((offset & 1) == 0);
+ return offset;
+ }
+
+ BufferOffset deadline(next_instr_offset.getOffset() +
+ ImmBranchMaxForwardOffset(bits));
+ DEBUG_PRINTF("\tregisterBranchDeadline %d type %d\n", deadline.getOffset(),
+ OffsetSizeToImmBranchRangeType(bits));
+ m_buffer.registerBranchDeadline(OffsetSizeToImmBranchRangeType(bits),
+ deadline);
+ // The label is unbound and previously unused: Store the offset in the label
+ // itself for patching by bind().
+ if (!L->used()) {
JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
next_instr_offset.getOffset());
L->use(next_instr_offset.getOffset());
- bool ok = label_cache_.putNew(L->offset(), next_instr_offset);
- if (!ok) {
+ if (!label_cache_.putNew(L->offset(), next_instr_offset)) {
NoEnoughLabelCache();
}
DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
@@ -1230,11 +1239,51 @@ int32_t Assembler::branchOffsetHelper(Label* L, OffsetSize bits) {
return kEndOfJumpChain;
}
- int32_t offset = target_pos - next_instr_offset.getOffset();
- DEBUG_PRINTF("\toffset = %d\n", offset);
- MOZ_ASSERT(is_intn(offset, bits));
- MOZ_ASSERT((offset & 1) == 0);
- return offset;
+ // The label is unbound and has multiple users. Create a linked list between
+ // the branches, and update the linked list head in the label struct. This is
+ // not always trivial since the branches in the linked list have limited
+ // ranges.
+
+ LabelCache::Ptr p = label_cache_.lookup(L->offset());
+ MOZ_ASSERT(p);
+ MOZ_ASSERT(p->key() == L->offset());
+ const int32_t target_pos = p->value().getOffset();
+
+ // If the existing instruction at the head of the list is within reach of the
+ // new branch, we can simply insert the new branch at the front of the list.
+ if (jumpChainPutTargetAt(BufferOffset(target_pos), next_instr_offset)) {
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ if (!label_cache_.put(L->offset(), next_instr_offset)) {
+ NoEnoughLabelCache();
+ }
+ } else {
+ DEBUG_PRINTF("\tLabel %p can't be added to link: %d -> %d\n", L,
+ BufferOffset(target_pos).getOffset(),
+ next_instr_offset.getOffset());
+
+ // The label already has a linked list of uses, but we can't reach the head
+ // of the list with the allowed branch range. Insert this branch at a
+ // different position in the list. We need to find an existing branch
+ // `exbr`.
+ //
+ // In particular, the end of the list is always a viable candidate, so we'll
+ // just get that.
+ //
+ // See also vixl::MozBaseAssembler::LinkAndGetOffsetTo.
+
+ BufferOffset next(L);
+ BufferOffset exbr;
+ do {
+ exbr = next;
+ const int link = jumpChainTargetAt(next, false);
+ next = link == kEndOfChain ? BufferOffset() : BufferOffset(link);
+ } while (next.assigned());
+ mozilla::DebugOnly<bool> ok = jumpChainPutTargetAt(exbr, next_instr_offset);
+ MOZ_ASSERT(ok, "Still can't reach list head");
+ }
+
+ return kEndOfJumpChain;
}
Assembler::Condition Assembler::InvertCondition(Condition cond) {
diff --git a/js/src/jit/riscv64/Assembler-riscv64.h b/js/src/jit/riscv64/Assembler-riscv64.h
@@ -372,7 +372,7 @@ class Assembler : public AssemblerShared,
bool jumpChainPutTargetAt(BufferOffset pos, BufferOffset target_pos,
bool trampoline = false);
int32_t branchOffsetHelper(Label* L, OffsetSize bits);
- int32_t branchLongOffset(Label* L);
+ int32_t branchLongOffsetHelper(Label* L);
// Determines if Label is bound and near enough so that branch instruction
// can be used to reach it, instead of jump instruction.
diff --git a/js/src/jit/riscv64/MacroAssembler-riscv64.cpp b/js/src/jit/riscv64/MacroAssembler-riscv64.cpp
@@ -4999,13 +4999,13 @@ void MacroAssemblerRiscv64::BranchLong(Label* L) {
// Generate position independent long branch.
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
- int32_t imm = branchLongOffset(L);
+ int32_t imm = branchLongOffsetHelper(L);
GenPCRelativeJump(scratch, imm);
}
CodeOffset MacroAssemblerRiscv64::BranchAndLinkLong(Label* L) {
// Generate position independent long branch and link.
- int32_t imm = branchLongOffset(L);
+ int32_t imm = branchLongOffsetHelper(L);
UseScratchRegisterScope temps(this);
Register scratch = temps.Acquire();
GenPCRelativeJumpAndLink(scratch, imm);
