commit 89305675eff9b4e86c90d2639364b59617f07dd5
parent b3599c5840c5e0edbc2952b2f03e3976cc897463
Author: Julien Pages <jpages@mozilla.com>
Date: Wed, 12 Nov 2025 14:52:26 +0000
Bug 1894123 - wasm: Rework branch hinting to move cold blocks out of line. r=rhunt
Added a mir property representing the likelihood of a basic block to be
executed at runtime. By default this is considered unknown and it has
no effect. If a basic block is set to unlikely, the block will be moved
out of line at the LIR level, after register allocation.
Setting a block to Likely has no effect for now, but something could be
done in the future for likely blocks.
Branch hinting was also modified to use this more general API. The goal
of this approach is to allow more passes to set the priority at the
basic blocks level.
Differential Revision: https://phabricator.services.mozilla.com/D247835
Diffstat:
11 files changed, 436 insertions(+), 121 deletions(-)
diff --git a/js/src/jit-test/tests/wasm/branch-hinting/complex_control_flow.js b/js/src/jit-test/tests/wasm/branch-hinting/complex_control_flow.js
@@ -36,3 +36,111 @@ let module = new WebAssembly.Module(wasmTextToBinary(`(module
assertEq(counter, 0);
assertEq(wasmParsedBranchHints(module), true);
+let instance = new WebAssembly.Instance(module, imports);
+instance.exports.run(42);
+
+module = new WebAssembly.Module(wasmTextToBinary(`
+(module
+(func $dummy)
+(func (export "nested") (param i32 i32) (result i32)
+ (if (result i32) (local.get 0)
+ (then
+ (if (local.get 1) (then (call $dummy) (block) (nop)))
+ (if (local.get 1) (then) (else (call $dummy) (block) (nop)))
+ (@metadata.code.branch_hint "\\01")
+ (if (result i32) (local.get 1)
+ (then (call $dummy) (i32.const 9))
+ (else (call $dummy) (i32.const 10))
+ )
+ )
+ (else
+ (if (local.get 1) (then (call $dummy) (block) (nop)))
+ (if (local.get 1) (then) (else (call $dummy) (block) (nop)))
+ (@metadata.code.branch_hint "\\00")
+ (if (result i32) (local.get 1)
+ (then (call $dummy) (i32.const 10))
+ (else (call $dummy) (i32.const 11))
+ )
+ )
+ )
+)
+)`));
+
+assertEq(wasmParsedBranchHints(module), true);
+instance = new WebAssembly.Instance(module, imports);
+instance.exports.nested(2, 5);
+
+// Test that branch hinting can propagate to successor blocks.
+module = new WebAssembly.Module(wasmTextToBinary(`
+(module
+ (func (export "main") (param $x i32) (result i32)
+ (if (result i32)
+ (i32.gt_s (local.get $x) (i32.const 0))
+ (then
+ (@metadata.code.branch_hint "\\00")
+ (if (result i32)
+ (i32.eq (local.get $x) (i32.const 1))
+ (then
+ (i32.const 10)
+ )
+ (else
+ (if (result i32)
+ (i32.eq (local.get $x) (i32.const 2))
+ (then
+ (i32.const 20)
+ )
+ (else
+ (i32.const 99)
+ )
+ )
+ )
+ )
+ )
+ (else
+ (i32.const 0)
+ )
+ )
+ )
+)`));
+
+assertEq(wasmParsedBranchHints(module), true);
+instance = new WebAssembly.Instance(module, imports);
+instance.exports.main(2);
+
+// Test that the nested if branch has the correct branch hinting likelyness.
+module = new WebAssembly.Module(wasmTextToBinary(`
+(module
+ (func (export "main") (param $x i32) (result i32)
+ (if (result i32)
+ (i32.gt_s (local.get $x) (i32.const 0))
+ (then
+ (@metadata.code.branch_hint "\\01")
+ (if (result i32)
+ (i32.eq (local.get $x) (i32.const 1))
+ (then
+ (i32.const 10)
+ )
+ (else
+ (@metadata.code.branch_hint "\\00")
+ (if (result i32)
+ (i32.eq (local.get $x) (i32.const 2))
+ (then
+ (i32.const 20)
+ )
+ (else
+ (i32.const 99)
+ )
+ )
+ )
+ )
+ )
+ (else
+ (i32.const 0)
+ )
+ )
+ )
+)`));
+
+assertEq(wasmParsedBranchHints(module), true);
+instance = new WebAssembly.Instance(module, imports);
+instance.exports.main(5);
diff --git a/js/src/jit-test/tests/wasm/branch-hinting/loops.js b/js/src/jit-test/tests/wasm/branch-hinting/loops.js
@@ -0,0 +1,90 @@
+
+// Move a nested if out of line.
+let module = new WebAssembly.Module(wasmTextToBinary(`
+(module
+ (type (;0;) (func))
+ (type (;1;) (func (param i32) (result i32)))
+ (type (;2;) (func (result i32)))
+ (func $__wasm_nullptr (type 0)
+ unreachable)
+ (func $main (type 2) (result i32)
+ (local i32 i32 i32 i32)
+ i32.const 0
+ local.tee 2
+ local.set 3
+ loop
+ local.get 2
+ i32.const 50000
+ i32.eq
+ (@metadata.code.branch_hint "\\00") if
+ i32.const 1
+ local.set 3
+ end
+ local.get 2
+ i32.const 1
+ i32.add
+ local.tee 2
+ i32.const 100000
+ i32.ne
+ (@metadata.code.branch_hint "\\01") br_if 0 (;@1;)
+ end
+ local.get 3)
+ (table (;0;) 1 1 funcref)
+ (memory (;0;) 17 128)
+ (global (;0;) (mut i32) (i32.const 42))
+ (export "memory" (memory 0))
+ (export "_main" (func $main))
+ (elem (;0;) (i32.const 0) func $__wasm_nullptr)
+ (type (;0;) (func (param i32)))
+)`));
+
+assertEq(wasmParsedBranchHints(module), true);
+let instance = new WebAssembly.Instance(module);
+instance.exports._main(10);
+
+
+// A loop with unlikely blocks in the middle.
+let m = new WebAssembly.Module(wasmTextToBinary(`
+(module
+ (type (;0;) (func (param i32)))
+ (memory (;0;) 1 1)
+ (func $test (param i32)
+ (local i32)
+ loop
+ local.get 1
+ local.get 0
+ i32.const 1
+ i32.sub
+ i32.eq
+ (@metadata.code.branch_hint "\\00")
+ if
+ local.get 1
+ call $test
+ end
+ local.get 1
+ local.get 0
+ i32.const 2
+ i32.sub
+ i32.eq
+ (@metadata.code.branch_hint "\\00")
+ if
+ local.get 1
+ call $test
+ end
+ local.get 1
+ i32.const 1
+ i32.add
+ local.tee 1
+ local.get 0
+ i32.lt_u
+ br_if 0
+ end
+ return
+ )
+ (export "test" (func $test))
+)`));
+
+assertEq(wasmParsedBranchHints(m), true);
+
+instance = new WebAssembly.Instance(m);
+instance.exports.test(10);
diff --git a/js/src/jit/BranchHinting.cpp b/js/src/jit/BranchHinting.cpp
@@ -16,45 +16,46 @@ using namespace js::jit;
// Implementation of the branch hinting proposal
// Some control instructions (if and br_if) can have a hint of the form
-// Likely or unlikely. That means a specific branch will be likely/unlikely
+// likely or unlikely. That means a specific branch will be likely/unlikely
// to be executed at runtime.
-// In a first pass, we tag the basic blocks if we have a hint.
-// In a Mir to Mir transformation, we read the hints and do something with it:
-// - Unlikely blocks are pushed to the end of the function.
-// Because of Ion's structure, we don't do that for blocks inside a loop.
-// - TODO: do something for likely blocks.
-// - TODO: register allocator can be tuned depending on the hints.
+// This pass will propagate the branch hints to successor blocks in the CFG.
+// Currently, we use the branch hinting information for the following:
+// - Unlikely blocks are moved out of line, this is done at the LIR level.
+// - TODO: use branch hinting to optimize likely blocks.
bool jit::BranchHinting(const MIRGenerator* mir, MIRGraph& graph) {
JitSpew(JitSpew_BranchHint, "Beginning BranchHinting pass");
- // Move towards the end all blocks marked as unlikely
- mozilla::Vector<MBasicBlock*, 0> toBeMoved;
-
- for (MBasicBlock* block : graph) {
- // If this block has a return instruction, it's safe to push it
- // to the end of the graph.
- // If the block doesn't contain a return, a backedge outside a loop will be
- // created, which would break ReversePostOrder assertions.
- // Avoid moving a block if it's in the middle of a loop as well.
- if (block->branchHintingUnlikely() && block->loopDepth() == 0 &&
- block->hasLastIns() && block->lastIns()->is<js::jit::MWasmReturn>()) {
- if (!toBeMoved.append(block)) {
- return false;
- }
+ /* This pass propagates branch hints across the control flow graph
+ using dominator information. Branch hints are read at compile-time for
+ specific basic blocks. This pass propagates this property to successor
+ blocks in a conservative way. The algorithm works as follows:
+ - The CFG is traversed in reverse-post-order (RPO). Dominator parents are
+ visited before the blocks they dominate.
+
+ - For each basic block, if we have a hint, it is propagated to the
+ blocks it immediately dominates (its children in the dominator tree).
+
+ - The pass will then continue to work its way through the CFG.
+
+ Because we only propagate along dominator-tree edges (parent -> child),
+ each block receives information from exactly one source. This avoids
+ conflicts that would otherwise arise at CFG join points.
+ */
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ if (block->isUnknownFrequency()) {
+ continue;
}
- }
-
- for (MBasicBlock* block : toBeMoved) {
-#ifdef JS_JITSPEW
- JitSpew(JitSpew_BranchHint, "Moving block%u to the end", block->id());
-#endif
- graph.moveBlockToEnd(block);
- }
- if (!toBeMoved.empty()) {
- // Renumber blocks after moving them around.
- RenumberBlocks(graph);
+ for (MBasicBlock** it = block->immediatelyDominatedBlocksBegin();
+ it != block->immediatelyDominatedBlocksEnd(); it++) {
+ // Don't propagate the information if this successor block has already
+ // some branch hints.
+ if ((*it)->isUnknownFrequency()) {
+ (*it)->setFrequency(block->getFrequency());
+ }
+ }
}
return true;
diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp
@@ -8369,7 +8369,7 @@ bool CodeGenerator::generateBody() {
AutoCreatedBy acb(masm, "CodeGenerator::generateBody");
JitSpew(JitSpew_Codegen, "==== BEGIN CodeGenerator::generateBody ====");
- IonScriptCounts* counts = maybeCreateScriptCounts();
+ counts_ = maybeCreateScriptCounts();
const bool compilingWasm = gen->compilingWasm();
@@ -8387,108 +8387,157 @@ bool CodeGenerator::generateBody() {
return false;
}
+ // Skip out of line blocks for now. They will be emitted in
+ // generateOutOfLineBlocks.
+ if (current->isOutOfLine()) {
+ continue;
+ }
+
+ // Generate a basic block
+ if (!generateBlock(current, i, counts_, compilingWasm)) {
+ return false;
+ }
+ }
+
+ JitSpew(JitSpew_Codegen, "==== END CodeGenerator::generateBody ====\n");
+ return true;
+}
+
+bool CodeGenerator::generateBlock(LBlock* current, size_t blockNumber,
+ IonScriptCounts* counts, bool compilingWasm) {
#ifdef JS_JITSPEW
- const char* filename = nullptr;
- size_t lineNumber = 0;
- JS::LimitedColumnNumberOneOrigin columnNumber;
- if (current->mir()->info().script()) {
- filename = current->mir()->info().script()->filename();
- if (current->mir()->pc()) {
- lineNumber = PCToLineNumber(current->mir()->info().script(),
- current->mir()->pc(), &columnNumber);
- }
+ const char* filename = nullptr;
+ size_t lineNumber = 0;
+ JS::LimitedColumnNumberOneOrigin columnNumber;
+ if (current->mir()->info().script()) {
+ filename = current->mir()->info().script()->filename();
+ if (current->mir()->pc()) {
+ lineNumber = PCToLineNumber(current->mir()->info().script(),
+ current->mir()->pc(), &columnNumber);
}
- JitSpew(JitSpew_Codegen, "--------------------------------");
- JitSpew(JitSpew_Codegen, "# block%zu %s:%zu:%u%s:", i,
- filename ? filename : "?", lineNumber,
- columnNumber.oneOriginValue(),
- current->mir()->isLoopHeader() ? " (loop header)" : "");
+ }
+ JitSpew(JitSpew_Codegen, "--------------------------------");
+ JitSpew(JitSpew_Codegen, "# block%zu %s:%zu:%u%s:", blockNumber,
+ filename ? filename : "?", lineNumber, columnNumber.oneOriginValue(),
+ current->mir()->isLoopHeader() ? " (loop header)" : "");
#endif
- if (current->mir()->isLoopHeader() && compilingWasm) {
- masm.nopAlign(CodeAlignment);
- }
+ if (current->mir()->isLoopHeader() && compilingWasm) {
+ masm.nopAlign(CodeAlignment);
+ }
- masm.bind(current->label());
+ masm.bind(current->label());
- mozilla::Maybe<ScriptCountBlockState> blockCounts;
- if (counts) {
- blockCounts.emplace(&counts->block(i), &masm);
- if (!blockCounts->init()) {
- return false;
- }
+ mozilla::Maybe<ScriptCountBlockState> blockCounts;
+ if (counts) {
+ blockCounts.emplace(&counts->block(blockNumber), &masm);
+ if (!blockCounts->init()) {
+ return false;
}
+ }
- for (LInstructionIterator iter = current->begin(); iter != current->end();
- iter++) {
- if (gen->shouldCancel("Generate Code (instruction loop)")) {
- return false;
- }
- if (!alloc().ensureBallast()) {
- return false;
- }
+ for (LInstructionIterator iter = current->begin(); iter != current->end();
+ iter++) {
+ if (gen->shouldCancel("Generate Code (instruction loop)")) {
+ return false;
+ }
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
- perfSpewer().recordInstruction(masm, *iter);
+ perfSpewer().recordInstruction(masm, *iter);
#ifdef JS_JITSPEW
- JitSpewStart(JitSpew_Codegen, " # LIR=%s",
- iter->opName());
- if (const char* extra = iter->getExtraName()) {
- JitSpewCont(JitSpew_Codegen, ":%s", extra);
- }
- JitSpewFin(JitSpew_Codegen);
+ JitSpewStart(JitSpew_Codegen, " # LIR=%s",
+ iter->opName());
+ if (const char* extra = iter->getExtraName()) {
+ JitSpewCont(JitSpew_Codegen, ":%s", extra);
+ }
+ JitSpewFin(JitSpew_Codegen);
#endif
- if (counts) {
- blockCounts->visitInstruction(*iter);
- }
+ if (counts) {
+ blockCounts->visitInstruction(*iter);
+ }
#ifdef CHECK_OSIPOINT_REGISTERS
- if (iter->safepoint() && !compilingWasm) {
- resetOsiPointRegs(iter->safepoint());
- }
+ if (iter->safepoint() && !compilingWasm) {
+ resetOsiPointRegs(iter->safepoint());
+ }
#endif
- if (!compilingWasm) {
- if (MDefinition* mir = iter->mirRaw()) {
- if (!addNativeToBytecodeEntry(mir->trackedSite())) {
- return false;
- }
+ if (!compilingWasm) {
+ if (MDefinition* mir = iter->mirRaw()) {
+ if (!addNativeToBytecodeEntry(mir->trackedSite())) {
+ return false;
}
}
+ }
- setElement(*iter); // needed to encode correct snapshot location.
+ setElement(*iter); // needed to encode correct snapshot location.
#ifdef DEBUG
- emitDebugForceBailing(*iter);
+ emitDebugForceBailing(*iter);
#endif
- switch (iter->op()) {
+ switch (iter->op()) {
#ifndef JS_CODEGEN_NONE
# define LIROP(op) \
case LNode::Opcode::op: \
visit##op(iter->to##op()); \
break;
- LIR_OPCODE_LIST(LIROP)
+ LIR_OPCODE_LIST(LIROP)
# undef LIROP
#endif
- case LNode::Opcode::Invalid:
- default:
- MOZ_CRASH("Invalid LIR op");
- }
+ case LNode::Opcode::Invalid:
+ default:
+ MOZ_CRASH("Invalid LIR op");
+ }
#ifdef DEBUG
- if (!counts) {
- emitDebugResultChecks(*iter);
- }
+ if (!counts) {
+ emitDebugResultChecks(*iter);
+ }
#endif
+ }
+
+ return !masm.oom();
+}
+
+bool CodeGenerator::generateOutOfLineBlocks() {
+ AutoCreatedBy acb(masm, "CodeGeneratorShared::generateOutOfLineBlocks");
+
+ // Generate out of line basic blocks.
+ // If we are generated some blocks at the end of the function, we need
+ // to adjust the frame depth.
+ if (!gen->branchHintingEnabled()) {
+ return true;
+ }
+ masm.setFramePushed(frameDepth_);
+
+ const bool compilingWasm = gen->compilingWasm();
+
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ current = graph.getBlock(i);
+
+ if (gen->shouldCancel("Generate Code (block loop)")) {
+ return false;
}
- if (masm.oom()) {
+
+ if (current->isTrivial()) {
+ continue;
+ }
+
+ // If this block is marked as out of line, we need to generate it now.
+ if (!current->isOutOfLine()) {
+ continue;
+ }
+
+ if (!generateBlock(current, i, counts_, compilingWasm)) {
return false;
}
}
- JitSpew(JitSpew_Codegen, "==== END CodeGenerator::generateBody ====\n");
- return true;
+ return !masm.oom();
}
void CodeGenerator::visitNewArrayCallVM(LNewArray* lir) {
@@ -17086,6 +17135,13 @@ bool CodeGenerator::generateWasm(wasm::CallIndirectId callIndirectId,
masm.bind(&returnLabel_);
wasm::GenerateFunctionEpilogue(masm, frameSize(), offsets);
+ perfSpewer().recordOffset(masm, "OOLBlocks");
+ // This must come before we generate OOL code, as OOL blocks may
+ // generate OOL code.
+ if (!generateOutOfLineBlocks()) {
+ return false;
+ }
+
perfSpewer().recordOffset(masm, "OOLCode");
if (!generateOutOfLineCode()) {
return false;
@@ -17197,6 +17253,13 @@ bool CodeGenerator::generate(const WarpSnapshot* snapshot) {
perfSpewer().recordOffset(masm, "InvalidateEpilogue");
generateInvalidateEpilogue();
+ perfSpewer().recordOffset(masm, "OOLBlocks");
+ // This must come before we generate OOL code, as OOL blocks may
+ // generate OOL code.
+ if (!generateOutOfLineBlocks()) {
+ return false;
+ }
+
// native => bytecode entries for OOL code will be added
// by CodeGeneratorShared::generateOutOfLineCode
perfSpewer().recordOffset(masm, "OOLCode");
@@ -20405,22 +20468,22 @@ void CodeGenerator::visitWasmParameter(LWasmParameter* lir) {}
void CodeGenerator::visitWasmParameterI64(LWasmParameterI64* lir) {}
void CodeGenerator::visitWasmReturn(LWasmReturn* lir) {
- // Don't emit a jump to the return label if this is the last block.
- if (current->mir() != *gen->graph().poBegin()) {
+ // Don't emit a jump to the return label if this is the last block.
+ if (current->mir() != *gen->graph().poBegin() || current->isOutOfLine()) {
masm.jump(&returnLabel_);
}
}
void CodeGenerator::visitWasmReturnI64(LWasmReturnI64* lir) {
// Don't emit a jump to the return label if this is the last block.
- if (current->mir() != *gen->graph().poBegin()) {
+ if (current->mir() != *gen->graph().poBegin() || current->isOutOfLine()) {
masm.jump(&returnLabel_);
}
}
void CodeGenerator::visitWasmReturnVoid(LWasmReturnVoid* lir) {
// Don't emit a jump to the return label if this is the last block.
- if (current->mir() != *gen->graph().poBegin()) {
+ if (current->mir() != *gen->graph().poBegin() || current->isOutOfLine()) {
masm.jump(&returnLabel_);
}
}
diff --git a/js/src/jit/CodeGenerator.h b/js/src/jit/CodeGenerator.h
@@ -55,6 +55,7 @@ class OutOfLineCallPostWriteElementBarrier;
class CodeGenerator final : public CodeGeneratorSpecific {
// Warp snapshot. This is nullptr for Wasm compilations.
const WarpSnapshot* snapshot_ = nullptr;
+ IonScriptCounts* counts_ = nullptr;
[[nodiscard]] bool generateBody();
@@ -99,7 +100,10 @@ class CodeGenerator final : public CodeGeneratorSpecific {
wasm::FuncOffsets* offsets,
wasm::StackMaps* stackMaps,
wasm::Decoder* decoder);
+ [[nodiscard]] bool generateBlock(LBlock* current, size_t blockNumber,
+ IonScriptCounts* counts, bool compilingWasm);
+ [[nodiscard]] bool generateOutOfLineBlocks();
[[nodiscard]] bool link(JSContext* cx);
void emitOOLTestObject(Register objreg, Label* ifTruthy, Label* ifFalsy,
diff --git a/js/src/jit/LIR.cpp b/js/src/jit/LIR.cpp
@@ -62,8 +62,17 @@ void LIRGraph::dump() {
#endif
LBlock::LBlock(MBasicBlock* from)
- : block_(from), entryMoveGroup_(nullptr), exitMoveGroup_(nullptr) {
+ : block_(from),
+ entryMoveGroup_(nullptr),
+ exitMoveGroup_(nullptr),
+ isOutOfLine_(false) {
from->assignLir(this);
+
+ // If branch hinting is enabled, and this block is unlikely to be executed,
+ // it will be generated out of line.
+ if (from->info().branchHintingEnabled() && from->isUnlikelyFrequency()) {
+ isOutOfLine_ = true;
+ }
}
bool LBlock::init(TempAllocator& alloc) {
diff --git a/js/src/jit/LIR.h b/js/src/jit/LIR.h
@@ -1118,6 +1118,8 @@ class LBlock {
LMoveGroup* entryMoveGroup_;
LMoveGroup* exitMoveGroup_;
Label label_;
+ // If true, this block will be generated out of line.
+ bool isOutOfLine_;
public:
explicit LBlock(MBasicBlock* block);
@@ -1131,6 +1133,7 @@ class LBlock {
LPhi* getPhi(size_t index) { return &phis_[index]; }
const LPhi* getPhi(size_t index) const { return &phis_[index]; }
MBasicBlock* mir() const { return block_; }
+ bool isOutOfLine() const { return isOutOfLine_; }
LInstructionIterator begin() { return instructions_.begin(); }
LInstructionIterator begin(LInstruction* at) {
return instructions_.begin(at);
diff --git a/js/src/jit/MIRGraph.h b/js/src/jit/MIRGraph.h
@@ -36,6 +36,12 @@ using MResumePointIterator = InlineForwardListIterator<MResumePoint>;
class LBlock;
+// Represents the likelihood of a basic block to be executed at runtime.
+// Unknown: default value.
+// Likely: Likely to be executed at runtime, hot block.
+// Unlikely: unlikely to be executed, cold block.
+enum class Frequency : uint8_t { Unknown = 0, Likely = 1, Unlikely = 2 };
+
class MBasicBlock : public TempObject, public InlineListNode<MBasicBlock> {
public:
enum Kind {
@@ -62,8 +68,9 @@ class MBasicBlock : public TempObject, public InlineListNode<MBasicBlock> {
// This block will unconditionally bail out.
bool alwaysBails_ = false;
- // Will be used for branch hinting in wasm.
- wasm::BranchHint branchHint_ = wasm::BranchHint::Invalid;
+ // Represents the execution frequency of this block, considered unknown by
+ // default. Various passes can use this information for optimizations.
+ Frequency frequency_ = Frequency::Unknown;
// Pushes a copy of a local variable or argument.
void pushVariable(uint32_t slot) { push(slots_[slot]); }
@@ -385,14 +392,14 @@ class MBasicBlock : public TempObject, public InlineListNode<MBasicBlock> {
uint32_t id() const { return id_; }
uint32_t numPredecessors() const { return predecessors_.length(); }
- bool branchHintingUnlikely() const {
- return branchHint_ == wasm::BranchHint::Unlikely;
- }
- bool branchHintingLikely() const {
- return branchHint_ == wasm::BranchHint::Likely;
- }
+ bool isUnknownFrequency() const { return frequency_ == Frequency::Unknown; }
+
+ bool isLikelyFrequency() const { return frequency_ == Frequency::Likely; }
+
+ bool isUnlikelyFrequency() const { return frequency_ == Frequency::Unlikely; }
- void setBranchHinting(wasm::BranchHint value) { branchHint_ = value; }
+ Frequency getFrequency() const { return frequency_; }
+ void setFrequency(Frequency value) { frequency_ = value; }
uint32_t domIndex() const {
MOZ_ASSERT(!isDead());
diff --git a/js/src/jit/loong64/CodeGenerator-loong64.h b/js/src/jit/loong64/CodeGenerator-loong64.h
@@ -67,7 +67,7 @@ class CodeGeneratorLOONG64 : public CodeGeneratorShared {
void bailoutFrom(Label* label, LSnapshot* snapshot);
void bailout(LSnapshot* snapshot);
- bool generateOutOfLineCode();
+ bool generateOutOfLineCode(CodeGenerator* codegen);
template <typename T>
void branchToBlock(Register lhs, T rhs, MBasicBlock* mir,
diff --git a/js/src/jit/shared/CodeGenerator-shared.h b/js/src/jit/shared/CodeGenerator-shared.h
@@ -261,17 +261,38 @@ class CodeGeneratorShared : public LElementVisitor {
// Test whether the given block can be reached via fallthrough from the
// current block.
inline bool isNextBlock(LBlock* block) {
- uint32_t target = skipTrivialBlocks(block->mir())->id();
- uint32_t i = current->mir()->id() + 1;
- if (target < i) {
+ uint32_t targetId = skipTrivialBlocks(block->mir())->id();
+
+ // If the target is before next, then it's not next.
+ if (targetId < current->mir()->id() + 1) {
+ return false;
+ }
+
+ if (current->isOutOfLine() != graph.getBlock(targetId)->isOutOfLine()) {
return false;
}
- // Trivial blocks can be crossed via fallthrough.
- for (; i != target; ++i) {
- if (!graph.getBlock(i)->isTrivial()) {
- return false;
+
+ // Scan through blocks until the target to see if we can fallthrough them.
+ for (uint32_t nextId = current->mir()->id() + 1; nextId != targetId;
+ ++nextId) {
+ LBlock* nextBlock = graph.getBlock(nextId);
+
+ // If the next block is generated in a different section than this
+ // one, then we don't need to consider it for fallthrough.
+ if (nextBlock->isOutOfLine() != graph.getBlock(targetId)->isOutOfLine()) {
+ continue;
}
+
+ // If the next block is trivial, no code will be generated and we don't
+ // need to consider it for fallthrough.
+ if (nextBlock->isTrivial()) {
+ continue;
+ }
+
+ // Otherwise this is a real block that will prevent fallthrough.
+ return false;
}
+
return true;
}
diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp
@@ -5862,7 +5862,12 @@ class FunctionCompiler {
// Use branch hinting information if any.
if (pendingBlocks_[absolute].hint != BranchHint::Invalid) {
- join->setBranchHinting(pendingBlocks_[absolute].hint);
+ BranchHint hint = pendingBlocks_[absolute].hint;
+ if (hint == BranchHint::Likely) {
+ join->setFrequency(Frequency::Likely);
+ } else if (hint == BranchHint::Unlikely) {
+ join->setFrequency(Frequency::Unlikely);
+ }
}
pred->mark();
@@ -6186,7 +6191,11 @@ bool FunctionCompiler::emitIf() {
// Store the branch hint in the basic block.
if (!inDeadCode() && branchHint != BranchHint::Invalid) {
- getCurBlock()->setBranchHinting(branchHint);
+ if (branchHint == BranchHint::Likely) {
+ getCurBlock()->setFrequency(Frequency::Likely);
+ } else if (branchHint == BranchHint::Unlikely) {
+ getCurBlock()->setFrequency(Frequency::Unlikely);
+ }
}
iter().controlItem().block = elseBlock;
