commit 94e910dc62290e54846cf2362828a10d096fad49
parent 2c5382eac078cdf19a1d0d1b5059f96a0aeee393
Author: Iain Ireland <iireland@mozilla.com>
Date: Thu, 27 Nov 2025 21:39:56 +0000
Bug 2000328: Implement MacroAssembler::emitValueReadBarrierFastPath r=jandem,jonco
This is used in the next patch.
Differential Revision: https://phabricator.services.mozilla.com/D273673
Diffstat:
8 files changed, 149 insertions(+), 36 deletions(-)
diff --git a/js/public/HeapAPI.h b/js/public/HeapAPI.h
@@ -315,8 +315,8 @@ class alignas(TypicalCacheLineSize) ChunkMarkBitmap
MOZ_ALWAYS_INLINE void getMarkWordAndMask(const void* cell, ColorBit colorBit,
Word** wordp, uintptr_t* maskp) {
- // Note: the JIT pre-barrier trampolines inline this code. Update
- // MacroAssembler::emitPreBarrierFastPath code too when making changes here!
+ // Note: the JIT inlines this code. Update MacroAssembler::loadMarkBits and
+ // its callers when making changes here!
MOZ_ASSERT(size_t(colorBit) < MarkBitsPerCell);
@@ -868,6 +868,9 @@ namespace gc {
extern JS_PUBLIC_API void PerformIncrementalReadBarrier(JS::GCCellPtr thing);
static MOZ_ALWAYS_INLINE void ExposeGCThingToActiveJS(JS::GCCellPtr thing) {
+ // js::jit::ReadBarrier is a specialized version of this function designed to
+ // be called from jitcode. If this code is changed, it should be kept in sync.
+
// TODO: I'd like to assert !RuntimeHeapIsBusy() here but this gets
// called while we are tracing the heap, e.g. during memory reporting
// (see bug 1313318).
diff --git a/js/src/jit/ABIFunctionList-inl.h b/js/src/jit/ABIFunctionList-inl.h
@@ -178,6 +178,7 @@ namespace jit {
_(js::jit::PreserveWrapper) \
_(js::jit::Printf0) \
_(js::jit::Printf1) \
+ _(js::jit::ReadBarrier) \
_(js::jit::StringFromCharCodeNoGC) \
_(js::jit::StringTrimEndIndex) \
_(js::jit::StringTrimStartIndex) \
diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp
@@ -8067,6 +8067,47 @@ void MacroAssembler::nopPatchableToCall(const wasm::CallSiteDesc& desc) {
append(desc, offset);
}
+// Given a cell and the chunk containing that cell, load the word containing
+// the mark bits for that cell, and compute the bitIndex for a particular mark
+// color.
+void MacroAssembler::loadMarkBits(Register cell, Register chunk,
+ Register markWord, Register bitIndex,
+ Register temp, gc::ColorBit color) {
+ MOZ_ASSERT(temp != bitIndex);
+ MOZ_ASSERT(temp != chunk);
+ MOZ_ASSERT(chunk != bitIndex);
+
+ // Determine the bit index and store in bitIndex.
+ //
+ // bit = (addr & js::gc::ChunkMask) / js::gc::CellBytesPerMarkBit +
+ // static_cast<uint32_t>(colorBit);
+ static_assert(gc::CellBytesPerMarkBit == 8,
+ "Calculation below relies on this");
+ andPtr(Imm32(gc::ChunkMask), cell, bitIndex);
+ rshiftPtr(Imm32(3), bitIndex);
+ if (int32_t(color) != 0) {
+ addPtr(Imm32(int32_t(color)), bitIndex);
+ }
+
+ static_assert(gc::ChunkMarkBitmap::BitsPerWord == JS_BITS_PER_WORD,
+ "Calculation below relies on this");
+
+ // Load the mark word
+ //
+ // word = chunk.bitmap[bit / WordBits];
+
+ // Fold the adjustment for the fact that arenas don't start at the beginning
+ // of the chunk into the offset to the chunk bitmap.
+ const size_t firstArenaAdjustment =
+ gc::ChunkMarkBitmap::FirstThingAdjustmentBits / CHAR_BIT;
+ const intptr_t offset =
+ intptr_t(gc::ChunkMarkBitmapOffset) - intptr_t(firstArenaAdjustment);
+
+ uint8_t shift = mozilla::FloorLog2Size(JS_BITS_PER_WORD);
+ rshiftPtr(Imm32(shift), bitIndex, temp);
+ loadPtr(BaseIndex(chunk, temp, ScalePointer, offset), markWord);
+}
+
void MacroAssembler::emitPreBarrierFastPath(MIRType type, Register temp1,
Register temp2, Register temp3,
Label* noBarrier) {
@@ -8117,42 +8158,11 @@ void MacroAssembler::emitPreBarrierFastPath(MIRType type, Register temp1,
#endif
}
- // Determine the bit index and store in temp1.
- //
- // bit = (addr & js::gc::ChunkMask) / js::gc::CellBytesPerMarkBit +
- // static_cast<uint32_t>(colorBit);
- static_assert(gc::CellBytesPerMarkBit == 8,
- "Calculation below relies on this");
- static_assert(size_t(gc::ColorBit::BlackBit) == 0,
- "Calculation below relies on this");
- andPtr(Imm32(gc::ChunkMask), temp1);
- rshiftPtr(Imm32(3), temp1);
-
- static_assert(gc::ChunkMarkBitmap::BitsPerWord == JS_BITS_PER_WORD,
- "Calculation below relies on this");
-
- // Load the bitmap word in temp2.
- //
- // word = chunk.bitmap[bit / WordBits];
-
- // Fold the adjustment for the fact that arenas don't start at the beginning
- // of the chunk into the offset to the chunk bitmap.
- const size_t firstArenaAdjustment =
- gc::ChunkMarkBitmap::FirstThingAdjustmentBits / CHAR_BIT;
- const intptr_t offset =
- intptr_t(gc::ChunkMarkBitmapOffset) - intptr_t(firstArenaAdjustment);
-
- movePtr(temp1, temp3);
-#if JS_BITS_PER_WORD == 64
- rshiftPtr(Imm32(6), temp1);
- loadPtr(BaseIndex(temp2, temp1, TimesEight, offset), temp2);
-#else
- rshiftPtr(Imm32(5), temp1);
- loadPtr(BaseIndex(temp2, temp1, TimesFour, offset), temp2);
-#endif
+ // Check if the cell is marked black.
+ // Load the bitindex into temp3 and the bitmap word into temp2.
+ loadMarkBits(temp1, temp2, temp2, temp3, temp1, gc::ColorBit::BlackBit);
// Load the mask in temp1.
- //
// mask = uintptr_t(1) << (bit % WordBits);
andPtr(Imm32(gc::ChunkMarkBitmap::BitsPerWord - 1), temp3);
move32(Imm32(1), temp1);
@@ -8162,6 +8172,64 @@ void MacroAssembler::emitPreBarrierFastPath(MIRType type, Register temp1,
branchTestPtr(Assembler::NonZero, temp2, temp1, noBarrier);
}
+void MacroAssembler::emitValueReadBarrierFastPath(
+ ValueOperand value, Register cell, Register temp1, Register temp2,
+ Register temp3, Register temp4, Label* barrier) {
+ Label done;
+
+ // No barrier needed for non-GC types
+ branchTestGCThing(Assembler::NotEqual, value, &done);
+
+ // Load the GC thing in `cell`.
+ unboxGCThingForGCBarrier(value, cell);
+
+ // Load the chunk address.
+ Register chunk = temp1;
+ andPtr(Imm32(int32_t(~gc::ChunkMask)), cell, chunk);
+
+ // If the GC thing is in the nursery, we don't need to barrier it.
+ branchPtr(Assembler::NotEqual, Address(chunk, gc::ChunkStoreBufferOffset),
+ ImmWord(0), &done);
+
+ // Load the mark word and bit index for the black bit.
+ Register markWord = temp2;
+ Register bitIndex = temp3;
+ loadMarkBits(cell, chunk, markWord, bitIndex, temp4, gc::ColorBit::BlackBit);
+
+ // The mask for the black bit is 1 << (bitIndex % WordBits).
+ Register mask = temp4;
+ andPtr(Imm32(gc::ChunkMarkBitmap::BitsPerWord - 1), bitIndex);
+ move32(Imm32(1), mask);
+ flexibleLshiftPtr(bitIndex, mask);
+
+ // No barrier is needed if the black bit is set.
+ branchTestPtr(Assembler::NonZero, markWord, mask, &done);
+
+ // The bit index for the gray bit is bitIndex + 1. If the black bit
+ // is any bit except the highest bit of the mark word, we can reuse
+ // the mark word we've already loaded, and simply shift the mask by
+ // 1.
+ Label noMaskOverflow;
+ lshiftPtr(Imm32(1), mask);
+ branchTestPtr(Assembler::NonZero, mask, mask, &noMaskOverflow);
+
+ // If the black bit was the high bit of the mark word, we need to load
+ // a new mark word. In this case the mask for the gray bit will always
+ // be 1 (the lowest bit of the next word).
+ loadMarkBits(cell, chunk, markWord, bitIndex, temp4,
+ gc::ColorBit::GrayOrBlackBit);
+ move32(Imm32(1), mask);
+ bind(&noMaskOverflow);
+
+ // If the gray bit is set, then we *do* need a barrier.
+ branchTestPtr(Assembler::NonZero, markWord, mask, barrier);
+
+ // Otherwise, we don't need a barrier unless we're in the middle of
+ // an incremental GC.
+ branchTestNeedsIncrementalBarrierAnyZone(Assembler::NonZero, barrier, temp1);
+ bind(&done);
+}
+
// ========================================================================
// JS atomic operations.
diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h
@@ -4193,6 +4193,14 @@ class MacroAssembler : public MacroAssemblerSpecific {
void emitPreBarrierFastPath(MIRType type, Register temp1, Register temp2,
Register temp3, Label* noBarrier);
+ void emitValueReadBarrierFastPath(ValueOperand value, Register cell,
+ Register temp1, Register temp2,
+ Register temp3, Register temp4,
+ Label* barrier);
+
+ private:
+ void loadMarkBits(Register cell, Register chunk, Register markWord,
+ Register bitIndex, Register temp, gc::ColorBit color);
public:
// ========================================================================
diff --git a/js/src/jit/VMFunctions.cpp b/js/src/jit/VMFunctions.cpp
@@ -3319,6 +3319,27 @@ void AssertPropertyLookup(NativeObject* obj, PropertyKey id, uint32_t slot) {
#endif
}
+// This is a specialized version of ExposeJSThingToActiveJS
+void ReadBarrier(gc::Cell* cell) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!JS::RuntimeHeapIsCollecting());
+ MOZ_ASSERT(!gc::IsInsideNursery(cell));
+
+ gc::TenuredCell* tenured = &cell->asTenured();
+ MOZ_ASSERT(!gc::detail::TenuredCellIsMarkedBlack(tenured));
+
+ Zone* zone = tenured->zone();
+ if (zone->needsIncrementalBarrier()) {
+ gc::PerformIncrementalReadBarrier(tenured);
+ } else if (!zone->isGCPreparing() &&
+ gc::detail::NonBlackCellIsMarkedGray(tenured)) {
+ gc::UnmarkGrayGCThingRecursively(tenured);
+ }
+ MOZ_ASSERT_IF(!zone->isGCPreparing(),
+ !gc::detail::TenuredCellIsMarkedGray(tenured));
+}
+
void AssumeUnreachable(const char* output) {
MOZ_ReportAssertionFailure(output, __FILE__, __LINE__);
}
diff --git a/js/src/jit/VMFunctions.h b/js/src/jit/VMFunctions.h
@@ -733,6 +733,8 @@ void AssertMapObjectHash(JSContext* cx, MapObject* obj, const Value* value,
void AssertPropertyLookup(NativeObject* obj, PropertyKey id, uint32_t slot);
+void ReadBarrier(gc::Cell* cell);
+
// Functions used when JS_MASM_VERBOSE is enabled.
void AssumeUnreachable(const char* output);
void Printf0(const char* output);
diff --git a/js/src/jit/arm/MacroAssembler-arm.h b/js/src/jit/arm/MacroAssembler-arm.h
@@ -875,6 +875,11 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM {
void unboxGCThingForGCBarrier(const Address& src, Register dest) {
load32(ToPayload(src), dest);
}
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ if (src.payloadReg() != dest) {
+ ma_mov(src.payloadReg(), dest);
+ }
+ }
void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
load32(ToPayload(src), dest);
diff --git a/js/src/jit/x86/MacroAssembler-x86.h b/js/src/jit/x86/MacroAssembler-x86.h
@@ -918,6 +918,11 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared {
void unboxGCThingForGCBarrier(const Address& src, Register dest) {
movl(payloadOf(src), dest);
}
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ if (src.payloadReg() != dest) {
+ movl(src.payloadReg(), dest);
+ }
+ }
void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
movl(ImmWord(wasm::AnyRef::GCThingMask), dest);
