commit 2be25aae43d145fc7e28633e4045d9eced869fcb
parent 55dcc8c0cf229f3958e575ab3041a98e93f46a14
Author: Julian Seward <jseward@acm.org>
Date: Wed, 17 Dec 2025 10:27:24 +0000
Bug 2005437 - Allow movement of Wasm{Struct,Array}Object OOL blocks during GC. r=rhunt.
Bug 1995106 introduced suitable plumbing that allows stackmap entries for OOL
storage of Wasm{Struct,Array}Objects to flow through to the GC, but it does not
make use of them. This patch completes the job, by giving the GC the
opportunity of moving those blocks, recording their new locations in the
Nursery's forwarded-block table, and later using those entries to fix up
on-stack pointers to such blocks.
There is no change to the layout of either WasmStructObject or WasmArrayObject.
Changes:
* WasmArrayObject::obj_moved, WasmStructObject::obj_moved: completely
rewritten. More assertions, and more clarity about the {Nursery,Tenured} ->
{Nursery,Tenured} transitions. Allows the GC to move OOL blocks if required,
and if so adds their new addresses to the Nursery's forwarded-block table.
* Instance::updateFrameForMovingGC: completely rewritten. Use the
forwarded-block table to update OOL pointers as required.
* A bit of tidying of routines associated with WasmArrayObject::DataHeader.
Creates DataHeader::dataHeaderToDataPointer and makes that be more obviously
the "inverse" to ::dataHeaderFromDataPointer.
Differential Revision: https://phabricator.services.mozilla.com/D276007
Diffstat:
7 files changed, 201 insertions(+), 75 deletions(-)
diff --git a/js/src/jit/JitFrames.cpp b/js/src/jit/JitFrames.cpp
@@ -1506,6 +1506,7 @@ void TraceWeakJitActivationsInSweepingZones(JSContext* cx, JSTracer* trc) {
void UpdateJitActivationsForMinorGC(JSRuntime* rt) {
MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
+ Nursery& nursery = rt->gc.nursery();
JSContext* cx = rt->mainContextFromOwnThread();
for (JitActivationIterator activations(cx); !activations.done();
++activations) {
@@ -1518,7 +1519,7 @@ void UpdateJitActivationsForMinorGC(JSRuntime* rt) {
} else if (iter.isWasm()) {
const wasm::WasmFrameIter& frame = iter.asWasm();
frame.instance()->updateFrameForMovingGC(
- frame, frame.resumePCinCurrentFrame());
+ frame, frame.resumePCinCurrentFrame(), nursery);
}
}
}
@@ -1526,6 +1527,7 @@ void UpdateJitActivationsForMinorGC(JSRuntime* rt) {
void UpdateJitActivationsForCompactingGC(JSRuntime* rt) {
MOZ_ASSERT(JS::RuntimeHeapIsMajorCollecting());
+ Nursery& nursery = rt->gc.nursery();
JSContext* cx = rt->mainContextFromOwnThread();
for (JitActivationIterator activations(cx); !activations.done();
++activations) {
@@ -1533,7 +1535,7 @@ void UpdateJitActivationsForCompactingGC(JSRuntime* rt) {
if (iter.isWasm()) {
const wasm::WasmFrameIter& frame = iter.asWasm();
frame.instance()->updateFrameForMovingGC(
- frame, frame.resumePCinCurrentFrame());
+ frame, frame.resumePCinCurrentFrame(), nursery);
}
}
}
diff --git a/js/src/jit/MIR-wasm.h b/js/src/jit/MIR-wasm.h
@@ -2585,13 +2585,6 @@ class MWasmLoadField : public MBinaryInstruction, public NoTypePolicy::Data {
setMovable();
}
initWasmRefType(maybeRefType);
- if (aliases_.flags() ==
- AliasSet::Load(AliasSet::WasmStructOutlineDataPointer).flags() ||
- aliases_.flags() ==
- AliasSet::Load(AliasSet::WasmArrayDataPointer).flags()) {
- aliases_ = AliasSet::Store(AliasSet::Any);
- setNotMovableUnchecked();
- }
}
public:
diff --git a/js/src/wasm/WasmGcObject-inl.h b/js/src/wasm/WasmGcObject-inl.h
@@ -210,8 +210,8 @@ MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArrayOOL(
}
DataHeader* outlineHeader = (DataHeader*)outlineAlloc;
- uint8_t* outlineData = (uint8_t*)(outlineHeader + 1);
*outlineHeader = DataIsOOL;
+ uint8_t* outlineData = dataHeaderToDataPointer(outlineHeader);
arrayObj->initShape(typeDefData->shape);
arrayObj->superTypeVector_ = typeDefData->superTypeVector;
diff --git a/js/src/wasm/WasmGcObject.cpp b/js/src/wasm/WasmGcObject.cpp
@@ -287,37 +287,80 @@ void WasmArrayObject::obj_trace(JSTracer* trc, JSObject* object) {
}
/* static */
-size_t WasmArrayObject::obj_moved(JSObject* obj, JSObject* old) {
- // Moving inline arrays requires us to update the data pointer.
- WasmArrayObject& arrayObj = obj->as<WasmArrayObject>();
- WasmArrayObject& oldArrayObj = old->as<WasmArrayObject>();
- if (oldArrayObj.isDataInline()) {
- // The old array had inline storage, which has been copied.
- // Fix up the data pointer on the new array to point to it.
- arrayObj.data_ = WasmArrayObject::addressOfInlineData(&arrayObj);
+size_t WasmArrayObject::obj_moved(JSObject* objNew, JSObject* objOld) {
+ // This gets called for array objects, both with and without OOL areas.
+ // Dealing with the no-OOL case is simple. Thereafter, the logic for the OOL
+ // case is essentially the same as for WasmStructObject::obj_moved, since
+ // that routine is only used for WasmStructObjects that have OOL storage.
+ MOZ_ASSERT(objNew != objOld);
+
+ WasmArrayObject& arrayNew = objNew->as<WasmArrayObject>();
+ WasmArrayObject& arrayOld = objOld->as<WasmArrayObject>();
+
+ const TypeDef* typeDefNew = &arrayNew.typeDef();
+ mozilla::DebugOnly<const TypeDef*> typeDefOld = &arrayOld.typeDef();
+ MOZ_ASSERT(typeDefNew->isArrayType());
+ MOZ_ASSERT(typeDefOld == typeDefNew);
+
+ // At this point, the object has been copied, but the OOL storage area, if
+ // any, has not been copied, nor has the data_ pointer been updated. Hence:
+ MOZ_ASSERT(arrayNew.data_ == arrayOld.data_);
+
+ if (arrayOld.isDataInline()) {
+ // The old array had inline storage, which has been copied. Fix up the
+ // data pointer in the new array to point to it, and we're done.
+ arrayNew.data_ = WasmArrayObject::addressOfInlineData(&arrayNew);
+ MOZ_ASSERT(arrayNew.isDataInline());
+ return 0;
}
- MOZ_ASSERT(arrayObj.isDataInline() == oldArrayObj.isDataInline());
-
- if (IsInsideNursery(old)) {
- Nursery& nursery = obj->runtimeFromMainThread()->gc.nursery();
- // It's been tenured.
- if (!arrayObj.isDataInline()) {
- const TypeDef& typeDef = arrayObj.typeDef();
- MOZ_ASSERT(typeDef.isArrayType());
- // arrayObj.numElements_ was validated not to overflow when constructing
- // the array
- size_t trailerSize = calcStorageBytesUnchecked(
- typeDef.arrayType().elementType().size(), arrayObj.numElements_);
- // Ensured by WasmArrayObject::createArrayOOL.
- MOZ_RELEASE_ASSERT(trailerSize <= size_t(MaxArrayPayloadBytes));
- uint8_t* outlineAlloc =
- (uint8_t*)dataHeaderFromDataPointer(arrayObj.data_);
- uint8_t* prior = outlineAlloc;
- nursery.maybeMoveBufferOnPromotion(&outlineAlloc, obj, trailerSize);
- if (outlineAlloc != prior) {
- arrayObj.data_ = (uint8_t*)(((DataHeader*)outlineAlloc) + 1);
- }
- }
+
+ // The array has OOL storage. This means the logic that follows is similar
+ // to that for WasmStructObject::obj_moved, since that routine is only used
+ // for WasmStructObjects that have OOL storage.
+
+ bool newIsInNursery = IsInsideNursery(objNew);
+ bool oldIsInNursery = IsInsideNursery(objOld);
+
+ // Tenured -> Tenured
+ if (!oldIsInNursery && !newIsInNursery) {
+ // The object already was in the tenured heap and has merely been moved
+ // somewhere else in the the tenured heap. This isn't interesting to us.
+ return 0;
+ }
+
+ // Tenured -> Nursery: this transition isn't possible.
+ MOZ_RELEASE_ASSERT(oldIsInNursery);
+
+ // Nursery -> Nursery and Nursery -> Tenured
+ // The object is being moved, either within the nursery or from the nursery
+ // to the tenured heap. Either way, we have to ask the nursery if it wants
+ // to move the OOL block too, and if so set up a forwarding record for it.
+
+ // arrayNew.numElements_ was validated not to overflow when constructing
+ // the array.
+ size_t oolBlockSize = calcStorageBytesUnchecked(
+ typeDefNew->arrayType().elementType().size(), arrayNew.numElements_);
+ // Ensured by WasmArrayObject::createArrayOOL.
+ MOZ_RELEASE_ASSERT(oolBlockSize <= size_t(MaxArrayPayloadBytes) +
+ sizeof(WasmArrayObject::DataHeader));
+
+ // Ask the nursery if it wants to relocate the OOL block, and if so capture
+ // its new location in `oolHeaderNew`. Note, at this point `arrayNew.data_`
+ // has not been updated; hence the computation for `oolHeaderOld` is correct.
+ DataHeader* oolHeaderOld = dataHeaderFromDataPointer(arrayNew.data_);
+ DataHeader* oolHeaderNew = oolHeaderOld;
+ Nursery& nursery = objNew->runtimeFromMainThread()->gc.nursery();
+ nursery.maybeMoveBufferOnPromotion(&oolHeaderNew, objNew, oolBlockSize);
+
+ if (oolHeaderNew != oolHeaderOld) {
+ // The OOL block has been moved. Fix up the data pointer in the new
+ // object.
+ arrayNew.data_ = dataHeaderToDataPointer(oolHeaderNew);
+ // Set up forwarding for the OOL block. Use indirect forwarding.
+ // Unfortunately, if the call to `.setForwardingPointer..` OOMs, there's no
+ // way to recover.
+ nursery.setForwardingPointerWhileTenuring(oolHeaderOld, oolHeaderNew,
+ /*direct=*/false);
}
return 0;
@@ -443,20 +486,60 @@ void WasmStructObject::obj_trace(JSTracer* trc, JSObject* object) {
}
/* static */
-size_t WasmStructObject::obj_moved(JSObject* obj, JSObject* old) {
- // See also, corresponding comments in WasmArrayObject::obj_moved.
- if (IsInsideNursery(old)) {
- Nursery& nursery = obj->runtimeFromMainThread()->gc.nursery();
- WasmStructObject& structObj = obj->as<WasmStructObject>();
- const TypeDef& typeDef = structObj.typeDef();
- MOZ_ASSERT(typeDef.isStructType());
- uint32_t totalBytes = typeDef.structType().size_;
- uint32_t inlineBytes, outlineBytes;
- WasmStructObject::getDataByteSizes(totalBytes, &inlineBytes, &outlineBytes);
- MOZ_ASSERT(inlineBytes == WasmStructObject_MaxInlineBytes);
- MOZ_ASSERT(outlineBytes > 0);
- nursery.maybeMoveBufferOnPromotion(&structObj.outlineData_, obj,
- outlineBytes);
+size_t WasmStructObject::obj_moved(JSObject* objNew, JSObject* objOld) {
+ // This gets called only for struct objects that have an OOL area. Compare
+ // WasmStructObjectInlineClassExt vs WasmStructObjectOutlineClassExt below.
+ MOZ_ASSERT(objNew != objOld);
+
+ WasmStructObject& structNew = objNew->as<WasmStructObject>();
+ WasmStructObject& structOld = objOld->as<WasmStructObject>();
+ MOZ_ASSERT(structNew.outlineData_ && structOld.outlineData_);
+
+ const TypeDef* typeDefNew = &structNew.typeDef();
+ mozilla::DebugOnly<const TypeDef*> typeDefOld = &structOld.typeDef();
+ MOZ_ASSERT(typeDefNew->isStructType());
+ MOZ_ASSERT(typeDefOld == typeDefNew);
+
+ // At this point, the object has been copied, but the OOL storage area has
+ // not been copied, nor has the outlineData_ pointer been updated. Hence:
+ MOZ_ASSERT(structNew.outlineData_ == structOld.outlineData_);
+
+ bool newIsInNursery = IsInsideNursery(objNew);
+ bool oldIsInNursery = IsInsideNursery(objOld);
+
+ // Tenured -> Tenured
+ if (!oldIsInNursery && !newIsInNursery) {
+ // The object already was in the tenured heap and has merely been moved
+ // somewhere else in the the tenured heap. This isn't interesting to us.
+ return 0;
+ }
+
+ // Tenured -> Nursery: this transition isn't possible.
+ MOZ_RELEASE_ASSERT(oldIsInNursery);
+
+ // Nursery -> Nursery and Nursery -> Tenured
+ // The object is being moved, either within the nursery or from the nursery
+ // to the tenured heap. Either way, we have to ask the nursery if it wants
+ // to move the OOL block too, and if so set up a forwarding record for it.
+
+ uint32_t totalBytes = typeDefNew->structType().size_;
+ uint32_t inlineBytes, outlineBytes;
+ WasmStructObject::getDataByteSizes(totalBytes, &inlineBytes, &outlineBytes);
+ MOZ_ASSERT(inlineBytes == WasmStructObject_MaxInlineBytes);
+ MOZ_ASSERT(outlineBytes > 0);
+
+ // Ask the nursery if it wants to relocate the OOL area, and if so capture
+ // its new location in `structNew.outlineData_`.
+ Nursery& nursery = structNew.runtimeFromMainThread()->gc.nursery();
+ nursery.maybeMoveBufferOnPromotion(&structNew.outlineData_, objNew,
+ outlineBytes);
+ // Set up forwarding for the OOL area. Use indirect forwarding. As in
+ // WasmArrayObject::obj_moved, if the call to `.setForwardingPointer..` OOMs,
+ // there's no way to recover.
+ if (structOld.outlineData_ != structNew.outlineData_) {
+ nursery.setForwardingPointerWhileTenuring(structOld.outlineData_,
+ structNew.outlineData_,
+ /*direct=*/false);
}
return 0;
diff --git a/js/src/wasm/WasmGcObject.h b/js/src/wasm/WasmGcObject.h
@@ -207,9 +207,12 @@ class WasmArrayObject : public WasmGcObject,
size_t sizeOfExcludingThis() const;
+ // These constants can be anything, so long as they are not the same. Use
+ // small but unlikely values in the hope of getting more value from
+ // assertions involving them.
using DataHeader = uintptr_t;
- static const DataHeader DataIsIL = 0;
- static const DataHeader DataIsOOL = 1;
+ static const DataHeader DataIsIL = 0x37;
+ static const DataHeader DataIsOOL = 0x71;
// Creates a new array object with out-of-line storage. Reports an error on
// OOM. The element type, shape, class pointer, alloc site and alloc kind are
@@ -260,19 +263,29 @@ class WasmArrayObject : public WasmGcObject,
// Tracing and finalization
static void obj_trace(JSTracer* trc, JSObject* object);
static void obj_finalize(JS::GCContext* gcx, JSObject* object);
- static size_t obj_moved(JSObject* obj, JSObject* old);
+ static size_t obj_moved(JSObject* objNew, JSObject* objOld);
void storeVal(const wasm::Val& val, uint32_t itemIndex);
void fillVal(const wasm::Val& val, uint32_t itemIndex, uint32_t len);
- static DataHeader* dataHeaderFromDataPointer(const uint8_t* data) {
+ static inline DataHeader* dataHeaderFromDataPointer(const uint8_t* data) {
MOZ_ASSERT(data);
- return (DataHeader*)data - 1;
+ DataHeader* header = (DataHeader*)data;
+ header--;
+ MOZ_ASSERT(*header == DataIsIL || *header == DataIsOOL);
+ return header;
}
DataHeader* dataHeader() const {
return WasmArrayObject::dataHeaderFromDataPointer(data_);
}
+ static inline uint8_t* dataHeaderToDataPointer(const DataHeader* header) {
+ MOZ_ASSERT(header);
+ MOZ_ASSERT(*header == DataIsIL || *header == DataIsOOL);
+ header++;
+ return (uint8_t*)header;
+ }
+
static bool isDataInline(uint8_t* data) {
const DataHeader* header = dataHeaderFromDataPointer(data);
MOZ_ASSERT(*header == DataIsIL || *header == DataIsOOL);
@@ -406,7 +419,7 @@ class WasmStructObject : public WasmGcObject,
// Tracing and finalization
static void obj_trace(JSTracer* trc, JSObject* object);
static void obj_finalize(JS::GCContext* gcx, JSObject* object);
- static size_t obj_moved(JSObject* obj, JSObject* old);
+ static size_t obj_moved(JSObject* objNew, JSObject* objOld);
void storeVal(const wasm::Val& val, uint32_t fieldIndex);
};
diff --git a/js/src/wasm/WasmInstance.cpp b/js/src/wasm/WasmInstance.cpp
@@ -3271,7 +3271,7 @@ uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
}
void Instance::updateFrameForMovingGC(const wasm::WasmFrameIter& wfi,
- uint8_t* nextPC) {
+ uint8_t* nextPC, Nursery& nursery) {
const StackMap* map = code().lookupStackMap(nextPC);
if (!map) {
return;
@@ -3279,21 +3279,55 @@ void Instance::updateFrameForMovingGC(const wasm::WasmFrameIter& wfi,
Frame* frame = wfi.frame();
uintptr_t* stackWords = GetFrameScanStartForStackMap(frame, map, nullptr);
- // Update interior array data pointers for any inline-storage arrays that
- // moved.
+ // Update array data pointers, both IL and OOL, and struct data pointers,
+ // which are only OOL, for any such data areas that moved. Note, the
+ // remapping info consulted by the calls to Nursery::forwardBufferPointer is
+ // what previous calls to Nursery::setForwardingPointerWhileTenuring in
+ // Wasm{Struct,Array}Object::obj_moved set up.
+
for (uint32_t i = 0; i < map->header.numMappedWords; i++) {
- if (map->get(i) != StackMap::Kind::ArrayDataPointer) {
- continue;
- }
+ StackMap::Kind kind = map->get(i);
+
+ switch (kind) {
+ case StackMap::Kind::ArrayDataPointer: {
+ // Make oldDataPointer point at the storage array in the old object.
+ uint8_t* oldDataPointer = (uint8_t*)stackWords[i];
+ if (WasmArrayObject::isDataInline(oldDataPointer)) {
+ // It's a pointer into the object itself. Figure out where the old
+ // object is, ask where it got moved to, and fish out the updated
+ // value from the new object.
+ WasmArrayObject* oldArray =
+ WasmArrayObject::fromInlineDataPointer(oldDataPointer);
+ WasmArrayObject* newArray =
+ (WasmArrayObject*)gc::MaybeForwarded(oldArray);
+ if (newArray != oldArray) {
+ stackWords[i] =
+ uintptr_t(WasmArrayObject::addressOfInlineData(newArray));
+ MOZ_ASSERT(WasmArrayObject::isDataInline((uint8_t*)stackWords[i]));
+ }
+ } else {
+ WasmArrayObject::DataHeader* oldHeader =
+ WasmArrayObject::dataHeaderFromDataPointer(oldDataPointer);
+ WasmArrayObject::DataHeader* newHeader = oldHeader;
+ nursery.forwardBufferPointer((uintptr_t*)&newHeader);
+ if (newHeader != oldHeader) {
+ stackWords[i] =
+ uintptr_t(WasmArrayObject::dataHeaderToDataPointer(newHeader));
+ MOZ_ASSERT(!WasmArrayObject::isDataInline((uint8_t*)stackWords[i]));
+ }
+ }
+ break;
+ }
- uint8_t** addressOfArrayDataPointer = (uint8_t**)&stackWords[i];
- if (WasmArrayObject::isDataInline(*addressOfArrayDataPointer)) {
- WasmArrayObject* oldArray =
- WasmArrayObject::fromInlineDataPointer(*addressOfArrayDataPointer);
- WasmArrayObject* newArray =
- (WasmArrayObject*)gc::MaybeForwarded(oldArray);
- *addressOfArrayDataPointer =
- WasmArrayObject::addressOfInlineData(newArray);
+ case StackMap::Kind::StructDataPointer: {
+ // It's an unmodified pointer from BufferAllocator, so this is simple.
+ nursery.forwardBufferPointer(&stackWords[i]);
+ break;
+ }
+
+ default: {
+ break;
+ }
}
}
}
diff --git a/js/src/wasm/WasmInstance.h b/js/src/wasm/WasmInstance.h
@@ -280,7 +280,8 @@ class alignas(16) Instance {
uintptr_t traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
uint8_t* nextPC,
uintptr_t highestByteVisitedInPrevFrame);
- void updateFrameForMovingGC(const wasm::WasmFrameIter& wfi, uint8_t* nextPC);
+ void updateFrameForMovingGC(const wasm::WasmFrameIter& wfi, uint8_t* nextPC,
+ Nursery& nursery);
static constexpr size_t offsetOfMemory0Base() {
return offsetof(Instance, memory0Base_);
