commit b90c10909db17cd4d89c5ce56016fbf07a25f2f6
parent d8491f82adc36bfa37686e026f73a63a4ad47780
Author: Hannes Verschore <hv1989@gmail.com>
Date: Thu, 11 Dec 2025 22:43:11 +0000
Bug 1999828: Part 1 - Split TryFoldingStubs into two passes. One to detect the shape (and later slot offset) and another to gather the values. r=iain
Preparation to being able to fold GuardShape followed by {Load|Store}{Fixed|Dynamic}Slot.
For this we split the detection of stubs with different shapes (and in the future slots) and gathering.
This patch does this in two passes. One to detect the availability to fold (detect shape and slot change at same offset).
The second pass is used to make sure all Stub information is the same and saves the different shapes (and slots).
The other changes are to support the subsequent patches.
Differential Revision: https://phabricator.services.mozilla.com/D274201
Diffstat:
1 file changed, 153 insertions(+), 88 deletions(-)
diff --git a/js/src/jit/StubFolding.cpp b/js/src/jit/StubFolding.cpp
@@ -24,19 +24,18 @@
using namespace js;
using namespace js::jit;
-bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
- JSScript* script, ICScript* icScript) {
+static bool TryFoldingGuardShapes(JSContext* cx, ICFallbackStub* fallback,
+ JSScript* script, ICScript* icScript) {
+ // Try folding similar stubs with GuardShapes
+ // into GuardMultipleShapes
+
ICEntry* icEntry = icScript->icEntryForStub(fallback);
ICStub* entryStub = icEntry->firstStub();
-
- // Don't fold unless there are at least two stubs.
- if (entryStub == fallback) {
- return true;
- }
ICCacheIRStub* firstStub = entryStub->toCacheIRStub();
- if (firstStub->next()->isFallback()) {
- return true;
- }
+
+ // The caller guarantees that there are at least two stubs.
+ MOZ_ASSERT(entryStub != fallback);
+ MOZ_ASSERT(!firstStub->next()->isFallback());
const uint8_t* firstStubData = firstStub->stubDataStart();
const CacheIRStubInfo* stubInfo = firstStub->stubInfo();
@@ -53,14 +52,16 @@ bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
// GuardMultipleShapes.
uint32_t numActive = 0;
- mozilla::Maybe<uint32_t> foldableFieldOffset;
+ mozilla::Maybe<uint32_t> foldableShapeOffset;
GCVector<Value, 8> shapeList(cx);
- // Try to add a shape to the list. Can fail on OOM or for cross-realm shapes.
+ // Helper function: Keep list of different shapes.
+ // Can fail on OOM or for cross-realm shapes.
// Returns true if the shape was successfully added to the list, and false
// (with no pending exception) otherwise.
auto addShape = [&shapeList, cx](uintptr_t rawShape) -> bool {
Shape* shape = reinterpret_cast<Shape*>(rawShape);
+
// Only add same realm shapes.
if (shape->realm() != cx->realm()) {
return false;
@@ -75,72 +76,106 @@ bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
return true;
};
+ // Find the offset of the first Shape that differs.
for (ICCacheIRStub* other = firstStub->nextCacheIR(); other;
other = other->nextCacheIR()) {
// Verify that the stubs share the same code.
if (other->stubInfo() != stubInfo) {
return true;
}
- const uint8_t* otherStubData = other->stubDataStart();
if (other->enteredCount() > 0) {
numActive++;
}
+ if (foldableShapeOffset.isSome()) {
+ // Already found.
+ // Continue through all stubs to run above code.
+ continue;
+ }
+
+ const uint8_t* otherStubData = other->stubDataStart();
uint32_t fieldIndex = 0;
size_t offset = 0;
while (stubInfo->fieldType(fieldIndex) != StubField::Type::Limit) {
StubField::Type fieldType = stubInfo->fieldType(fieldIndex);
- if (StubField::sizeIsWord(fieldType)) {
- uintptr_t firstRaw = stubInfo->getStubRawWord(firstStubData, offset);
- uintptr_t otherRaw = stubInfo->getStubRawWord(otherStubData, offset);
-
- if (firstRaw != otherRaw) {
- if (fieldType != StubField::Type::WeakShape) {
- // Case 1: a field differs that is not a Shape. We only support
- // folding GuardShape to GuardMultipleShapes.
- return true;
- }
- if (foldableFieldOffset.isNothing()) {
- // Case 2: this is the first field where the stub data differs.
- foldableFieldOffset.emplace(offset);
- if (!addShape(firstRaw) || !addShape(otherRaw)) {
- return true;
- }
- } else if (*foldableFieldOffset == offset) {
- // Case 3: this is the corresponding offset in a different stub.
- if (!addShape(otherRaw)) {
- return true;
- }
- } else {
- // Case 4: we have found more than one field that differs.
- return true;
- }
+ // Continue if the fields have same value.
+ if (StubField::sizeIsInt64(fieldType)) {
+ if (stubInfo->getStubRawInt64(firstStubData, offset) ==
+ stubInfo->getStubRawInt64(otherStubData, offset)) {
+ offset += StubField::sizeInBytes(fieldType);
+ fieldIndex++;
+ continue;
}
} else {
- MOZ_ASSERT(StubField::sizeIsInt64(fieldType));
-
- // We do not support folding any ops with int64-sized fields.
- if (stubInfo->getStubRawInt64(firstStubData, offset) !=
- stubInfo->getStubRawInt64(otherStubData, offset)) {
- return true;
+ MOZ_ASSERT(StubField::sizeIsWord(fieldType));
+ if (stubInfo->getStubRawWord(firstStubData, offset) ==
+ stubInfo->getStubRawWord(otherStubData, offset)) {
+ offset += StubField::sizeInBytes(fieldType);
+ fieldIndex++;
+ continue;
}
}
- offset += StubField::sizeInBytes(fieldType);
- fieldIndex++;
+ // Early abort if it is a non-shape field that differs.
+ if (fieldType != StubField::Type::WeakShape) {
+ return true;
+ }
+
+ // Save the offset
+ foldableShapeOffset.emplace(offset);
+ break;
}
+ }
- // We should never attach two completely identical stubs.
- MOZ_ASSERT(foldableFieldOffset.isSome());
+ if (foldableShapeOffset.isNothing()) {
+ return true;
}
if (numActive == 0) {
return true;
}
- // Clone the CacheIR, replacing GuardShape with GuardMultipleShapes.
+ // Make sure the shape is the only value that differ.
+ // Collect the shape values at the same time.
+ for (ICCacheIRStub* stub = firstStub; stub; stub = stub->nextCacheIR()) {
+ const uint8_t* stubData = stub->stubDataStart();
+ uint32_t fieldIndex = 0;
+ size_t offset = 0;
+
+ while (stubInfo->fieldType(fieldIndex) != StubField::Type::Limit) {
+ StubField::Type fieldType = stubInfo->fieldType(fieldIndex);
+ if (offset == *foldableShapeOffset) {
+ // Save the shapes of all stubs.
+ MOZ_ASSERT(fieldType == StubField::Type::WeakShape);
+ uintptr_t raw = stubInfo->getStubRawWord(stubData, offset);
+ if (!addShape(raw)) {
+ return true;
+ }
+ } else {
+ // Check all other fields are the same.
+ if (StubField::sizeIsInt64(fieldType)) {
+ if (stubInfo->getStubRawInt64(firstStubData, offset) !=
+ stubInfo->getStubRawInt64(stubData, offset)) {
+ return true;
+ }
+ } else {
+ MOZ_ASSERT(StubField::sizeIsWord(fieldType));
+ if (stubInfo->getStubRawWord(firstStubData, offset) !=
+ stubInfo->getStubRawWord(stubData, offset)) {
+ return true;
+ }
+ }
+ }
+
+ offset += StubField::sizeInBytes(fieldType);
+ fieldIndex++;
+ }
+ }
+
+ // Clone the CacheIR and replace
+ // - specific GuardShape with GuardMultipleShapes.
CacheIRWriter writer(cx);
CacheIRReader reader(stubInfo);
CacheIRCloner cloner(firstStub);
@@ -151,40 +186,45 @@ bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
writer.setInputOperandId(i);
}
- bool success = false;
+ // Create the shapeList to bake in the new stub.
+ Rooted<ListObject*> shapeObj(cx);
+ {
+ gc::AutoSuppressGC suppressGC(cx);
+
+ shapeObj.set(ShapeListObject::create(cx));
+
+ if (!shapeObj) {
+ return false;
+ }
+
+ for (uint32_t i = 0; i < shapeList.length(); i++) {
+ if (!shapeObj->append(cx, shapeList[i])) {
+ cx->recoverFromOutOfMemory();
+ return false;
+ }
+
+ MOZ_ASSERT(static_cast<Shape*>(shapeList[i].toPrivate())->realm() ==
+ shapeObj->realm());
+ }
+ }
+
+ bool shapeSuccess = false;
while (reader.more()) {
CacheOp op = reader.readOp();
switch (op) {
case CacheOp::GuardShape: {
auto [objId, shapeOffset] = reader.argsForGuardShape();
- if (shapeOffset == *foldableFieldOffset) {
- // Ensure that the allocation of the ShapeListObject doesn't trigger a
- // GC and free the stubInfo we're currently reading. Note that
- // AutoKeepJitScripts isn't sufficient, because optimized stubs can be
- // discarded even if the JitScript is preserved.
- gc::AutoSuppressGC suppressGC(cx);
-
- Rooted<ShapeListObject*> shapeObj(cx, ShapeListObject::create(cx));
- if (!shapeObj) {
- return false;
- }
- for (uint32_t i = 0; i < shapeList.length(); i++) {
- if (!shapeObj->append(cx, shapeList[i])) {
- return false;
- }
-
- MOZ_ASSERT(static_cast<Shape*>(shapeList[i].toPrivate())->realm() ==
- shapeObj->realm());
- }
-
- writer.guardMultipleShapes(objId, shapeObj);
- success = true;
- } else {
+ if (shapeOffset != *foldableShapeOffset) {
+ // Unrelated GuardShape.
WeakHeapPtr<Shape*>& ptr =
stubInfo->getStubField<StubField::Type::WeakShape>(firstStub,
shapeOffset);
writer.guardShape(objId, ptr.unbarrieredGet());
+ break;
}
+
+ writer.guardMultipleShapes(objId, shapeObj);
+ shapeSuccess = true;
break;
}
default:
@@ -192,7 +232,8 @@ bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
break;
}
}
- if (!success) {
+
+ if (!shapeSuccess) {
// If the shape field that differed was not part of a GuardShape,
// we can't fold these stubs together.
return true;
@@ -216,6 +257,27 @@ bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
script->column().oneOriginValue());
fallback->setMayHaveFoldedStub();
+
+ return true;
+}
+
+bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
+ JSScript* script, ICScript* icScript) {
+ ICEntry* icEntry = icScript->icEntryForStub(fallback);
+ ICStub* entryStub = icEntry->firstStub();
+
+ // Don't fold unless there are at least two stubs.
+ if (entryStub == fallback) {
+ return true;
+ }
+
+ ICCacheIRStub* firstStub = entryStub->toCacheIRStub();
+ if (firstStub->next()->isFallback()) {
+ return true;
+ }
+
+ if (!TryFoldingGuardShapes(cx, fallback, script, icScript)) return false;
+
return true;
}
@@ -238,7 +300,7 @@ bool js::jit::AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer,
mozilla::Maybe<uint32_t> shapeFieldOffset;
RootedValue newShape(cx);
- Rooted<ShapeListObject*> foldedShapes(cx);
+ Rooted<ListObject*> shapeList(cx);
CacheIRReader stubReader(stubInfo);
CacheIRReader newReader(writer);
@@ -251,19 +313,19 @@ bool js::jit::AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer,
if (newOp != CacheOp::GuardShape) {
return false;
}
-
// Check that the object being guarded is the same.
if (newReader.objOperandId() != stubReader.objOperandId()) {
return false;
}
- // Check that the field offset is the same.
+ // Check that the shape offset is the same.
uint32_t newShapeOffset = newReader.stubOffset();
uint32_t stubShapesOffset = stubReader.stubOffset();
if (newShapeOffset != stubShapesOffset) {
return false;
}
- MOZ_ASSERT(shapeFieldOffset.isNothing());
+
+ MOZ_ASSERT(shapeList == nullptr);
shapeFieldOffset.emplace(newShapeOffset);
// Get the shape from the new stub
@@ -273,10 +335,10 @@ bool js::jit::AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer,
newShape = PrivateValue(shape);
// Get the shape array from the old stub.
- JSObject* shapeList = stubInfo->getStubField<StubField::Type::JSObject>(
+ JSObject* obj = stubInfo->getStubField<StubField::Type::JSObject>(
stub, stubShapesOffset);
- foldedShapes = &shapeList->as<ShapeListObject>();
- MOZ_ASSERT(foldedShapes->compartment() == shape->compartment());
+ shapeList = &obj->as<ShapeListObject>();
+ MOZ_ASSERT(shapeList->compartment() == shape->compartment());
// Don't add a shape if it's from a different realm than the first
// shape.
@@ -288,9 +350,11 @@ bool js::jit::AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer,
// The assert verifies this property by checking the first element has
// the same realm (and since everything in the list has the same realm,
// checking the first element suffices)
- Realm* shapesRealm = foldedShapes->realm();
- MOZ_ASSERT_IF(!foldedShapes->isEmpty(),
- foldedShapes->getUnbarriered(0)->realm() == shapesRealm);
+ Realm* shapesRealm = shapeList->realm();
+ MOZ_ASSERT_IF(
+ !shapeList->isEmpty(),
+ shapeList->as<ShapeListObject>().getUnbarriered(0)->realm() ==
+ shapesRealm);
if (shapesRealm != shape->realm()) {
return false;
}
@@ -320,27 +384,28 @@ bool js::jit::AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer,
return false;
}
- // Check to verify that all the other stub fields are the same.
if (!writer.stubDataEqualsIgnoring(stubData, *shapeFieldOffset)) {
return false;
}
+ ShapeListObject* obj = &shapeList->as<ShapeListObject>();
+
// Limit the maximum number of shapes we will add before giving up.
// If we give up, transition the stub.
- if (foldedShapes->length() == ShapeListObject::MaxLength) {
+ if (obj->length() == ShapeListObject::MaxLength) {
MOZ_ASSERT(fallback->state().mode() != ICState::Mode::Generic);
fallback->state().forceTransition();
fallback->discardStubs(cx->zone(), icEntry);
return false;
}
- if (!foldedShapes->append(cx, newShape)) {
+ if (!obj->append(cx, newShape)) {
cx->recoverFromOutOfMemory();
return false;
}
JitSpew(JitSpew_StubFolding, "ShapeListObject %p: new length: %u",
- foldedShapes.get(), foldedShapes->length());
+ shapeList.get(), shapeList->length());
return true;
}
