commit f8b8182ee6960e6ebc5d883ca30cc27a56a12fba
parent a5a3b6609dec82004b0047d40a83dab0031e16ab
Author: Ryan Hunt <rhunt@eqrion.net>
Date: Thu, 18 Dec 2025 16:45:27 +0000
Bug 2002625 - wasm: Merge SuspenderObject and SuspenderObjectData. r=yury
Now that we don't require a SuspenderObjectData to be a linked list
element, we can merge it with SuspenderObject. This reduces the number
of registers we need to for operations, and indirections to use.
The JIT routines are largely untouched and can be simplified in a
future patch.
Differential Revision: https://phabricator.services.mozilla.com/D274190
Diffstat:
8 files changed, 269 insertions(+), 284 deletions(-)
diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp
@@ -10610,20 +10610,18 @@ void CodeGenerator::visitWasmStackSwitchToSuspendable(
masm.framePushed() - sizeof(wasm::Frame), WasmStackAlignment);
masm.reserveStack(reserve);
- masm.loadPrivate(Address(SuspenderReg, NativeObject::getFixedSlotOffset(
- wasm::SuspenderObjectDataSlot)),
- SuspenderDataReg);
+ masm.movePtr(SuspenderReg, SuspenderDataReg);
// Switch stacks to suspendable, keep original FP to maintain
// frames chain between main and suspendable stack segments.
- masm.storeStackPtr(
- Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainSP()));
- masm.storePtr(
+ masm.storeStackPtrToPrivateValue(
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainSP()));
+ masm.storePrivateValue(
FramePointer,
- Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainFP()));
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainFP()));
- masm.loadStackPtr(Address(
- SuspenderDataReg, wasm::SuspenderObjectData::offsetOfSuspendableSP()));
+ masm.loadStackPtrFromPrivateValue(Address(
+ SuspenderDataReg, wasm::SuspenderObject::offsetOfSuspendableSP()));
masm.assertStackAlignment(WasmStackAlignment);
@@ -10655,10 +10653,9 @@ void CodeGenerator::visitWasmStackSwitchToSuspendable(
masm.computeEffectiveAddress(
Address(masm.getStackPointer(), -int32_t(sizeof(wasm::Frame))),
ScratchReg2);
- masm.storePtr(
- ScratchReg2,
- Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendableExitFP()));
+ masm.storePrivateValue(
+ ScratchReg2, Address(SuspenderDataReg,
+ wasm::SuspenderObject::offsetOfSuspendableExitFP()));
masm.mov(&returnCallsite, ReturnAddressReg);
@@ -10762,21 +10759,19 @@ void CodeGenerator::visitWasmStackSwitchToMain(LWasmStackSwitchToMain* lir) {
masm.framePushed() - sizeof(wasm::Frame), WasmStackAlignment);
masm.reserveStack(reserve);
- masm.loadPrivate(Address(SuspenderReg, NativeObject::getFixedSlotOffset(
- wasm::SuspenderObjectDataSlot)),
- SuspenderDataReg);
+ masm.movePtr(SuspenderReg, SuspenderDataReg);
// Switch stacks to main.
- masm.storeStackPtr(Address(
- SuspenderDataReg, wasm::SuspenderObjectData::offsetOfSuspendableSP()));
- masm.storePtr(FramePointer,
- Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendableFP()));
-
- masm.loadStackPtr(
- Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainSP()));
- masm.loadPtr(
- Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainFP()),
+ masm.storeStackPtrToPrivateValue(Address(
+ SuspenderDataReg, wasm::SuspenderObject::offsetOfSuspendableSP()));
+ masm.storePrivateValue(
+ FramePointer, Address(SuspenderDataReg,
+ wasm::SuspenderObject::offsetOfSuspendableFP()));
+
+ masm.loadStackPtrFromPrivateValue(
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainSP()));
+ masm.loadPrivate(
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainFP()),
FramePointer);
// Set main_ra field to returnCallsite.
@@ -10785,18 +10780,18 @@ void CodeGenerator::visitWasmStackSwitchToMain(LWasmStackSwitchToMain* lir) {
MOZ_ASSERT(ScratchReg1 == SuspenderDataReg);
masm.push(DataReg);
masm.mov(&returnCallsite, DataReg);
- masm.storePtr(
+ masm.storePrivateValue(
DataReg,
Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendedReturnAddress()));
+ wasm::SuspenderObject::offsetOfSuspendedReturnAddress()));
masm.pop(DataReg);
# else
MOZ_ASSERT(ScratchReg1 != SuspenderDataReg);
masm.mov(&returnCallsite, ScratchReg1);
- masm.storePtr(
+ masm.storePrivateValue(
ScratchReg1,
Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendedReturnAddress()));
+ wasm::SuspenderObject::offsetOfSuspendedReturnAddress()));
# endif
masm.assertStackAlignment(WasmStackAlignment);
@@ -10829,14 +10824,14 @@ void CodeGenerator::visitWasmStackSwitchToMain(LWasmStackSwitchToMain* lir) {
masm.computeEffectiveAddress(
Address(masm.getStackPointer(), -int32_t(sizeof(wasm::Frame))),
ScratchReg2);
- masm.storePtr(ScratchReg2,
- Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfMainExitFP()));
+ masm.storePrivateValue(
+ ScratchReg2,
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainExitFP()));
// Load InstanceReg from suspendable stack exit frame.
- masm.loadPtr(Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendableExitFP()),
- ScratchReg2);
+ masm.loadPrivate(Address(SuspenderDataReg,
+ wasm::SuspenderObject::offsetOfSuspendableExitFP()),
+ ScratchReg2);
masm.loadPtr(
Address(ScratchReg2, wasm::FrameWithInstances::callerInstanceOffset()),
ScratchReg2);
@@ -10844,9 +10839,9 @@ void CodeGenerator::visitWasmStackSwitchToMain(LWasmStackSwitchToMain* lir) {
WasmCallerInstanceOffsetBeforeCall));
// Load RA from suspendable stack exit frame.
- masm.loadPtr(Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendableExitFP()),
- ScratchReg1);
+ masm.loadPrivate(Address(SuspenderDataReg,
+ wasm::SuspenderObject::offsetOfSuspendableExitFP()),
+ ScratchReg1);
masm.loadPtr(Address(ScratchReg1, wasm::Frame::returnAddressOffset()),
ReturnAddressReg);
@@ -10950,19 +10945,17 @@ void CodeGenerator::visitWasmStackContinueOnSuspendable(
masm.framePushed() - sizeof(wasm::Frame), WasmStackAlignment);
masm.reserveStack(reserve);
- masm.loadPrivate(Address(SuspenderReg, NativeObject::getFixedSlotOffset(
- wasm::SuspenderObjectDataSlot)),
- SuspenderDataReg);
- masm.storeStackPtr(
- Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainSP()));
- masm.storePtr(
+ masm.movePtr(SuspenderReg, SuspenderDataReg);
+ masm.storeStackPtrToPrivateValue(
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainSP()));
+ masm.storePrivateValue(
FramePointer,
- Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainFP()));
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfMainFP()));
// Adjust exit frame FP.
- masm.loadPtr(Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendableExitFP()),
- ScratchReg1);
+ masm.loadPrivate(Address(SuspenderDataReg,
+ wasm::SuspenderObject::offsetOfSuspendableExitFP()),
+ ScratchReg1);
masm.storePtr(FramePointer,
Address(ScratchReg1, wasm::Frame::callerFPOffset()));
@@ -10977,11 +10970,11 @@ void CodeGenerator::visitWasmStackContinueOnSuspendable(
Address(ScratchReg1, wasm::FrameWithInstances::callerInstanceOffset()));
// Switch stacks to suspendable.
- masm.loadStackPtr(Address(
- SuspenderDataReg, wasm::SuspenderObjectData::offsetOfSuspendableSP()));
- masm.loadPtr(Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendableFP()),
- FramePointer);
+ masm.loadStackPtrFromPrivateValue(Address(
+ SuspenderDataReg, wasm::SuspenderObject::offsetOfSuspendableSP()));
+ masm.loadPrivate(
+ Address(SuspenderDataReg, wasm::SuspenderObject::offsetOfSuspendableFP()),
+ FramePointer);
masm.assertStackAlignment(WasmStackAlignment);
@@ -11013,9 +11006,9 @@ void CodeGenerator::visitWasmStackContinueOnSuspendable(
const Register ReturnAddressReg = ScratchReg1;
// Pretend we just returned from the function.
- masm.loadPtr(
+ masm.loadPrivate(
Address(SuspenderDataReg,
- wasm::SuspenderObjectData::offsetOfSuspendedReturnAddress()),
+ wasm::SuspenderObject::offsetOfSuspendedReturnAddress()),
ReturnAddressReg);
masm.jump(ReturnAddressReg);
diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h
@@ -5924,6 +5924,15 @@ class MacroAssembler : public MacroAssemblerSpecific {
storePtr(getStackPointer(), t);
}
+ template <typename T>
+ void loadStackPtrFromPrivateValue(T t) {
+ loadStackPtr(t);
+ }
+ template <typename T>
+ void storeStackPtrToPrivateValue(T t) {
+ storeStackPtr(t);
+ }
+
// StackPointer testing functions.
// On ARM64, sp can function as the zero register depending on context.
// Code shared across platforms must use these functions to be valid.
diff --git a/js/src/jit/arm64/MacroAssembler-arm64-inl.h b/js/src/jit/arm64/MacroAssembler-arm64-inl.h
@@ -4265,6 +4265,16 @@ void MacroAssemblerCompat::storeStackPtr(const Address& dest) {
}
}
+void MacroAssemblerCompat::loadStackPtrFromPrivateValue(const Address& src) {
+ // On ARM64, a private value is stored the same as any pointer.
+ loadStackPtr(src);
+}
+
+void MacroAssemblerCompat::storeStackPtrToPrivateValue(const Address& dest) {
+ // On ARM64, a private value is stored the same as any pointer.
+ storeStackPtr(dest);
+}
+
void MacroAssemblerCompat::branchTestStackPtr(Condition cond, Imm32 rhs,
Label* label) {
if (sp.Is(GetStackPointer64())) {
diff --git a/js/src/jit/arm64/MacroAssembler-arm64.h b/js/src/jit/arm64/MacroAssembler-arm64.h
@@ -933,6 +933,9 @@ class MacroAssemblerCompat : public vixl::MacroAssembler {
inline void loadStackPtr(const Address& src);
inline void storeStackPtr(const Address& dest);
+ inline void loadStackPtrFromPrivateValue(const Address& src);
+ inline void storeStackPtrToPrivateValue(const Address& dest);
+
// StackPointer testing functions.
inline void branchTestStackPtr(Condition cond, Imm32 rhs, Label* label);
inline void branchStackPtr(Condition cond, Register rhs, Label* label);
diff --git a/js/src/wasm/WasmContext.cpp b/js/src/wasm/WasmContext.cpp
@@ -89,7 +89,7 @@ void Context::traceRoots(JSTracer* trc) {
for (auto iter = suspenders_.iter(); !iter.done(); iter.next()) {
SuspenderObject* object = iter.get();
if (object->state() == SuspenderState::Suspended) {
- TraceSuspendableStack(trc, *object->data());
+ TraceSuspendableStack(trc, object);
}
}
}
diff --git a/js/src/wasm/WasmContext.h b/js/src/wasm/WasmContext.h
@@ -29,7 +29,6 @@ namespace js::wasm {
#ifdef ENABLE_WASM_JSPI
class SuspenderObject;
-class SuspenderObjectData;
using SuspenderObjectSet =
HashSet<SuspenderObject*, PointerHasher<SuspenderObject*>,
SystemAllocPolicy>;
diff --git a/js/src/wasm/WasmPI.cpp b/js/src/wasm/WasmPI.cpp
@@ -58,24 +58,18 @@ using namespace js::jit;
#ifdef ENABLE_WASM_JSPI
namespace js::wasm {
-SuspenderObjectData::SuspenderObjectData(void* stackMemory)
- : stackMemory_(stackMemory),
- suspendableFP_(nullptr),
- suspendableSP_(static_cast<uint8_t*>(stackMemory) +
- SuspendableStackPlusRedZoneSize),
- state_(SuspenderState::Initial) {}
-
-void SuspenderObjectData::releaseStackMemory() {
- MOZ_ASSERT(isMoribund() == !stackMemory_);
- if (stackMemory_) {
- js_free(stackMemory_);
- stackMemory_ = nullptr;
+void SuspenderObject::releaseStackMemory() {
+ void* memory = stackMemory();
+ MOZ_ASSERT(isMoribund() == !memory);
+ if (memory) {
+ js_free(memory);
+ setStackMemory(nullptr);
setState(SuspenderState::Moribund);
}
}
# if defined(JS_SIMULATOR_ARM64)
-void SuspenderObjectData::switchSimulatorToMain() {
+void SuspenderObject::switchSimulatorToMain() {
auto* sim = Simulator::Current();
suspendableSP_ = (void*)sim->xreg(Registers::sp, vixl::Reg31IsStackPointer);
suspendableFP_ = (void*)sim->xreg(Registers::fp);
@@ -84,7 +78,7 @@ void SuspenderObjectData::switchSimulatorToMain() {
sim->set_xreg(Registers::fp, (int64_t)mainFP_);
}
-void SuspenderObjectData::switchSimulatorToSuspendable() {
+void SuspenderObject::switchSimulatorToSuspendable() {
auto* sim = Simulator::Current();
mainSP_ = (void*)sim->xreg(Registers::sp, vixl::Reg31IsStackPointer);
mainFP_ = (void*)sim->xreg(Registers::fp);
@@ -94,14 +88,14 @@ void SuspenderObjectData::switchSimulatorToSuspendable() {
}
# elif defined(JS_SIMULATOR_ARM)
-void SuspenderObjectData::switchSimulatorToMain() {
+void SuspenderObject::switchSimulatorToMain() {
suspendableSP_ = (void*)Simulator::Current()->get_register(Simulator::sp);
suspendableFP_ = (void*)Simulator::Current()->get_register(Simulator::fp);
Simulator::Current()->set_register(Simulator::sp, (int)mainSP_);
Simulator::Current()->set_register(Simulator::fp, (int)mainFP_);
}
-void SuspenderObjectData::switchSimulatorToSuspendable() {
+void SuspenderObject::switchSimulatorToSuspendable() {
mainSP_ = (void*)Simulator::Current()->get_register(Simulator::sp);
mainFP_ = (void*)Simulator::Current()->get_register(Simulator::fp);
Simulator::Current()->set_register(Simulator::sp, (int)suspendableSP_);
@@ -110,7 +104,7 @@ void SuspenderObjectData::switchSimulatorToSuspendable() {
# elif defined(JS_SIMULATOR_RISCV64) || defined(JS_SIMULATOR_LOONG64) || \
defined(JS_SIMULATOR_MIPS64)
-void SuspenderObjectData::switchSimulatorToMain() {
+void SuspenderObject::switchSimulatorToMain() {
suspendableSP_ = (void*)Simulator::Current()->getRegister(Simulator::sp);
suspendableFP_ = (void*)Simulator::Current()->getRegister(Simulator::fp);
Simulator::Current()->setRegister(
@@ -121,7 +115,7 @@ void SuspenderObjectData::switchSimulatorToMain() {
static_cast<int64_t>(reinterpret_cast<uintptr_t>(mainFP_)));
}
-void SuspenderObjectData::switchSimulatorToSuspendable() {
+void SuspenderObject::switchSimulatorToSuspendable() {
mainSP_ = (void*)Simulator::Current()->getRegister(Simulator::sp);
mainFP_ = (void*)Simulator::Current()->getRegister(Simulator::fp);
Simulator::Current()->setRegister(
@@ -140,7 +134,7 @@ const size_t CONTINUE_ON_SUSPENDABLE_SLOT = 1;
const size_t PROMISE_SLOT = 2;
static JitActivation* FindSuspendableStackActivation(
- JSTracer* trc, const SuspenderObjectData& data) {
+ JSTracer* trc, SuspenderObject* suspender) {
// The jitActivation.refNoCheck() can be used since during trace/marking
// the main thread will be paused.
JitActivation* activation =
@@ -152,7 +146,7 @@ static JitActivation* FindSuspendableStackActivation(
// Scan all JitActivations to find one that starts with suspended stack
// frame pointer.
WasmFrameIter iter(activation);
- if (!iter.done() && data.hasFramePointer(iter.frame())) {
+ if (!iter.done() && suspender->hasFramePointer(iter.frame())) {
return activation;
}
}
@@ -161,22 +155,23 @@ static JitActivation* FindSuspendableStackActivation(
MOZ_CRASH("Suspendable stack activation not found");
}
-void TraceSuspendableStack(JSTracer* trc, const SuspenderObjectData& data) {
- MOZ_ASSERT(data.isTraceable());
- void* exitFP = data.suspendableExitFP();
+void TraceSuspendableStack(JSTracer* trc, SuspenderObject* suspender) {
+ MOZ_ASSERT(suspender->isTraceable());
+ void* exitFP = suspender->suspendableExitFP();
// Create and iterator for wasm frames:
- // - If a stack entry for suspended stack exists, the data.suspendableFP()
- // and data.suspendedReturnAddress() provide start of the frames.
+ // - If a stack entry for suspended stack exists, the
+ // suspender->suspendableFP()
+ // and suspender->suspendedReturnAddress() provide start of the frames.
// - Otherwise, the stack is the part of the main stack, the context
// JitActivation frames will be used to trace.
WasmFrameIter iter =
- data.isSuspended()
+ suspender->isSuspended()
? WasmFrameIter(
- static_cast<FrameWithInstances*>(data.suspendableFP()),
- data.suspendedReturnAddress())
- : WasmFrameIter(FindSuspendableStackActivation(trc, data));
- MOZ_ASSERT_IF(data.isSuspended(), iter.currentFrameStackSwitched());
+ static_cast<FrameWithInstances*>(suspender->suspendableFP()),
+ suspender->suspendedReturnAddress())
+ : WasmFrameIter(FindSuspendableStackActivation(trc, suspender));
+ MOZ_ASSERT_IF(suspender->isSuspended(), iter.currentFrameStackSwitched());
uintptr_t highestByteVisitedInPrevWasmFrame = 0;
while (true) {
MOZ_ASSERT(!iter.done());
@@ -211,6 +206,20 @@ SuspenderObject* SuspenderObject::create(JSContext* cx) {
return nullptr;
}
+ // Initialize all of the slots
+ suspender->initFixedSlot(StateSlot, Int32Value(SuspenderState::Moribund));
+ suspender->initFixedSlot(PromisingPromiseSlot, NullValue());
+ suspender->initFixedSlot(SuspendingReturnTypeSlot,
+ Int32Value(int32_t(ReturnType::Unknown)));
+ suspender->initFixedSlot(StackMemorySlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(MainFPSlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(MainSPSlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(SuspendableFPSlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(SuspendableSPSlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(SuspendableExitFPSlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(SuspendedRASlot, PrivateValue(nullptr));
+ suspender->initFixedSlot(MainExitFPSlot, PrivateValue(nullptr));
+
void* stackMemory = js_malloc(SuspendableStackPlusRedZoneSize);
if (!stackMemory) {
ReportOutOfMemory(cx);
@@ -223,36 +232,16 @@ SuspenderObject* SuspenderObject::create(JSContext* cx) {
return nullptr;
}
- SuspenderObjectData* data = js_new<SuspenderObjectData>(stackMemory);
- if (!data) {
- cx->wasm().suspenders_.remove(suspender);
- js_free(stackMemory);
- ReportOutOfMemory(cx);
- return nullptr;
- }
- MOZ_RELEASE_ASSERT(data->state() != SuspenderState::Moribund);
+ // We are now fully constructed and can transition states
+ suspender->setStackMemory(stackMemory);
+ suspender->setFixedSlot(SuspendableSPSlot,
+ PrivateValue(static_cast<uint8_t*>(stackMemory) +
+ SuspendableStackPlusRedZoneSize));
+ suspender->setState(SuspenderState::Initial);
- suspender->initReservedSlot(DataSlot, PrivateValue(data));
- suspender->initReservedSlot(PromisingPromiseSlot, NullValue());
- suspender->initReservedSlot(SuspendingReturnTypeSlot,
- Int32Value(int32_t(ReturnType::Unknown)));
return suspender;
}
-JS::NativeStackLimit SuspenderObject::stackMemoryBase() const {
- return ((uintptr_t)data()->stackMemory()) + SuspendableStackPlusRedZoneSize;
-}
-
-JS::NativeStackLimit SuspenderObject::stackMemoryLimitForSystem() const {
- return JS::NativeStackLimit(data()->stackMemory());
-}
-
-JS::NativeStackLimit SuspenderObject::stackMemoryLimitForJit() const {
- return stackMemoryLimitForSystem() + SuspendableRedZoneSize;
-}
-
-static_assert(SuspenderObjectDataSlot == SuspenderObject::DataSlot);
-
const JSClass SuspenderObject::class_ = {
"SuspenderObject",
JSCLASS_HAS_RESERVED_SLOTS(SlotCount) | JSCLASS_FOREGROUND_FINALIZE,
@@ -281,33 +270,24 @@ const ClassExtension SuspenderObject::classExt_ = {
/* static */
void SuspenderObject::finalize(JS::GCContext* gcx, JSObject* obj) {
SuspenderObject& suspender = obj->as<SuspenderObject>();
- if (!suspender.hasData()) {
- return;
- }
- SuspenderObjectData* data = suspender.data();
- if (!data->isMoribund()) {
+ if (!suspender.isMoribund()) {
gcx->runtime()->mainContextFromOwnThread()->wasm().suspenders_.remove(
&suspender);
}
- data->releaseStackMemory();
- MOZ_ASSERT(data->isMoribund());
- js_free(data);
+ suspender.releaseStackMemory();
+ MOZ_ASSERT(suspender.isMoribund());
}
/* static */
void SuspenderObject::trace(JSTracer* trc, JSObject* obj) {
SuspenderObject& suspender = obj->as<SuspenderObject>();
- if (!suspender.hasData()) {
- return;
- }
- SuspenderObjectData& data = *suspender.data();
- // The SuspenderObjectData refers stacks frames that need to be traced
+ // The SuspenderObject refers stacks frames that need to be traced
// only during major GC to determine if SuspenderObject content is
// reachable from JS.
- if (!data.isTraceable() || trc->isTenuringTracer()) {
+ if (!suspender.isTraceable() || trc->isTenuringTracer()) {
return;
}
- TraceSuspendableStack(trc, data);
+ TraceSuspendableStack(trc, &suspender);
}
/* static */
@@ -322,26 +302,25 @@ size_t SuspenderObject::moved(JSObject* obj, JSObject* old) {
void SuspenderObject::setMoribund(JSContext* cx) {
MOZ_ASSERT(state() == SuspenderState::Active);
cx->wasm().leaveSuspendableStack(cx);
- SuspenderObjectData* data = this->data();
- if (!data->isMoribund()) {
+ if (!this->isMoribund()) {
cx->wasm().suspenders_.remove(this);
}
- data->releaseStackMemory();
- MOZ_ASSERT(data->isMoribund());
+ this->releaseStackMemory();
+ MOZ_ASSERT(this->isMoribund());
}
void SuspenderObject::setActive(JSContext* cx) {
- data()->setState(SuspenderState::Active);
+ this->setState(SuspenderState::Active);
cx->wasm().enterSuspendableStack(cx, this);
}
void SuspenderObject::setSuspended(JSContext* cx) {
- data()->setState(SuspenderState::Suspended);
+ this->setState(SuspenderState::Suspended);
cx->wasm().leaveSuspendableStack(cx);
}
void SuspenderObject::setCalledOnMain(JSContext* cx) {
- data()->setState(SuspenderState::CalledOnMain);
+ this->setState(SuspenderState::CalledOnMain);
cx->wasm().leaveSuspendableStack(cx);
}
@@ -413,17 +392,15 @@ void SuspenderObject::leave(JSContext* cx) {
void SuspenderObject::forwardToSuspendable() {
// Injecting suspendable stack back into main one at the exit frame.
- SuspenderObjectData* data = this->data();
- uint8_t* mainExitFP = (uint8_t*)data->mainExitFP();
+ uint8_t* mainExitFP = (uint8_t*)this->mainExitFP();
*reinterpret_cast<void**>(mainExitFP + Frame::callerFPOffset()) =
- data->suspendableFP();
+ this->suspendableFP();
*reinterpret_cast<void**>(mainExitFP + Frame::returnAddressOffset()) =
- data->suspendedReturnAddress();
+ this->suspendedReturnAddress();
}
bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
Rooted<SuspenderObject*> suspender(cx, cx->wasm().activeSuspender());
- SuspenderObjectData* stacks = suspender->data();
MOZ_ASSERT(suspender->state() == SuspenderState::Active);
suspender->setCalledOnMain(cx);
@@ -434,24 +411,23 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
defined(JS_SIMULATOR_MIPS64)
// The simulator is using its own stack, however switching is needed for
// virtual registers.
- stacks->switchSimulatorToMain();
+ suspender->switchSimulatorToMain();
bool res = fn(data);
- stacks->switchSimulatorToSuspendable();
+ suspender->switchSimulatorToSuspendable();
# else
# error "not supported"
# endif
# else
// The platform specific code below inserts offsets as strings into inline
// assembly. CHECK_OFFSETS verifies the specified literals in macros below.
-# define CHECK_OFFSETS(MAIN_FP_OFFSET, MAIN_SP_OFFSET, \
- SUSPENDABLE_FP_OFFSET, SUSPENDABLE_SP_OFFSET) \
- static_assert( \
- (MAIN_FP_OFFSET) == SuspenderObjectData::offsetOfMainFP() && \
- (MAIN_SP_OFFSET) == SuspenderObjectData::offsetOfMainSP() && \
- (SUSPENDABLE_FP_OFFSET) == \
- SuspenderObjectData::offsetOfSuspendableFP() && \
- (SUSPENDABLE_SP_OFFSET) == \
- SuspenderObjectData::offsetOfSuspendableSP());
+# define CHECK_OFFSETS(MAIN_FP_OFFSET, MAIN_SP_OFFSET, \
+ SUSPENDABLE_FP_OFFSET, SUSPENDABLE_SP_OFFSET) \
+ static_assert((MAIN_FP_OFFSET) == SuspenderObject::offsetOfMainFP() && \
+ (MAIN_SP_OFFSET) == SuspenderObject::offsetOfMainSP() && \
+ (SUSPENDABLE_FP_OFFSET) == \
+ SuspenderObject::offsetOfSuspendableFP() && \
+ (SUSPENDABLE_SP_OFFSET) == \
+ SuspenderObject::offsetOfSuspendableSP());
// The following assembly code temporarily switches FP/SP pointers to be on
// main stack, while maintaining frames linking. After
@@ -466,6 +442,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
// `Registers::VolatileMask` (in the assembler complex) is useful in that it
// lists the caller-saved registers.
+ SuspenderObject* stacks = suspender.get();
uintptr_t res;
// clang-format off
@@ -509,7 +486,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: "x0", "x3", "x27", CALLER_SAVED_REGS, "cc", "memory")
- INLINED_ASM(8, 16, 24, 32);
+ INLINED_ASM(56, 64, 72, 80);
# elif defined(_WIN64) && defined(_M_X64)
# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
@@ -540,7 +517,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: "rcx", "rax", "cc", "memory")
- INLINED_ASM(8, 16, 24, 32);
+ INLINED_ASM(56, 64, 72, 80);
# elif defined(__x86_64__)
# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
@@ -571,7 +548,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: "rdi", "rax", "cc", "memory")
- INLINED_ASM(8, 16, 24, 32);
+ INLINED_ASM(56, 64, 72, 80);
# elif defined(__i386__) || defined(_M_IX86)
# define CALLER_SAVED_REGS "eax", "ecx", "edx"
# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
@@ -600,7 +577,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: CALLER_SAVED_REGS, "cc", "memory")
- INLINED_ASM(4, 8, 12, 16);
+ INLINED_ASM(48, 56, 64, 72);
# elif defined(__arm__)
# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
@@ -632,7 +609,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: "r0", "r1", "r2", "r3", "cc", "memory")
- INLINED_ASM(4, 8, 12, 16);
+ INLINED_ASM(48, 56, 64, 72);
#elif defined(__loongarch_lp64)
# define CALLER_SAVED_REGS \
@@ -669,7 +646,7 @@ bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: "$a0", "$a3", CALLER_SAVED_REGS, "cc", "memory")
-INLINED_ASM(8, 16, 24, 32);
+INLINED_ASM(56, 64, 72, 80);
# elif defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64)
# define CALLER_SAVED_REGS \
@@ -709,7 +686,7 @@ INLINED_ASM(8, 16, 24, 32);
: "=r"(res) \
: "r"(stacks), "r"(fn), "r"(data) \
: "ra", "a0", "a3", CALLER_SAVED_REGS, "cc", "memory")
-INLINED_ASM(8, 16, 24, 32);
+INLINED_ASM(56, 64, 72, 80);
# elif defined(__mips64)
# define CALLER_SAVED_REGS \
diff --git a/js/src/wasm/WasmPI.h b/js/src/wasm/WasmPI.h
@@ -112,14 +112,12 @@ class Context;
#ifdef ENABLE_WASM_JSPI
-static const uint32_t SuspenderObjectDataSlot = 0;
-
-enum SuspenderState {
- // The suspender's stack hasn't been entered yet.
- Initial,
- // The suspender's stack has returned from the root frame and has been
+enum SuspenderState : int32_t {
+ // The suspender's has no stack or the stack has finished and has been
// destroyed.
Moribund,
+ // The suspender's stack hasn't been entered yet.
+ Initial,
// The suspender's stack is active and is the currently active stack.
Active,
// The suspender's stack has been suspended and is no longer the active stack
@@ -130,179 +128,166 @@ enum SuspenderState {
CalledOnMain,
};
-class SuspenderObjectData {
- void* stackMemory_;
-
- // Stored main stack FP register.
- void* mainFP_;
+class SuspenderObject : public NativeObject {
+ public:
+ static const JSClass class_;
- // Stored main stack SP register.
- void* mainSP_;
+ enum {
+ StateSlot,
- // Stored suspendable stack FP register.
- void* suspendableFP_;
+ PromisingPromiseSlot,
+ SuspendingReturnTypeSlot,
- // Stored suspendable stack SP register.
- void* suspendableSP_;
+ StackMemorySlot,
+ MainFPSlot,
+ MainSPSlot,
+ SuspendableFPSlot,
+ SuspendableSPSlot,
+ SuspendableExitFPSlot,
+ SuspendedRASlot,
+ MainExitFPSlot,
- // Stored suspendable stack exit/bottom frame pointer.
- void* suspendableExitFP_;
+ SlotCount,
+ };
- // Stored return address for return to suspendable stack.
- void* suspendedReturnAddress_;
-
- // Stored main stack exit/top frame pointer.
- void* mainExitFP_;
-
- SuspenderState state_;
+ enum class ReturnType : int32_t { Unknown, Promise, Exception };
- public:
- explicit SuspenderObjectData(void* stackMemory);
+ static SuspenderObject* create(JSContext* cx);
- inline SuspenderState state() const { return state_; }
- void setState(SuspenderState state) { state_ = state; }
+ SuspenderState state() const {
+ return (SuspenderState)getFixedSlot(StateSlot).toInt32();
+ }
+ void setState(SuspenderState state) {
+ setFixedSlot(StateSlot, JS::Int32Value((int32_t)state));
+ }
// This suspender can be traced if it's not 'Initial' or 'Moribund'.
bool isTraceable() const {
- return state_ == SuspenderState::Active ||
- state_ == SuspenderState::Suspended ||
- state_ == SuspenderState::CalledOnMain;
+ SuspenderState current = state();
+ return current == SuspenderState::Active ||
+ current == SuspenderState::Suspended ||
+ current == SuspenderState::CalledOnMain;
+ }
+ bool isMoribund() const { return state() == SuspenderState::Moribund; }
+ bool isActive() const { return state() == SuspenderState::Active; }
+ bool isSuspended() const { return state() == SuspenderState::Suspended; }
+ bool isCalledOnMain() const {
+ return state() == SuspenderState::CalledOnMain;
}
- bool isMoribund() const { return state_ == SuspenderState::Moribund; }
- bool isActive() const { return state_ == SuspenderState::Active; }
- bool isSuspended() const { return state_ == SuspenderState::Suspended; }
- bool isCalledOnMain() const { return state_ == SuspenderState::CalledOnMain; }
- bool hasFramePointer(void* fp) const {
- MOZ_ASSERT(!isMoribund());
- return (uintptr_t)stackMemory_ <= (uintptr_t)fp &&
- (uintptr_t)fp <
- (uintptr_t)stackMemory_ + SuspendableStackPlusRedZoneSize;
+ PromiseObject* promisingPromise() const {
+ return &getFixedSlot(PromisingPromiseSlot).toObject().as<PromiseObject>();
+ }
+ void setPromisingPromise(Handle<PromiseObject*> promise) {
+ setFixedSlot(PromisingPromiseSlot, ObjectOrNullValue(promise));
}
+ ReturnType suspendingReturnType() const {
+ return ReturnType(getFixedSlot(SuspendingReturnTypeSlot).toInt32());
+ }
+ void setSuspendingReturnType(ReturnType type) {
+ // The SuspendingReturnTypeSlot will change after result is defined,
+ // and becomes invalid after GetSuspendingPromiseResult. The assert is
+ // checking if the result was processed by GetSuspendingPromiseResult.
+ MOZ_ASSERT((type == ReturnType::Unknown) !=
+ (suspendingReturnType() == ReturnType::Unknown));
+
+ setFixedSlot(SuspendingReturnTypeSlot, Int32Value(int32_t(type)));
+ }
+
+ // Pointer to the beginning of the stack memory allocation.
void* stackMemory() const {
+ return getFixedSlot(StackMemorySlot).toPrivate();
+ }
+ void setStackMemory(void* stackMemory) {
+ setFixedSlot(StackMemorySlot, PrivateValue(stackMemory));
+ }
+
+ // The logical beginning or bottom of the stack, which is the physically
+ // highest memory address in the stack allocation.
+ JS::NativeStackLimit stackMemoryBase() const {
+ return ((uintptr_t)stackMemory()) + SuspendableStackPlusRedZoneSize;
+ }
+ // The logical end or top of the stack for system code, which is the
+ // physically lowest memory address in the stack allocation. This does not
+ // include any 'red zone' space, and so it is not safe to use if a stub
+ // or OS interrupt handler could run on the stack. Use
+ // `stackMemoryLimitForJit` instead.
+ JS::NativeStackLimit stackMemoryLimitForSystem() const {
+ return JS::NativeStackLimit(stackMemory());
+ }
+ // The logical end or top of the stack for JIT code, which is the
+ // physically lowest memory address in the stack allocation. This does
+ // include 'red zone' space for running stubs or OS interrupt handlers.
+ JS::NativeStackLimit stackMemoryLimitForJit() const {
+ return stackMemoryLimitForSystem() + SuspendableRedZoneSize;
+ }
+
+ bool hasFramePointer(void* fp) const {
MOZ_ASSERT(!isMoribund());
- return stackMemory_;
+ void* base = stackMemory();
+ return (uintptr_t)base <= (uintptr_t)fp &&
+ (uintptr_t)fp < (uintptr_t)base + SuspendableStackPlusRedZoneSize;
}
+ // Stored main stack FP register.
void* mainFP() const {
MOZ_ASSERT(isActive());
- return mainFP_;
+ return getFixedSlot(MainFPSlot).toPrivate();
}
+ // Stored main stack SP register.
void* mainSP() const {
MOZ_ASSERT(isActive());
- return mainSP_;
+ return getFixedSlot(MainSPSlot).toPrivate();
}
+ // Stored main stack exit/top frame pointer.
void* mainExitFP() const {
MOZ_ASSERT(isSuspended());
- return mainExitFP_;
+ return getFixedSlot(MainExitFPSlot).toPrivate();
}
+ // Stored suspendable stack FP register.
void* suspendableFP() const {
MOZ_ASSERT(isSuspended());
- return suspendableFP_;
+ return getFixedSlot(SuspendableFPSlot).toPrivate();
}
+ // Stored suspendable stack SP register.
void* suspendableSP() const {
MOZ_ASSERT(isSuspended());
- return suspendableSP_;
+ return getFixedSlot(SuspendableSPSlot).toPrivate();
}
+ // Stored return address for return to suspendable stack.
void* suspendedReturnAddress() const {
MOZ_ASSERT(isSuspended());
- return suspendedReturnAddress_;
+ return getFixedSlot(SuspendedRASlot).toPrivate();
}
+ // Stored suspendable stack exit/bottom frame pointer.
void* suspendableExitFP() const {
// We always have the root frame when we've been entered into, which is
// when we're traceable.
MOZ_ASSERT(isTraceable());
- return suspendableExitFP_;
+ return getFixedSlot(SuspendableExitFPSlot).toPrivate();
}
- void releaseStackMemory();
-
-# if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \
- defined(JS_SIMULATOR_RISCV64) || defined(JS_SIMULATOR_LOONG64) || \
- defined(JS_SIMULATOR_MIPS64)
- void switchSimulatorToMain();
- void switchSimulatorToSuspendable();
-# endif
-
static constexpr size_t offsetOfMainFP() {
- return offsetof(SuspenderObjectData, mainFP_);
+ return getFixedSlotOffset(MainFPSlot);
}
-
static constexpr size_t offsetOfMainSP() {
- return offsetof(SuspenderObjectData, mainSP_);
+ return getFixedSlotOffset(MainSPSlot);
}
-
static constexpr size_t offsetOfSuspendableFP() {
- return offsetof(SuspenderObjectData, suspendableFP_);
+ return getFixedSlotOffset(SuspendableFPSlot);
}
-
static constexpr size_t offsetOfSuspendableSP() {
- return offsetof(SuspenderObjectData, suspendableSP_);
+ return getFixedSlotOffset(SuspendableSPSlot);
}
-
static constexpr size_t offsetOfSuspendableExitFP() {
- return offsetof(SuspenderObjectData, suspendableExitFP_);
+ return getFixedSlotOffset(SuspendableExitFPSlot);
}
-
static constexpr size_t offsetOfMainExitFP() {
- return offsetof(SuspenderObjectData, mainExitFP_);
+ return getFixedSlotOffset(MainExitFPSlot);
}
-
static constexpr size_t offsetOfSuspendedReturnAddress() {
- return offsetof(SuspenderObjectData, suspendedReturnAddress_);
- }
-};
-
-class SuspenderObject : public NativeObject {
- public:
- static const JSClass class_;
-
- enum { DataSlot, PromisingPromiseSlot, SuspendingReturnTypeSlot, SlotCount };
-
- enum class ReturnType : int32_t { Unknown, Promise, Exception };
-
- static SuspenderObject* create(JSContext* cx);
-
- PromiseObject* promisingPromise() const {
- return &getReservedSlot(PromisingPromiseSlot)
- .toObject()
- .as<PromiseObject>();
- }
-
- void setPromisingPromise(Handle<PromiseObject*> promise) {
- setReservedSlot(PromisingPromiseSlot, ObjectOrNullValue(promise));
- }
-
- ReturnType suspendingReturnType() const {
- return ReturnType(getReservedSlot(SuspendingReturnTypeSlot).toInt32());
- }
-
- void setSuspendingReturnType(ReturnType type) {
- // The SuspendingReturnTypeSlot will change after result is defined,
- // and becomes invalid after GetSuspendingPromiseResult. The assert is
- // checking if the result was processed by GetSuspendingPromiseResult.
- MOZ_ASSERT((type == ReturnType::Unknown) !=
- (suspendingReturnType() == ReturnType::Unknown));
-
- setReservedSlot(SuspendingReturnTypeSlot, Int32Value(int32_t(type)));
- }
-
- JS::NativeStackLimit stackMemoryBase() const;
- JS::NativeStackLimit stackMemoryLimitForSystem() const;
- JS::NativeStackLimit stackMemoryLimitForJit() const;
-
- SuspenderState state() { return data()->state(); }
-
- inline bool hasData() { return !getReservedSlot(DataSlot).isUndefined(); }
-
- inline SuspenderObjectData* data() {
- return static_cast<SuspenderObjectData*>(
- getReservedSlot(DataSlot).toPrivate());
- }
- inline const SuspenderObjectData* data() const {
- return static_cast<const SuspenderObjectData*>(
- getReservedSlot(DataSlot).toPrivate());
+ return getFixedSlotOffset(SuspendedRASlot);
}
void setMoribund(JSContext* cx);
@@ -315,6 +300,15 @@ class SuspenderObject : public NativeObject {
void resume(JSContext* cx);
void leave(JSContext* cx);
+ void releaseStackMemory();
+
+# if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \
+ defined(JS_SIMULATOR_RISCV64) || defined(JS_SIMULATOR_LOONG64) || \
+ defined(JS_SIMULATOR_MIPS64)
+ void switchSimulatorToMain();
+ void switchSimulatorToSuspendable();
+# endif
+
// Modifies frames to inject the suspendable stack back into the main one.
void forwardToSuspendable();
@@ -364,7 +358,7 @@ int32_t SetPromisingPromiseResults(Instance* instance,
void UpdateSuspenderState(Instance* instance, SuspenderObject* suspender,
UpdateSuspenderStateAction action);
-void TraceSuspendableStack(JSTracer* trc, const SuspenderObjectData& data);
+void TraceSuspendableStack(JSTracer* trc, SuspenderObject* suspender);
#endif // ENABLE_WASM_JSPI
