commit fea5740ed0c1b1a93c36219121a051482446a8e3
parent 9383bc69f7497336f6cdb4739ff80d3aa4b2d1e3
Author: Ryan Hunt <rhunt@eqrion.net>
Date: Thu, 18 Dec 2025 16:45:30 +0000
Bug 2002625 - wasm: Dynamically switch to main stack in traps. r=yury
Use the infrastructure added for builtin thunks to do this.
The trickiest part is that we need the InstanceReg in the trap
stub to know if we're on a suspendable stack or not, but don't
have one available. We fix this by using the trap handler to
find the correct Instance and stashing it on the most recent
wasm::Frame so that the trap stub can find it.
The unwinder code is updated to be aware of suspenders. Leaving
the last frame of a suspender will release it. Unwinding into the
middle of a suspender will resume it.
Remove all the CallOnMain code as it's no longer needed. All VM
code, like stack limit checking, now no longer needs to know
about wasm suspendable stacks.
Differential Revision: https://phabricator.services.mozilla.com/D274196
Diffstat:
15 files changed, 255 insertions(+), 554 deletions(-)
diff --git a/js/public/friend/StackLimits.h b/js/public/friend/StackLimits.h
@@ -201,20 +201,9 @@ MOZ_ALWAYS_INLINE bool AutoCheckRecursionLimit::checkLimitImpl(
#endif
}
-#ifdef ENABLE_WASM_JSPI
-bool IsSuspendableStackActive(JSContext* cx);
-JS::NativeStackLimit GetSuspendableStackLimit(JSContext* cx);
-#endif
-
MOZ_ALWAYS_INLINE JS::NativeStackLimit
AutoCheckRecursionLimit::getStackLimitSlow(JSContext* cx) const {
JS::StackKind kind = stackKindForCurrentPrincipal(cx);
-#ifdef ENABLE_WASM_JSPI
- if (IsSuspendableStackActive(cx)) {
- MOZ_RELEASE_ASSERT(kind == JS::StackForUntrustedScript);
- return GetSuspendableStackLimit(cx);
- }
-#endif
return getStackLimitHelper(cx, kind, 0);
}
diff --git a/js/src/vm/FrameIter.cpp b/js/src/vm/FrameIter.cpp
@@ -173,7 +173,13 @@ void JitFrameIter::settle() {
wasm::Frame* prevFP = (wasm::Frame*)jitFrame.prevFp();
if (mustUnwindActivation_) {
- act_->setWasmExitFP(prevFP);
+ // The JS-JIT exit frame doesn't support stack switching and will only be
+ // used if original wasm func was on the main stack.
+#ifdef ENABLE_WASM_JSPI
+ MOZ_ASSERT(!act_->cx()->wasm().findSuspenderForStackAddress(
+ prevFP->wasmCaller()));
+#endif
+ act_->setWasmExitFP(prevFP, nullptr);
}
iter_.destroy();
diff --git a/js/src/vm/JitActivation.cpp b/js/src/vm/JitActivation.cpp
@@ -246,7 +246,11 @@ void js::jit::JitActivation::startWasmTrap(wasm::Trap trap,
const wasm::Code& code = wasm::GetNearestEffectiveInstance(fp)->code();
MOZ_RELEASE_ASSERT(&code == wasm::LookupCode(pc));
- setWasmExitFP(fp);
+ wasm::SuspenderObject* suspender = nullptr;
+#ifdef ENABLE_WASM_JSPI
+ suspender = cx()->wasm().findSuspenderForStackAddress(fp);
+#endif
+ setWasmExitFP(fp, suspender);
wasmTrapData_.emplace();
wasmTrapData_->resumePC =
((uint8_t*)state.pc) + jit::WasmTrapInstructionLength;
@@ -269,9 +273,17 @@ void js::jit::JitActivation::startWasmTrap(wasm::Trap trap,
MOZ_ASSERT(isWasmTrapping());
}
-void js::jit::JitActivation::finishWasmTrap() {
+void js::jit::JitActivation::finishWasmTrap(bool isResuming) {
+ MOZ_ASSERT(hasWasmExitFP());
MOZ_ASSERT(isWasmTrapping());
- packedExitFP_ = nullptr;
wasmTrapData_.reset();
+ packedExitFP_ = nullptr;
+ // Don't clear the exit suspender if we're resuming, or else the trap stub
+ // won't know that it needs to switch back to the main stack.
+#ifdef ENABLE_WASM_JSPI
+ if (!isResuming) {
+ wasmExitSuspender_ = nullptr;
+ }
+#endif
MOZ_ASSERT(!isWasmTrapping());
}
diff --git a/js/src/vm/JitActivation.h b/js/src/vm/JitActivation.h
@@ -144,10 +144,14 @@ class JitActivation : public Activation {
MOZ_ASSERT(hasJSExitFP());
return packedExitFP_;
}
- void setJSExitFP(uint8_t* fp) { packedExitFP_ = fp; }
+ void setJSExitFP(uint8_t* fp) {
+ packedExitFP_ = fp;
+#ifdef ENABLE_WASM_JSPI
+ wasmExitSuspender_ = nullptr;
+#endif
+ }
uint8_t* packedExitFP() const { return packedExitFP_; }
- void setPackedExitFP(uint8_t* fp) { packedExitFP_ = fp; }
#ifdef CHECK_OSIPOINT_REGISTERS
void setCheckRegs(bool check) { checkRegs_ = check; }
@@ -224,13 +228,20 @@ class JitActivation : public Activation {
wasm::Instance* wasmExitInstance() const {
return wasm::GetNearestEffectiveInstance(wasmExitFP());
}
- void setWasmExitFP(const wasm::Frame* fp) {
+ void setWasmExitFP(const wasm::Frame* fp, wasm::SuspenderObject* suspender) {
if (fp) {
MOZ_ASSERT(!wasm::Frame::isExitFP(fp));
packedExitFP_ = wasm::Frame::addExitFPTag(fp);
+#ifdef ENABLE_WASM_JSPI
+ wasmExitSuspender_ = suspender;
+#endif
MOZ_ASSERT(hasWasmExitFP());
} else {
+ MOZ_ASSERT(!suspender);
packedExitFP_ = nullptr;
+#ifdef ENABLE_WASM_JSPI
+ wasmExitSuspender_ = nullptr;
+#endif
}
}
wasm::ExitReason wasmExitReason() const {
@@ -253,7 +264,7 @@ class JitActivation : public Activation {
void startWasmTrap(wasm::Trap trap, const wasm::TrapSite& trapSite,
const wasm::RegisterState& state);
- void finishWasmTrap();
+ void finishWasmTrap(bool isResuming);
bool isWasmTrapping() const { return !!wasmTrapData_; }
const wasm::TrapData& wasmTrapData() { return *wasmTrapData_; }
};
diff --git a/js/src/vm/Runtime.cpp b/js/src/vm/Runtime.cpp
@@ -415,19 +415,7 @@ static bool HandleInterrupt(JSContext* cx, bool invokeCallback) {
return true;
}
- bool stop;
-#ifdef ENABLE_WASM_JSPI
- if (IsSuspendableStackActive(cx)) {
- stop = wasm::CallOnMainStack(
- cx, reinterpret_cast<wasm::CallOnMainStackFn>(InvokeInterruptCallbacks),
- (void*)cx);
- } else
-#endif
- {
- stop = InvokeInterruptCallbacks(cx);
- }
-
- if (!stop) {
+ if (!InvokeInterruptCallbacks(cx)) {
// Debugger treats invoking the interrupt callback as a "step", so
// invoke the onStep handler.
if (cx->realm()->isDebuggee()) {
diff --git a/js/src/wasm/WasmBuiltins.cpp b/js/src/wasm/WasmBuiltins.cpp
@@ -527,25 +527,6 @@ static JitActivation* CallingActivation(JSContext* cx) {
return act->asJit();
}
-template <typename Fn, typename... Ts>
-static bool ForwardToMainStack(Fn fn, JSContext* cx, Ts... args) {
-#ifdef ENABLE_WASM_JSPI
- if (IsSuspendableStackActive(cx)) {
- struct InvokeContext {
- bool (*fn)(JSContext*, Ts...);
- JSContext* cx;
- std::tuple<Ts...> args;
- static bool Run(InvokeContext* data) {
- return data->fn(data->cx, std::get<Ts>(data->args)...);
- }
- } data = {fn, cx, std::make_tuple(args...)};
- return CallOnMainStack(
- cx, reinterpret_cast<CallOnMainStackFn>(InvokeContext::Run), &data);
- }
-#endif
- return fn(cx, args...);
-}
-
static bool WasmHandleDebugTrap() {
JSContext* cx = TlsContext.get(); // Cold code
JitActivation* activation = CallingActivation(cx);
@@ -797,6 +778,12 @@ void wasm::HandleExceptionWasm(JSContext* cx, JitFrameIter& iter,
MOZ_ASSERT(cx->activation()->asJit()->hasWasmExitFP());
MOZ_ASSERT(rfe->kind == ExceptionResumeKind::EntryFrame);
+#ifdef ENABLE_WASM_JSPI
+ // This should always run on the main stack. The throw stub should perform
+ // a stack switch if that's not the case.
+ MOZ_ASSERT(!cx->wasm().onSuspendableStack());
+#endif
+
// WasmFrameIter iterates down wasm frames in the activation starting at
// JitActivation::wasmExitFP(). Calling WasmFrameIter::startUnwinding pops
// JitActivation::wasmExitFP() once each time WasmFrameIter is incremented,
@@ -859,11 +846,22 @@ void wasm::HandleExceptionWasm(JSContext* cx, JitFrameIter& iter,
(uint8_t*)(rfe->framePointer - tryNote->landingPadFramePushed());
rfe->target = codeBlock->base() + tryNote->landingPadEntryPoint();
- // Make sure to clear trapping state if we got here due to a trap.
+#ifdef ENABLE_WASM_JSPI
+ wasm::SuspenderObject* destSuspender = activation->wasmExitSuspender();
+ if (destSuspender) {
+ destSuspender->enter(cx);
+ }
+#endif
+
+ // Maintain the invariant that trapping and exit frame state is always
+ // clear when we return back into wasm JIT code.
if (activation->isWasmTrapping()) {
- activation->finishWasmTrap();
+ // This will clear the exit fp and suspender state.
+ activation->finishWasmTrap(/*isResuming=*/false);
+ } else {
+ // We need to manually clear the exit fp and suspender state.
+ activation->setWasmExitFP(nullptr, nullptr);
}
- activation->setWasmExitFP(nullptr);
return;
}
}
@@ -878,8 +876,7 @@ void wasm::HandleExceptionWasm(JSContext* cx, JitFrameIter& iter,
// Assume ResumeMode::Terminate if no exception is pending --
// no onExceptionUnwind handlers must be fired.
if (cx->isExceptionPending()) {
- if (!ForwardToMainStack(DebugAPI::onExceptionUnwind, cx,
- AbstractFramePtr(frame))) {
+ if (!DebugAPI::onExceptionUnwind(cx, AbstractFramePtr(frame))) {
if (cx->isPropagatingForcedReturn()) {
cx->clearPropagatingForcedReturn();
// Unexpected trap return -- raising error since throw recovery
@@ -892,9 +889,8 @@ void wasm::HandleExceptionWasm(JSContext* cx, JitFrameIter& iter,
}
}
- bool ok =
- ForwardToMainStack(DebugAPI::onLeaveFrame, cx, AbstractFramePtr(frame),
- (const jsbytecode*)nullptr, false);
+ bool ok = DebugAPI::onLeaveFrame(cx, AbstractFramePtr(frame),
+ (const jsbytecode*)nullptr, false);
if (ok) {
// Unexpected success from the handler onLeaveFrame -- raising error
// since throw recovery is not yet implemented in the wasm baseline.
@@ -920,10 +916,13 @@ void wasm::HandleExceptionWasm(JSContext* cx, JitFrameIter& iter,
}
static void* WasmHandleThrow(jit::ResumeFromException* rfe) {
- jit::HandleException(rfe);
// Return a pointer to the exception handler trampoline code to jump to from
// the throw stub.
JSContext* cx = TlsContext.get();
+#ifdef ENABLE_WASM_JSPI
+ MOZ_ASSERT(!cx->wasm().onSuspendableStack());
+#endif
+ jit::HandleException(rfe);
return cx->runtime()->jitRuntime()->getExceptionTailReturnValueCheck().value;
}
@@ -936,7 +935,9 @@ static void* CheckInterrupt(JSContext* cx, JitActivation* activation) {
}
void* resumePC = activation->wasmTrapData().resumePC;
- activation->finishWasmTrap();
+ activation->finishWasmTrap(/*isResuming=*/true);
+ // Do not reset the exit frame pointer and suspender, or else we won't switch
+ // back to the main stack.
return resumePC;
}
@@ -949,6 +950,9 @@ static void* CheckInterrupt(JSContext* cx, JitActivation* activation) {
static void* WasmHandleTrap() {
JSContext* cx = TlsContext.get(); // Cold code
JitActivation* activation = CallingActivation(cx);
+#ifdef ENABLE_WASM_JSPI
+ MOZ_ASSERT(!cx->wasm().onSuspendableStack());
+#endif
switch (activation->wasmTrapData().trap) {
case Trap::Unreachable: {
diff --git a/js/src/wasm/WasmContext.cpp b/js/src/wasm/WasmContext.cpp
@@ -71,6 +71,19 @@ void Context::initStackLimit(JSContext* cx) {
}
#ifdef ENABLE_WASM_JSPI
+SuspenderObject* Context::findSuspenderForStackAddress(
+ const void* stackAddress) {
+ // TODO: add a fast path for the main stack that avoids linear search. We
+ // need an accurate main stack base/limit for that.
+ for (auto iter = suspenders_.iter(); !iter.done(); iter.next()) {
+ SuspenderObject* object = iter.get();
+ if (object->isActive() && object->hasStackAddress(stackAddress)) {
+ return object;
+ }
+ }
+ return nullptr;
+}
+
void Context::trace(JSTracer* trc) {
if (activeSuspender_) {
TraceEdge(trc, &activeSuspender_, "suspender");
@@ -128,13 +141,4 @@ void Context::leaveSuspendableStack(JSContext* cx) {
cx->runtime()->jitRuntime()->clearDisallowArbitraryCode();
# endif
}
-
-bool js::IsSuspendableStackActive(JSContext* cx) {
- return cx->wasm().onSuspendableStack();
-}
-
-JS::NativeStackLimit js::GetSuspendableStackLimit(JSContext* cx) {
- MOZ_ASSERT(IsSuspendableStackActive(cx));
- return cx->wasm().stackLimit;
-}
#endif
diff --git a/js/src/wasm/WasmContext.h b/js/src/wasm/WasmContext.h
@@ -75,6 +75,8 @@ class Context {
void enterSuspendableStack(JSContext* cx, SuspenderObject* suspender);
void leaveSuspendableStack(JSContext* cx);
+ SuspenderObject* findSuspenderForStackAddress(const void* stackAddress);
+
void trace(JSTracer* trc);
void traceRoots(JSTracer* trc);
#endif
diff --git a/js/src/wasm/WasmFrame.h b/js/src/wasm/WasmFrame.h
@@ -369,6 +369,10 @@ class FrameWithInstances
Instance* calleeInstance() { return calleeInstance_; }
Instance* callerInstance() { return callerInstance_; }
+ Instance* setCalleeInstance(Instance* instance) {
+ return calleeInstance_ = instance;
+ }
+
constexpr static uint32_t sizeOfInstanceFields() {
return sizeof(wasm::FrameWithInstances) - sizeof(wasm::Frame) -
js::jit::ShadowStackSpace;
diff --git a/js/src/wasm/WasmFrameIter.cpp b/js/src/wasm/WasmFrameIter.cpp
@@ -132,7 +132,7 @@ WasmFrameIter::WasmFrameIter(JitActivation* activation, wasm::Frame* fp)
// indicated by the returnAddress of the exit stub's frame. If the caller
// was Ion, we can just skip the wasm frames.
- popFrame();
+ popFrame(/*isLeavingFrame=*/false);
MOZ_ASSERT(!done() || unwoundCallerFP_);
}
@@ -172,25 +172,7 @@ bool WasmFrameIter::done() const {
void WasmFrameIter::operator++() {
MOZ_ASSERT(!done());
-
- // When the iterator is set to unwind, each time the iterator pops a frame,
- // the JitActivation is updated so that the just-popped frame is no longer
- // visible. This is necessary since Debugger::onLeaveFrame is called before
- // popping each frame and, once onLeaveFrame is called for a given frame,
- // that frame must not be visible to subsequent stack iteration (or it
- // could be added as a "new" frame just as it becomes garbage). When the
- // frame is trapping, then exitFP is included in the callstack (otherwise,
- // it is skipped, as explained above). So to unwind the innermost frame, we
- // just clear the trapping state.
-
- if (isLeavingFrames_) {
- if (activation_->isWasmTrapping()) {
- activation_->finishWasmTrap();
- }
- activation_->setWasmExitFP(fp_);
- }
-
- popFrame();
+ popFrame(/*isLeavingFrame=*/isLeavingFrames_);
}
static inline void AssertJitExitFrame(const void* fp,
@@ -208,10 +190,12 @@ static inline void AssertDirectJitCall(const void* fp) {
AssertJitExitFrame(fp, jit::ExitFrameType::DirectWasmJitCall);
}
-void WasmFrameIter::popFrame() {
+void WasmFrameIter::popFrame(bool isLeavingFrame) {
// If we're visiting inlined frames, see if this frame was inlined.
if (enableInlinedFrames_ && inlinedCallerOffsets_.size() > 0) {
- // We do not support inlining and debugging
+ // We do not support inlining and debugging. If we did we'd need to support
+ // `isLeavingFrame` here somehow to remove inlined frames from the
+ // JitActivation.
MOZ_ASSERT(!code_->debugEnabled());
// The inlined callee offsets are ordered so that our immediate caller is
@@ -244,6 +228,22 @@ void WasmFrameIter::popFrame() {
currentFrameStackSwitched_ = false;
#endif
+ // Track the current suspender if we are leaving frames.
+#ifdef ENABLE_WASM_JSPI
+ wasm::SuspenderObject* currentSuspender = nullptr;
+#endif
+ if (isLeavingFrame) {
+ MOZ_ASSERT(activation_->hasWasmExitFP());
+#ifdef ENABLE_WASM_JSPI
+ currentSuspender = activation_->wasmExitSuspender();
+#endif
+
+ // If we are trapping and leaving frames, then remove the trapping state.
+ if (activation_->isWasmTrapping()) {
+ activation_->finishWasmTrap(/*isResuming=*/false);
+ }
+ }
+
if (!code_) {
// This is a direct call from the jit into the wasm function's body. The
// call stack resembles this at this point:
@@ -264,7 +264,7 @@ void WasmFrameIter::popFrame() {
unwoundCallerFPIsJSJit_ = true;
unwoundAddressOfReturnAddress_ = fp_->addressOfReturnAddress();
- if (isLeavingFrames_) {
+ if (isLeavingFrame) {
activation_->setJSExitFP(unwoundCallerFP_);
}
@@ -297,10 +297,10 @@ void WasmFrameIter::popFrame() {
lineOrBytecode_ = UINT32_MAX;
inlinedCallerOffsets_ = BytecodeOffsetSpan();
- if (isLeavingFrames_) {
+ if (isLeavingFrame) {
// We're exiting via the interpreter entry; we can safely reset
// exitFP.
- activation_->setWasmExitFP(nullptr);
+ activation_->setWasmExitFP(nullptr, nullptr);
}
MOZ_ASSERT(done());
@@ -332,7 +332,7 @@ void WasmFrameIter::popFrame() {
lineOrBytecode_ = UINT32_MAX;
inlinedCallerOffsets_ = BytecodeOffsetSpan();
- if (isLeavingFrames_) {
+ if (isLeavingFrame) {
activation_->setJSExitFP(unwoundCallerFP());
}
@@ -361,6 +361,28 @@ void WasmFrameIter::popFrame() {
inlinedCallerOffsets_ = site.inlinedCallerOffsetsSpan();
failedUnwindSignatureMismatch_ = false;
+ if (isLeavingFrame) {
+#ifdef ENABLE_WASM_JSPI
+ wasm::SuspenderObject* newSuspender = currentSuspender;
+ // If we switched stacks, look up the new suspender using the new FP.
+ if (currentFrameStackSwitched_) {
+ newSuspender =
+ activation_->cx()->wasm().findSuspenderForStackAddress(fp_);
+ }
+
+ // If we are unwinding past a suspender, unwind it to release its
+ // resources.
+ if (newSuspender != currentSuspender) {
+ currentSuspender->unwind(activation_->cx());
+ }
+#else
+ wasm::SuspenderObject* newSuspender = nullptr;
+#endif
+ // Any future frame iteration will start by popping the exitFP, so setting
+ // it to `prevFP` ensures that frame iteration starts at our new `fp_`.
+ activation_->setWasmExitFP(prevFP, newSuspender);
+ }
+
MOZ_ASSERT(!done());
}
diff --git a/js/src/wasm/WasmFrameIter.h b/js/src/wasm/WasmFrameIter.h
@@ -100,7 +100,11 @@ class WasmFrameIter {
// Whether unwoundCallerFP_ is a JS JIT exit frame.
bool unwoundCallerFPIsJSJit_ = false;
- void popFrame();
+ // Pop the frame. `isLeavingFrame` indicates if we should update the
+ // JitActivation so that any other frame iteration doesn't see the frame we
+ // just popped. This is normally equal to `isLeavingFrames_`, but is
+ // different for the very first `popFrame` of a wasm exit frame.
+ void popFrame(bool isLeavingFrame);
public:
// See comment above this class definition.
diff --git a/js/src/wasm/WasmPI.cpp b/js/src/wasm/WasmPI.cpp
@@ -40,18 +40,6 @@
#include "wasm/WasmGcObject-inl.h"
#include "wasm/WasmInstance-inl.h"
-#if defined(JS_CODEGEN_ARM)
-# include "jit/arm/Simulator-arm.h"
-#elif defined(JS_CODEGEN_ARM64)
-# include "jit/arm64/vixl/Simulator-vixl.h"
-#elif defined(JS_CODEGEN_RISCV64)
-# include "jit/riscv64/Simulator-riscv64.h"
-#elif defined(JS_CODEGEN_LOONG64)
-# include "jit/loong64/Simulator-loong64.h"
-#elif defined(JS_CODEGEN_MIPS64)
-# include "jit/mips64/Simulator-mips64.h"
-#endif
-
using namespace js;
using namespace js::jit;
@@ -68,65 +56,6 @@ void SuspenderObject::releaseStackMemory() {
}
}
-# if defined(JS_SIMULATOR_ARM64)
-void SuspenderObject::switchSimulatorToMain() {
- auto* sim = Simulator::Current();
- suspendableSP_ = (void*)sim->xreg(Registers::sp, vixl::Reg31IsStackPointer);
- suspendableFP_ = (void*)sim->xreg(Registers::fp);
- sim->set_xreg(Registers::sp, (int64_t)mainSP_, vixl::Debugger::LogRegWrites,
- vixl::Reg31IsStackPointer);
- sim->set_xreg(Registers::fp, (int64_t)mainFP_);
-}
-
-void SuspenderObject::switchSimulatorToSuspendable() {
- auto* sim = Simulator::Current();
- mainSP_ = (void*)sim->xreg(Registers::sp, vixl::Reg31IsStackPointer);
- mainFP_ = (void*)sim->xreg(Registers::fp);
- sim->set_xreg(Registers::sp, (int64_t)suspendableSP_,
- vixl::Debugger::LogRegWrites, vixl::Reg31IsStackPointer);
- sim->set_xreg(Registers::fp, (int64_t)suspendableFP_);
-}
-
-# elif defined(JS_SIMULATOR_ARM)
-void SuspenderObject::switchSimulatorToMain() {
- suspendableSP_ = (void*)Simulator::Current()->get_register(Simulator::sp);
- suspendableFP_ = (void*)Simulator::Current()->get_register(Simulator::fp);
- Simulator::Current()->set_register(Simulator::sp, (int)mainSP_);
- Simulator::Current()->set_register(Simulator::fp, (int)mainFP_);
-}
-
-void SuspenderObject::switchSimulatorToSuspendable() {
- mainSP_ = (void*)Simulator::Current()->get_register(Simulator::sp);
- mainFP_ = (void*)Simulator::Current()->get_register(Simulator::fp);
- Simulator::Current()->set_register(Simulator::sp, (int)suspendableSP_);
- Simulator::Current()->set_register(Simulator::fp, (int)suspendableFP_);
-}
-
-# elif defined(JS_SIMULATOR_RISCV64) || defined(JS_SIMULATOR_LOONG64) || \
- defined(JS_SIMULATOR_MIPS64)
-void SuspenderObject::switchSimulatorToMain() {
- suspendableSP_ = (void*)Simulator::Current()->getRegister(Simulator::sp);
- suspendableFP_ = (void*)Simulator::Current()->getRegister(Simulator::fp);
- Simulator::Current()->setRegister(
- Simulator::sp,
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(mainSP_)));
- Simulator::Current()->setRegister(
- Simulator::fp,
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(mainFP_)));
-}
-
-void SuspenderObject::switchSimulatorToSuspendable() {
- mainSP_ = (void*)Simulator::Current()->getRegister(Simulator::sp);
- mainFP_ = (void*)Simulator::Current()->getRegister(Simulator::fp);
- Simulator::Current()->setRegister(
- Simulator::sp,
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(suspendableSP_)));
- Simulator::Current()->setRegister(
- Simulator::fp,
- static_cast<int64_t>(reinterpret_cast<uintptr_t>(suspendableFP_)));
-}
-# endif
-
// Slots that used in various JSFunctionExtended below.
const size_t SUSPENDER_SLOT = 0;
const size_t WRAPPED_FN_SLOT = 1;
@@ -146,7 +75,7 @@ static JitActivation* FindSuspendableStackActivation(
// Scan all JitActivations to find one that starts with suspended stack
// frame pointer.
WasmFrameIter iter(activation);
- if (!iter.done() && suspender->hasFramePointer(iter.frame())) {
+ if (!iter.done() && suspender->hasStackAddress(iter.frame())) {
return activation;
}
}
@@ -319,13 +248,12 @@ void SuspenderObject::setSuspended(JSContext* cx) {
cx->wasm().leaveSuspendableStack(cx);
}
-void SuspenderObject::setCalledOnMain(JSContext* cx) {
- this->setState(SuspenderState::CalledOnMain);
- cx->wasm().leaveSuspendableStack(cx);
-}
-
void SuspenderObject::enter(JSContext* cx) {
- MOZ_ASSERT(state() == SuspenderState::Initial);
+ // We can enter a suspender normally from Initial, or through unwinding when
+ // are in the 'CalledOnMain' or 'Suspended' states.
+ MOZ_ASSERT(state() == SuspenderState::Initial ||
+ state() == SuspenderState::CalledOnMain ||
+ state() == SuspenderState::Suspended);
setActive(cx);
}
@@ -390,6 +318,22 @@ void SuspenderObject::leave(JSContext* cx) {
}
}
+void SuspenderObject::unwind(JSContext* cx) {
+ switch (state()) {
+ case SuspenderState::Suspended:
+ case SuspenderState::CalledOnMain: {
+ cx->wasm().suspenders_.remove(this);
+ this->releaseStackMemory();
+ MOZ_ASSERT(this->isMoribund());
+ break;
+ }
+ case SuspenderState::Active:
+ case SuspenderState::Initial:
+ case SuspenderState::Moribund:
+ MOZ_CRASH();
+ }
+}
+
void SuspenderObject::forwardToSuspendable() {
// Injecting suspendable stack back into main one at the exit frame.
uint8_t* mainExitFP = (uint8_t*)this->mainExitFP();
@@ -399,357 +343,6 @@ void SuspenderObject::forwardToSuspendable() {
this->suspendedReturnAddress();
}
-bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data) {
- Rooted<SuspenderObject*> suspender(cx, cx->wasm().activeSuspender());
-
- MOZ_ASSERT(suspender->state() == SuspenderState::Active);
- suspender->setCalledOnMain(cx);
-
-# ifdef JS_SIMULATOR
-# if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \
- defined(JS_SIMULATOR_RISCV64) || defined(JS_SIMULATOR_LOONG64) || \
- defined(JS_SIMULATOR_MIPS64)
- // The simulator is using its own stack, however switching is needed for
- // virtual registers.
- suspender->switchSimulatorToMain();
- bool res = fn(data);
- suspender->switchSimulatorToSuspendable();
-# else
-# error "not supported"
-# endif
-# else
- // The platform specific code below inserts offsets as strings into inline
- // assembly. CHECK_OFFSETS verifies the specified literals in macros below.
-# define CHECK_OFFSETS(MAIN_FP_OFFSET, MAIN_SP_OFFSET, \
- SUSPENDABLE_FP_OFFSET, SUSPENDABLE_SP_OFFSET) \
- static_assert((MAIN_FP_OFFSET) == SuspenderObject::offsetOfMainFP() && \
- (MAIN_SP_OFFSET) == SuspenderObject::offsetOfMainSP() && \
- (SUSPENDABLE_FP_OFFSET) == \
- SuspenderObject::offsetOfSuspendableFP() && \
- (SUSPENDABLE_SP_OFFSET) == \
- SuspenderObject::offsetOfSuspendableSP());
-
- // The following assembly code temporarily switches FP/SP pointers to be on
- // main stack, while maintaining frames linking. After
- // `CallImportData::Call` execution suspendable stack FP/SP will be restored.
- //
- // Because the assembly sequences contain a call, the trashed-register list
- // must contain all the caller saved registers. They must also contain "cc"
- // and "memory" since both of those state elements could be modified by the
- // call. They also need a "volatile" qualifier to ensure that the they don't
- // get optimised out or otherwise messed with by clang/gcc.
- //
- // `Registers::VolatileMask` (in the assembler complex) is useful in that it
- // lists the caller-saved registers.
-
- SuspenderObject* stacks = suspender.get();
- uintptr_t res;
-
- // clang-format off
-#if defined(_M_ARM64) || defined(__aarch64__)
-# define CALLER_SAVED_REGS \
- "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9", "x10", \
- "x11", "x12", "x13", "x14", "x15", "x16", "x17", /* "x18", */ \
- "x19", "x20", /* it's unclear who saves these two, so be safe */ \
- /* claim that all the vector regs are caller-saved, for safety */ \
- "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", \
- "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", \
- "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", \
- "v29", "v30", "v31"
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm volatile( \
- "\n mov x0, %1" \
- "\n mov x27, sp " \
- "\n str x29, [x0, #" #SUSPENDABLE_FP "] " \
- "\n str x27, [x0, #" #SUSPENDABLE_SP "] " \
- \
- "\n ldr x29, [x0, #" #MAIN_FP "] " \
- "\n ldr x27, [x0, #" #MAIN_SP "] " \
- "\n mov sp, x27 " \
- \
- "\n stp x0, x27, [sp, #-16]! " \
- \
- "\n mov x0, %3" \
- "\n blr %2 " \
- \
- "\n ldp x3, x27, [sp], #16 " \
- \
- "\n mov x27, sp " \
- "\n str x29, [x3, #" #MAIN_FP "] " \
- "\n str x27, [x3, #" #MAIN_SP "] " \
- \
- "\n ldr x29, [x3, #" #SUSPENDABLE_FP "] " \
- "\n ldr x27, [x3, #" #SUSPENDABLE_SP "] " \
- "\n mov sp, x27 " \
- "\n mov %0, x0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : "x0", "x3", "x27", CALLER_SAVED_REGS, "cc", "memory")
- INLINED_ASM(56, 64, 72, 80);
-
-# elif defined(_WIN64) && defined(_M_X64)
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm("\n mov %1, %%rcx" \
- "\n mov %%rbp, " #SUSPENDABLE_FP "(%%rcx)" \
- "\n mov %%rsp, " #SUSPENDABLE_SP "(%%rcx)" \
- \
- "\n mov " #MAIN_FP "(%%rcx), %%rbp" \
- "\n mov " #MAIN_SP "(%%rcx), %%rsp" \
- \
- "\n push %%rcx" \
- "\n push %%rdx" \
- \
- "\n mov %3, %%rcx" \
- "\n call *%2" \
- \
- "\n pop %%rdx" \
- "\n pop %%rcx" \
- \
- "\n mov %%rbp, " #MAIN_FP "(%%rcx)" \
- "\n mov %%rsp, " #MAIN_SP "(%%rcx)" \
- \
- "\n mov " #SUSPENDABLE_FP "(%%rcx), %%rbp" \
- "\n mov " #SUSPENDABLE_SP "(%%rcx), %%rsp" \
- \
- "\n movq %%rax, %0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : "rcx", "rax", "cc", "memory")
- INLINED_ASM(56, 64, 72, 80);
-
-# elif defined(__x86_64__)
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm("\n mov %1, %%rdi" \
- "\n mov %%rbp, " #SUSPENDABLE_FP "(%%rdi)" \
- "\n mov %%rsp, " #SUSPENDABLE_SP "(%%rdi)" \
- \
- "\n mov " #MAIN_FP "(%%rdi), %%rbp" \
- "\n mov " #MAIN_SP "(%%rdi), %%rsp" \
- \
- "\n push %%rdi" \
- "\n push %%rdx" \
- \
- "\n mov %3, %%rdi" \
- "\n call *%2" \
- \
- "\n pop %%rdx" \
- "\n pop %%rdi" \
- \
- "\n mov %%rbp, " #MAIN_FP "(%%rdi)" \
- "\n mov %%rsp, " #MAIN_SP "(%%rdi)" \
- \
- "\n mov " #SUSPENDABLE_FP "(%%rdi), %%rbp" \
- "\n mov " #SUSPENDABLE_SP "(%%rdi), %%rsp" \
- \
- "\n movq %%rax, %0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : "rdi", "rax", "cc", "memory")
- INLINED_ASM(56, 64, 72, 80);
-# elif defined(__i386__) || defined(_M_IX86)
-# define CALLER_SAVED_REGS "eax", "ecx", "edx"
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm("\n mov %1, %%edx" \
- "\n mov %%ebp, " #SUSPENDABLE_FP "(%%edx)" \
- "\n mov %%esp, " #SUSPENDABLE_SP "(%%edx)" \
- \
- "\n mov " #MAIN_FP "(%%edx), %%ebp" \
- "\n mov " #MAIN_SP "(%%edx), %%esp" \
- \
- "\n push %%edx" \
- "\n sub $8, %%esp" \
- "\n push %3" \
- "\n call *%2" \
- "\n add $12, %%esp" \
- "\n pop %%edx" \
- \
- "\n mov %%ebp, " #MAIN_FP "(%%edx)" \
- "\n mov %%esp, " #MAIN_SP "(%%edx)" \
- \
- "\n mov " #SUSPENDABLE_FP "(%%edx), %%ebp" \
- "\n mov " #SUSPENDABLE_SP "(%%edx), %%esp" \
- \
- "\n mov %%eax, %0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : CALLER_SAVED_REGS, "cc", "memory")
- INLINED_ASM(48, 56, 64, 72);
-
-# elif defined(__arm__)
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm("\n mov r0, %1" \
- "\n mov r1, sp" \
- "\n str r11, [r0, #" #SUSPENDABLE_FP "]" \
- "\n str r1, [r0, #" #SUSPENDABLE_SP "]" \
- \
- "\n ldr r11, [r0, #" #MAIN_FP "]" \
- "\n ldr r1, [r0, #" #MAIN_SP "]" \
- "\n mov sp, r1" \
- \
- "\n str r0, [sp, #-8]! " \
- \
- "\n mov r0, %3" \
- "\n blx %2" \
- \
- "\n ldr r2, [sp], #8 " \
- \
- "\n mov r1, sp" \
- "\n str r11, [r2, #" #MAIN_FP "]" \
- "\n str r1, [r2, #" #MAIN_SP "]" \
- \
- "\n ldr r11, [r2, #" #SUSPENDABLE_FP "]" \
- "\n ldr r1, [r2, #" #SUSPENDABLE_SP "]" \
- "\n mov sp, r1" \
- "\n mov %0, r0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : "r0", "r1", "r2", "r3", "cc", "memory")
- INLINED_ASM(48, 56, 64, 72);
-
-#elif defined(__loongarch_lp64)
-# define CALLER_SAVED_REGS \
- "$ra", "$a0", "$a1", "$a2", "$a3", "$a4", "$a5", "$a6", "$a7", \
- "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8", \
- "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", \
- "$f9", "$f10", "$f11", "$f12", "$f13", "$f14", "$f15", "$f16", \
- "$f17", "$f18", "$f19", "$f20", "$f21", "$f22", "$f23"
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm volatile( \
- "\n move $a0, %1" \
- "\n st.d $fp, $a0, " #SUSPENDABLE_FP \
- "\n st.d $sp, $a0, " #SUSPENDABLE_SP \
- \
- "\n ld.d $fp, $a0, " #MAIN_FP \
- "\n ld.d $sp, $a0, " #MAIN_SP \
- \
- "\n addi.d $sp, $sp, -16" \
- "\n st.d $a0, $sp, 8" \
- \
- "\n move $a0, %3" \
- "\n jirl $ra, %2, 0" \
- \
- "\n ld.d $a3, $sp, 8" \
- "\n addi.d $sp, $sp, 16" \
- \
- "\n st.d $fp, $a3, " #MAIN_FP \
- "\n st.d $sp, $a3, " #MAIN_SP \
- \
- "\n ld.d $fp, $a3, " #SUSPENDABLE_FP \
- "\n ld.d $sp, $a3, " #SUSPENDABLE_SP \
- "\n move %0, $a0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : "$a0", "$a3", CALLER_SAVED_REGS, "cc", "memory")
-INLINED_ASM(56, 64, 72, 80);
-
-# elif defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64)
-# define CALLER_SAVED_REGS \
- "ra", \
- "t0", "t1", "t2", "t3", "t4", "t5", "t6", \
- "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", \
- "ft0", "ft1", "ft2", "ft3", "ft4", "ft5", "ft6", "ft7", \
- "ft8", "ft9", "ft10", "ft11", \
- "fa0", "fa1", "fa2", "fa3", "fa4", "fa5", "fa6", "fa7", \
- "fs0", "fs1", "fs2", "fs3", "fs4", "fs5", "fs6", "fs7", \
- "fs8", "fs9", "fs10", "fs11"
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm volatile( \
- "\n mv a0, %1" \
- "\n sd fp, " #SUSPENDABLE_FP "(a0)" \
- "\n sd sp, " #SUSPENDABLE_SP "(a0)" \
- \
- "\n ld fp, " #MAIN_FP "(a0)" \
- "\n ld sp, " #MAIN_SP "(a0)" \
- \
- "\n addi sp, sp, -16" \
- "\n sd a0, 8(sp)" \
- \
- "\n mv a0, %3" \
- "\n jalr ra, 0(%2)" \
- \
- "\n ld a3, 8(sp)" \
- "\n addi sp, sp, 16" \
- \
- "\n sd fp, " #MAIN_FP "(a3)" \
- "\n sd sp, " #MAIN_SP "(a3)" \
- \
- "\n ld fp, " #SUSPENDABLE_FP "(a3)" \
- "\n ld sp, " #SUSPENDABLE_SP "(a3)" \
- "\n mv %0, a0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : "ra", "a0", "a3", CALLER_SAVED_REGS, "cc", "memory")
-INLINED_ASM(56, 64, 72, 80);
-
-# elif defined(__mips64)
-# define CALLER_SAVED_REGS \
- "at", "v0", "v1", \
- "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", \
- "t0", "t1", "t2", "t3", /* Assemblers don't recognize t4-t7 */ \
- "t8", "t9", "ra", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", \
- "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f12", "$f13", \
- "$f14", "$f15", "$f16", "$f17", "$f18", "$f19", "$f20", "$f21", \
- "$f22", "$f23"
-# define INLINED_ASM(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP) \
- CHECK_OFFSETS(MAIN_FP, MAIN_SP, SUSPENDABLE_FP, SUSPENDABLE_SP); \
- asm volatile( \
- "\n move $a0, %1" \
- "\n move $t9, %2" /* PIC function entry */ \
- "\n sd $fp, " #SUSPENDABLE_FP "($a0)" \
- "\n sd $sp, " #SUSPENDABLE_SP "($a0)" \
- \
- "\n ld $fp, " #MAIN_FP "($a0)" \
- "\n ld $sp, " #MAIN_SP "($a0)" \
- \
- "\n daddiu $sp, $sp, -16" \
- "\n sd $a0, 8($sp)" \
- \
- "\n move $a0, %3" \
- "\n jalr $t9" \
- \
- "\n ld $a3, 8($sp)" \
- "\n daddiu $sp, $sp, 16" \
- \
- "\n sd $fp, " #MAIN_FP "($a3)" \
- "\n sd $sp, " #MAIN_SP "($a3)" \
- \
- "\n ld $fp, " #SUSPENDABLE_FP "($a3)" \
- "\n ld $sp, " #SUSPENDABLE_SP "($a3)" \
- "\n move %0, $v0" \
- : "=r"(res) \
- : "r"(stacks), "r"(fn), "r"(data) \
- : CALLER_SAVED_REGS, "cc", "memory")
- INLINED_ASM(24, 32, 40, 48);
-
-# else
- MOZ_CRASH("Not supported for this platform");
-# endif
- // clang-format on
-# endif // JS_SIMULATOR
-
- bool ok = (res & 255) != 0; // need only low byte
- suspender->setActive(cx);
-
-# undef INLINED_ASM
-# undef CHECK_OFFSETS
-# undef CALLER_SAVED_REGS
-
- return ok;
-}
-
-static void CleanupActiveSuspender(JSContext* cx) {
- SuspenderObject* suspender = cx->wasm().activeSuspender();
- MOZ_ASSERT(suspender);
- suspender->setMoribund(cx);
-}
-
// Suspending
// Builds a wasm module with following structure:
@@ -1203,8 +796,10 @@ static bool WasmPISuspendTaskContinue(JSContext* cx, unsigned argc, Value* vp) {
return true;
}
- // The stack was unwound during exception -- time to release resources.
- CleanupActiveSuspender(cx);
+ // The stack was unwound during exception.
+ MOZ_RELEASE_ASSERT(!cx->wasm().activeSuspender());
+ MOZ_RELEASE_ASSERT(
+ suspender.toObject().as<wasm::SuspenderObject>().isMoribund());
if (cx->isThrowingOutOfMemory()) {
return false;
@@ -1590,12 +1185,9 @@ static bool WasmPIPromisingFunction(JSContext* cx, unsigned argc, Value* vp) {
return true;
}
- // During an exception the stack was unwound -- time to release resources.
- // At this point, the suspender might be null, if that's the case
- // don't try to clean up.
- if (cx->wasm().activeSuspender() != nullptr) {
- CleanupActiveSuspender(cx);
- }
+ // The stack was unwound during exception. There should be no active
+ // suspender.
+ MOZ_RELEASE_ASSERT(!cx->wasm().activeSuspender());
if (cx->isThrowingOutOfMemory()) {
return false;
diff --git a/js/src/wasm/WasmPI.h b/js/src/wasm/WasmPI.h
@@ -223,11 +223,12 @@ class SuspenderObject : public NativeObject {
return stackMemoryLimitForSystem() + SuspendableRedZoneSize;
}
- bool hasFramePointer(void* fp) const {
+ bool hasStackAddress(const void* stackAddress) const {
MOZ_ASSERT(!isMoribund());
void* base = stackMemory();
- return (uintptr_t)base <= (uintptr_t)fp &&
- (uintptr_t)fp < (uintptr_t)base + SuspendableStackPlusRedZoneSize;
+ return (uintptr_t)base <= (uintptr_t)stackAddress &&
+ (uintptr_t)stackAddress <
+ (uintptr_t)base + SuspendableStackPlusRedZoneSize;
}
// Stored main stack FP register.
@@ -299,22 +300,15 @@ class SuspenderObject : public NativeObject {
void setMoribund(JSContext* cx);
void setActive(JSContext* cx);
void setSuspended(JSContext* cx);
- void setCalledOnMain(JSContext* cx);
void enter(JSContext* cx);
void suspend(JSContext* cx);
void resume(JSContext* cx);
void leave(JSContext* cx);
+ void unwind(JSContext* cx);
void releaseStackMemory();
-# if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \
- defined(JS_SIMULATOR_RISCV64) || defined(JS_SIMULATOR_LOONG64) || \
- defined(JS_SIMULATOR_MIPS64)
- void switchSimulatorToMain();
- void switchSimulatorToSuspendable();
-# endif
-
// Modifies frames to inject the suspendable stack back into the main one.
void forwardToSuspendable();
@@ -327,9 +321,6 @@ class SuspenderObject : public NativeObject {
static size_t moved(JSObject* obj, JSObject* old);
};
-using CallOnMainStackFn = bool (*)(void* data);
-bool CallOnMainStack(JSContext* cx, CallOnMainStackFn fn, void* data);
-
JSFunction* WasmSuspendingFunctionCreate(JSContext* cx, HandleObject func,
wasm::ValTypeVector&& params,
wasm::ValTypeVector&& results);
@@ -366,6 +357,12 @@ void UpdateSuspenderState(Instance* instance, SuspenderObject* suspender,
void TraceSuspendableStack(JSTracer* trc, SuspenderObject* suspender);
+#else
+
+// Provide an empty forward declaration to simplify some conditional
+// compilation code.
+class SuspenderObject;
+
#endif // ENABLE_WASM_JSPI
} // namespace wasm
diff --git a/js/src/wasm/WasmSignalHandlers.cpp b/js/src/wasm/WasmSignalHandlers.cpp
@@ -544,6 +544,10 @@ struct AutoHandlingTrap {
MOZ_RELEASE_ASSERT(&instance->code() == codeBlock->code ||
trap == Trap::IndirectCallBadSig);
+ // Ensure the active FP has a valid instance pointer that the trap stub can
+ // use.
+ ((FrameWithInstances*)frame)->setCalleeInstance(instance);
+
JSContext* cx =
instance->realm()->runtimeFromAnyThread()->mainContextFromAnyThread();
MOZ_RELEASE_ASSERT(!assertCx || cx == assertCx);
@@ -1011,10 +1015,16 @@ bool wasm::MemoryAccessTraps(const RegisterState& regs, uint8_t* addr,
return false;
}
- const Instance& instance =
- *GetNearestEffectiveInstance(Frame::fromUntaggedWasmExitFP(regs.fp));
+ // This is a safe and expected wasm trap. This guarantees that FP is pointing
+ // at a wasm frame.
+ FrameWithInstances* frame = (FrameWithInstances*)(regs.fp);
+ Instance& instance = *GetNearestEffectiveInstance(frame);
MOZ_ASSERT(&instance.code() == codeBlock->code);
+ // Ensure the active FP has a valid instance pointer that the trap stub can
+ // use.
+ frame->setCalleeInstance(&instance);
+
switch (trap) {
case Trap::OutOfBounds:
if (!instance.memoryAccessInGuardRegion((uint8_t*)addr, numBytes)) {
@@ -1064,6 +1074,16 @@ bool wasm::HandleIllegalInstruction(const RegisterState& regs,
return false;
}
+ // This is a safe and expected wasm trap. This guarantees that FP is pointing
+ // at a wasm frame.
+ FrameWithInstances* frame = (FrameWithInstances*)(regs.fp);
+ Instance& instance = *GetNearestEffectiveInstance(frame);
+ MOZ_ASSERT(&instance.code() == codeBlock->code);
+
+ // Ensure the active FP has a valid instance pointer that the trap stub can
+ // use.
+ frame->setCalleeInstance(&instance);
+
JSContext* cx = TlsContext.get(); // Cold simulator helper function
jit::JitActivation* activation = cx->activation()->asJit();
activation->startWasmTrap(trap, trapSite, regs);
diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp
@@ -2717,26 +2717,72 @@ static bool GenerateTrapExit(MacroAssembler& masm, Label* throwLabel,
masm.PushRegsInMask(RegsToPreserve);
unsigned offsetOfReturnWord = masm.framePushed() - framePushedBeforePreserve;
+ // Load the instance register from the wasm::FrameWithInstances. Normally we
+ // are only guaranteed to have a valid instance there if the frame was a
+ // cross-instance call, however wasm::HandleTrap in the signal handler is
+ // kind enough to store the active instance into that slot for us.
+ masm.loadPtr(
+ Address(FramePointer, wasm::FrameWithInstances::calleeInstanceOffset()),
+ InstanceReg);
+
+ // Grab the stack pointer before we do any stack switches or dynamic
+ // alignment. Store it in a register that won't be used in the stack switch
+ // operation.
+ Register originalStackPointer = ABINonArgReg3;
+ masm.moveStackPtrTo(originalStackPointer);
+
+#ifdef ENABLE_WASM_JSPI
+ GenerateExitPrologueMainStackSwitch(masm, InstanceReg, ABINonArgReg0,
+ ABINonArgReg1, ABINonArgReg2);
+#endif
+
// We know that StackPointer is word-aligned, but not necessarily
- // stack-aligned, so we need to align it dynamically.
- Register preAlignStackPointer = ABINonVolatileReg;
- masm.moveStackPtrTo(preAlignStackPointer);
+ // stack-aligned, so we need to align it dynamically. After we've aligned the
+ // stack, we store the original stack pointer in a slot on the stack.
+ // We're careful to not break stack alignment with that slot.
masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1)));
+ masm.reserveStack(ABIStackAlignment);
+ masm.storePtr(originalStackPointer, Address(masm.getStackPointer(), 0));
+
+ // Push the shadow stack space for the call if we need to. This won't break
+ // stack alignment.
if (ShadowStackSpace) {
masm.subFromStackPtr(Imm32(ShadowStackSpace));
}
+ // Call the WasmHandleTrap function.
masm.assertStackAlignment(ABIStackAlignment);
masm.call(SymbolicAddress::HandleTrap);
// WasmHandleTrap returns null if control should transfer to the throw stub.
+ // That will unwind the stack, and so we don't need to pop anything from the
+ // stack ourselves.
masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
+ // Remove the shadow stack space that we added.
+ if (ShadowStackSpace) {
+ masm.addToStackPtr(Imm32(ShadowStackSpace));
+ }
+
+#ifdef ENABLE_WASM_JSPI
+ // We don't need to reload the InstanceReg because it is non-volatile in the
+ // system ABI.
+ MOZ_ASSERT(NonVolatileRegs.has(InstanceReg));
+ LoadActivation(masm, InstanceReg, ABINonArgReturnReg0);
+ GenerateExitEpilogueMainStackReturn(masm, InstanceReg, ABINonArgReturnReg0,
+ ABINonArgReturnReg1);
+#endif
+
+ // Get the original stack pointer back for before we dynamically aligned it.
+ // This will switch the SP back to the original stack we were on. Be careful
+ // not to use the return register for this, which is live.
+ masm.loadPtr(Address(masm.getStackPointer(), 0), ABINonArgReturnReg0);
+ masm.moveToStackPtr(ABINonArgReturnReg0);
+
// Otherwise, the return value is the TrapData::resumePC we must jump to.
// We must restore register state before jumping, which will clobber
// ReturnReg, so store ReturnReg in the above-reserved stack slot which we
// use to jump to via ret.
- masm.moveToStackPtr(preAlignStackPointer);
masm.storePtr(ReturnReg, Address(masm.getStackPointer(), offsetOfReturnWord));
masm.PopRegsInMask(RegsToPreserve);
#ifdef JS_CODEGEN_ARM64
