TrampolineNatives.cpp (13427B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "jit/TrampolineNatives.h" 8 9 #include "jit/CalleeToken.h" 10 #include "jit/Ion.h" 11 #include "jit/JitCommon.h" 12 #include "jit/JitRuntime.h" 13 #include "jit/MacroAssembler.h" 14 #include "jit/PerfSpewer.h" 15 #include "js/CallArgs.h" 16 #include "js/experimental/JitInfo.h" 17 #include "vm/TypedArrayObject.h" 18 19 #include "jit/MacroAssembler-inl.h" 20 #include "vm/Activation-inl.h" 21 22 using namespace js; 23 using namespace js::jit; 24 25 #define ADD_NATIVE(native) \ 26 const JSJitInfo js::jit::JitInfo_##native{ \ 27 {nullptr}, \ 28 {uint16_t(TrampolineNative::native)}, \ 29 {0}, \ 30 JSJitInfo::TrampolineNative}; 31 TRAMPOLINE_NATIVE_LIST(ADD_NATIVE) 32 #undef ADD_NATIVE 33 34 void js::jit::SetTrampolineNativeJitEntry(JSContext* cx, JSFunction* fun, 35 TrampolineNative native) { 36 if (!cx->runtime()->jitRuntime()) { 37 // No JIT support so there's no trampoline. 38 return; 39 } 40 void** entry = cx->runtime()->jitRuntime()->trampolineNativeJitEntry(native); 41 MOZ_ASSERT(entry); 42 MOZ_ASSERT(*entry); 43 fun->setTrampolineNativeJitEntry(entry); 44 } 45 46 uint32_t JitRuntime::generateArraySortTrampoline(MacroAssembler& masm, 47 ArraySortKind kind) { 48 AutoCreatedBy acb(masm, "JitRuntime::generateArraySortTrampoline"); 49 50 const uint32_t offset = startTrampolineCode(masm); 51 52 // The stack for the trampoline frame will look like this: 53 // 54 // [TrampolineNativeFrameLayout] 55 // * this and arguments passed by the caller 56 // * CalleeToken 57 // * Descriptor 58 // * Return Address 59 // * Saved frame pointer <= FramePointer 60 // [ArraySortData] 61 // * ... 62 // * Comparator this + argument Values --+ -> comparator JitFrameLayout 63 // * Comparator (CalleeToken) | 64 // * Descriptor ----+ <= StackPointer 65 // 66 // The call to the comparator pushes the return address and the frame pointer, 67 // so we check the alignment after pushing these two pointers. 68 constexpr size_t FrameSize = sizeof(ArraySortData); 69 constexpr size_t PushedByCall = 2 * sizeof(void*); 70 static_assert((FrameSize + PushedByCall) % JitStackAlignment == 0); 71 72 // Assert ArraySortData comparator data matches JitFrameLayout. 73 static_assert(PushedByCall + ArraySortData::offsetOfDescriptor() == 74 JitFrameLayout::offsetOfDescriptor()); 75 static_assert(PushedByCall + ArraySortData::offsetOfComparator() == 76 JitFrameLayout::offsetOfCalleeToken()); 77 static_assert(PushedByCall + ArraySortData::offsetOfComparatorThis() == 78 JitFrameLayout::offsetOfThis()); 79 static_assert(PushedByCall + ArraySortData::offsetOfComparatorArgs() == 80 JitFrameLayout::offsetOfActualArgs()); 81 static_assert(CalleeToken_Function == 0, 82 "JSFunction* is valid CalleeToken for non-constructor calls"); 83 84 // Compute offsets from FramePointer. 85 constexpr int32_t ComparatorOffset = 86 -int32_t(FrameSize) + ArraySortData::offsetOfComparator(); 87 constexpr int32_t RvalOffset = 88 -int32_t(FrameSize) + ArraySortData::offsetOfComparatorReturnValue(); 89 constexpr int32_t DescriptorOffset = 90 -int32_t(FrameSize) + ArraySortData::offsetOfDescriptor(); 91 constexpr int32_t ComparatorThisOffset = 92 -int32_t(FrameSize) + ArraySortData::offsetOfComparatorThis(); 93 constexpr int32_t ComparatorArgsOffset = 94 -int32_t(FrameSize) + ArraySortData::offsetOfComparatorArgs(); 95 96 #ifdef JS_USE_LINK_REGISTER 97 masm.pushReturnAddress(); 98 #endif 99 masm.push(FramePointer); 100 masm.moveStackPtrTo(FramePointer); 101 102 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); 103 regs.takeUnchecked(ReturnReg); 104 regs.takeUnchecked(JSReturnOperand); 105 Register temp0 = regs.takeAny(); 106 Register temp1 = regs.takeAny(); 107 Register temp2 = regs.takeAny(); 108 109 // Reserve space and check alignment of the comparator frame. 110 masm.reserveStack(FrameSize); 111 masm.assertStackAlignment(JitStackAlignment, PushedByCall); 112 113 // Trampoline control flow looks like this: 114 // 115 // call ArraySortFromJit or TypedArraySortFromJit 116 // goto checkReturnValue 117 // call_comparator: 118 // call comparator 119 // call ArraySortData::sortArrayWithComparator or 120 // ArraySortData::sortTypedArrayWithComparator 121 // checkReturnValue: 122 // check return value, jump to call_comparator if needed 123 // return rval 124 125 auto pushExitFrame = [&](Register cxReg, Register scratchReg) { 126 MOZ_ASSERT(masm.framePushed() == FrameSize); 127 masm.Push(FrameDescriptor(FrameType::TrampolineNative)); 128 masm.Push(ImmWord(0)); // Fake return address. 129 masm.Push(FramePointer); 130 masm.enterFakeExitFrame(cxReg, scratchReg, ExitFrameType::Bare); 131 }; 132 133 // Call {Typed}ArraySortFromJit. 134 using Fn1 = ArraySortResult (*)(JSContext* cx, 135 jit::TrampolineNativeFrameLayout* frame); 136 masm.loadJSContext(temp0); 137 pushExitFrame(temp0, temp1); 138 masm.setupAlignedABICall(); 139 masm.passABIArg(temp0); 140 masm.passABIArg(FramePointer); 141 switch (kind) { 142 case ArraySortKind::Array: 143 masm.callWithABI<Fn1, ArraySortFromJit>( 144 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); 145 break; 146 case ArraySortKind::TypedArray: 147 masm.callWithABI<Fn1, TypedArraySortFromJit>( 148 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); 149 break; 150 } 151 152 // Check return value. 153 Label checkReturnValue; 154 masm.jump(&checkReturnValue); 155 masm.setFramePushed(FrameSize); 156 157 // Call the comparator. Store the frame descriptor before each call to ensure 158 // the HasCachedSavedFrame flag from a previous call is cleared. 159 uintptr_t jitCallDescriptor = MakeFrameDescriptorForJitCall( 160 jit::FrameType::TrampolineNative, ArraySortData::ComparatorActualArgs); 161 Label callDone, jitCallFast, jitCallSlow; 162 masm.bind(&jitCallFast); 163 { 164 masm.storeValue(UndefinedValue(), 165 Address(FramePointer, ComparatorThisOffset)); 166 masm.storePtr(ImmWord(jitCallDescriptor), 167 Address(FramePointer, DescriptorOffset)); 168 masm.loadPtr(Address(FramePointer, ComparatorOffset), temp0); 169 masm.loadJitCodeRaw(temp0, temp1); 170 masm.callJit(temp1); 171 masm.jump(&callDone); 172 } 173 masm.bind(&jitCallSlow); 174 { 175 masm.storeValue(UndefinedValue(), 176 Address(FramePointer, ComparatorThisOffset)); 177 masm.storePtr(ImmWord(jitCallDescriptor), 178 Address(FramePointer, DescriptorOffset)); 179 masm.loadPtr(Address(FramePointer, ComparatorOffset), temp0); 180 masm.loadJitCodeRaw(temp0, temp1); 181 masm.switchToObjectRealm(temp0, temp2); 182 183 // Handle arguments underflow. 184 Label noUnderflow, restoreRealm; 185 masm.loadFunctionArgCount(temp0, temp0); 186 masm.branch32(Assembler::BelowOrEqual, temp0, 187 Imm32(ArraySortData::ComparatorActualArgs), &noUnderflow); 188 { 189 // If the comparator expects more than two arguments, we must 190 // push additional undefined values. 191 if (JitStackValueAlignment > 1) { 192 static_assert(!(ArraySortData::ComparatorActualArgs & 1)); 193 static_assert(sizeof(JitFrameLayout) % JitStackAlignment == 0, 194 "JitFrameLayout doesn't affect stack alignment"); 195 MOZ_ASSERT(JitStackValueAlignment == 2); 196 // We're currently aligned so that we'll be aligned to JitStackAlignment 197 // after pushing PushedByCall bytes. Before we do the call, we will be 198 // pushing nargs arguments, `this`, a callee token, and a descriptor. 199 // This is (nargs + 1) * sizeof(Value) + 2 * sizeof(uintptr_t) bytes. 200 // We want to push a multiple of JitStackAlignment bytes, which may 201 // necessitate 8 bytes of padding, depending on the parity of nargs. 202 203 bool evenNargsNeedsAlignment = 204 (sizeof(Value) + 2 * sizeof(uintptr_t)) % JitStackAlignment != 0; 205 Assembler::Condition cond = 206 evenNargsNeedsAlignment ? Assembler::NonZero : Assembler::Zero; 207 208 Label aligned; 209 masm.branchTest32(cond, temp0, Imm32(1), &aligned); 210 masm.subFromStackPtr(Imm32(sizeof(Value))); 211 masm.bind(&aligned); 212 } 213 214 // Push `undefined` arguments. 215 Label loop; 216 masm.bind(&loop); 217 masm.pushValue(UndefinedValue()); 218 masm.sub32(Imm32(1), temp0); 219 masm.branch32(Assembler::GreaterThan, temp0, 220 Imm32(ArraySortData::ComparatorActualArgs), &loop); 221 222 // Copy the existing arguments, this, callee, and descriptor, then call. 223 masm.pushValue( 224 Address(FramePointer, ComparatorArgsOffset + int32_t(sizeof(Value)))); 225 masm.pushValue(Address(FramePointer, ComparatorArgsOffset)); 226 masm.pushValue(Address(FramePointer, ComparatorThisOffset)); 227 masm.push(Address(FramePointer, ComparatorOffset)); 228 masm.push(Address(FramePointer, DescriptorOffset)); 229 masm.callJit(temp1); 230 231 // Restore the expected stack pointer. 232 masm.computeEffectiveAddress(Address(FramePointer, -int32_t(FrameSize)), 233 temp0); 234 masm.moveToStackPtr(temp0); 235 236 masm.jump(&restoreRealm); 237 } 238 masm.bind(&noUnderflow); 239 masm.callJit(temp1); 240 241 masm.bind(&restoreRealm); 242 Address calleeToken(FramePointer, 243 TrampolineNativeFrameLayout::offsetOfCalleeToken()); 244 masm.loadFunctionFromCalleeToken(calleeToken, temp0); 245 masm.switchToObjectRealm(temp0, temp1); 246 } 247 248 // Store the comparator's return value. 249 masm.bind(&callDone); 250 masm.storeValue(JSReturnOperand, Address(FramePointer, RvalOffset)); 251 252 // Call ArraySortData::sort{Typed}ArrayWithComparator. 253 using Fn2 = ArraySortResult (*)(ArraySortData* data); 254 masm.moveStackPtrTo(temp2); 255 masm.loadJSContext(temp0); 256 pushExitFrame(temp0, temp1); 257 masm.setupAlignedABICall(); 258 masm.passABIArg(temp2); 259 switch (kind) { 260 case ArraySortKind::Array: 261 masm.callWithABI<Fn2, ArraySortData::sortArrayWithComparator>( 262 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); 263 break; 264 case ArraySortKind::TypedArray: 265 masm.callWithABI<Fn2, ArraySortData::sortTypedArrayWithComparator>( 266 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); 267 break; 268 } 269 270 // Check return value. 271 masm.bind(&checkReturnValue); 272 masm.branch32(Assembler::Equal, ReturnReg, 273 Imm32(int32_t(ArraySortResult::Failure)), masm.failureLabel()); 274 masm.freeStack(ExitFrameLayout::SizeWithFooter()); 275 masm.branch32(Assembler::Equal, ReturnReg, 276 Imm32(int32_t(ArraySortResult::CallJSSameRealmNoUnderflow)), 277 &jitCallFast); 278 masm.branch32(Assembler::Equal, ReturnReg, 279 Imm32(int32_t(ArraySortResult::CallJS)), &jitCallSlow); 280 #ifdef DEBUG 281 Label ok; 282 masm.branch32(Assembler::Equal, ReturnReg, 283 Imm32(int32_t(ArraySortResult::Done)), &ok); 284 masm.assumeUnreachable("Unexpected return value"); 285 masm.bind(&ok); 286 #endif 287 288 masm.loadValue(Address(FramePointer, RvalOffset), JSReturnOperand); 289 masm.moveToStackPtr(FramePointer); 290 masm.pop(FramePointer); 291 masm.ret(); 292 293 return offset; 294 } 295 296 void JitRuntime::generateTrampolineNatives( 297 MacroAssembler& masm, TrampolineNativeJitEntryOffsets& offsets, 298 PerfSpewerRangeRecorder& rangeRecorder) { 299 offsets[TrampolineNative::ArraySort] = 300 generateArraySortTrampoline(masm, ArraySortKind::Array); 301 rangeRecorder.recordOffset("Trampoline: ArraySort"); 302 303 offsets[TrampolineNative::TypedArraySort] = 304 generateArraySortTrampoline(masm, ArraySortKind::TypedArray); 305 rangeRecorder.recordOffset("Trampoline: TypedArraySort"); 306 } 307 308 bool jit::CallTrampolineNativeJitCode(JSContext* cx, TrampolineNative native, 309 CallArgs& args) { 310 // Use the EnterJit trampoline to enter the native's trampoline code. 311 312 AutoCheckRecursionLimit recursion(cx); 313 if (!recursion.check(cx)) { 314 return false; 315 } 316 317 MOZ_ASSERT(!args.isConstructing()); 318 CalleeToken calleeToken = CalleeToToken(&args.callee().as<JSFunction>(), 319 /* constructing = */ false); 320 321 Value* maxArgv = args.array(); 322 size_t maxArgc = args.length(); 323 324 Rooted<Value> result(cx, Int32Value(args.length())); 325 326 AssertRealmUnchanged aru(cx); 327 JitActivation activation(cx); 328 329 EnterJitCode enter = cx->runtime()->jitRuntime()->enterJit(); 330 void* code = *cx->runtime()->jitRuntime()->trampolineNativeJitEntry(native); 331 332 CALL_GENERATED_CODE(enter, code, maxArgc, maxArgv, /* osrFrame = */ nullptr, 333 calleeToken, /* envChain = */ nullptr, 334 /* osrNumStackValues = */ 0, result.address()); 335 336 // Ensure the counter was reset to zero after exiting from JIT code. 337 MOZ_ASSERT(!cx->isInUnsafeRegion()); 338 339 // Release temporary buffer used for OSR into Ion. 340 cx->runtime()->jitRuntime()->freeIonOsrTempData(); 341 342 if (result.isMagic()) { 343 MOZ_ASSERT(result.isMagic(JS_ION_ERROR)); 344 return false; 345 } 346 347 args.rval().set(result); 348 return true; 349 }