CodeGenerator.h (20642B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_CodeGenerator_h 8 #define jit_CodeGenerator_h 9 10 #include "jit/PerfSpewer.h" 11 #include "js/Prefs.h" 12 #include "js/ScalarType.h" // js::Scalar::Type 13 14 #if defined(JS_CODEGEN_X86) 15 # include "jit/x86/CodeGenerator-x86.h" 16 #elif defined(JS_CODEGEN_X64) 17 # include "jit/x64/CodeGenerator-x64.h" 18 #elif defined(JS_CODEGEN_ARM) 19 # include "jit/arm/CodeGenerator-arm.h" 20 #elif defined(JS_CODEGEN_ARM64) 21 # include "jit/arm64/CodeGenerator-arm64.h" 22 #elif defined(JS_CODEGEN_MIPS64) 23 # include "jit/mips64/CodeGenerator-mips64.h" 24 #elif defined(JS_CODEGEN_LOONG64) 25 # include "jit/loong64/CodeGenerator-loong64.h" 26 #elif defined(JS_CODEGEN_RISCV64) 27 # include "jit/riscv64/CodeGenerator-riscv64.h" 28 #elif defined(JS_CODEGEN_WASM32) 29 # include "jit/wasm32/CodeGenerator-wasm32.h" 30 #elif defined(JS_CODEGEN_NONE) 31 # include "jit/none/CodeGenerator-none.h" 32 #else 33 # error "Unknown architecture!" 34 #endif 35 36 namespace js { 37 38 namespace wasm { 39 class Decoder; 40 class StackMaps; 41 } // namespace wasm 42 43 namespace jit { 44 45 class WarpSnapshot; 46 47 template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo> 48 class OutOfLineCallVM; 49 50 class OutOfLineTestObject; 51 class OutOfLineICFallback; 52 class OutOfLineCallPostWriteBarrier; 53 class OutOfLineCallPostWriteElementBarrier; 54 55 class CodeGenerator final : public CodeGeneratorSpecific { 56 // Warp snapshot. This is nullptr for Wasm compilations. 57 const WarpSnapshot* snapshot_ = nullptr; 58 IonScriptCounts* counts_ = nullptr; 59 60 [[nodiscard]] bool generateBody(); 61 62 ConstantOrRegister toConstantOrRegister(LInstruction* lir, size_t n, 63 MIRType type); 64 65 #ifdef CHECK_OSIPOINT_REGISTERS 66 void resetOsiPointRegs(LSafepoint* safepoint); 67 bool shouldVerifyOsiPointRegs(LSafepoint* safepoint); 68 void verifyOsiPointRegs(LSafepoint* safepoint); 69 #endif 70 71 void callVMInternal(VMFunctionId id, LInstruction* ins); 72 73 template <typename Fn, Fn fn> 74 void callVM(LInstruction* ins); 75 76 template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo> 77 inline OutOfLineCode* oolCallVM(LInstruction* ins, const ArgSeq& args, 78 const StoreOutputTo& out); 79 80 template <typename LCallIns> 81 void emitCallNative(LCallIns* call, JSNative native, Register argContextReg, 82 Register argUintNReg, Register argVpReg, Register tempReg, 83 uint32_t unusedStack); 84 85 template <typename LCallIns> 86 void emitCallNative(LCallIns* call, JSNative native); 87 88 public: 89 CodeGenerator(MIRGenerator* gen, LIRGraph* graph, 90 MacroAssembler* masm = nullptr, 91 const wasm::CodeMetadata* wasmCodeMeta = nullptr); 92 ~CodeGenerator(); 93 94 [[nodiscard]] bool generate(const WarpSnapshot* snapshot); 95 [[nodiscard]] bool generateWasm(wasm::CallIndirectId callIndirectId, 96 const wasm::TrapSiteDesc& entryTrapSiteDesc, 97 const wasm::ArgTypeVector& argTys, 98 const RegisterOffsets& trapExitLayout, 99 size_t trapExitLayoutNumWords, 100 wasm::FuncOffsets* offsets, 101 wasm::StackMaps* stackMaps, 102 wasm::Decoder* decoder); 103 [[nodiscard]] bool generateBlock(LBlock* current, size_t blockNumber, 104 IonScriptCounts* counts, bool compilingWasm); 105 106 [[nodiscard]] bool generateOutOfLineBlocks(); 107 [[nodiscard]] bool link(JSContext* cx); 108 109 void emitOOLTestObject(Register objreg, Label* ifTruthy, Label* ifFalsy, 110 Register scratch); 111 112 void emitTypeOfCheck(JSValueType type, Register tag, Register output, 113 Label* done, Label* oolObject); 114 void emitTypeOfJSType(JSValueType type, Register output); 115 void emitTypeOfObject(Register obj, Register output, Label* done); 116 void emitTypeOfIsObject(MTypeOfIs* mir, Register obj, Register output, 117 Label* success, Label* fail, Label* slowCheck); 118 void emitTypeOfIsObjectOOL(MTypeOfIs* mir, Register obj, Register output); 119 120 template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo> 121 void visitOutOfLineCallVM( 122 OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool); 123 124 void emitIsCallableOOL(Register object, Register output); 125 126 void emitResumableWasmTrapOOL(LInstruction* lir, size_t framePushed, 127 const wasm::TrapSiteDesc& trapSiteDesc, 128 wasm::Trap trap); 129 130 void visitOutOfLineICFallback(OutOfLineICFallback* ool); 131 132 void visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool); 133 void visitOutOfLineCallPostWriteElementBarrier( 134 OutOfLineCallPostWriteElementBarrier* ool); 135 136 void callWasmStructAllocFun(LInstruction* lir, wasm::SymbolicAddress fun, 137 Register typeDefData, Register allocSite, 138 Register output, 139 const wasm::TrapSiteDesc& trapSiteDesc); 140 141 void callWasmArrayAllocFun(LInstruction* lir, wasm::SymbolicAddress fun, 142 Register numElements, Register typeDefData, 143 Register allocSite, Register output, 144 const wasm::TrapSiteDesc& trapSiteDesc); 145 146 #ifdef ENABLE_WASM_JSPI 147 void callWasmUpdateSuspenderState(wasm::UpdateSuspenderStateAction kind, 148 Register suspender, Register temp); 149 // Stack switching trampoline requires two arguments (suspender and data) to 150 // be passed. The function prepares stack and registers according Wasm ABI. 151 void prepareWasmStackSwitchTrampolineCall(Register suspender, Register data); 152 #endif 153 154 void setCompilationTime(mozilla::TimeDuration duration) { 155 compileTime_ = duration; 156 } 157 mozilla::TimeDuration getCompilationTime() const { return compileTime_; } 158 159 private: 160 void emitPostWriteBarrier(const LAllocation* obj); 161 void emitPostWriteBarrier(Register objreg); 162 void emitPostWriteBarrierS(Address address, Register prev, Register next); 163 164 void emitElementPostWriteBarrier(MInstruction* mir, 165 const LiveRegisterSet& liveVolatileRegs, 166 Register obj, Register index, 167 Register scratch, 168 const ConstantOrRegister& val, 169 int32_t indexDiff = 0); 170 171 template <class LPostBarrierType, MIRType nurseryType> 172 void visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool); 173 template <class LPostBarrierType> 174 void visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool); 175 void visitLoadSlotByIteratorIndexCommon(Register object, 176 Register indexScratch, 177 Register kindScratch, 178 ValueOperand result); 179 void visitStoreSlotByIteratorIndexCommon(Register object, 180 Register indexScratch, 181 Register kindScratch, 182 ValueOperand value); 183 184 void emitCallInvokeFunction(LInstruction* call, Register callereg, 185 bool isConstructing, bool ignoresReturnValue, 186 uint32_t argc, uint32_t unusedStack); 187 template <typename T> 188 void emitApplyGeneric(T* apply); 189 template <typename T> 190 void emitCallInvokeFunction(T* apply); 191 template <typename T> 192 void emitAllocateSpaceForApply(T* apply, Register calleeReg, Register argcreg, 193 Register scratch); 194 template <typename T> 195 void emitAllocateSpaceForConstructAndPushNewTarget( 196 T* apply, Register calleeReg, Register argcreg, 197 Register newTargetAndScratch); 198 void emitCopyValuesForApply(Register argvSrcBase, Register argvIndex, 199 Register copyreg, size_t argvSrcOffset, 200 size_t argvDstOffset); 201 void emitRestoreStackPointerFromFP(); 202 void emitPushArguments(Register argcreg, Register scratch, Register copyreg, 203 uint32_t extraFormals); 204 void emitPushArrayAsArguments(Register tmpArgc, Register srcBaseAndArgc, 205 Register scratch, size_t argvSrcOffset); 206 void emitPushArguments(LApplyArgsGeneric* apply); 207 void emitPushArguments(LApplyArgsObj* apply); 208 void emitPushArguments(LApplyArrayGeneric* apply); 209 void emitPushArguments(LConstructArgsGeneric* construct); 210 void emitPushArguments(LConstructArrayGeneric* construct); 211 212 template <typename T> 213 void emitApplyNative(T* apply); 214 template <typename T> 215 void emitAlignStackForApplyNative(T* apply, Register argc); 216 template <typename T> 217 void emitPushNativeArguments(T* apply); 218 template <typename T> 219 void emitPushArrayAsNativeArguments(T* apply); 220 void emitPushArguments(LApplyArgsNative* apply); 221 void emitPushArguments(LApplyArgsObjNative* apply); 222 void emitPushArguments(LApplyArrayNative* apply); 223 void emitPushArguments(LConstructArgsNative* construct); 224 void emitPushArguments(LConstructArrayNative* construct); 225 226 template <typename T> 227 void emitApplyArgsGuard(T* apply); 228 229 template <typename T> 230 void emitApplyArgsObjGuard(T* apply); 231 232 template <typename T> 233 void emitApplyArrayGuard(T* apply); 234 235 template <class GetInlinedArgument> 236 void emitGetInlinedArgument(GetInlinedArgument* lir, Register index, 237 ValueOperand output); 238 239 void emitMaybeAtomizeSlot(LInstruction* ins, Register stringReg, 240 Address slotAddr, TypedOrValueRegister dest); 241 242 void emitWeakMapLookupObject(Register weakMap, Register obj, 243 Register hashTable, Register hashCode, 244 Register scratch, Register scratch2, 245 Register scratch3, Register scratch4, 246 Register scratch5, Label* found, Label* missing); 247 248 using RegisterOrInt32 = mozilla::Variant<Register, int32_t>; 249 250 static RegisterOrInt32 ToRegisterOrInt32(const LAllocation* allocation); 251 252 using AddressOrBaseIndex = mozilla::Variant<Address, BaseIndex>; 253 254 static AddressOrBaseIndex ToAddressOrBaseIndex(Register elements, 255 const LAllocation* index, 256 Scalar::Type type); 257 258 using AddressOrBaseObjectElementIndex = 259 mozilla::Variant<Address, BaseObjectElementIndex>; 260 261 static AddressOrBaseObjectElementIndex ToAddressOrBaseObjectElementIndex( 262 Register elements, const LAllocation* index); 263 264 #ifdef DEBUG 265 void emitAssertArgumentsSliceBounds(const RegisterOrInt32& begin, 266 const RegisterOrInt32& count, 267 Register numActualArgs); 268 #endif 269 270 template <class ArgumentsSlice> 271 void emitNewArray(ArgumentsSlice* lir, const RegisterOrInt32& count, 272 Register output, Register temp); 273 274 void visitNewArrayCallVM(LNewArray* lir); 275 void visitNewObjectVMCall(LNewObject* lir); 276 277 void emitConcat(LInstruction* lir, Register lhs, Register rhs, 278 Register output); 279 280 void emitInstanceOf(LInstruction* ins, Register protoReg); 281 282 void emitIteratorHasIndicesAndBranch(Register iterator, Register object, 283 Register temp, Register temp2, 284 Label* ifFalse); 285 286 #ifdef DEBUG 287 void emitAssertResultV(const ValueOperand output, const MDefinition* mir); 288 void emitAssertGCThingResult(Register input, const MDefinition* mir); 289 #endif 290 291 #ifdef DEBUG 292 void emitDebugForceBailing(LInstruction* lir); 293 #endif 294 295 IonScriptCounts* extractScriptCounts() { 296 IonScriptCounts* counts = scriptCounts_; 297 scriptCounts_ = nullptr; // prevent delete in dtor 298 return counts; 299 } 300 301 void addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, 302 TypedOrValueRegister value, 303 const ConstantOrRegister& id, ValueOperand output); 304 void addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, 305 Register objReg, Register temp, 306 const ConstantOrRegister& id, 307 const ConstantOrRegister& value, bool strict); 308 309 template <class IteratorObject, class TableObject> 310 void emitGetNextEntryForIterator(LGetNextEntryForIterator* lir); 311 312 template <class TableObject> 313 void emitLoadIteratorValues(Register result, Register temp, Register front); 314 315 void emitStringToInt64(LInstruction* lir, Register input, Register64 output); 316 317 OutOfLineCode* createBigIntOutOfLine(LInstruction* lir, Scalar::Type type, 318 Register64 input, Register output); 319 320 void emitCreateBigInt(LInstruction* lir, Scalar::Type type, Register64 input, 321 Register output, Register maybeTemp, 322 Register64 maybeTemp64 = Register64::Invalid()); 323 324 void emitCallMegamorphicGetter(LInstruction* lir, 325 ValueOperand accessorAndOutput, Register obj, 326 Register calleeScratch, Register argcScratch, 327 Label* nullGetter); 328 329 template <size_t NumDefs> 330 void emitIonToWasmCallBase(LIonToWasmCallBase<NumDefs>* lir); 331 332 IonScriptCounts* maybeCreateScriptCounts(); 333 334 template <typename InstructionWithMaybeTrapSite, class AddressOrBaseIndexT> 335 void emitWasmValueLoad(InstructionWithMaybeTrapSite* ins, MIRType type, 336 MWideningOp wideningOp, AddressOrBaseIndexT addr, 337 AnyRegister dst); 338 template <typename InstructionWithMaybeTrapSite, class AddressOrBaseIndexT> 339 void emitWasmValueStore(InstructionWithMaybeTrapSite* ins, MIRType type, 340 MNarrowingOp narrowingOp, AnyRegister src, 341 AddressOrBaseIndexT addr); 342 343 void testValueTruthyForType(JSValueType type, ScratchTagScope& tag, 344 const ValueOperand& value, Register tempToUnbox, 345 Register temp, FloatRegister floatTemp, 346 Label* ifTruthy, Label* ifFalsy, 347 OutOfLineTestObject* ool, bool skipTypeTest); 348 349 // Test whether value is truthy or not and jump to the corresponding label. 350 // The control flow falls through when the object is truthy, as an 351 // optimization. 352 void testValueTruthy(const ValueOperand& value, Register tempToUnbox, 353 Register temp, FloatRegister floatTemp, 354 const TypeDataList& observedTypes, Label* ifTruthy, 355 Label* ifFalsy, OutOfLineTestObject* ool); 356 357 // This function behaves like testObjectEmulatesUndefined with the exception 358 // that it can choose to let control flow fall through when the object 359 // doesn't emulate undefined, as an optimization. Use the regular 360 // testObjectEmulatesUndefined when it's required to branch to one of the 361 // two labels. 362 void testObjectEmulatesUndefinedKernel(Register objreg, 363 Label* ifEmulatesUndefined, 364 Label* ifDoesntEmulateUndefined, 365 Register scratch, 366 OutOfLineTestObject* ool); 367 368 // Test whether an object emulates |undefined|. If it does, jump to 369 // |ifEmulatesUndefined|; the caller is responsible for binding this label. 370 // If it doesn't, fall through; the label |ifDoesntEmulateUndefined| (which 371 // must be initially unbound) will be bound at this point. 372 void branchTestObjectEmulatesUndefined(Register objreg, 373 Label* ifEmulatesUndefined, 374 Label* ifDoesntEmulateUndefined, 375 Register scratch, 376 OutOfLineTestObject* ool); 377 378 // Test whether an object emulates |undefined|, and jump to the 379 // corresponding label. 380 // 381 // This method should be used when subsequent code can't be laid out in a 382 // straight line; if it can, branchTest* should be used instead. 383 void testObjectEmulatesUndefined(Register objreg, Label* ifEmulatesUndefined, 384 Label* ifDoesntEmulateUndefined, 385 Register scratch, OutOfLineTestObject* ool); 386 387 // Bailout if an element about to be written to is a hole. 388 void emitStoreHoleCheck(Address dest, LSnapshot* snapshot); 389 void emitStoreHoleCheck(BaseObjectElementIndex dest, LSnapshot* snapshot); 390 391 void emitAssertRangeI(MIRType type, const Range* r, Register input); 392 void emitAssertRangeD(const Range* r, FloatRegister input, 393 FloatRegister temp); 394 395 void maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal, 396 OutOfLineCode* ool); 397 398 void incrementWarmUpCounter(AbsoluteAddress warmUpCount, JSScript* script, 399 Register tmp); 400 401 Vector<CodeOffset, 0, JitAllocPolicy> ionScriptLabels_; 402 403 // Used to bake in a pointer into the IonScript's list of nursery objects, for 404 // MNurseryObject codegen. 405 struct NurseryObjectLabel { 406 CodeOffset offset; 407 uint32_t nurseryIndex; 408 NurseryObjectLabel(CodeOffset offset, uint32_t nurseryIndex) 409 : offset(offset), nurseryIndex(nurseryIndex) {} 410 }; 411 Vector<NurseryObjectLabel, 0, JitAllocPolicy> nurseryObjectLabels_; 412 413 // Like NurseryObjectLabel but for Values. The Values in the IonScript will be 414 // stored in the constants-list so this also stores the constantPoolIndex for 415 // that list. 416 struct NurseryValueLabel { 417 CodeOffset offset; 418 uint32_t nurseryIndex; 419 uint32_t constantPoolIndex = UINT32_MAX; 420 NurseryValueLabel(CodeOffset offset, uint32_t nurseryIndex) 421 : offset(offset), nurseryIndex(nurseryIndex) {} 422 }; 423 Vector<NurseryValueLabel, 0, JitAllocPolicy> nurseryValueLabels_; 424 425 Address getNurseryValueAddress(ValueOrNurseryValueIndex val, Register reg); 426 427 void branchIfInvalidated(Register temp, Label* invalidated); 428 429 #ifdef DEBUG 430 void emitDebugResultChecks(LInstruction* ins); 431 void emitGCThingResultChecks(LInstruction* lir, MDefinition* mir); 432 void emitValueResultChecks(LInstruction* lir, MDefinition* mir); 433 void emitWasmAnyrefResultChecks(LInstruction* lir, MDefinition* mir); 434 #endif 435 436 // Script counts created during code generation. 437 IonScriptCounts* scriptCounts_; 438 439 // Total Ion compilation time. 440 mozilla::TimeDuration compileTime_; 441 442 #ifdef FUZZING_JS_FUZZILLI 443 void emitFuzzilliHashObject(LInstruction* lir, Register obj, Register output); 444 void emitFuzzilliHashBigInt(LInstruction* lir, Register bigInt, 445 Register output); 446 #endif 447 448 #define LIR_OP(op) void visit##op(L##op* ins); 449 LIR_OPCODE_LIST(LIR_OP) 450 #undef LIR_OP 451 452 // In debug mode, we need to validate that we've not made a mistake with the 453 // fuse. 454 void assertObjectDoesNotEmulateUndefined(Register input, Register temp, 455 const MInstruction* mir); 456 457 // Register a dependency on the HasSeenObjectEmulateUndefined fuse. 458 bool addHasSeenObjectEmulateUndefinedFuseDependency(); 459 460 // Register a dependency on the HasSeenArrayExceedsInt32Length fuse. 461 bool addHasSeenArrayExceedsInt32LengthFuseDependency(); 462 463 // Return true if the fuse is intact, and if the fuse is intact note the 464 // dependency 465 bool hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted() { 466 bool intact = gen->outerInfo().hasSeenObjectEmulateUndefinedFuseIntact(); 467 if (intact) { 468 bool tryToAdd = addHasSeenObjectEmulateUndefinedFuseDependency(); 469 // If we oom, just pretend that the fuse is popped. 470 return tryToAdd; 471 } 472 return false; 473 } 474 475 bool hasSeenArrayExceedsInt32LengthFuseIntactAndDependencyNoted() { 476 bool intact = gen->outerInfo().hasSeenArrayExceedsInt32LengthFuseIntact(); 477 if (intact) { 478 return addHasSeenArrayExceedsInt32LengthFuseDependency(); 479 } 480 return false; 481 } 482 }; 483 484 } // namespace jit 485 } // namespace js 486 487 #endif /* jit_CodeGenerator_h */