WarpOracle.cpp (61151B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "jit/WarpOracle.h" 8 9 #include "mozilla/ScopeExit.h" 10 #include "mozilla/Try.h" 11 12 #include <algorithm> 13 14 #include "jit/CacheIR.h" 15 #include "jit/CacheIRCompiler.h" 16 #include "jit/CacheIRReader.h" 17 #include "jit/CompileInfo.h" 18 #include "jit/InlineScriptTree.h" 19 #include "jit/JitHints.h" 20 #include "jit/JitRuntime.h" 21 #include "jit/JitScript.h" 22 #include "jit/JitSpewer.h" 23 #include "jit/JitZone.h" 24 #include "jit/MIRGenerator.h" 25 #include "jit/ShapeList.h" 26 #include "jit/StubFolding.h" 27 #include "jit/TrialInlining.h" 28 #include "jit/TypeData.h" 29 #include "jit/WarpBuilder.h" 30 #include "js/ColumnNumber.h" // JS::LimitedColumnNumberOneOrigin 31 #include "util/DifferentialTesting.h" 32 #include "vm/BuiltinObjectKind.h" 33 #include "vm/BytecodeIterator.h" 34 #include "vm/BytecodeLocation.h" 35 36 #include "jit/InlineScriptTree-inl.h" 37 #include "vm/BytecodeIterator-inl.h" 38 #include "vm/BytecodeLocation-inl.h" 39 #include "vm/EnvironmentObject-inl.h" 40 #include "vm/Interpreter-inl.h" 41 42 using namespace js; 43 using namespace js::jit; 44 45 using mozilla::Maybe; 46 47 // WarpScriptOracle creates a WarpScriptSnapshot for a single JSScript. Note 48 // that a single WarpOracle can use multiple WarpScriptOracles when scripts are 49 // inlined. 50 class MOZ_STACK_CLASS WarpScriptOracle { 51 JSContext* cx_; 52 WarpOracle* oracle_; 53 MIRGenerator& mirGen_; 54 TempAllocator& alloc_; 55 HandleScript script_; 56 const CompileInfo* info_; 57 ICScript* icScript_; 58 59 // Index of the next ICEntry for getICEntry. This assumes the script's 60 // bytecode is processed from first to last instruction. 61 uint32_t icEntryIndex_ = 0; 62 63 template <typename... Args> 64 mozilla::GenericErrorResult<AbortReason> abort(Args&&... args) { 65 return oracle_->abort(script_, args...); 66 } 67 68 WarpEnvironment createEnvironment(); 69 AbortReasonOr<Ok> maybeInlineIC(WarpOpSnapshotList& snapshots, 70 BytecodeLocation loc); 71 AbortReasonOr<bool> maybeInlineCall(WarpOpSnapshotList& snapshots, 72 BytecodeLocation loc, ICCacheIRStub* stub, 73 ICFallbackStub* fallbackStub, 74 uint8_t* stubDataCopy); 75 AbortReasonOr<bool> maybeInlinePolymorphicTypes(WarpOpSnapshotList& snapshots, 76 BytecodeLocation loc, 77 ICCacheIRStub* firstStub, 78 ICFallbackStub* fallbackStub); 79 [[nodiscard]] bool replaceNurseryAndAllocSitePointers( 80 ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo, 81 uint8_t* stubDataCopy); 82 bool maybeReplaceNurseryPointer(const CacheIRStubInfo* stubInfo, 83 uint8_t* stubDataCopy, JSObject* obj, 84 size_t offset); 85 bool maybeReplaceNurseryPointer(const CacheIRStubInfo* stubInfo, 86 uint8_t* stubDataCopy, Value v, 87 size_t offset); 88 89 public: 90 WarpScriptOracle(JSContext* cx, WarpOracle* oracle, HandleScript script, 91 const CompileInfo* info, ICScript* icScript) 92 : cx_(cx), 93 oracle_(oracle), 94 mirGen_(oracle->mirGen()), 95 alloc_(mirGen_.alloc()), 96 script_(script), 97 info_(info), 98 icScript_(icScript) {} 99 100 AbortReasonOr<WarpScriptSnapshot*> createScriptSnapshot(); 101 102 ICEntry& getICEntryAndFallback(BytecodeLocation loc, 103 ICFallbackStub** fallback); 104 }; 105 106 WarpOracle::WarpOracle(JSContext* cx, MIRGenerator& mirGen, 107 HandleScript outerScript) 108 : cx_(cx), 109 mirGen_(mirGen), 110 alloc_(mirGen.alloc()), 111 outerScript_(outerScript) {} 112 113 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script, 114 AbortReason r) { 115 auto res = mirGen_.abort(r); 116 JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename()); 117 return res; 118 } 119 120 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script, 121 AbortReason r, 122 const char* message, 123 ...) { 124 va_list ap; 125 va_start(ap, message); 126 auto res = mirGen_.abortFmt(r, message, ap); 127 va_end(ap); 128 JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename()); 129 return res; 130 } 131 132 void WarpOracle::addScriptSnapshot(WarpScriptSnapshot* scriptSnapshot, 133 ICScript* icScript, size_t bytecodeLength) { 134 scriptSnapshots_.insertBack(scriptSnapshot); 135 accumulatedBytecodeSize_ += bytecodeLength; 136 #ifdef DEBUG 137 runningScriptHash_ = 138 mozilla::AddToHash(runningScriptHash_, icScript->hash(cx_)); 139 #endif 140 } 141 142 AbortReasonOr<WarpSnapshot*> WarpOracle::createSnapshot() { 143 #ifdef JS_JITSPEW 144 const char* mode; 145 if (outerScript_->hasIonScript()) { 146 mode = "Recompiling"; 147 } else { 148 mode = "Compiling"; 149 } 150 JitSpew(JitSpew_IonScripts, 151 "Warp %s script %s:%u:%u (%p) (warmup-counter=%" PRIu32 ",%s%s)", 152 mode, outerScript_->filename(), outerScript_->lineno(), 153 outerScript_->column().oneOriginValue(), 154 static_cast<JSScript*>(outerScript_), outerScript_->getWarmUpCount(), 155 outerScript_->isGenerator() ? " isGenerator" : "", 156 outerScript_->isAsync() ? " isAsync" : ""); 157 #endif 158 159 accumulatedBytecodeSize_ = outerScript_->length(); 160 161 MOZ_ASSERT(outerScript_->hasJitScript()); 162 ICScript* icScript = outerScript_->jitScript()->icScript(); 163 WarpScriptOracle scriptOracle(cx_, this, outerScript_, &mirGen_.outerInfo(), 164 icScript); 165 166 WarpScriptSnapshot* scriptSnapshot = 167 MOZ_TRY(scriptOracle.createScriptSnapshot()); 168 169 // Insert the outermost scriptSnapshot at the front of the list. 170 scriptSnapshots_.insertFront(scriptSnapshot); 171 172 bool recordFinalWarmUpCount = false; 173 #ifdef JS_CACHEIR_SPEW 174 recordFinalWarmUpCount = outerScript_->needsFinalWarmUpCount(); 175 #endif 176 177 auto* snapshot = new (alloc_.fallible()) 178 WarpSnapshot(cx_, alloc_, std::move(scriptSnapshots_), zoneStubs_, 179 bailoutInfo_, recordFinalWarmUpCount); 180 if (!snapshot) { 181 return abort(outerScript_, AbortReason::Alloc); 182 } 183 184 if (!snapshot->nurseryObjects().appendAll(nurseryObjects_)) { 185 return abort(outerScript_, AbortReason::Alloc); 186 } 187 if (!snapshot->nurseryValues().appendAll(nurseryValues_)) { 188 return abort(outerScript_, AbortReason::Alloc); 189 } 190 191 #ifdef JS_JITSPEW 192 if (JitSpewEnabled(JitSpew_WarpSnapshots)) { 193 Fprinter& out = JitSpewPrinter(); 194 snapshot->dump(out); 195 } 196 #endif 197 198 #ifdef DEBUG 199 // When transpiled CacheIR bails out, we do not want to recompile 200 // with the exact same data and get caught in an invalidation loop. 201 // 202 // To avoid this, we store a hash of the stub pointers and entry 203 // counts in this snapshot, save that hash in the JitScript if we 204 // have a TranspiledCacheIR or MonomorphicInlinedStubFolding bailout, 205 // and assert that the hash has changed when we recompile. 206 // 207 // Note: this assertion catches potential performance issues. 208 // Failing this assertion is not a correctness/security problem. 209 // We therefore ignore cases involving resource exhaustion (OOM, 210 // stack overflow, etc), or stubs purged by GC. 211 HashNumber hash = mozilla::AddToHash(icScript->hash(cx_), runningScriptHash_); 212 if (outerScript_->jitScript()->hasFailedICHash()) { 213 HashNumber oldHash = outerScript_->jitScript()->getFailedICHash(); 214 MOZ_ASSERT_IF(hash == oldHash && !js::SupportDifferentialTesting(), 215 cx_->hadResourceExhaustion()); 216 } 217 snapshot->setICHash(hash); 218 #endif 219 220 return snapshot; 221 } 222 223 template <typename T, typename... Args> 224 [[nodiscard]] static bool AddOpSnapshot(TempAllocator& alloc, 225 WarpOpSnapshotList& snapshots, 226 uint32_t offset, Args&&... args) { 227 T* snapshot = new (alloc.fallible()) T(offset, std::forward<Args>(args)...); 228 if (!snapshot) { 229 return false; 230 } 231 232 snapshots.insertBack(snapshot); 233 return true; 234 } 235 236 [[nodiscard]] static bool AddWarpGetImport(TempAllocator& alloc, 237 WarpOpSnapshotList& snapshots, 238 uint32_t offset, JSScript* script, 239 PropertyName* name) { 240 ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script); 241 MOZ_ASSERT(env); 242 243 mozilla::Maybe<PropertyInfo> prop; 244 ModuleEnvironmentObject* targetEnv; 245 MOZ_ALWAYS_TRUE(env->lookupImport(NameToId(name), &targetEnv, &prop)); 246 247 uint32_t numFixedSlots = targetEnv->numFixedSlots(); 248 uint32_t slot = prop->slot(); 249 250 // In the rare case where this import hasn't been initialized already (we have 251 // an import cycle where modules reference each other's imports), we need a 252 // check. 253 bool needsLexicalCheck = 254 targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL); 255 256 return AddOpSnapshot<WarpGetImport>(alloc, snapshots, offset, targetEnv, 257 numFixedSlots, slot, needsLexicalCheck); 258 } 259 260 ICEntry& WarpScriptOracle::getICEntryAndFallback(BytecodeLocation loc, 261 ICFallbackStub** fallback) { 262 const uint32_t offset = loc.bytecodeToOffset(script_); 263 264 do { 265 *fallback = icScript_->fallbackStub(icEntryIndex_); 266 icEntryIndex_++; 267 } while ((*fallback)->pcOffset() < offset); 268 269 MOZ_ASSERT((*fallback)->pcOffset() == offset); 270 return icScript_->icEntry(icEntryIndex_ - 1); 271 } 272 273 WarpEnvironment WarpScriptOracle::createEnvironment() { 274 // Don't do anything if the script doesn't use the environment chain. 275 if (!script_->jitScript()->usesEnvironmentChain()) { 276 return WarpEnvironment(NoEnvironment()); 277 } 278 279 if (script_->isModule()) { 280 ModuleObject* module = script_->module(); 281 JSObject* obj = &module->initialEnvironment(); 282 return WarpEnvironment(ConstantObjectEnvironment(obj)); 283 } 284 285 JSFunction* fun = script_->function(); 286 if (!fun) { 287 // For global scripts without a non-syntactic global scope, the environment 288 // chain is the global lexical environment. 289 MOZ_ASSERT(!script_->isForEval()); 290 MOZ_ASSERT(!script_->hasNonSyntacticScope()); 291 JSObject* obj = &script_->global().lexicalEnvironment(); 292 return WarpEnvironment(ConstantObjectEnvironment(obj)); 293 } 294 295 auto [callObjectTemplate, namedLambdaTemplate] = 296 script_->jitScript()->functionEnvironmentTemplates(fun); 297 298 gc::Heap initialHeap = gc::Heap::Default; 299 JitScript* jitScript = script_->jitScript(); 300 if (jitScript->hasEnvAllocSite()) { 301 initialHeap = jitScript->icScript()->maybeEnvAllocSite()->initialHeap(); 302 } 303 304 return WarpEnvironment(FunctionEnvironment(callObjectTemplate, 305 namedLambdaTemplate, initialHeap)); 306 } 307 308 AbortReasonOr<WarpScriptSnapshot*> WarpScriptOracle::createScriptSnapshot() { 309 MOZ_ASSERT(script_->hasJitScript()); 310 311 if (!script_->jitScript()->ensureHasCachedIonData(cx_, script_)) { 312 return abort(AbortReason::Error); 313 } 314 315 if (script_->failedBoundsCheck()) { 316 oracle_->bailoutInfo().setFailedBoundsCheck(); 317 } 318 if (script_->failedLexicalCheck()) { 319 oracle_->bailoutInfo().setFailedLexicalCheck(); 320 } 321 322 WarpEnvironment environment = createEnvironment(); 323 324 // Unfortunately LinkedList<> asserts the list is empty in its destructor. 325 // Clear the list if we abort compilation. 326 WarpOpSnapshotList opSnapshots; 327 auto autoClearOpSnapshots = 328 mozilla::MakeScopeExit([&] { opSnapshots.clear(); }); 329 330 ModuleObject* moduleObject = nullptr; 331 332 // Analyze the bytecode. Abort compilation for unsupported ops and create 333 // WarpOpSnapshots. 334 for (BytecodeLocation loc : AllBytecodesIterable(script_)) { 335 JSOp op = loc.getOp(); 336 uint32_t offset = loc.bytecodeToOffset(script_); 337 switch (op) { 338 case JSOp::Arguments: { 339 MOZ_ASSERT(script_->needsArgsObj()); 340 bool mapped = script_->hasMappedArgsObj(); 341 ArgumentsObject* templateObj = 342 script_->global().maybeArgumentsTemplateObject(mapped); 343 if (!AddOpSnapshot<WarpArguments>(alloc_, opSnapshots, offset, 344 templateObj)) { 345 return abort(AbortReason::Alloc); 346 } 347 break; 348 } 349 case JSOp::RegExp: { 350 bool hasShared = loc.getRegExp(script_)->hasShared(); 351 if (!AddOpSnapshot<WarpRegExp>(alloc_, opSnapshots, offset, 352 hasShared)) { 353 return abort(AbortReason::Alloc); 354 } 355 break; 356 } 357 358 case JSOp::FunctionThis: 359 if (!script_->strict() && script_->hasNonSyntacticScope()) { 360 // Abort because MBoxNonStrictThis doesn't support non-syntactic 361 // scopes (a deprecated SpiderMonkey mechanism). If this becomes an 362 // issue we could support it by refactoring GetFunctionThis to not 363 // take a frame pointer and then call that. 364 return abort(AbortReason::Disable, 365 "JSOp::FunctionThis with non-syntactic scope"); 366 } 367 break; 368 369 case JSOp::GlobalThis: 370 MOZ_ASSERT(!script_->hasNonSyntacticScope()); 371 break; 372 373 case JSOp::BuiltinObject: { 374 // If we already resolved this built-in we can bake it in. 375 auto kind = loc.getBuiltinObjectKind(); 376 if (JSObject* proto = MaybeGetBuiltinObject(cx_->global(), kind)) { 377 if (!AddOpSnapshot<WarpBuiltinObject>(alloc_, opSnapshots, offset, 378 proto)) { 379 return abort(AbortReason::Alloc); 380 } 381 } 382 break; 383 } 384 385 case JSOp::GetIntrinsic: { 386 // If we already cloned this intrinsic we can bake it in. 387 // NOTE: When the initializer runs in a content global, we also have to 388 // worry about nursery objects. These quickly tenure and stay that 389 // way so this is only a temporary problem. 390 PropertyName* name = loc.getPropertyName(script_); 391 Value val; 392 if (cx_->global()->maybeGetIntrinsicValue(name, &val, cx_) && 393 JS::GCPolicy<Value>::isTenured(val)) { 394 if (!AddOpSnapshot<WarpGetIntrinsic>(alloc_, opSnapshots, offset, 395 val)) { 396 return abort(AbortReason::Alloc); 397 } 398 } 399 break; 400 } 401 402 case JSOp::ImportMeta: { 403 if (!moduleObject) { 404 moduleObject = GetModuleObjectForScript(script_); 405 MOZ_ASSERT(moduleObject->isTenured()); 406 } 407 break; 408 } 409 410 case JSOp::GetImport: { 411 PropertyName* name = loc.getPropertyName(script_); 412 if (!AddWarpGetImport(alloc_, opSnapshots, offset, script_, name)) { 413 return abort(AbortReason::Alloc); 414 } 415 break; 416 } 417 418 case JSOp::Lambda: { 419 JSFunction* fun = loc.getFunction(script_); 420 if (IsAsmJSModule(fun)) { 421 return abort(AbortReason::Disable, "asm.js module function lambda"); 422 } 423 MOZ_TRY(maybeInlineIC(opSnapshots, loc)); 424 break; 425 } 426 427 case JSOp::GetElemSuper: { 428 #if defined(JS_CODEGEN_X86) 429 // x86 does not have enough registers. 430 return abort(AbortReason::Disable, 431 "GetElemSuper is not supported on x86"); 432 #else 433 MOZ_TRY(maybeInlineIC(opSnapshots, loc)); 434 break; 435 #endif 436 } 437 438 case JSOp::Rest: { 439 if (Shape* shape = 440 script_->global().maybeArrayShapeWithDefaultProto()) { 441 if (!AddOpSnapshot<WarpRest>(alloc_, opSnapshots, offset, shape)) { 442 return abort(AbortReason::Alloc); 443 } 444 } 445 break; 446 } 447 448 case JSOp::BindUnqualifiedGName: { 449 GlobalObject* global = &script_->global(); 450 PropertyName* name = loc.getPropertyName(script_); 451 if (JSObject* env = 452 MaybeOptimizeBindUnqualifiedGlobalName(global, name)) { 453 MOZ_ASSERT(env->isTenured()); 454 if (!AddOpSnapshot<WarpBindUnqualifiedGName>(alloc_, opSnapshots, 455 offset, env)) { 456 return abort(AbortReason::Alloc); 457 } 458 } else { 459 MOZ_TRY(maybeInlineIC(opSnapshots, loc)); 460 } 461 break; 462 } 463 464 case JSOp::PushVarEnv: { 465 Rooted<VarScope*> scope(cx_, &loc.getScope(script_)->as<VarScope>()); 466 467 auto* templateObj = 468 VarEnvironmentObject::createTemplateObject(cx_, scope); 469 if (!templateObj) { 470 return abort(AbortReason::Alloc); 471 } 472 MOZ_ASSERT(templateObj->isTenured()); 473 474 if (!AddOpSnapshot<WarpVarEnvironment>(alloc_, opSnapshots, offset, 475 templateObj)) { 476 return abort(AbortReason::Alloc); 477 } 478 break; 479 } 480 481 case JSOp::PushLexicalEnv: 482 case JSOp::FreshenLexicalEnv: 483 case JSOp::RecreateLexicalEnv: { 484 Rooted<LexicalScope*> scope(cx_, 485 &loc.getScope(script_)->as<LexicalScope>()); 486 487 auto* templateObj = 488 BlockLexicalEnvironmentObject::createTemplateObject(cx_, scope); 489 if (!templateObj) { 490 return abort(AbortReason::Alloc); 491 } 492 MOZ_ASSERT(templateObj->isTenured()); 493 494 if (!AddOpSnapshot<WarpLexicalEnvironment>(alloc_, opSnapshots, offset, 495 templateObj)) { 496 return abort(AbortReason::Alloc); 497 } 498 break; 499 } 500 501 case JSOp::PushClassBodyEnv: { 502 Rooted<ClassBodyScope*> scope( 503 cx_, &loc.getScope(script_)->as<ClassBodyScope>()); 504 505 auto* templateObj = 506 ClassBodyLexicalEnvironmentObject::createTemplateObject(cx_, scope); 507 if (!templateObj) { 508 return abort(AbortReason::Alloc); 509 } 510 MOZ_ASSERT(templateObj->isTenured()); 511 512 if (!AddOpSnapshot<WarpClassBodyEnvironment>(alloc_, opSnapshots, 513 offset, templateObj)) { 514 return abort(AbortReason::Alloc); 515 } 516 break; 517 } 518 519 case JSOp::String: 520 if (!loc.atomizeString(cx_, script_)) { 521 return abort(AbortReason::Alloc); 522 } 523 break; 524 case JSOp::GetName: 525 case JSOp::GetGName: 526 case JSOp::GetProp: 527 case JSOp::GetElem: 528 case JSOp::SetProp: 529 case JSOp::StrictSetProp: 530 case JSOp::Call: 531 case JSOp::CallContent: 532 case JSOp::CallIgnoresRv: 533 case JSOp::CallIter: 534 case JSOp::CallContentIter: 535 case JSOp::New: 536 case JSOp::NewContent: 537 case JSOp::SuperCall: 538 case JSOp::SpreadCall: 539 case JSOp::SpreadNew: 540 case JSOp::SpreadSuperCall: 541 case JSOp::ToNumeric: 542 case JSOp::Pos: 543 case JSOp::Inc: 544 case JSOp::Dec: 545 case JSOp::Neg: 546 case JSOp::BitNot: 547 case JSOp::Iter: 548 case JSOp::Eq: 549 case JSOp::Ne: 550 case JSOp::Lt: 551 case JSOp::Le: 552 case JSOp::Gt: 553 case JSOp::Ge: 554 case JSOp::StrictEq: 555 case JSOp::StrictNe: 556 case JSOp::BindName: 557 case JSOp::BindUnqualifiedName: 558 case JSOp::GetBoundName: 559 case JSOp::Add: 560 case JSOp::Sub: 561 case JSOp::Mul: 562 case JSOp::Div: 563 case JSOp::Mod: 564 case JSOp::Pow: 565 case JSOp::BitAnd: 566 case JSOp::BitOr: 567 case JSOp::BitXor: 568 case JSOp::Lsh: 569 case JSOp::Rsh: 570 case JSOp::Ursh: 571 case JSOp::In: 572 case JSOp::HasOwn: 573 case JSOp::CheckPrivateField: 574 case JSOp::Instanceof: 575 case JSOp::GetPropSuper: 576 case JSOp::InitProp: 577 case JSOp::InitLockedProp: 578 case JSOp::InitHiddenProp: 579 case JSOp::InitElem: 580 case JSOp::InitHiddenElem: 581 case JSOp::InitLockedElem: 582 case JSOp::InitElemInc: 583 case JSOp::SetName: 584 case JSOp::StrictSetName: 585 case JSOp::SetGName: 586 case JSOp::StrictSetGName: 587 case JSOp::InitGLexical: 588 case JSOp::SetElem: 589 case JSOp::StrictSetElem: 590 case JSOp::ToPropertyKey: 591 case JSOp::OptimizeSpreadCall: 592 case JSOp::Typeof: 593 case JSOp::TypeofExpr: 594 case JSOp::TypeofEq: 595 case JSOp::NewObject: 596 case JSOp::NewInit: 597 case JSOp::NewArray: 598 case JSOp::JumpIfFalse: 599 case JSOp::JumpIfTrue: 600 case JSOp::And: 601 case JSOp::Or: 602 case JSOp::Not: 603 case JSOp::CloseIter: 604 case JSOp::OptimizeGetIterator: 605 MOZ_TRY(maybeInlineIC(opSnapshots, loc)); 606 break; 607 608 case JSOp::Nop: 609 case JSOp::NopDestructuring: 610 case JSOp::NopIsAssignOp: 611 case JSOp::TryDestructuring: 612 case JSOp::Lineno: 613 case JSOp::DebugLeaveLexicalEnv: 614 case JSOp::Undefined: 615 case JSOp::Void: 616 case JSOp::Null: 617 case JSOp::Hole: 618 case JSOp::Uninitialized: 619 case JSOp::IsConstructing: 620 case JSOp::False: 621 case JSOp::True: 622 case JSOp::Zero: 623 case JSOp::One: 624 case JSOp::Int8: 625 case JSOp::Uint16: 626 case JSOp::Uint24: 627 case JSOp::Int32: 628 case JSOp::Double: 629 case JSOp::BigInt: 630 case JSOp::Symbol: 631 case JSOp::Pop: 632 case JSOp::PopN: 633 case JSOp::Dup: 634 case JSOp::Dup2: 635 case JSOp::DupAt: 636 case JSOp::Swap: 637 case JSOp::Pick: 638 case JSOp::Unpick: 639 case JSOp::GetLocal: 640 case JSOp::SetLocal: 641 case JSOp::InitLexical: 642 case JSOp::GetArg: 643 case JSOp::GetFrameArg: 644 case JSOp::SetArg: 645 case JSOp::ArgumentsLength: 646 case JSOp::GetActualArg: 647 case JSOp::JumpTarget: 648 case JSOp::LoopHead: 649 case JSOp::Case: 650 case JSOp::Default: 651 case JSOp::Coalesce: 652 case JSOp::Goto: 653 case JSOp::DebugCheckSelfHosted: 654 case JSOp::DynamicImport: 655 case JSOp::ToString: 656 case JSOp::GlobalOrEvalDeclInstantiation: 657 case JSOp::BindVar: 658 case JSOp::MutateProto: 659 case JSOp::Callee: 660 case JSOp::ToAsyncIter: 661 case JSOp::ObjWithProto: 662 case JSOp::GetAliasedVar: 663 case JSOp::SetAliasedVar: 664 case JSOp::InitAliasedLexical: 665 case JSOp::EnvCallee: 666 case JSOp::MoreIter: 667 case JSOp::EndIter: 668 case JSOp::IsNoIter: 669 case JSOp::IsNullOrUndefined: 670 case JSOp::DelProp: 671 case JSOp::StrictDelProp: 672 case JSOp::DelElem: 673 case JSOp::StrictDelElem: 674 case JSOp::SetFunName: 675 case JSOp::PopLexicalEnv: 676 case JSOp::ImplicitThis: 677 case JSOp::CheckClassHeritage: 678 case JSOp::CheckThis: 679 case JSOp::CheckThisReinit: 680 case JSOp::Generator: 681 case JSOp::AfterYield: 682 case JSOp::FinalYieldRval: 683 case JSOp::AsyncResolve: 684 case JSOp::AsyncReject: 685 case JSOp::CheckResumeKind: 686 case JSOp::CanSkipAwait: 687 case JSOp::MaybeExtractAwaitValue: 688 case JSOp::AsyncAwait: 689 case JSOp::Await: 690 case JSOp::CheckReturn: 691 case JSOp::CheckLexical: 692 case JSOp::CheckAliasedLexical: 693 case JSOp::InitHomeObject: 694 case JSOp::SuperBase: 695 case JSOp::SuperFun: 696 case JSOp::InitElemArray: 697 case JSOp::InitPropGetter: 698 case JSOp::InitPropSetter: 699 case JSOp::InitHiddenPropGetter: 700 case JSOp::InitHiddenPropSetter: 701 case JSOp::InitElemGetter: 702 case JSOp::InitElemSetter: 703 case JSOp::InitHiddenElemGetter: 704 case JSOp::InitHiddenElemSetter: 705 case JSOp::NewTarget: 706 case JSOp::Object: 707 case JSOp::CallSiteObj: 708 case JSOp::CheckIsObj: 709 case JSOp::CheckObjCoercible: 710 case JSOp::FunWithProto: 711 case JSOp::Debugger: 712 case JSOp::TableSwitch: 713 case JSOp::Exception: 714 case JSOp::ExceptionAndStack: 715 case JSOp::Throw: 716 case JSOp::ThrowWithStack: 717 case JSOp::ThrowSetConst: 718 case JSOp::SetRval: 719 case JSOp::GetRval: 720 case JSOp::Return: 721 case JSOp::RetRval: 722 case JSOp::InitialYield: 723 case JSOp::Yield: 724 case JSOp::ResumeKind: 725 case JSOp::ThrowMsg: 726 case JSOp::Try: 727 case JSOp::Finally: 728 case JSOp::NewPrivateName: 729 case JSOp::StrictConstantEq: 730 case JSOp::StrictConstantNe: 731 #ifdef ENABLE_EXPLICIT_RESOURCE_MANAGEMENT 732 case JSOp::AddDisposable: 733 case JSOp::TakeDisposeCapability: 734 case JSOp::CreateSuppressedError: 735 #endif 736 // Supported by WarpBuilder. Nothing to do. 737 break; 738 739 // Unsupported ops. Don't use a 'default' here, we want to trigger a 740 // compiler warning when adding a new JSOp. 741 #define DEF_CASE(OP) case JSOp::OP: 742 WARP_UNSUPPORTED_OPCODE_LIST(DEF_CASE) 743 #undef DEF_CASE 744 #ifdef DEBUG 745 return abort(AbortReason::Disable, "Unsupported opcode: %s", 746 CodeName(op)); 747 #else 748 return abort(AbortReason::Disable, "Unsupported opcode: %u", 749 uint8_t(op)); 750 #endif 751 } 752 } 753 754 auto* scriptSnapshot = new (alloc_.fallible()) WarpScriptSnapshot( 755 script_, environment, std::move(opSnapshots), moduleObject); 756 if (!scriptSnapshot) { 757 return abort(AbortReason::Alloc); 758 } 759 760 autoClearOpSnapshots.release(); 761 return scriptSnapshot; 762 } 763 764 static void LineNumberAndColumn(HandleScript script, BytecodeLocation loc, 765 unsigned* line, 766 JS::LimitedColumnNumberOneOrigin* column) { 767 #ifdef DEBUG 768 *line = PCToLineNumber(script, loc.toRawBytecode(), column); 769 #else 770 *line = script->lineno(); 771 *column = script->column(); 772 #endif 773 } 774 775 static void MaybeSetInliningStateFromJitHints(JSContext* cx, 776 ICFallbackStub* fallbackStub, 777 JSScript* script, 778 BytecodeLocation loc) { 779 // Only update the state if it has already been marked as a candidate. 780 if (fallbackStub->trialInliningState() != TrialInliningState::Candidate) { 781 return; 782 } 783 784 // Make sure the op is inlineable. 785 if (!TrialInliner::IsValidInliningOp(loc.getOp())) { 786 return; 787 } 788 789 if (!cx->runtime()->jitRuntime()->hasJitHintsMap()) { 790 return; 791 } 792 793 JitHintsMap* jitHints = cx->runtime()->jitRuntime()->getJitHintsMap(); 794 uint32_t offset = loc.bytecodeToOffset(script); 795 796 if (jitHints->hasMonomorphicInlineHintAtOffset(script, offset)) { 797 fallbackStub->setTrialInliningState(TrialInliningState::MonomorphicInlined); 798 } 799 } 800 801 template <auto FuseMember, CompilationDependency::Type DepType> 802 struct RealmFuseDependency final : public CompilationDependency { 803 RealmFuseDependency() : CompilationDependency(DepType) {} 804 805 virtual bool registerDependency(JSContext* cx, 806 const IonScriptKey& ionScript) override { 807 MOZ_ASSERT(checkDependency(cx)); 808 809 return (cx->realm()->realmFuses.*FuseMember) 810 .addFuseDependency(cx, ionScript); 811 } 812 813 virtual CompilationDependency* clone(TempAllocator& alloc) const override { 814 return new (alloc.fallible()) RealmFuseDependency<FuseMember, DepType>(); 815 } 816 817 virtual bool checkDependency(JSContext* cx) const override { 818 return (cx->realm()->realmFuses.*FuseMember).intact(); 819 } 820 821 virtual HashNumber hash() const override { 822 return mozilla::HashGeneric(type); 823 } 824 virtual bool operator==(const CompilationDependency& dep) const override { 825 return dep.type == type; 826 } 827 }; 828 829 bool WarpOracle::addFuseDependency(RealmFuses::FuseIndex fuseIndex, 830 bool* stillValid) { 831 auto addIfStillValid = [&](const auto& dep) { 832 if (!dep.checkDependency(cx_)) { 833 *stillValid = false; 834 return true; 835 } 836 *stillValid = true; 837 return mirGen().tracker.addDependency(alloc_, dep); 838 }; 839 840 // Register a compilation dependency for all invalidating fuses that are still 841 // valid. 842 switch (fuseIndex) { 843 case RealmFuses::FuseIndex::OptimizeGetIteratorFuse: { 844 using Dependency = 845 RealmFuseDependency<&RealmFuses::optimizeGetIteratorFuse, 846 CompilationDependency::Type::GetIterator>; 847 return addIfStillValid(Dependency()); 848 } 849 case RealmFuses::FuseIndex::OptimizeArraySpeciesFuse: { 850 using Dependency = 851 RealmFuseDependency<&RealmFuses::optimizeArraySpeciesFuse, 852 CompilationDependency::Type::ArraySpecies>; 853 return addIfStillValid(Dependency()); 854 } 855 case RealmFuses::FuseIndex::OptimizeTypedArraySpeciesFuse: { 856 using Dependency = 857 RealmFuseDependency<&RealmFuses::optimizeTypedArraySpeciesFuse, 858 CompilationDependency::Type::TypedArraySpecies>; 859 return addIfStillValid(Dependency()); 860 } 861 case RealmFuses::FuseIndex::OptimizeRegExpPrototypeFuse: { 862 using Dependency = 863 RealmFuseDependency<&RealmFuses::optimizeRegExpPrototypeFuse, 864 CompilationDependency::Type::RegExpPrototype>; 865 return addIfStillValid(Dependency()); 866 } 867 default: 868 MOZ_ASSERT(!RealmFuses::isInvalidatingFuse(fuseIndex)); 869 *stillValid = true; 870 return true; 871 } 872 } 873 874 template <auto FuseMember, CompilationDependency::Type DepType> 875 struct RuntimeFuseDependency final : public CompilationDependency { 876 explicit RuntimeFuseDependency() : CompilationDependency(DepType) {} 877 878 bool registerDependency(JSContext* cx, 879 const IonScriptKey& ionScript) override { 880 MOZ_ASSERT(checkDependency(cx)); 881 return (cx->runtime()->runtimeFuses.ref().*FuseMember) 882 .addFuseDependency(cx, ionScript); 883 } 884 885 CompilationDependency* clone(TempAllocator& alloc) const override { 886 return new (alloc.fallible()) RuntimeFuseDependency<FuseMember, DepType>(); 887 } 888 889 bool checkDependency(JSContext* cx) const override { 890 return (cx->runtime()->runtimeFuses.ref().*FuseMember).intact(); 891 } 892 893 HashNumber hash() const override { return mozilla::HashGeneric(type); } 894 895 bool operator==(const CompilationDependency& dep) const override { 896 // Since this dependency is runtime wide, they are all equal. 897 return dep.type == type; 898 } 899 }; 900 901 bool WarpOracle::addFuseDependency(RuntimeFuses::FuseIndex fuseIndex, 902 bool* stillValid) { 903 MOZ_ASSERT(RuntimeFuses::isInvalidatingFuse(fuseIndex), 904 "All current runtime fuses are invalidating"); 905 906 auto addIfStillValid = [&](const auto& dep) { 907 if (!dep.checkDependency(cx_)) { 908 *stillValid = false; 909 return true; 910 } 911 *stillValid = true; 912 return mirGen().tracker.addDependency(alloc_, dep); 913 }; 914 915 // Register a compilation dependency for all fuses that are still valid. 916 switch (fuseIndex) { 917 case RuntimeFuses::FuseIndex::HasSeenObjectEmulateUndefinedFuse: { 918 using Dependency = RuntimeFuseDependency< 919 &RuntimeFuses::hasSeenObjectEmulateUndefinedFuse, 920 CompilationDependency::Type::EmulatesUndefined>; 921 return addIfStillValid(Dependency()); 922 } 923 case RuntimeFuses::FuseIndex::HasSeenArrayExceedsInt32LengthFuse: { 924 using Dependency = RuntimeFuseDependency< 925 &RuntimeFuses::hasSeenArrayExceedsInt32LengthFuse, 926 CompilationDependency::Type::ArrayExceedsInt32Length>; 927 return addIfStillValid(Dependency()); 928 } 929 case RuntimeFuses::FuseIndex::DefaultLocaleHasDefaultCaseMappingFuse: { 930 using Dependency = RuntimeFuseDependency< 931 &RuntimeFuses::defaultLocaleHasDefaultCaseMappingFuse, 932 CompilationDependency::Type::DefaultCaseMapping>; 933 return addIfStillValid(Dependency()); 934 } 935 case RuntimeFuses::FuseIndex::LastFuseIndex: 936 break; 937 } 938 MOZ_CRASH("invalid runtime fuse index"); 939 } 940 941 class ObjectPropertyFuseDependency final : public CompilationDependency { 942 ObjectFuse* fuse_; 943 uint32_t expectedGeneration_; 944 uint32_t propSlot_; 945 946 public: 947 ObjectPropertyFuseDependency(ObjectFuse* fuse, uint32_t expectedGeneration, 948 uint32_t propSlot) 949 : CompilationDependency(CompilationDependency::Type::ObjectFuseProperty), 950 fuse_(fuse), 951 expectedGeneration_(expectedGeneration), 952 propSlot_(propSlot) {} 953 954 virtual bool registerDependency(JSContext* cx, 955 const IonScriptKey& ionScript) override { 956 MOZ_ASSERT(checkDependency(cx)); 957 return fuse_->addDependency(propSlot_, ionScript); 958 } 959 960 virtual CompilationDependency* clone(TempAllocator& alloc) const override { 961 return new (alloc.fallible()) 962 ObjectPropertyFuseDependency(fuse_, expectedGeneration_, propSlot_); 963 } 964 965 virtual bool checkDependency(JSContext* cx) const override { 966 return fuse_->checkPropertyIsConstant(expectedGeneration_, propSlot_); 967 } 968 969 virtual HashNumber hash() const override { 970 return mozilla::HashGeneric(type, fuse_, expectedGeneration_, propSlot_); 971 } 972 virtual bool operator==(const CompilationDependency& dep) const override { 973 if (dep.type != type) { 974 return false; 975 } 976 auto& other = static_cast<const ObjectPropertyFuseDependency&>(dep); 977 return fuse_ == other.fuse_ && 978 expectedGeneration_ == other.expectedGeneration_ && 979 propSlot_ == other.propSlot_; 980 } 981 }; 982 983 AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots, 984 BytecodeLocation loc) { 985 // Do one of the following: 986 // 987 // * If the Baseline IC has a single ICStub we can inline, add a WarpCacheIR 988 // snapshot to transpile it to MIR. 989 // 990 // * If that single ICStub is a call IC with a known target, instead add a 991 // WarpInline snapshot to transpile the guards to MIR and inline the target. 992 // 993 // * If that single ICStub has a CacheIR fuse guard for an invalidating fuse, 994 // add a compilation dependency for this fuse. If the fuse is no longer 995 // valid, add a WarpBailout snapshot to avoid throwing away the JIT code 996 // immediately after compilation. 997 // 998 // * If the Baseline IC is cold (never executed), add a WarpBailout snapshot 999 // so that we can collect information in Baseline. 1000 // 1001 // * Else, don't add a snapshot and rely on WarpBuilder adding an Ion IC. 1002 1003 MOZ_ASSERT(loc.opHasIC()); 1004 1005 // Don't create snapshots when testing ICs. 1006 if (JitOptions.forceInlineCaches) { 1007 return Ok(); 1008 } 1009 1010 ICFallbackStub* fallbackStub; 1011 const ICEntry& entry = getICEntryAndFallback(loc, &fallbackStub); 1012 1013 uint32_t offset = loc.bytecodeToOffset(script_); 1014 1015 // Set the trial inlining state directly if there is a hint cached from a 1016 // previous compilation. 1017 MaybeSetInliningStateFromJitHints(cx_, fallbackStub, script_, loc); 1018 1019 // Clear the used-by-transpiler flag on the IC. It can still be set from a 1020 // previous compilation because we don't clear the flag on every IC when 1021 // invalidating. 1022 fallbackStub->clearUsedByTranspiler(); 1023 1024 if (entry.firstStub() == fallbackStub) { 1025 [[maybe_unused]] unsigned line; 1026 [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column; 1027 LineNumberAndColumn(script_, loc, &line, &column); 1028 1029 // No optimized stubs. 1030 JitSpew(JitSpew_WarpTranspiler, 1031 "fallback stub (entered-count: %" PRIu32 1032 ") for JSOp::%s @ %s:%u:%u", 1033 fallbackStub->enteredCount(), CodeName(loc.getOp()), 1034 script_->filename(), line, column.oneOriginValue()); 1035 1036 // If the fallback stub was used but there's no optimized stub, use an IC. 1037 if (fallbackStub->enteredCount() != 0) { 1038 return Ok(); 1039 } 1040 1041 // Cold IC. Bailout to collect information. 1042 if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) { 1043 return abort(AbortReason::Alloc); 1044 } 1045 return Ok(); 1046 } 1047 1048 // Try to fold stubs, in case new stubs were added after trial inlining. 1049 if (!TryFoldingStubs(cx_, fallbackStub, script_, icScript_)) { 1050 return abort(AbortReason::Error); 1051 } 1052 1053 // Don't transpile if this IC ever encountered a case where it had 1054 // no stub to attach. 1055 if (fallbackStub->state().hasFailures()) { 1056 [[maybe_unused]] unsigned line; 1057 [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column; 1058 LineNumberAndColumn(script_, loc, &line, &column); 1059 1060 JitSpew(JitSpew_WarpTranspiler, "Failed to attach for JSOp::%s @ %s:%u:%u", 1061 CodeName(loc.getOp()), script_->filename(), line, 1062 column.oneOriginValue()); 1063 return Ok(); 1064 } 1065 1066 ICCacheIRStub* stub = entry.firstStub()->toCacheIRStub(); 1067 1068 // Don't transpile if there are other stubs with entered-count > 0. Counters 1069 // are reset when a new stub is attached so this means the stub that was added 1070 // most recently didn't handle all cases. 1071 // If this code is changed, ICScript::hash may also need changing. 1072 bool firstStubHandlesAllCases = true; 1073 for (ICStub* next = stub->next(); next; next = next->maybeNext()) { 1074 if (next->enteredCount() != 0) { 1075 firstStubHandlesAllCases = false; 1076 break; 1077 } 1078 } 1079 1080 if (!firstStubHandlesAllCases) { 1081 // In some polymorphic cases, we can generate better code than the 1082 // default fallback if we know the observed types of the operands 1083 // and their relative frequency. 1084 if (ICSupportsPolymorphicTypeData(loc.getOp()) && 1085 fallbackStub->enteredCount() == 0) { 1086 bool inlinedPolymorphicTypes = MOZ_TRY( 1087 maybeInlinePolymorphicTypes(snapshots, loc, stub, fallbackStub)); 1088 if (inlinedPolymorphicTypes) { 1089 return Ok(); 1090 } 1091 } 1092 1093 [[maybe_unused]] unsigned line; 1094 [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column; 1095 LineNumberAndColumn(script_, loc, &line, &column); 1096 1097 JitSpew(JitSpew_WarpTranspiler, 1098 "multiple active stubs for JSOp::%s @ %s:%u:%u", 1099 CodeName(loc.getOp()), script_->filename(), line, 1100 column.oneOriginValue()); 1101 return Ok(); 1102 } 1103 1104 const CacheIRStubInfo* stubInfo = stub->stubInfo(); 1105 const uint8_t* stubData = stub->stubDataStart(); 1106 1107 // List of shapes for a GuardMultipleShapes op with a small number of shapes. 1108 mozilla::Maybe<ShapeListSnapshot> shapeList; 1109 mozilla::Maybe<ShapeListWithOffsetsSnapshot> shapeListWithOffsets; 1110 1111 // Only create a snapshot if all opcodes are supported by the transpiler. 1112 CacheIRReader reader(stubInfo); 1113 bool hasInvalidFuseGuard = false; 1114 while (reader.more()) { 1115 CacheOp op = reader.readOp(); 1116 CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)]; 1117 1118 if (!opInfo.transpile) { 1119 [[maybe_unused]] unsigned line; 1120 [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column; 1121 LineNumberAndColumn(script_, loc, &line, &column); 1122 1123 MOZ_ASSERT( 1124 fallbackStub->trialInliningState() != TrialInliningState::Inlined, 1125 "Trial-inlined stub not supported by transpiler"); 1126 1127 // Unsupported CacheIR opcode. 1128 JitSpew(JitSpew_WarpTranspiler, 1129 "unsupported CacheIR opcode %s for JSOp::%s @ %s:%u:%u", 1130 CacheIROpNames[size_t(op)], CodeName(loc.getOp()), 1131 script_->filename(), line, column.oneOriginValue()); 1132 return Ok(); 1133 } 1134 1135 // While on the main thread, ensure code stubs exist for ops that require 1136 // them. 1137 switch (op) { 1138 case CacheOp::CallRegExpMatcherResult: 1139 reader.argsForCallRegExpMatcherResult(); // Unused. 1140 if (!oracle_->snapshotJitZoneStub(JitZone::StubKind::RegExpMatcher)) { 1141 return abort(AbortReason::Error); 1142 } 1143 break; 1144 case CacheOp::CallRegExpSearcherResult: 1145 reader.argsForCallRegExpSearcherResult(); // Unused. 1146 if (!oracle_->snapshotJitZoneStub(JitZone::StubKind::RegExpSearcher)) { 1147 return abort(AbortReason::Error); 1148 } 1149 break; 1150 case CacheOp::RegExpBuiltinExecMatchResult: 1151 reader.argsForRegExpBuiltinExecMatchResult(); // Unused. 1152 if (!oracle_->snapshotJitZoneStub(JitZone::StubKind::RegExpExecMatch)) { 1153 return abort(AbortReason::Error); 1154 } 1155 break; 1156 case CacheOp::RegExpBuiltinExecTestResult: 1157 reader.argsForRegExpBuiltinExecTestResult(); // Unused. 1158 if (!oracle_->snapshotJitZoneStub(JitZone::StubKind::RegExpExecTest)) { 1159 return abort(AbortReason::Error); 1160 } 1161 break; 1162 case CacheOp::ConcatStringsResult: 1163 reader.argsForConcatStringsResult(); // Unused. 1164 if (!oracle_->snapshotJitZoneStub(JitZone::StubKind::StringConcat)) { 1165 return abort(AbortReason::Error); 1166 } 1167 break; 1168 case CacheOp::GuardFuse: { 1169 auto [fuseIndex] = reader.argsForGuardFuse(); 1170 bool stillValid; 1171 if (!oracle_->addFuseDependency(fuseIndex, &stillValid)) { 1172 return abort(AbortReason::Alloc); 1173 } 1174 if (!stillValid) { 1175 hasInvalidFuseGuard = true; 1176 } 1177 break; 1178 } 1179 case CacheOp::GuardRuntimeFuse: { 1180 auto [fuseIndex] = reader.argsForGuardRuntimeFuse(); 1181 bool stillValid; 1182 if (!oracle_->addFuseDependency(fuseIndex, &stillValid)) { 1183 return abort(AbortReason::Alloc); 1184 } 1185 if (!stillValid) { 1186 hasInvalidFuseGuard = true; 1187 } 1188 break; 1189 } 1190 case CacheOp::GuardObjectFuseProperty: { 1191 auto args = reader.argsForGuardObjectFuseProperty(); 1192 ObjectFuse* fuse = reinterpret_cast<ObjectFuse*>( 1193 stubInfo->getStubRawWord(stubData, args.objFuseOffset)); 1194 uint32_t generation = 1195 stubInfo->getStubRawInt32(stubData, args.expectedGenerationOffset); 1196 uint32_t propIndex = 1197 stubInfo->getStubRawInt32(stubData, args.propIndexOffset); 1198 uint32_t propMask = 1199 stubInfo->getStubRawInt32(stubData, args.propMaskOffset); 1200 uint32_t propSlot = 1201 ObjectFuse::propertySlotFromIndexAndMask(propIndex, propMask); 1202 ObjectPropertyFuseDependency dep(fuse, generation, propSlot); 1203 if (dep.checkDependency(cx_)) { 1204 if (!oracle_->mirGen().tracker.addDependency(alloc_, dep)) { 1205 return abort(AbortReason::Alloc); 1206 } 1207 } else { 1208 hasInvalidFuseGuard = true; 1209 } 1210 break; 1211 } 1212 case CacheOp::GuardMultipleShapes: { 1213 auto args = reader.argsForGuardMultipleShapes(); 1214 JSObject* shapes = stubInfo->getStubField<StubField::Type::JSObject>( 1215 stub, args.shapesOffset); 1216 auto* shapesObject = &shapes->as<ShapeListObject>(); 1217 MOZ_ASSERT(shapeList.isNothing()); 1218 size_t numShapes = shapesObject->length(); 1219 if (ShapeListSnapshot::shouldSnapshot(numShapes)) { 1220 shapeList.emplace(); 1221 for (size_t i = 0; i < numShapes; i++) { 1222 shapeList->init(i, shapesObject->get(i)); 1223 } 1224 } 1225 break; 1226 } 1227 case CacheOp::GuardMultipleShapesToOffset: { 1228 auto args = reader.argsForGuardMultipleShapesToOffset(); 1229 JSObject* shapes = stubInfo->getStubField<StubField::Type::JSObject>( 1230 stub, args.shapesOffset); 1231 auto* shapesObject = &shapes->as<ShapeListWithOffsetsObject>(); 1232 MOZ_ASSERT(shapeListWithOffsets.isNothing()); 1233 size_t numShapes = shapesObject->numShapes(); 1234 if (ShapeListSnapshot::shouldSnapshot(numShapes)) { 1235 shapeListWithOffsets.emplace(); 1236 for (size_t i = 0; i < numShapes; i++) { 1237 shapeListWithOffsets->init(i, shapesObject->getShape(i), 1238 shapesObject->getOffset(i)); 1239 } 1240 } 1241 break; 1242 } 1243 default: 1244 reader.skip(opInfo.argLength); 1245 break; 1246 } 1247 } 1248 1249 // Insert a bailout if the stub has a guard for an invalidating fuse that's 1250 // no longer intact. 1251 if (hasInvalidFuseGuard) { 1252 if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) { 1253 return abort(AbortReason::Alloc); 1254 } 1255 return Ok(); 1256 } 1257 1258 // Check GC is not possible between updating stub pointers and creating the 1259 // snapshot. 1260 JS::AutoAssertNoGC nogc; 1261 1262 // Copy the ICStub data to protect against the stub being unlinked or mutated. 1263 // We don't need to copy the CacheIRStubInfo: because we store and trace the 1264 // stub's JitCode*, the baselineCacheIRStubCodes_ map in JitZone will keep it 1265 // alive. 1266 uint8_t* stubDataCopy = nullptr; 1267 size_t bytesNeeded = stubInfo->stubDataSize(); 1268 if (bytesNeeded > 0) { 1269 stubDataCopy = alloc_.allocateArray<uint8_t>(bytesNeeded); 1270 if (!stubDataCopy) { 1271 return abort(AbortReason::Alloc); 1272 } 1273 1274 // Note: nursery pointers are handled below and the read barrier for weak 1275 // pointers is handled above so we can do a bitwise copy here. 1276 std::copy_n(stubData, bytesNeeded, stubDataCopy); 1277 1278 if (!replaceNurseryAndAllocSitePointers(stub, stubInfo, stubDataCopy)) { 1279 return abort(AbortReason::Alloc); 1280 } 1281 } 1282 1283 JitCode* jitCode = stub->jitCode(); 1284 1285 if (fallbackStub->trialInliningState() == TrialInliningState::Inlined || 1286 fallbackStub->trialInliningState() == 1287 TrialInliningState::MonomorphicInlined) { 1288 bool inlinedCall = MOZ_TRY( 1289 maybeInlineCall(snapshots, loc, stub, fallbackStub, stubDataCopy)); 1290 if (inlinedCall) { 1291 return Ok(); 1292 } 1293 } 1294 1295 if (shapeList.isSome()) { 1296 MOZ_ASSERT(shapeListWithOffsets.isNothing()); 1297 if (!AddOpSnapshot<WarpCacheIRWithShapeList>(alloc_, snapshots, offset, 1298 jitCode, stubInfo, 1299 stubDataCopy, *shapeList)) { 1300 return abort(AbortReason::Alloc); 1301 } 1302 } else if (shapeListWithOffsets.isSome()) { 1303 if (!AddOpSnapshot<WarpCacheIRWithShapeListAndOffsets>( 1304 alloc_, snapshots, offset, jitCode, stubInfo, stubDataCopy, 1305 *shapeListWithOffsets)) { 1306 return abort(AbortReason::Alloc); 1307 } 1308 } else { 1309 if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, 1310 stubInfo, stubDataCopy)) { 1311 return abort(AbortReason::Alloc); 1312 } 1313 } 1314 1315 fallbackStub->setUsedByTranspiler(); 1316 1317 return Ok(); 1318 } 1319 1320 AbortReasonOr<bool> WarpScriptOracle::maybeInlineCall( 1321 WarpOpSnapshotList& snapshots, BytecodeLocation loc, ICCacheIRStub* stub, 1322 ICFallbackStub* fallbackStub, uint8_t* stubDataCopy) { 1323 Maybe<InlinableOpData> inlineData = FindInlinableOpData(stub, loc); 1324 if (inlineData.isNothing()) { 1325 return false; 1326 } 1327 1328 RootedScript targetScript(cx_, inlineData->target); 1329 if (!TrialInliner::canInline(cx_, targetScript, script_, loc)) { 1330 return false; 1331 } 1332 1333 // We can speculatively inline scripts while they're in blinterp, 1334 // but by the time we actually Ion-compile the outer script, the 1335 // callee should have at least reached baseline. 1336 if (!targetScript->hasBaselineScript()) { 1337 return false; 1338 } 1339 1340 bool isTrialInlined = 1341 fallbackStub->trialInliningState() == TrialInliningState::Inlined; 1342 MOZ_ASSERT_IF(!isTrialInlined, fallbackStub->trialInliningState() == 1343 TrialInliningState::MonomorphicInlined); 1344 1345 ICScript* icScript = nullptr; 1346 if (isTrialInlined) { 1347 icScript = inlineData->icScript; 1348 } else { 1349 JitScript* jitScript = targetScript->jitScript(); 1350 icScript = jitScript->icScript(); 1351 } 1352 1353 if (!icScript) { 1354 return false; 1355 } 1356 1357 // This is just a cheap check to limit the damage we can do to ourselves if 1358 // we try to monomorphically inline an indirectly recursive call. 1359 if (!isTrialInlined && 1360 info_->inlineScriptTree()->depth() > InlineScriptTree::MaxDepth) { 1361 return false; 1362 } 1363 1364 // And this is a second cheap check to ensure monomorphic inlining doesn't 1365 // cause us to blow past our script size budget. 1366 if (oracle_->accumulatedBytecodeSize() + targetScript->length() > 1367 JitOptions.ionMaxScriptSize) { 1368 return false; 1369 } 1370 1371 // Add the inlined script to the inline script tree. 1372 LifoAlloc* lifoAlloc = alloc_.lifoAlloc(); 1373 InlineScriptTree* inlineScriptTree = info_->inlineScriptTree()->addCallee( 1374 &alloc_, loc.toRawBytecode(), targetScript, !isTrialInlined); 1375 if (!inlineScriptTree) { 1376 return abort(AbortReason::Alloc); 1377 } 1378 1379 // Create a CompileInfo for the inlined script. 1380 jsbytecode* osrPc = nullptr; 1381 bool needsArgsObj = targetScript->needsArgsObj(); 1382 CompileInfo* info = lifoAlloc->new_<CompileInfo>( 1383 mirGen_.runtime, targetScript, osrPc, needsArgsObj, inlineScriptTree); 1384 if (!info) { 1385 return abort(AbortReason::Alloc); 1386 } 1387 1388 // Take a snapshot of the CacheIR. 1389 uint32_t offset = loc.bytecodeToOffset(script_); 1390 JitCode* jitCode = stub->jitCode(); 1391 const CacheIRStubInfo* stubInfo = stub->stubInfo(); 1392 WarpCacheIR* cacheIRSnapshot = new (alloc_.fallible()) 1393 WarpCacheIR(offset, jitCode, stubInfo, stubDataCopy); 1394 if (!cacheIRSnapshot) { 1395 return abort(AbortReason::Alloc); 1396 } 1397 1398 // Read barrier for weak stub data copied into the snapshot. 1399 Zone* zone = jitCode->zone(); 1400 if (zone->needsIncrementalBarrier()) { 1401 TraceWeakCacheIRStub(zone->barrierTracer(), stub, stub->stubInfo()); 1402 } 1403 1404 // Take a snapshot of the inlined script (which may do more 1405 // inlining recursively). 1406 WarpScriptOracle scriptOracle(cx_, oracle_, targetScript, info, icScript); 1407 1408 AbortReasonOr<WarpScriptSnapshot*> maybeScriptSnapshot = 1409 scriptOracle.createScriptSnapshot(); 1410 1411 if (maybeScriptSnapshot.isErr()) { 1412 JitSpew(JitSpew_WarpTranspiler, "Can't create snapshot for JSOp::%s", 1413 CodeName(loc.getOp())); 1414 1415 switch (maybeScriptSnapshot.unwrapErr()) { 1416 case AbortReason::Disable: { 1417 // If the target script can't be warp-compiled, mark it as 1418 // uninlineable, clean up, and fall through to the non-inlined path. 1419 1420 // If we monomorphically inline mutually recursive functions, 1421 // we can reach this point more than once for the same stub. 1422 // We should only unlink the stub once. 1423 ICEntry* entry = icScript_->icEntryForStub(fallbackStub); 1424 MOZ_ASSERT_IF(entry->firstStub() != stub, 1425 entry->firstStub() == stub->next()); 1426 if (entry->firstStub() == stub) { 1427 fallbackStub->unlinkStub(cx_->zone(), entry, /*prev=*/nullptr, stub); 1428 } 1429 targetScript->setUninlineable(); 1430 info_->inlineScriptTree()->removeCallee(inlineScriptTree); 1431 if (isTrialInlined) { 1432 icScript_->removeInlinedChild(loc.bytecodeToOffset(script_)); 1433 } 1434 fallbackStub->setTrialInliningState(TrialInliningState::Failure); 1435 oracle_->ignoreFailedICHash(); 1436 return false; 1437 } 1438 case AbortReason::Error: 1439 case AbortReason::Alloc: 1440 return Err(maybeScriptSnapshot.unwrapErr()); 1441 default: 1442 MOZ_CRASH("Unexpected abort reason"); 1443 } 1444 } 1445 1446 WarpScriptSnapshot* scriptSnapshot = maybeScriptSnapshot.unwrap(); 1447 oracle_->addScriptSnapshot(scriptSnapshot, icScript, targetScript->length()); 1448 #ifdef DEBUG 1449 if (!isTrialInlined && targetScript->jitScript()->hasPurgedStubs()) { 1450 oracle_->ignoreFailedICHash(); 1451 } 1452 #endif 1453 1454 if (!AddOpSnapshot<WarpInlinedCall>(alloc_, snapshots, offset, 1455 cacheIRSnapshot, scriptSnapshot, info)) { 1456 return abort(AbortReason::Alloc); 1457 } 1458 fallbackStub->setUsedByTranspiler(); 1459 1460 // Store the location of this monomorphic inline as a hint for future 1461 // compilations. 1462 if (!isTrialInlined && cx_->runtime()->jitRuntime()->hasJitHintsMap()) { 1463 JitHintsMap* jitHints = cx_->runtime()->jitRuntime()->getJitHintsMap(); 1464 if (!jitHints->addMonomorphicInlineLocation(script_, loc)) { 1465 return abort(AbortReason::Alloc); 1466 } 1467 } 1468 1469 return true; 1470 } 1471 1472 bool WarpOracle::snapshotJitZoneStub(JitZone::StubKind kind) { 1473 if (zoneStubs_[kind]) { 1474 return true; 1475 } 1476 JitCode* stub = cx_->zone()->jitZone()->ensureStubExists(cx_, kind); 1477 if (!stub) { 1478 return false; 1479 } 1480 zoneStubs_[kind] = stub; 1481 return true; 1482 } 1483 1484 void WarpOracle::ignoreFailedICHash() { 1485 outerScript_->jitScript()->notePurgedStubs(); 1486 } 1487 1488 struct TypeFrequency { 1489 TypeData typeData_; 1490 uint32_t successCount_; 1491 TypeFrequency(TypeData typeData, uint32_t successCount) 1492 : typeData_(typeData), successCount_(successCount) {} 1493 1494 // Sort highest frequency first. 1495 bool operator<(const TypeFrequency& other) const { 1496 return other.successCount_ < successCount_; 1497 } 1498 }; 1499 1500 AbortReasonOr<bool> WarpScriptOracle::maybeInlinePolymorphicTypes( 1501 WarpOpSnapshotList& snapshots, BytecodeLocation loc, 1502 ICCacheIRStub* firstStub, ICFallbackStub* fallbackStub) { 1503 MOZ_ASSERT(ICSupportsPolymorphicTypeData(loc.getOp())); 1504 1505 // We use polymorphic type data if there are multiple active stubs, 1506 // all of which have type data available. 1507 Vector<TypeFrequency, 6, SystemAllocPolicy> candidates; 1508 for (ICStub* stub = firstStub; !stub->isFallback(); 1509 stub = stub->maybeNext()) { 1510 ICCacheIRStub* cacheIRStub = stub->toCacheIRStub(); 1511 uint32_t successCount = 1512 cacheIRStub->enteredCount() - cacheIRStub->next()->enteredCount(); 1513 if (successCount == 0) { 1514 continue; 1515 } 1516 TypeData types = cacheIRStub->typeData(); 1517 if (!types.hasData()) { 1518 return false; 1519 } 1520 if (!candidates.append(TypeFrequency(types, successCount))) { 1521 return abort(AbortReason::Alloc); 1522 } 1523 } 1524 if (candidates.length() < 2) { 1525 return false; 1526 } 1527 1528 // Sort candidates by success frequency. 1529 std::sort(candidates.begin(), candidates.end()); 1530 1531 TypeDataList list; 1532 for (auto& candidate : candidates) { 1533 list.addTypeData(candidate.typeData_); 1534 } 1535 1536 uint32_t offset = loc.bytecodeToOffset(script_); 1537 if (!AddOpSnapshot<WarpPolymorphicTypes>(alloc_, snapshots, offset, list)) { 1538 return abort(AbortReason::Alloc); 1539 } 1540 1541 return true; 1542 } 1543 1544 bool WarpScriptOracle::replaceNurseryAndAllocSitePointers( 1545 ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo, 1546 uint8_t* stubDataCopy) { 1547 // If the stub data contains nursery object pointers, replace them with the 1548 // corresponding nursery index. See WarpObjectField. 1549 // 1550 // If the stub data contains allocation site pointers replace them with the 1551 // initial heap to use, because the site's state may be mutated by the main 1552 // thread while we are compiling. 1553 // 1554 // If the stub data contains weak pointers then trigger a read barrier. This 1555 // is necessary as these will now be strong references in the snapshot. 1556 // 1557 // If the stub data contains strings then atomize them. This ensures we don't 1558 // try to access potentially unstable characters from a background thread and 1559 // also facilitates certain optimizations. 1560 // 1561 // Also asserts non-object fields don't contain nursery pointers. 1562 1563 uint32_t field = 0; 1564 size_t offset = 0; 1565 while (true) { 1566 StubField::Type fieldType = stubInfo->fieldType(field); 1567 switch (fieldType) { 1568 case StubField::Type::RawInt32: 1569 case StubField::Type::RawPointer: 1570 case StubField::Type::RawInt64: 1571 case StubField::Type::Double: 1572 break; 1573 case StubField::Type::Shape: 1574 static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>, 1575 "Code assumes shapes are tenured"); 1576 break; 1577 case StubField::Type::WeakShape: { 1578 static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>, 1579 "Code assumes shapes are tenured"); 1580 stubInfo->getStubField<StubField::Type::WeakShape>(stub, offset).get(); 1581 break; 1582 } 1583 case StubField::Type::Symbol: 1584 static_assert(std::is_convertible_v<JS::Symbol*, gc::TenuredCell*>, 1585 "Code assumes symbols are tenured"); 1586 break; 1587 case StubField::Type::WeakBaseScript: { 1588 static_assert(std::is_convertible_v<BaseScript*, gc::TenuredCell*>, 1589 "Code assumes scripts are tenured"); 1590 stubInfo->getStubField<StubField::Type::WeakBaseScript>(stub, offset) 1591 .get(); 1592 break; 1593 } 1594 case StubField::Type::JitCode: 1595 static_assert(std::is_convertible_v<JitCode*, gc::TenuredCell*>, 1596 "Code assumes JitCodes are tenured"); 1597 break; 1598 case StubField::Type::JSObject: { 1599 JSObject* obj = 1600 stubInfo->getStubField<StubField::Type::JSObject>(stub, offset); 1601 if (!maybeReplaceNurseryPointer(stubInfo, stubDataCopy, obj, offset)) { 1602 return false; 1603 } 1604 break; 1605 } 1606 case StubField::Type::WeakObject: { 1607 JSObject* obj = 1608 stubInfo->getStubField<StubField::Type::WeakObject>(stub, offset); 1609 if (!maybeReplaceNurseryPointer(stubInfo, stubDataCopy, obj, offset)) { 1610 return false; 1611 } 1612 break; 1613 } 1614 case StubField::Type::String: { 1615 uintptr_t oldWord = stubInfo->getStubRawWord(stub, offset); 1616 JSString* str = reinterpret_cast<JSString*>(oldWord); 1617 MOZ_ASSERT(!IsInsideNursery(str)); 1618 JSAtom* atom = AtomizeString(cx_, str); 1619 if (!atom) { 1620 return false; 1621 } 1622 if (atom != str) { 1623 uintptr_t newWord = reinterpret_cast<uintptr_t>(atom); 1624 stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord); 1625 } 1626 break; 1627 } 1628 case StubField::Type::Id: { 1629 #ifdef DEBUG 1630 // jsid never contains nursery-allocated things. 1631 jsid id = stubInfo->getStubField<StubField::Type::Id>(stub, offset); 1632 MOZ_ASSERT_IF(id.isGCThing(), 1633 !IsInsideNursery(id.toGCCellPtr().asCell())); 1634 #endif 1635 break; 1636 } 1637 case StubField::Type::Value: 1638 case StubField::Type::WeakValue: { 1639 Value v; 1640 if (fieldType == StubField::Type::Value) { 1641 v = stubInfo->getStubField<StubField::Type::Value>(stub, offset) 1642 .get(); 1643 } else { 1644 MOZ_ASSERT(fieldType == StubField::Type::WeakValue); 1645 v = stubInfo->getStubField<StubField::Type::WeakValue>(stub, offset) 1646 .get(); 1647 } 1648 if (v.isString()) { 1649 Value newVal; 1650 JSAtom* atom = AtomizeString(cx_, v.toString()); 1651 if (!atom) { 1652 return false; 1653 } 1654 newVal.setString(atom); 1655 stubInfo->replaceStubRawValueBits(stubDataCopy, offset, v.asRawBits(), 1656 newVal.asRawBits()); 1657 } else { 1658 if (!maybeReplaceNurseryPointer(stubInfo, stubDataCopy, v, offset)) { 1659 return false; 1660 } 1661 } 1662 break; 1663 } 1664 case StubField::Type::AllocSite: { 1665 uintptr_t oldWord = stubInfo->getStubRawWord(stub, offset); 1666 auto* site = reinterpret_cast<gc::AllocSite*>(oldWord); 1667 gc::Heap initialHeap = site->initialHeap(); 1668 uintptr_t newWord = uintptr_t(initialHeap); 1669 stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord); 1670 break; 1671 } 1672 case StubField::Type::Limit: 1673 return true; // Done. 1674 } 1675 field++; 1676 offset += StubField::sizeInBytes(fieldType); 1677 } 1678 } 1679 1680 bool WarpScriptOracle::maybeReplaceNurseryPointer( 1681 const CacheIRStubInfo* stubInfo, uint8_t* stubDataCopy, JSObject* obj, 1682 size_t offset) { 1683 if (!IsInsideNursery(obj)) { 1684 return true; 1685 } 1686 1687 uint32_t nurseryIndex; 1688 if (!oracle_->registerNurseryObject(obj, &nurseryIndex)) { 1689 return false; 1690 } 1691 1692 uintptr_t oldWord = WarpObjectField::fromObject(obj).rawData(); 1693 uintptr_t newWord = WarpObjectField::fromNurseryIndex(nurseryIndex).rawData(); 1694 stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord); 1695 return true; 1696 } 1697 1698 bool WarpScriptOracle::maybeReplaceNurseryPointer( 1699 const CacheIRStubInfo* stubInfo, uint8_t* stubDataCopy, Value v, 1700 size_t offset) { 1701 // ValueOrNurseryValueIndex uses MagicValueUint32 to encode nursery indexes. 1702 // Assert this doesn't conflict with |v|. 1703 MOZ_ASSERT(ValueOrNurseryValueIndex::fromValue(v).isValue()); 1704 1705 if (!v.isGCThing() || !IsInsideNursery(v.toGCThing())) { 1706 return true; 1707 } 1708 1709 uint32_t nurseryIndex; 1710 if (!oracle_->registerNurseryValue(v, &nurseryIndex)) { 1711 return false; 1712 } 1713 1714 auto newValue = ValueOrNurseryValueIndex::fromNurseryIndex(nurseryIndex); 1715 stubInfo->replaceStubRawValueBits(stubDataCopy, offset, v.asRawBits(), 1716 newValue.asRawBits()); 1717 return true; 1718 } 1719 1720 bool WarpOracle::registerNurseryObject(JSObject* obj, uint32_t* nurseryIndex) { 1721 MOZ_ASSERT(IsInsideNursery(obj)); 1722 1723 auto p = nurseryObjectsMap_.lookupForAdd(obj); 1724 if (p) { 1725 *nurseryIndex = p->value(); 1726 return true; 1727 } 1728 1729 if (!nurseryObjects_.append(obj)) { 1730 return false; 1731 } 1732 *nurseryIndex = nurseryObjects_.length() - 1; 1733 return nurseryObjectsMap_.add(p, obj, *nurseryIndex); 1734 } 1735 1736 bool WarpOracle::registerNurseryValue(Value v, uint32_t* nurseryIndex) { 1737 gc::Cell* cell = v.toGCThing(); 1738 MOZ_ASSERT(IsInsideNursery(cell)); 1739 1740 auto p = nurseryValuesMap_.lookupForAdd(cell); 1741 if (p) { 1742 *nurseryIndex = p->value(); 1743 return true; 1744 } 1745 1746 if (!nurseryValues_.append(v)) { 1747 return false; 1748 } 1749 *nurseryIndex = nurseryValues_.length() - 1; 1750 return nurseryValuesMap_.add(p, cell, *nurseryIndex); 1751 }