MacroAssembler-inl.h (39124B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_MacroAssembler_inl_h 8 #define jit_MacroAssembler_inl_h 9 10 #include "jit/MacroAssembler.h" 11 12 #include "mozilla/FloatingPoint.h" 13 #include "mozilla/MathAlgorithms.h" 14 15 #include "gc/Zone.h" 16 #include "jit/CalleeToken.h" 17 #include "jit/CompileWrappers.h" 18 #include "jit/JitFrames.h" 19 #include "jit/JSJitFrameIter.h" 20 #include "js/Prefs.h" 21 #include "vm/BigIntType.h" 22 #include "vm/JSObject.h" 23 #include "vm/ProxyObject.h" 24 #include "vm/Runtime.h" 25 #include "vm/StringType.h" 26 27 #include "jit/ABIFunctionList-inl.h" 28 29 #if defined(JS_CODEGEN_X86) 30 # include "jit/x86/MacroAssembler-x86-inl.h" 31 #elif defined(JS_CODEGEN_X64) 32 # include "jit/x64/MacroAssembler-x64-inl.h" 33 #elif defined(JS_CODEGEN_ARM) 34 # include "jit/arm/MacroAssembler-arm-inl.h" 35 #elif defined(JS_CODEGEN_ARM64) 36 # include "jit/arm64/MacroAssembler-arm64-inl.h" 37 #elif defined(JS_CODEGEN_MIPS64) 38 # include "jit/mips64/MacroAssembler-mips64-inl.h" 39 #elif defined(JS_CODEGEN_LOONG64) 40 # include "jit/loong64/MacroAssembler-loong64-inl.h" 41 #elif defined(JS_CODEGEN_RISCV64) 42 # include "jit/riscv64/MacroAssembler-riscv64-inl.h" 43 #elif defined(JS_CODEGEN_WASM32) 44 # include "jit/wasm32/MacroAssembler-wasm32-inl.h" 45 #elif !defined(JS_CODEGEN_NONE) 46 # error "Unknown architecture!" 47 #endif 48 49 #include "wasm/WasmBuiltins.h" 50 51 namespace js { 52 namespace jit { 53 54 template <typename Sig> 55 DynFn DynamicFunction(Sig fun) { 56 ABIFunctionSignature<Sig> sig; 57 return DynFn{sig.address(fun)}; 58 } 59 60 // Helper for generatePreBarrier. 61 inline DynFn JitPreWriteBarrier(MIRType type) { 62 switch (type) { 63 case MIRType::Value: { 64 using Fn = void (*)(JSRuntime* rt, Value* vp); 65 return DynamicFunction<Fn>(JitValuePreWriteBarrier); 66 } 67 case MIRType::String: { 68 using Fn = void (*)(JSRuntime* rt, JSString** stringp); 69 return DynamicFunction<Fn>(JitStringPreWriteBarrier); 70 } 71 case MIRType::Object: { 72 using Fn = void (*)(JSRuntime* rt, JSObject** objp); 73 return DynamicFunction<Fn>(JitObjectPreWriteBarrier); 74 } 75 case MIRType::Shape: { 76 using Fn = void (*)(JSRuntime* rt, Shape** shapep); 77 return DynamicFunction<Fn>(JitShapePreWriteBarrier); 78 } 79 case MIRType::WasmAnyRef: { 80 using Fn = void (*)(JSRuntime* rt, wasm::AnyRef* refp); 81 return DynamicFunction<Fn>(JitWasmAnyRefPreWriteBarrier); 82 } 83 default: 84 MOZ_CRASH(); 85 } 86 } 87 88 //{{{ check_macroassembler_style 89 // =============================================================== 90 // Stack manipulation functions. 91 92 CodeOffset MacroAssembler::PushWithPatch(ImmWord word) { 93 framePushed_ += sizeof(word.value); 94 return pushWithPatch(word); 95 } 96 97 CodeOffset MacroAssembler::PushWithPatch(ImmPtr imm) { 98 return PushWithPatch(ImmWord(uintptr_t(imm.value))); 99 } 100 101 // =============================================================== 102 // Simple call functions. 103 104 void MacroAssembler::call(TrampolinePtr code) { call(ImmPtr(code.value)); } 105 106 CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc, 107 const Register reg) { 108 CodeOffset l = call(reg); 109 append(desc, l); 110 return l; 111 } 112 113 CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc, 114 uint32_t funcIndex) { 115 CodeOffset l = callWithPatch(); 116 append(desc, l, funcIndex); 117 return l; 118 } 119 120 void MacroAssembler::call(const wasm::CallSiteDesc& desc, wasm::Trap trap) { 121 CodeOffset l = callWithPatch(); 122 append(desc, l, trap); 123 } 124 125 CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc, 126 wasm::SymbolicAddress imm) { 127 CodeOffset raOffset = call(imm); 128 append(desc, raOffset); 129 return raOffset; 130 } 131 132 // =============================================================== 133 // ABI function calls. 134 135 void MacroAssembler::passABIArg(Register reg) { 136 passABIArg(MoveOperand(reg), ABIType::General); 137 } 138 139 void MacroAssembler::passABIArg(FloatRegister reg, ABIType type) { 140 passABIArg(MoveOperand(reg), type); 141 } 142 143 void MacroAssembler::callWithABI(DynFn fun, ABIType result, 144 CheckUnsafeCallWithABI check) { 145 AutoProfilerCallInstrumentation profiler(*this); 146 callWithABINoProfiler(fun.address, result, check); 147 } 148 149 template <typename Sig, Sig fun> 150 void MacroAssembler::callWithABI(ABIType result, CheckUnsafeCallWithABI check) { 151 ABIFunction<Sig, fun> abiFun; 152 AutoProfilerCallInstrumentation profiler(*this); 153 callWithABINoProfiler(abiFun.address(), result, check); 154 } 155 156 void MacroAssembler::callWithABI(Register fun, ABIType result) { 157 AutoProfilerCallInstrumentation profiler(*this); 158 callWithABINoProfiler(fun, result); 159 } 160 161 void MacroAssembler::callWithABI(const Address& fun, ABIType result) { 162 AutoProfilerCallInstrumentation profiler(*this); 163 callWithABINoProfiler(fun, result); 164 } 165 166 void MacroAssembler::appendSignatureType(ABIType type) { 167 #ifdef JS_SIMULATOR 168 signature_ <<= ABITypeArgShift; 169 signature_ |= uint32_t(type); 170 #endif 171 } 172 173 ABIFunctionType MacroAssembler::signature() const { 174 #ifdef JS_SIMULATOR 175 # ifdef DEBUG 176 switch (signature_) { 177 case Args_General0: 178 case Args_General1: 179 case Args_General2: 180 case Args_General3: 181 case Args_General4: 182 case Args_General5: 183 case Args_General6: 184 case Args_General7: 185 case Args_General8: 186 case Args_Double_None: 187 case Args_Int_Double: 188 case Args_Float32_Float32: 189 case Args_Float32_Float64: 190 case Args_Float32_General: 191 case Args_Float32_Int32: 192 case Args_Int_Float32: 193 case Args_Int32_Float32: 194 case Args_Double_Double: 195 case Args_Double_Int: 196 case Args_Double_DoubleInt: 197 case Args_Double_DoubleDouble: 198 case Args_Double_IntDouble: 199 case Args_Int_IntDouble: 200 case Args_Int_DoubleInt: 201 case Args_Int_DoubleIntInt: 202 case Args_Int_IntDoubleIntInt: 203 case Args_Double_DoubleDoubleDouble: 204 case Args_Double_DoubleDoubleDoubleDouble: 205 case Args_Int64_GeneralGeneral: 206 case Args_General_GeneralInt64GeneralGeneral: 207 case Args_General_GeneralFloat32GeneralGeneral: 208 break; 209 default: 210 MOZ_CRASH("Unexpected type"); 211 } 212 # endif // DEBUG 213 214 return ABIFunctionType(signature_); 215 #else 216 // No simulator enabled. 217 MOZ_CRASH("Only available for making calls within a simulator."); 218 #endif 219 } 220 221 // =============================================================== 222 // Jit Frames. 223 224 uint32_t MacroAssembler::callJitNoProfiler(Register callee) { 225 #ifdef JS_USE_LINK_REGISTER 226 // The return address is pushed by the callee. 227 call(callee); 228 #else 229 callAndPushReturnAddress(callee); 230 #endif 231 return currentOffset(); 232 } 233 234 uint32_t MacroAssembler::callJit(Register callee) { 235 AutoProfilerCallInstrumentation profiler(*this); 236 uint32_t ret = callJitNoProfiler(callee); 237 return ret; 238 } 239 240 uint32_t MacroAssembler::callJit(JitCode* callee) { 241 AutoProfilerCallInstrumentation profiler(*this); 242 call(callee); 243 return currentOffset(); 244 } 245 246 uint32_t MacroAssembler::callJit(TrampolinePtr code) { 247 AutoProfilerCallInstrumentation profiler(*this); 248 call(code); 249 return currentOffset(); 250 } 251 252 uint32_t MacroAssembler::callJit(ImmPtr callee) { 253 AutoProfilerCallInstrumentation profiler(*this); 254 call(callee); 255 return currentOffset(); 256 } 257 258 void MacroAssembler::push(FrameDescriptor descriptor) { 259 push(Imm32(descriptor.value())); 260 } 261 262 void MacroAssembler::Push(FrameDescriptor descriptor) { 263 Push(Imm32(descriptor.value())); 264 } 265 266 void MacroAssembler::makeFrameDescriptorForJitCall(FrameType type, 267 Register argc, Register dest, 268 bool hasInlineICScript) { 269 lshift32(Imm32(FrameDescriptor::NumActualArgsShift), argc, dest); 270 FrameDescriptor base(type, 0, hasInlineICScript); 271 if (base.value()) { 272 or32(Imm32(base.value()), dest); 273 } 274 } 275 276 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type, 277 Register argc, 278 Register scratch, 279 bool hasInlineICScript) { 280 makeFrameDescriptorForJitCall(type, argc, scratch, hasInlineICScript); 281 push(scratch); 282 } 283 284 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type, 285 Register argc, 286 Register scratch, 287 bool hasInlineICScript) { 288 pushFrameDescriptorForJitCall(type, argc, scratch, hasInlineICScript); 289 framePushed_ += sizeof(uintptr_t); 290 } 291 292 void MacroAssembler::loadNumActualArgs(Register framePtr, Register dest) { 293 loadPtr(Address(framePtr, JitFrameLayout::offsetOfDescriptor()), dest); 294 rshift32(Imm32(FrameDescriptor::NumActualArgsShift), dest); 295 } 296 297 void MacroAssembler::PushCalleeToken(Register callee, bool constructing) { 298 if (constructing) { 299 orPtr(Imm32(CalleeToken_FunctionConstructing), callee); 300 Push(callee); 301 andPtr(Imm32(uint32_t(CalleeTokenMask)), callee); 302 } else { 303 static_assert(CalleeToken_Function == 0, 304 "Non-constructing call requires no tagging"); 305 Push(callee); 306 } 307 } 308 309 void MacroAssembler::loadFunctionFromCalleeToken(Address token, Register dest) { 310 #ifdef DEBUG 311 Label ok; 312 loadPtr(token, dest); 313 andPtr(Imm32(uint32_t(~CalleeTokenMask)), dest); 314 branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_Function), &ok); 315 branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_FunctionConstructing), 316 &ok); 317 assumeUnreachable("Unexpected CalleeToken tag"); 318 bind(&ok); 319 #endif 320 loadPtr(token, dest); 321 andPtr(Imm32(uint32_t(CalleeTokenMask)), dest); 322 } 323 324 uint32_t MacroAssembler::buildFakeExitFrame(Register scratch) { 325 mozilla::DebugOnly<uint32_t> initialDepth = framePushed(); 326 327 Push(FrameDescriptor(FrameType::IonJS)); 328 uint32_t retAddr = pushFakeReturnAddress(scratch); 329 Push(FramePointer); 330 331 MOZ_ASSERT(framePushed() == initialDepth + ExitFrameLayout::Size()); 332 return retAddr; 333 } 334 335 // =============================================================== 336 // Exit frame footer. 337 338 void MacroAssembler::enterExitFrame(Register cxreg, Register scratch, 339 VMFunctionId f) { 340 linkExitFrame(cxreg, scratch); 341 // Push `ExitFrameType::VMFunction + VMFunctionId`, for marking the arguments. 342 // See ExitFooterFrame::data_. 343 uintptr_t type = uintptr_t(ExitFrameType::VMFunction) + uintptr_t(f); 344 MOZ_ASSERT(type <= INT32_MAX); 345 Push(Imm32(type)); 346 } 347 348 void MacroAssembler::enterFakeExitFrame(Register cxreg, Register scratch, 349 ExitFrameType type) { 350 linkExitFrame(cxreg, scratch); 351 Push(Imm32(int32_t(type))); 352 } 353 354 void MacroAssembler::enterFakeExitFrameForNative(Register cxreg, 355 Register scratch, 356 bool isConstructing) { 357 enterFakeExitFrame(cxreg, scratch, 358 isConstructing ? ExitFrameType::ConstructNative 359 : ExitFrameType::CallNative); 360 } 361 362 void MacroAssembler::leaveExitFrame(size_t extraFrame) { 363 freeStack(ExitFooterFrame::Size() + extraFrame); 364 } 365 366 // =============================================================== 367 // Move instructions 368 369 void MacroAssembler::moveValue(const ConstantOrRegister& src, 370 const ValueOperand& dest) { 371 if (src.constant()) { 372 moveValue(src.value(), dest); 373 return; 374 } 375 376 moveValue(src.reg(), dest); 377 } 378 379 // =============================================================== 380 // Copy instructions 381 382 void MacroAssembler::copy64(const Address& src, const Address& dest, 383 Register scratch) { 384 #if JS_BITS_PER_WORD == 32 385 MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch); 386 load32(LowWord(src), scratch); 387 store32(scratch, LowWord(dest)); 388 load32(HighWord(src), scratch); 389 store32(scratch, HighWord(dest)); 390 #else 391 Register64 scratch64(scratch); 392 load64(src, scratch64); 393 store64(scratch64, dest); 394 #endif 395 } 396 397 // =============================================================== 398 // Arithmetic functions 399 400 void MacroAssembler::addPtr(ImmPtr imm, Register dest) { 401 addPtr(ImmWord(uintptr_t(imm.value)), dest); 402 } 403 404 // =============================================================== 405 // Branch functions 406 407 void MacroAssembler::branchTest64(Condition cond, Register64 lhs, 408 Register64 rhs, Label* success, Label* fail) { 409 branchTest64(cond, lhs, rhs, InvalidReg, success, fail); 410 } 411 412 void MacroAssembler::branchIfFalseBool(Register reg, Label* label) { 413 // Note that C++ bool is only 1 byte, so ignore the higher-order bits. 414 branchTest32(Assembler::Zero, reg, Imm32(0xFF), label); 415 } 416 417 void MacroAssembler::branchIfTrueBool(Register reg, Label* label) { 418 // Note that C++ bool is only 1 byte, so ignore the higher-order bits. 419 branchTest32(Assembler::NonZero, reg, Imm32(0xFF), label); 420 } 421 422 void MacroAssembler::branchIfNotNullOrUndefined(ValueOperand val, 423 Label* label) { 424 Label nullOrUndefined; 425 ScratchTagScope tag(*this, val); 426 splitTagForTest(val, tag); 427 branchTestNull(Assembler::Equal, tag, &nullOrUndefined); 428 branchTestUndefined(Assembler::NotEqual, tag, label); 429 bind(&nullOrUndefined); 430 } 431 432 void MacroAssembler::branchIfRope(Register str, Label* label) { 433 Address flags(str, JSString::offsetOfFlags()); 434 branchTest32(Assembler::Zero, flags, Imm32(JSString::LINEAR_BIT), label); 435 } 436 437 void MacroAssembler::branchIfNotRope(Register str, Label* label) { 438 Address flags(str, JSString::offsetOfFlags()); 439 branchTest32(Assembler::NonZero, flags, Imm32(JSString::LINEAR_BIT), label); 440 } 441 442 void MacroAssembler::branchLatin1String(Register string, Label* label) { 443 branchTest32(Assembler::NonZero, Address(string, JSString::offsetOfFlags()), 444 Imm32(JSString::LATIN1_CHARS_BIT), label); 445 } 446 447 void MacroAssembler::branchTwoByteString(Register string, Label* label) { 448 branchTest32(Assembler::Zero, Address(string, JSString::offsetOfFlags()), 449 Imm32(JSString::LATIN1_CHARS_BIT), label); 450 } 451 452 void MacroAssembler::branchIfBigIntIsNegative(Register bigInt, Label* label) { 453 branchTest32(Assembler::NonZero, Address(bigInt, BigInt::offsetOfFlags()), 454 Imm32(BigInt::signBitMask()), label); 455 } 456 457 void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt, 458 Label* label) { 459 branchTest32(Assembler::Zero, Address(bigInt, BigInt::offsetOfFlags()), 460 Imm32(BigInt::signBitMask()), label); 461 } 462 463 void MacroAssembler::branchIfBigIntIsZero(Register bigInt, Label* label) { 464 branch32(Assembler::Equal, Address(bigInt, BigInt::offsetOfLength()), 465 Imm32(0), label); 466 } 467 468 void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt, Label* label) { 469 branch32(Assembler::NotEqual, Address(bigInt, BigInt::offsetOfLength()), 470 Imm32(0), label); 471 } 472 473 void MacroAssembler::branchTestFunctionFlags(Register fun, uint32_t flags, 474 Condition cond, Label* label) { 475 Address address(fun, JSFunction::offsetOfFlagsAndArgCount()); 476 branchTest32(cond, address, Imm32(flags), label); 477 } 478 479 void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun, 480 Register scratch, 481 Label* label) { 482 // Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT 483 // have the SELF_HOSTED flag. 484 // This is equivalent to JSFunction::isNonBuiltinConstructor. 485 constexpr int32_t mask = FunctionFlags::BASESCRIPT | 486 FunctionFlags::SELF_HOSTED | 487 FunctionFlags::CONSTRUCTOR; 488 constexpr int32_t expected = 489 FunctionFlags::BASESCRIPT | FunctionFlags::CONSTRUCTOR; 490 491 load32(Address(fun, JSFunction::offsetOfFlagsAndArgCount()), scratch); 492 and32(Imm32(mask), scratch); 493 branch32(Assembler::NotEqual, scratch, Imm32(expected), label); 494 } 495 496 void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun, Label* label) { 497 uint16_t flags = FunctionFlags::HasJitEntryFlags(); 498 branchTestFunctionFlags(fun, flags, Assembler::Zero, label); 499 } 500 501 void MacroAssembler::branchIfFunctionHasJitEntry(Register fun, Label* label) { 502 uint16_t flags = FunctionFlags::HasJitEntryFlags(); 503 branchTestFunctionFlags(fun, flags, Assembler::NonZero, label); 504 } 505 506 void MacroAssembler::branchIfScriptHasJitScript(Register script, Label* label) { 507 static_assert(ScriptWarmUpData::JitScriptTag == 0, 508 "Code below depends on tag value"); 509 branchTestPtr(Assembler::Zero, 510 Address(script, JSScript::offsetOfWarmUpData()), 511 Imm32(ScriptWarmUpData::TagMask), label); 512 } 513 514 void MacroAssembler::branchIfScriptHasNoJitScript(Register script, 515 Label* label) { 516 static_assert(ScriptWarmUpData::JitScriptTag == 0, 517 "Code below depends on tag value"); 518 static_assert(BaseScript::offsetOfWarmUpData() == 519 SelfHostedLazyScript::offsetOfWarmUpData(), 520 "SelfHostedLazyScript and BaseScript must use same layout for " 521 "warmUpData_"); 522 branchTestPtr(Assembler::NonZero, 523 Address(script, JSScript::offsetOfWarmUpData()), 524 Imm32(ScriptWarmUpData::TagMask), label); 525 } 526 527 void MacroAssembler::loadJitScript(Register script, Register dest) { 528 #ifdef DEBUG 529 Label ok; 530 branchIfScriptHasJitScript(script, &ok); 531 assumeUnreachable("Script has no JitScript!"); 532 bind(&ok); 533 #endif 534 535 static_assert(ScriptWarmUpData::JitScriptTag == 0, 536 "Code below depends on tag value"); 537 loadPtr(Address(script, JSScript::offsetOfWarmUpData()), dest); 538 } 539 540 void MacroAssembler::loadFunctionArgCount(Register func, Register output) { 541 load32(Address(func, JSFunction::offsetOfFlagsAndArgCount()), output); 542 rshift32(Imm32(JSFunction::ArgCountShift), output); 543 } 544 545 void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg, 546 Register scratch, 547 Label* slowCheck, 548 Label* label) { 549 MOZ_ASSERT(objReg != scratch); 550 551 Label done; 552 553 loadRuntimeFuse(RuntimeFuses::FuseIndex::HasSeenObjectEmulateUndefinedFuse, 554 scratch); 555 branchPtr(Assembler::Equal, scratch, ImmPtr(nullptr), &done); 556 557 loadObjClassUnsafe(objReg, scratch); 558 559 Address flags(scratch, JSClass::offsetOfFlags()); 560 branchTest32(Assembler::NonZero, flags, Imm32(JSCLASS_EMULATES_UNDEFINED), 561 label); 562 563 // Call into C++ if the object is a wrapper. 564 branchTestClassIsProxy(false, scratch, &done); 565 branchTestProxyHandlerFamily(Assembler::Equal, objReg, scratch, 566 &Wrapper::family, slowCheck); 567 568 bind(&done); 569 } 570 571 void MacroAssembler::branchFunctionKind(Condition cond, 572 FunctionFlags::FunctionKind kind, 573 Register fun, Register scratch, 574 Label* label) { 575 Address address(fun, JSFunction::offsetOfFlagsAndArgCount()); 576 load32(address, scratch); 577 and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK), scratch); 578 branch32(cond, scratch, Imm32(kind), label); 579 } 580 581 void MacroAssembler::branchTestObjClass(Condition cond, Register obj, 582 const JSClass* clasp, Register scratch, 583 Register spectreRegToZero, 584 Label* label) { 585 MOZ_ASSERT(obj != scratch); 586 MOZ_ASSERT(scratch != spectreRegToZero); 587 588 loadPtr(Address(obj, JSObject::offsetOfShape()), scratch); 589 loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch); 590 branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp), 591 label); 592 593 if (JitOptions.spectreObjectMitigations) { 594 spectreZeroRegister(cond, scratch, spectreRegToZero); 595 } 596 } 597 598 void MacroAssembler::branchTestObjClassNoSpectreMitigations( 599 Condition cond, Register obj, const JSClass* clasp, Register scratch, 600 Label* label) { 601 loadPtr(Address(obj, JSObject::offsetOfShape()), scratch); 602 loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch); 603 branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp), 604 label); 605 } 606 607 void MacroAssembler::branchTestObjClass(Condition cond, Register obj, 608 const Address& clasp, Register scratch, 609 Register spectreRegToZero, 610 Label* label) { 611 MOZ_ASSERT(obj != scratch); 612 MOZ_ASSERT(scratch != spectreRegToZero); 613 614 loadObjClassUnsafe(obj, scratch); 615 branchPtr(cond, clasp, scratch, label); 616 617 if (JitOptions.spectreObjectMitigations) { 618 spectreZeroRegister(cond, scratch, spectreRegToZero); 619 } 620 } 621 622 void MacroAssembler::branchTestObjClassNoSpectreMitigations( 623 Condition cond, Register obj, const Address& clasp, Register scratch, 624 Label* label) { 625 MOZ_ASSERT(obj != scratch); 626 loadObjClassUnsafe(obj, scratch); 627 branchPtr(cond, clasp, scratch, label); 628 } 629 630 void MacroAssembler::branchTestObjClass(Condition cond, Register obj, 631 Register clasp, Register scratch, 632 Register spectreRegToZero, 633 Label* label) { 634 MOZ_ASSERT(obj != scratch); 635 MOZ_ASSERT(scratch != spectreRegToZero); 636 637 loadObjClassUnsafe(obj, scratch); 638 branchPtr(cond, clasp, scratch, label); 639 640 if (JitOptions.spectreObjectMitigations) { 641 spectreZeroRegister(cond, scratch, spectreRegToZero); 642 } 643 } 644 645 void MacroAssembler::branchTestClassIsFunction(Condition cond, Register clasp, 646 Label* label) { 647 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); 648 649 if (cond == Assembler::Equal) { 650 branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), label); 651 branchPtr(Assembler::Equal, clasp, ImmPtr(&ExtendedFunctionClass), label); 652 return; 653 } 654 655 Label isClass; 656 branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), &isClass); 657 branchPtr(Assembler::NotEqual, clasp, ImmPtr(&ExtendedFunctionClass), label); 658 bind(&isClass); 659 } 660 661 void MacroAssembler::branchTestObjIsFunction(Condition cond, Register obj, 662 Register scratch, 663 Register spectreRegToZero, 664 Label* label) { 665 MOZ_ASSERT(scratch != spectreRegToZero); 666 667 branchTestObjIsFunctionNoSpectreMitigations(cond, obj, scratch, label); 668 669 if (JitOptions.spectreObjectMitigations) { 670 spectreZeroRegister(cond, scratch, spectreRegToZero); 671 } 672 } 673 674 void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations( 675 Condition cond, Register obj, Register scratch, Label* label) { 676 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); 677 MOZ_ASSERT(obj != scratch); 678 679 loadObjClassUnsafe(obj, scratch); 680 branchTestClassIsFunction(cond, scratch, label); 681 } 682 683 void MacroAssembler::branchTestObjShape(Condition cond, Register obj, 684 const Shape* shape, Register scratch, 685 Register spectreRegToZero, 686 Label* label) { 687 MOZ_ASSERT(obj != scratch); 688 MOZ_ASSERT(spectreRegToZero != scratch); 689 690 if (JitOptions.spectreObjectMitigations) { 691 move32(Imm32(0), scratch); 692 } 693 694 branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape), 695 label); 696 697 if (JitOptions.spectreObjectMitigations) { 698 spectreMovePtr(cond, scratch, spectreRegToZero); 699 } 700 } 701 702 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond, 703 Register obj, 704 const Shape* shape, 705 Label* label) { 706 branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape), 707 label); 708 } 709 710 void MacroAssembler::branchTestObjShape(Condition cond, Register obj, 711 Register shape, Register scratch, 712 Register spectreRegToZero, 713 Label* label) { 714 MOZ_ASSERT(obj != scratch); 715 MOZ_ASSERT(obj != shape); 716 MOZ_ASSERT(spectreRegToZero != scratch); 717 718 if (JitOptions.spectreObjectMitigations) { 719 move32(Imm32(0), scratch); 720 } 721 722 branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label); 723 724 if (JitOptions.spectreObjectMitigations) { 725 spectreMovePtr(cond, scratch, spectreRegToZero); 726 } 727 } 728 729 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond, 730 Register obj, 731 Register shape, 732 Label* label) { 733 branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label); 734 } 735 736 void MacroAssembler::branchTestObjShapeUnsafe(Condition cond, Register obj, 737 Register shape, Label* label) { 738 branchTestObjShapeNoSpectreMitigations(cond, obj, shape, label); 739 } 740 741 void MacroAssembler::branchTestClassIsProxy(bool proxy, Register clasp, 742 Label* label) { 743 branchTest32(proxy ? Assembler::NonZero : Assembler::Zero, 744 Address(clasp, JSClass::offsetOfFlags()), 745 Imm32(JSCLASS_IS_PROXY), label); 746 } 747 748 void MacroAssembler::branchTestObjectIsProxy(bool proxy, Register object, 749 Register scratch, Label* label) { 750 constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift()); 751 static_assert(uint32_t(Shape::Kind::Proxy) == 0, 752 "branchTest32 below depends on proxy kind being 0"); 753 loadPtr(Address(object, JSObject::offsetOfShape()), scratch); 754 branchTest32(proxy ? Assembler::Zero : Assembler::NonZero, 755 Address(scratch, Shape::offsetOfImmutableFlags()), 756 Imm32(ShiftedMask), label); 757 } 758 759 void MacroAssembler::branchTestProxyHandlerFamily(Condition cond, 760 Register proxy, 761 Register scratch, 762 const void* handlerp, 763 Label* label) { 764 #ifdef DEBUG 765 Label ok; 766 branchTestObjectIsProxy(true, proxy, scratch, &ok); 767 assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily"); 768 bind(&ok); 769 #endif 770 771 Address handlerAddr(proxy, ProxyObject::offsetOfHandler()); 772 loadPtr(handlerAddr, scratch); 773 Address familyAddr(scratch, BaseProxyHandler::offsetOfFamily()); 774 branchPtr(cond, familyAddr, ImmPtr(handlerp), label); 775 } 776 777 void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond, 778 Label* label) { 779 MOZ_ASSERT(cond == Zero || cond == NonZero); 780 CompileZone* zone = realm()->zone(); 781 const uint32_t* needsBarrierAddr = zone->addressOfNeedsIncrementalBarrier(); 782 branchTest32(cond, AbsoluteAddress(needsBarrierAddr), Imm32(0x1), label); 783 } 784 785 void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone( 786 Condition cond, Label* label, Register scratch) { 787 MOZ_ASSERT(cond == Zero || cond == NonZero); 788 if (maybeRealm_) { 789 branchTestNeedsIncrementalBarrier(cond, label); 790 } else { 791 // We are compiling the interpreter or another runtime-wide trampoline, so 792 // we have to load cx->zone. 793 loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch); 794 Address needsBarrierAddr(scratch, Zone::offsetOfNeedsIncrementalBarrier()); 795 branchTest32(cond, needsBarrierAddr, Imm32(0x1), label); 796 } 797 } 798 799 void MacroAssembler::branchTestMagicValue(Condition cond, 800 const ValueOperand& val, 801 JSWhyMagic why, Label* label) { 802 MOZ_ASSERT(cond == Equal || cond == NotEqual); 803 branchTestValue(cond, val, MagicValue(why), label); 804 } 805 806 void MacroAssembler::branchDoubleNotInInt64Range(Address src, Register temp, 807 Label* fail) { 808 using mozilla::FloatingPoint; 809 810 // Tests if double is in [INT64_MIN; INT64_MAX] range 811 uint32_t EXPONENT_MASK = 0x7ff00000; 812 uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32; 813 uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 63) 814 << EXPONENT_SHIFT; 815 816 load32(Address(src.base, src.offset + sizeof(int32_t)), temp); 817 and32(Imm32(EXPONENT_MASK), temp); 818 branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail); 819 } 820 821 void MacroAssembler::branchDoubleNotInUInt64Range(Address src, Register temp, 822 Label* fail) { 823 using mozilla::FloatingPoint; 824 825 // Note: returns failure on -0.0 826 // Tests if double is in [0; UINT64_MAX] range 827 // Take the sign also in the equation. That way we can compare in one test? 828 uint32_t EXPONENT_MASK = 0xfff00000; 829 uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32; 830 uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 64) 831 << EXPONENT_SHIFT; 832 833 load32(Address(src.base, src.offset + sizeof(int32_t)), temp); 834 and32(Imm32(EXPONENT_MASK), temp); 835 branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail); 836 } 837 838 void MacroAssembler::branchFloat32NotInInt64Range(Address src, Register temp, 839 Label* fail) { 840 using mozilla::FloatingPoint; 841 842 // Tests if float is in [INT64_MIN; INT64_MAX] range 843 uint32_t EXPONENT_MASK = 0x7f800000; 844 uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift; 845 uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 63) 846 << EXPONENT_SHIFT; 847 848 load32(src, temp); 849 and32(Imm32(EXPONENT_MASK), temp); 850 branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail); 851 } 852 853 void MacroAssembler::branchFloat32NotInUInt64Range(Address src, Register temp, 854 Label* fail) { 855 using mozilla::FloatingPoint; 856 857 // Note: returns failure on -0.0 858 // Tests if float is in [0; UINT64_MAX] range 859 // Take the sign also in the equation. That way we can compare in one test? 860 uint32_t EXPONENT_MASK = 0xff800000; 861 uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift; 862 uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 64) 863 << EXPONENT_SHIFT; 864 865 load32(src, temp); 866 and32(Imm32(EXPONENT_MASK), temp); 867 branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail); 868 } 869 870 // ======================================================================== 871 // Canonicalization primitives. 872 void MacroAssembler::canonicalizeFloat(FloatRegister reg) { 873 Label notNaN; 874 branchFloat(DoubleOrdered, reg, reg, ¬NaN); 875 loadConstantFloat32(float(JS::GenericNaN()), reg); 876 bind(¬NaN); 877 } 878 879 void MacroAssembler::canonicalizeDouble(FloatRegister reg) { 880 Label notNaN; 881 branchDouble(DoubleOrdered, reg, reg, ¬NaN); 882 loadConstantDouble(JS::GenericNaN(), reg); 883 bind(¬NaN); 884 } 885 886 // ======================================================================== 887 // Memory access primitives. 888 889 template <class T> 890 void MacroAssembler::boxDouble(FloatRegister src, const T& dest) { 891 storeDouble(src, dest); 892 } 893 894 template <typename T> 895 void MacroAssembler::fallibleUnboxInt32(const T& src, Register dest, 896 Label* fail) { 897 // Int32Value can be unboxed efficiently with unboxInt32, so use that. 898 branchTestInt32(Assembler::NotEqual, src, fail); 899 unboxInt32(src, dest); 900 } 901 902 template <typename T> 903 void MacroAssembler::fallibleUnboxBoolean(const T& src, Register dest, 904 Label* fail) { 905 // BooleanValue can be unboxed efficiently with unboxBoolean, so use that. 906 branchTestBoolean(Assembler::NotEqual, src, fail); 907 unboxBoolean(src, dest); 908 } 909 910 template <typename T> 911 void MacroAssembler::fallibleUnboxObject(const T& src, Register dest, 912 Label* fail) { 913 fallibleUnboxPtr(src, dest, JSVAL_TYPE_OBJECT, fail); 914 } 915 916 template <typename T> 917 void MacroAssembler::fallibleUnboxString(const T& src, Register dest, 918 Label* fail) { 919 fallibleUnboxPtr(src, dest, JSVAL_TYPE_STRING, fail); 920 } 921 922 template <typename T> 923 void MacroAssembler::fallibleUnboxSymbol(const T& src, Register dest, 924 Label* fail) { 925 fallibleUnboxPtr(src, dest, JSVAL_TYPE_SYMBOL, fail); 926 } 927 928 template <typename T> 929 void MacroAssembler::fallibleUnboxBigInt(const T& src, Register dest, 930 Label* fail) { 931 fallibleUnboxPtr(src, dest, JSVAL_TYPE_BIGINT, fail); 932 } 933 934 //}}} check_macroassembler_style 935 // =============================================================== 936 937 void MacroAssembler::ensureDouble(const ValueOperand& source, 938 FloatRegister dest, Label* failure) { 939 Label isDouble, done; 940 941 { 942 ScratchTagScope tag(*this, source); 943 splitTagForTest(source, tag); 944 branchTestDouble(Assembler::Equal, tag, &isDouble); 945 branchTestInt32(Assembler::NotEqual, tag, failure); 946 } 947 948 convertInt32ToDouble(source.payloadOrValueReg(), dest); 949 jump(&done); 950 951 bind(&isDouble); 952 unboxDouble(source, dest); 953 954 bind(&done); 955 } 956 957 #ifndef JS_CODEGEN_ARM64 958 959 template <typename T> 960 void MacroAssembler::branchTestStackPtr(Condition cond, T t, Label* label) { 961 branchTestPtr(cond, getStackPointer(), t, label); 962 } 963 964 template <typename T> 965 void MacroAssembler::branchStackPtr(Condition cond, T rhs, Label* label) { 966 branchPtr(cond, getStackPointer(), rhs, label); 967 } 968 969 template <typename T> 970 void MacroAssembler::branchStackPtrRhs(Condition cond, T lhs, Label* label) { 971 branchPtr(cond, lhs, getStackPointer(), label); 972 } 973 974 template <typename T> 975 void MacroAssembler::addToStackPtr(T t) { 976 addPtr(t, getStackPointer()); 977 } 978 979 template <typename T> 980 void MacroAssembler::addStackPtrTo(T t) { 981 addPtr(getStackPointer(), t); 982 } 983 984 void MacroAssembler::reserveStack(uint32_t amount) { 985 subFromStackPtr(Imm32(amount)); 986 adjustFrame(amount); 987 } 988 #endif // !JS_CODEGEN_ARM64 989 990 void MacroAssembler::loadObjClassUnsafe(Register obj, Register dest) { 991 loadPtr(Address(obj, JSObject::offsetOfShape()), dest); 992 loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest); 993 loadPtr(Address(dest, BaseShape::offsetOfClasp()), dest); 994 } 995 996 void MacroAssembler::loadObjShapeUnsafe(Register obj, Register dest) { 997 loadPtr(Address(obj, JSObject::offsetOfShape()), dest); 998 } 999 1000 template <typename EmitPreBarrier> 1001 void MacroAssembler::storeObjShape(Register shape, Register obj, 1002 EmitPreBarrier emitPreBarrier) { 1003 MOZ_ASSERT(shape != obj); 1004 Address shapeAddr(obj, JSObject::offsetOfShape()); 1005 emitPreBarrier(*this, shapeAddr); 1006 storePtr(shape, shapeAddr); 1007 } 1008 1009 template <typename EmitPreBarrier> 1010 void MacroAssembler::storeObjShape(Shape* shape, Register obj, 1011 EmitPreBarrier emitPreBarrier) { 1012 Address shapeAddr(obj, JSObject::offsetOfShape()); 1013 emitPreBarrier(*this, shapeAddr); 1014 storePtr(ImmGCPtr(shape), shapeAddr); 1015 } 1016 1017 void MacroAssembler::loadObjProto(Register obj, Register dest) { 1018 loadPtr(Address(obj, JSObject::offsetOfShape()), dest); 1019 loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest); 1020 loadPtr(Address(dest, BaseShape::offsetOfProto()), dest); 1021 } 1022 1023 void MacroAssembler::loadStringLength(Register str, Register dest) { 1024 load32(Address(str, JSString::offsetOfLength()), dest); 1025 } 1026 1027 template <typename Table, typename Match> 1028 void MacroAssembler::lookupMFBT(Register hashTable, Register hashCode, 1029 Register scratch, Register scratch2, 1030 Register scratch3, Register scratch4, 1031 Register scratch5, Label* missing, 1032 Match match) { 1033 // Inline implementation of |lookup| for mozilla::detail::HashTable 1034 1035 // If the hashtable is empty, we won't find an entry. 1036 branch32(Assembler::Equal, Address(hashTable, Table::offsetOfEntryCount()), 1037 Imm32(0), missing); 1038 1039 // Compute the primary hash address: 1040 // HashNumber h1 = hash1(aKeyHash); 1041 Register hash1 = scratch5; 1042 computeHash1MFBT<Table>(hashTable, hashCode, hash1, scratch); 1043 1044 Label primaryCollision; 1045 checkForMatchMFBT<Table>(hashTable, hash1, hashCode, scratch, scratch2, 1046 missing, &primaryCollision); 1047 match(); 1048 bind(&primaryCollision); 1049 1050 // Otherwise, we've had a collision. Double-hash. 1051 Register hash2 = scratch4; 1052 Register sizeMask = scratch3; 1053 computeHash2MFBT<Table>(hashTable, hashCode, hash2, sizeMask, scratch); 1054 1055 Label loop; 1056 bind(&loop); 1057 1058 applyDoubleHashMFBT(hash1, hash2, sizeMask); 1059 checkForMatchMFBT<Table>(hashTable, hash1, hashCode, scratch, scratch2, 1060 missing, &loop); 1061 match(); 1062 jump(&loop); 1063 } 1064 1065 void MacroAssembler::assertStackAlignment(uint32_t alignment, 1066 int32_t offset /* = 0 */) { 1067 #ifdef DEBUG 1068 Label ok, bad; 1069 MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment)); 1070 1071 // Wrap around the offset to be a non-negative number. 1072 offset %= alignment; 1073 if (offset < 0) { 1074 offset += alignment; 1075 } 1076 1077 // Test if each bit from offset is set. 1078 uint32_t off = offset; 1079 while (off) { 1080 uint32_t lowestBit = 1 << mozilla::CountTrailingZeroes32(off); 1081 branchTestStackPtr(Assembler::Zero, Imm32(lowestBit), &bad); 1082 off ^= lowestBit; 1083 } 1084 1085 // Check that all remaining bits are zero. 1086 branchTestStackPtr(Assembler::Zero, Imm32((alignment - 1) ^ offset), &ok); 1087 1088 bind(&bad); 1089 breakpoint(); 1090 bind(&ok); 1091 #endif 1092 } 1093 1094 void MacroAssembler::storeCallBoolResult(Register reg) { 1095 convertBoolToInt32(ReturnReg, reg); 1096 } 1097 1098 void MacroAssembler::storeCallInt32Result(Register reg) { 1099 #if JS_BITS_PER_WORD == 32 1100 storeCallPointerResult(reg); 1101 #else 1102 // Ensure the upper 32 bits are cleared. 1103 move32(ReturnReg, reg); 1104 #endif 1105 } 1106 1107 void MacroAssembler::storeCallResultValue(AnyRegister dest, JSValueType type) { 1108 unboxValue(JSReturnOperand, dest, type); 1109 } 1110 1111 void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest) { 1112 if (dest.hasValue()) { 1113 storeCallResultValue(dest.valueReg()); 1114 } else { 1115 storeCallResultValue(dest.typedReg(), ValueTypeFromMIRType(dest.type())); 1116 } 1117 } 1118 1119 } // namespace jit 1120 } // namespace js 1121 1122 #endif /* jit_MacroAssembler_inl_h */