Assembler-x64.h (44614B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_x64_Assembler_x64_h 8 #define jit_x64_Assembler_x64_h 9 10 #include "jit/JitCode.h" 11 #include "jit/shared/Assembler-shared.h" 12 13 namespace js { 14 namespace jit { 15 16 static constexpr Register rax{X86Encoding::rax}; 17 static constexpr Register rbx{X86Encoding::rbx}; 18 static constexpr Register rcx{X86Encoding::rcx}; 19 static constexpr Register rdx{X86Encoding::rdx}; 20 static constexpr Register rsi{X86Encoding::rsi}; 21 static constexpr Register rdi{X86Encoding::rdi}; 22 static constexpr Register rbp{X86Encoding::rbp}; 23 static constexpr Register r8{X86Encoding::r8}; 24 static constexpr Register r9{X86Encoding::r9}; 25 static constexpr Register r10{X86Encoding::r10}; 26 static constexpr Register r11{X86Encoding::r11}; 27 static constexpr Register r12{X86Encoding::r12}; 28 static constexpr Register r13{X86Encoding::r13}; 29 static constexpr Register r14{X86Encoding::r14}; 30 static constexpr Register r15{X86Encoding::r15}; 31 static constexpr Register rsp{X86Encoding::rsp}; 32 33 static constexpr FloatRegister xmm0 = 34 FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); 35 static constexpr FloatRegister xmm1 = 36 FloatRegister(X86Encoding::xmm1, FloatRegisters::Double); 37 static constexpr FloatRegister xmm2 = 38 FloatRegister(X86Encoding::xmm2, FloatRegisters::Double); 39 static constexpr FloatRegister xmm3 = 40 FloatRegister(X86Encoding::xmm3, FloatRegisters::Double); 41 static constexpr FloatRegister xmm4 = 42 FloatRegister(X86Encoding::xmm4, FloatRegisters::Double); 43 static constexpr FloatRegister xmm5 = 44 FloatRegister(X86Encoding::xmm5, FloatRegisters::Double); 45 static constexpr FloatRegister xmm6 = 46 FloatRegister(X86Encoding::xmm6, FloatRegisters::Double); 47 static constexpr FloatRegister xmm7 = 48 FloatRegister(X86Encoding::xmm7, FloatRegisters::Double); 49 static constexpr FloatRegister xmm8 = 50 FloatRegister(X86Encoding::xmm8, FloatRegisters::Double); 51 static constexpr FloatRegister xmm9 = 52 FloatRegister(X86Encoding::xmm9, FloatRegisters::Double); 53 static constexpr FloatRegister xmm10 = 54 FloatRegister(X86Encoding::xmm10, FloatRegisters::Double); 55 static constexpr FloatRegister xmm11 = 56 FloatRegister(X86Encoding::xmm11, FloatRegisters::Double); 57 static constexpr FloatRegister xmm12 = 58 FloatRegister(X86Encoding::xmm12, FloatRegisters::Double); 59 static constexpr FloatRegister xmm13 = 60 FloatRegister(X86Encoding::xmm13, FloatRegisters::Double); 61 static constexpr FloatRegister xmm14 = 62 FloatRegister(X86Encoding::xmm14, FloatRegisters::Double); 63 static constexpr FloatRegister xmm15 = 64 FloatRegister(X86Encoding::xmm15, FloatRegisters::Double); 65 66 // Vector registers fixed for use with some instructions, e.g. PBLENDVB. 67 static constexpr FloatRegister vmm0 = 68 FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128); 69 70 // X86-common synonyms. 71 static constexpr Register eax = rax; 72 static constexpr Register ebx = rbx; 73 static constexpr Register ecx = rcx; 74 static constexpr Register edx = rdx; 75 static constexpr Register esi = rsi; 76 static constexpr Register edi = rdi; 77 static constexpr Register ebp = rbp; 78 static constexpr Register esp = rsp; 79 80 static constexpr Register InvalidReg{X86Encoding::invalid_reg}; 81 static constexpr FloatRegister InvalidFloatReg = FloatRegister(); 82 83 static constexpr Register StackPointer = rsp; 84 static constexpr Register FramePointer = rbp; 85 static constexpr Register JSReturnReg = rcx; 86 // Avoid, except for assertions. 87 static constexpr Register JSReturnReg_Type = JSReturnReg; 88 static constexpr Register JSReturnReg_Data = JSReturnReg; 89 90 static constexpr Register ScratchReg = r11; 91 92 // Helper class for ScratchRegister usage. Asserts that only one piece 93 // of code thinks it has exclusive ownership of the scratch register. 94 struct ScratchRegisterScope : public AutoRegisterScope { 95 explicit ScratchRegisterScope(MacroAssembler& masm) 96 : AutoRegisterScope(masm, ScratchReg) {} 97 }; 98 99 static constexpr Register ReturnReg = rax; 100 static constexpr Register64 ReturnReg64(rax); 101 static constexpr FloatRegister ReturnFloat32Reg = 102 FloatRegister(X86Encoding::xmm0, FloatRegisters::Single); 103 static constexpr FloatRegister ReturnDoubleReg = 104 FloatRegister(X86Encoding::xmm0, FloatRegisters::Double); 105 static constexpr FloatRegister ReturnSimd128Reg = 106 FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128); 107 static constexpr FloatRegister ScratchFloat32Reg_ = 108 FloatRegister(X86Encoding::xmm15, FloatRegisters::Single); 109 static constexpr FloatRegister ScratchDoubleReg_ = 110 FloatRegister(X86Encoding::xmm15, FloatRegisters::Double); 111 static constexpr FloatRegister ScratchSimd128Reg = 112 FloatRegister(X86Encoding::xmm15, FloatRegisters::Simd128); 113 114 // Avoid rbp, which is the FramePointer, which is unavailable in some modes. 115 static constexpr Register CallTempReg0 = rax; 116 static constexpr Register CallTempReg1 = rdi; 117 static constexpr Register CallTempReg2 = rbx; 118 static constexpr Register CallTempReg3 = rcx; 119 static constexpr Register CallTempReg4 = rsi; 120 static constexpr Register CallTempReg5 = rdx; 121 122 // Different argument registers for WIN64 123 #if defined(_WIN64) 124 static constexpr Register IntArgReg0 = rcx; 125 static constexpr Register IntArgReg1 = rdx; 126 static constexpr Register IntArgReg2 = r8; 127 static constexpr Register IntArgReg3 = r9; 128 static constexpr uint32_t NumIntArgRegs = 4; 129 static constexpr Register IntArgRegs[NumIntArgRegs] = {rcx, rdx, r8, r9}; 130 131 static constexpr Register CallTempNonArgRegs[] = {rax, rdi, rbx, rsi}; 132 static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); 133 134 static constexpr FloatRegister FloatArgReg0 = xmm0; 135 static constexpr FloatRegister FloatArgReg1 = xmm1; 136 static constexpr FloatRegister FloatArgReg2 = xmm2; 137 static constexpr FloatRegister FloatArgReg3 = xmm3; 138 static constexpr uint32_t NumFloatArgRegs = 4; 139 static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {xmm0, xmm1, 140 xmm2, xmm3}; 141 #else 142 static constexpr Register IntArgReg0 = rdi; 143 static constexpr Register IntArgReg1 = rsi; 144 static constexpr Register IntArgReg2 = rdx; 145 static constexpr Register IntArgReg3 = rcx; 146 static constexpr Register IntArgReg4 = r8; 147 static constexpr Register IntArgReg5 = r9; 148 static constexpr uint32_t NumIntArgRegs = 6; 149 static constexpr Register IntArgRegs[NumIntArgRegs] = {rdi, rsi, rdx, 150 rcx, r8, r9}; 151 152 static constexpr Register CallTempNonArgRegs[] = {rax, rbx}; 153 static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); 154 155 static constexpr FloatRegister FloatArgReg0 = xmm0; 156 static constexpr FloatRegister FloatArgReg1 = xmm1; 157 static constexpr FloatRegister FloatArgReg2 = xmm2; 158 static constexpr FloatRegister FloatArgReg3 = xmm3; 159 static constexpr FloatRegister FloatArgReg4 = xmm4; 160 static constexpr FloatRegister FloatArgReg5 = xmm5; 161 static constexpr FloatRegister FloatArgReg6 = xmm6; 162 static constexpr FloatRegister FloatArgReg7 = xmm7; 163 static constexpr uint32_t NumFloatArgRegs = 8; 164 static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = { 165 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7}; 166 #endif 167 168 // Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use 169 // JSReturnOperand). 170 static constexpr Register RegExpMatcherRegExpReg = CallTempReg0; 171 static constexpr Register RegExpMatcherStringReg = CallTempReg1; 172 static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2; 173 174 // Registers used by RegExpExecTest stub (do not use ReturnReg). 175 static constexpr Register RegExpExecTestRegExpReg = CallTempReg1; 176 static constexpr Register RegExpExecTestStringReg = CallTempReg2; 177 178 // Registers used by RegExpSearcher stub (do not use ReturnReg). 179 static constexpr Register RegExpSearcherRegExpReg = CallTempReg1; 180 static constexpr Register RegExpSearcherStringReg = CallTempReg2; 181 static constexpr Register RegExpSearcherLastIndexReg = CallTempReg3; 182 183 class ABIArgGenerator : public ABIArgGeneratorShared { 184 #if defined(XP_WIN) 185 unsigned regIndex_; 186 #else 187 unsigned intRegIndex_; 188 unsigned floatRegIndex_; 189 #endif 190 ABIArg current_; 191 192 public: 193 explicit ABIArgGenerator(ABIKind kind); 194 ABIArg next(MIRType argType); 195 ABIArg& current() { return current_; } 196 }; 197 198 // See "ABI special registers" in Assembler-shared.h for more information. 199 // Avoid r11, which is the MacroAssembler's ScratchReg. 200 static constexpr Register ABINonArgReg0 = rax; 201 static constexpr Register ABINonArgReg1 = rbx; 202 static constexpr Register ABINonArgReg2 = r10; 203 static constexpr Register ABINonArgReg3 = r12; 204 205 // See "ABI special registers" in Assembler-shared.h for more information. 206 // Avoid xmm15 which is the ScratchDoubleReg. 207 static constexpr FloatRegister ABINonArgDoubleReg = 208 FloatRegister(X86Encoding::xmm8, FloatRegisters::Double); 209 210 // See "ABI special registers" in Assembler-shared.h for more information. 211 static constexpr Register ABINonArgReturnReg0 = r10; 212 static constexpr Register ABINonArgReturnReg1 = r12; 213 static constexpr Register ABINonVolatileReg = r13; 214 215 // See "ABI special registers" in Assembler-shared.h for more information. 216 static constexpr Register ABINonArgReturnVolatileReg = r10; 217 218 // See "ABI special registers" in Assembler-shared.h, and "The WASM ABIs" in 219 // WasmFrame.h for more information. 220 static constexpr Register InstanceReg = r14; 221 static constexpr Register HeapReg = r15; 222 223 // Registers used for asm.js/wasm table calls. These registers must be disjoint 224 // from the ABI argument registers, InstanceReg and each other. 225 static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0; 226 static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1; 227 static constexpr Register WasmTableCallSigReg = ABINonArgReg2; 228 static constexpr Register WasmTableCallIndexReg = ABINonArgReg3; 229 230 // Registers used for ref calls. 231 static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0; 232 static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1; 233 static constexpr Register WasmCallRefCallScratchReg2 = ABINonArgReg2; 234 static constexpr Register WasmCallRefReg = ABINonArgReg3; 235 236 // Registers used for wasm tail calls operations. 237 static constexpr Register WasmTailCallInstanceScratchReg = ABINonArgReg1; 238 static constexpr Register WasmTailCallRAScratchReg = ABINonArgReg2; 239 static constexpr Register WasmTailCallFPScratchReg = ABINonArgReg3; 240 241 // Register used as a scratch along the return path in the fast js -> wasm stub 242 // code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg. 243 // It must be a volatile register. 244 static constexpr Register WasmJitEntryReturnScratch = rbx; 245 246 static constexpr Register OsrFrameReg = IntArgReg3; 247 248 static constexpr Register PreBarrierReg = rdx; 249 250 static constexpr Register InterpreterPCReg = r14; 251 252 static constexpr uint32_t ABIStackAlignment = 16; 253 static constexpr uint32_t CodeAlignment = 16; 254 static constexpr uint32_t JitStackAlignment = 16; 255 256 static constexpr uint32_t JitStackValueAlignment = 257 JitStackAlignment / sizeof(Value); 258 static_assert(JitStackAlignment % sizeof(Value) == 0 && 259 JitStackValueAlignment >= 1, 260 "Stack alignment should be a non-zero multiple of sizeof(Value)"); 261 262 static constexpr uint32_t SimdMemoryAlignment = 16; 263 264 static_assert(CodeAlignment % SimdMemoryAlignment == 0, 265 "Code alignment should be larger than any of the alignments " 266 "which are used for " 267 "the constant sections of the code buffer. Thus it should be " 268 "larger than the " 269 "alignment for SIMD constants."); 270 271 static_assert(JitStackAlignment % SimdMemoryAlignment == 0, 272 "Stack alignment should be larger than any of the alignments " 273 "which are used for " 274 "spilled values. Thus it should be larger than the alignment " 275 "for SIMD accesses."); 276 277 static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment; 278 static constexpr uint32_t WasmTrapInstructionLength = 2; 279 280 // See comments in wasm::GenerateFunctionPrologue. The difference between these 281 // is the size of the largest callable prologue on the platform. 282 static constexpr uint32_t WasmCheckedCallEntryOffset = 0u; 283 284 static constexpr Scale ScalePointer = TimesEight; 285 286 } // namespace jit 287 } // namespace js 288 289 #include "jit/x86-shared/Assembler-x86-shared.h" 290 291 namespace js { 292 namespace jit { 293 294 // Return operand from a JS -> JS call. 295 static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg); 296 297 class Assembler : public AssemblerX86Shared { 298 // x64 jumps may need extra bits of relocation, because a jump may extend 299 // beyond the signed 32-bit range. To account for this we add an extended 300 // jump table at the bottom of the instruction stream, and if a jump 301 // overflows its range, it will redirect here. 302 // 303 // Each entry in this table is a jmp [rip], followed by a ud2 to hint to the 304 // hardware branch predictor that there is no fallthrough, followed by the 305 // eight bytes containing an immediate address. This comes out to 16 bytes. 306 // +1 byte for opcode 307 // +1 byte for mod r/m 308 // +4 bytes for rip-relative offset (2) 309 // +2 bytes for ud2 instruction 310 // +8 bytes for 64-bit address 311 // 312 static const uint32_t SizeOfExtendedJump = 1 + 1 + 4 + 2 + 8; 313 static const uint32_t SizeOfJumpTableEntry = 16; 314 315 // Two kinds of jumps on x64: 316 // 317 // * codeJumps_ tracks jumps with target within the executable code region 318 // for the process. These jumps don't need entries in the extended jump 319 // table because source and target must be within 2 GB of each other. 320 // 321 // * extendedJumps_ tracks jumps with target outside the executable code 322 // region. These jumps need entries in the extended jump table described 323 // above. 324 using PendingJumpVector = Vector<RelativePatch, 8, SystemAllocPolicy>; 325 PendingJumpVector codeJumps_; 326 PendingJumpVector extendedJumps_; 327 328 uint32_t extendedJumpTable_; 329 330 static JitCode* CodeFromJump(JitCode* code, uint8_t* jump); 331 332 private: 333 void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind reloc); 334 335 public: 336 using AssemblerX86Shared::j; 337 using AssemblerX86Shared::jmp; 338 using AssemblerX86Shared::pop; 339 using AssemblerX86Shared::push; 340 using AssemblerX86Shared::vmovq; 341 342 Assembler() : extendedJumpTable_(0) {} 343 344 static void TraceJumpRelocations(JSTracer* trc, JitCode* code, 345 CompactBufferReader& reader); 346 347 // The buffer is about to be linked, make sure any constant pools or excess 348 // bookkeeping has been flushed to the instruction stream. 349 void finish(); 350 351 // Copy the assembly code to the given buffer, and perform any pending 352 // relocations relying on the target address. 353 void executableCopy(uint8_t* buffer); 354 355 void assertNoGCThings() const { 356 #ifdef DEBUG 357 MOZ_ASSERT(dataRelocations_.length() == 0); 358 for (auto& j : codeJumps_) { 359 MOZ_ASSERT(j.kind == RelocationKind::HARDCODED); 360 } 361 for (auto& j : extendedJumps_) { 362 MOZ_ASSERT(j.kind == RelocationKind::HARDCODED); 363 } 364 #endif 365 } 366 367 // Actual assembly emitting functions. 368 369 void push(const ImmGCPtr ptr) { 370 movq(ptr, ScratchReg); 371 push(ScratchReg); 372 } 373 void push(const ImmWord ptr) { 374 // We often end up with ImmWords that actually fit into int32. 375 // Be aware of the sign extension behavior. 376 if (intptr_t(ptr.value) == intptr_t(int32_t(ptr.value))) { 377 push(Imm32(ptr.value)); 378 } else { 379 movq(ptr, ScratchReg); 380 push(ScratchReg); 381 } 382 } 383 void push(ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); } 384 void push(FloatRegister src) { 385 // We allocate space for double even when storing a float. 386 subq(Imm32(sizeof(double)), StackPointer); 387 if (src.isDouble()) { 388 vmovsd(src, Address(StackPointer, 0)); 389 } else { 390 MOZ_ASSERT(src.isSingle(), "simd128 is not supported"); 391 vmovss(src, Address(StackPointer, 0)); 392 } 393 } 394 CodeOffset pushWithPatch(ImmWord word) { 395 CodeOffset label = movWithPatch(word, ScratchReg); 396 push(ScratchReg); 397 return label; 398 } 399 400 void pop(FloatRegister src) { 401 if (src.isDouble()) { 402 vmovsd(Address(StackPointer, 0), src); 403 } else { 404 MOZ_ASSERT(src.isSingle(), "simd128 is not supported"); 405 vmovss(Address(StackPointer, 0), src); 406 } 407 // We free space for double even when storing a float. 408 addq(Imm32(sizeof(double)), StackPointer); 409 } 410 411 CodeOffset movWithPatch(ImmWord word, Register dest) { 412 masm.movq_i64r(word.value, dest.encoding()); 413 return CodeOffset(masm.currentOffset()); 414 } 415 CodeOffset movWithPatch(ImmPtr imm, Register dest) { 416 return movWithPatch(ImmWord(uintptr_t(imm.value)), dest); 417 } 418 419 // This is for patching during code generation, not after. 420 void patchAddq(CodeOffset offset, int32_t n) { 421 unsigned char* code = masm.data(); 422 X86Encoding::SetInt32(code + offset.offset(), n); 423 } 424 425 // Load an ImmWord value into a register. Note that this instruction will 426 // attempt to optimize its immediate field size. When a full 64-bit 427 // immediate is needed for a relocation, use movWithPatch. 428 void movq(ImmWord word, Register dest) { 429 // Load a 64-bit immediate into a register. If the value falls into 430 // certain ranges, we can use specialized instructions which have 431 // smaller encodings. 432 if (word.value <= UINT32_MAX) { 433 // movl has a 32-bit unsigned (effectively) immediate field. 434 masm.movl_i32r((uint32_t)word.value, dest.encoding()); 435 } else if ((intptr_t)word.value >= INT32_MIN && 436 (intptr_t)word.value <= INT32_MAX) { 437 // movq has a 32-bit signed immediate field. 438 masm.movq_i32r((int32_t)(intptr_t)word.value, dest.encoding()); 439 } else { 440 // Otherwise use movabs. 441 masm.movq_i64r(word.value, dest.encoding()); 442 } 443 } 444 void movq(ImmPtr imm, Register dest) { 445 movq(ImmWord(uintptr_t(imm.value)), dest); 446 } 447 void movq(ImmGCPtr ptr, Register dest) { 448 masm.movq_i64r(uintptr_t(ptr.value), dest.encoding()); 449 writeDataRelocation(ptr); 450 } 451 void movq(const Operand& src, Register dest) { 452 switch (src.kind()) { 453 case Operand::REG: 454 masm.movq_rr(src.reg(), dest.encoding()); 455 break; 456 case Operand::MEM_REG_DISP: 457 masm.movq_mr(src.disp(), src.base(), dest.encoding()); 458 break; 459 case Operand::MEM_SCALE: 460 masm.movq_mr(src.disp(), src.base(), src.index(), src.scale(), 461 dest.encoding()); 462 break; 463 case Operand::MEM_ADDRESS32: 464 masm.movq_mr(src.address(), dest.encoding()); 465 break; 466 default: 467 MOZ_CRASH("unexpected operand kind"); 468 } 469 } 470 void movq(Register src, const Operand& dest) { 471 switch (dest.kind()) { 472 case Operand::REG: 473 masm.movq_rr(src.encoding(), dest.reg()); 474 break; 475 case Operand::MEM_REG_DISP: 476 masm.movq_rm(src.encoding(), dest.disp(), dest.base()); 477 break; 478 case Operand::MEM_SCALE: 479 masm.movq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 480 dest.scale()); 481 break; 482 case Operand::MEM_ADDRESS32: 483 masm.movq_rm(src.encoding(), dest.address()); 484 break; 485 default: 486 MOZ_CRASH("unexpected operand kind"); 487 } 488 } 489 void movq(Imm32 imm32, const Operand& dest) { 490 switch (dest.kind()) { 491 case Operand::REG: 492 masm.movl_i32r(imm32.value, dest.reg()); 493 break; 494 case Operand::MEM_REG_DISP: 495 masm.movq_i32m(imm32.value, dest.disp(), dest.base()); 496 break; 497 case Operand::MEM_SCALE: 498 masm.movq_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), 499 dest.scale()); 500 break; 501 case Operand::MEM_ADDRESS32: 502 masm.movq_i32m(imm32.value, dest.address()); 503 break; 504 default: 505 MOZ_CRASH("unexpected operand kind"); 506 } 507 } 508 void vmovq(Register src, FloatRegister dest) { 509 masm.vmovq_rr(src.encoding(), dest.encoding()); 510 } 511 void vmovq(FloatRegister src, Register dest) { 512 masm.vmovq_rr(src.encoding(), dest.encoding()); 513 } 514 void movq(Register src, Register dest) { 515 masm.movq_rr(src.encoding(), dest.encoding()); 516 } 517 518 void cmovCCq(Condition cond, const Operand& src, Register dest) { 519 X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond); 520 switch (src.kind()) { 521 case Operand::REG: 522 masm.cmovCCq_rr(cc, src.reg(), dest.encoding()); 523 break; 524 case Operand::MEM_REG_DISP: 525 masm.cmovCCq_mr(cc, src.disp(), src.base(), dest.encoding()); 526 break; 527 case Operand::MEM_SCALE: 528 masm.cmovCCq_mr(cc, src.disp(), src.base(), src.index(), src.scale(), 529 dest.encoding()); 530 break; 531 default: 532 MOZ_CRASH("unexpected operand kind"); 533 } 534 } 535 void cmovCCq(Condition cond, Register src, Register dest) { 536 X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond); 537 masm.cmovCCq_rr(cc, src.encoding(), dest.encoding()); 538 } 539 540 void cmovzq(const Operand& src, Register dest) { 541 cmovCCq(Condition::Zero, src, dest); 542 } 543 void cmovnzq(const Operand& src, Register dest) { 544 cmovCCq(Condition::NonZero, src, dest); 545 } 546 547 template <typename T> 548 void lock_addq(T src, const Operand& op) { 549 masm.prefix_lock(); 550 addq(src, op); 551 } 552 template <typename T> 553 void lock_subq(T src, const Operand& op) { 554 masm.prefix_lock(); 555 subq(src, op); 556 } 557 template <typename T> 558 void lock_andq(T src, const Operand& op) { 559 masm.prefix_lock(); 560 andq(src, op); 561 } 562 template <typename T> 563 void lock_orq(T src, const Operand& op) { 564 masm.prefix_lock(); 565 orq(src, op); 566 } 567 template <typename T> 568 void lock_xorq(T src, const Operand& op) { 569 masm.prefix_lock(); 570 xorq(src, op); 571 } 572 573 void lock_cmpxchgq(Register src, const Operand& mem) { 574 masm.prefix_lock(); 575 switch (mem.kind()) { 576 case Operand::MEM_REG_DISP: 577 masm.cmpxchgq(src.encoding(), mem.disp(), mem.base()); 578 break; 579 case Operand::MEM_SCALE: 580 masm.cmpxchgq(src.encoding(), mem.disp(), mem.base(), mem.index(), 581 mem.scale()); 582 break; 583 default: 584 MOZ_CRASH("unexpected operand kind"); 585 } 586 } 587 588 void xchgq(Register src, Register dest) { 589 masm.xchgq_rr(src.encoding(), dest.encoding()); 590 } 591 592 void xchgq(Register src, const Operand& mem) { 593 switch (mem.kind()) { 594 case Operand::MEM_REG_DISP: 595 masm.xchgq_rm(src.encoding(), mem.disp(), mem.base()); 596 break; 597 case Operand::MEM_SCALE: 598 masm.xchgq_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), 599 mem.scale()); 600 break; 601 default: 602 MOZ_CRASH("unexpected operand kind"); 603 } 604 } 605 606 void lock_xaddq(Register srcdest, const Operand& mem) { 607 switch (mem.kind()) { 608 case Operand::MEM_REG_DISP: 609 masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base()); 610 break; 611 case Operand::MEM_SCALE: 612 masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base(), 613 mem.index(), mem.scale()); 614 break; 615 default: 616 MOZ_CRASH("unexpected operand kind"); 617 } 618 } 619 620 void movsbq(const Operand& src, Register dest) { 621 switch (src.kind()) { 622 case Operand::REG: 623 masm.movsbq_rr(src.reg(), dest.encoding()); 624 break; 625 case Operand::MEM_REG_DISP: 626 masm.movsbq_mr(src.disp(), src.base(), dest.encoding()); 627 break; 628 case Operand::MEM_SCALE: 629 masm.movsbq_mr(src.disp(), src.base(), src.index(), src.scale(), 630 dest.encoding()); 631 break; 632 default: 633 MOZ_CRASH("unexpected operand kind"); 634 } 635 } 636 637 void movzbq(const Operand& src, Register dest) { 638 // movzbl zero-extends to 64 bits and is one byte smaller, so use that 639 // instead. 640 movzbl(src, dest); 641 } 642 643 void movswq(const Operand& src, Register dest) { 644 switch (src.kind()) { 645 case Operand::REG: 646 masm.movswq_rr(src.reg(), dest.encoding()); 647 break; 648 case Operand::MEM_REG_DISP: 649 masm.movswq_mr(src.disp(), src.base(), dest.encoding()); 650 break; 651 case Operand::MEM_SCALE: 652 masm.movswq_mr(src.disp(), src.base(), src.index(), src.scale(), 653 dest.encoding()); 654 break; 655 default: 656 MOZ_CRASH("unexpected operand kind"); 657 } 658 } 659 660 void movzwq(const Operand& src, Register dest) { 661 // movzwl zero-extends to 64 bits and is one byte smaller, so use that 662 // instead. 663 movzwl(src, dest); 664 } 665 666 void movslq(Register src, Register dest) { 667 masm.movslq_rr(src.encoding(), dest.encoding()); 668 } 669 void movslq(const Operand& src, Register dest) { 670 switch (src.kind()) { 671 case Operand::REG: 672 masm.movslq_rr(src.reg(), dest.encoding()); 673 break; 674 case Operand::MEM_REG_DISP: 675 masm.movslq_mr(src.disp(), src.base(), dest.encoding()); 676 break; 677 case Operand::MEM_SCALE: 678 masm.movslq_mr(src.disp(), src.base(), src.index(), src.scale(), 679 dest.encoding()); 680 break; 681 default: 682 MOZ_CRASH("unexpected operand kind"); 683 } 684 } 685 686 void andq(Register src, Register dest) { 687 masm.andq_rr(src.encoding(), dest.encoding()); 688 } 689 void andq(Imm32 imm, Register dest) { 690 masm.andq_ir(imm.value, dest.encoding()); 691 } 692 void andq(const Operand& src, Register dest) { 693 switch (src.kind()) { 694 case Operand::REG: 695 masm.andq_rr(src.reg(), dest.encoding()); 696 break; 697 case Operand::MEM_REG_DISP: 698 masm.andq_mr(src.disp(), src.base(), dest.encoding()); 699 break; 700 case Operand::MEM_SCALE: 701 masm.andq_mr(src.disp(), src.base(), src.index(), src.scale(), 702 dest.encoding()); 703 break; 704 case Operand::MEM_ADDRESS32: 705 masm.andq_mr(src.address(), dest.encoding()); 706 break; 707 default: 708 MOZ_CRASH("unexpected operand kind"); 709 } 710 } 711 void andq(Register src, const Operand& dest) { 712 switch (dest.kind()) { 713 case Operand::REG: 714 masm.andq_rr(src.encoding(), dest.reg()); 715 break; 716 case Operand::MEM_REG_DISP: 717 masm.andq_rm(src.encoding(), dest.disp(), dest.base()); 718 break; 719 case Operand::MEM_SCALE: 720 masm.andq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 721 dest.scale()); 722 break; 723 default: 724 MOZ_CRASH("unexpected operand kind"); 725 } 726 } 727 728 void andnq(Register src1, Register src2, Register dest) { 729 MOZ_ASSERT(HasBMI1()); 730 masm.andnq_rrr(src1.encoding(), src2.encoding(), dest.encoding()); 731 } 732 733 void addq(Imm32 imm, Register dest) { 734 masm.addq_ir(imm.value, dest.encoding()); 735 } 736 CodeOffset addqWithPatch(Imm32 imm, Register dest) { 737 masm.addq_i32r(imm.value, dest.encoding()); 738 return CodeOffset(masm.currentOffset()); 739 } 740 void addq(Imm32 imm, const Operand& dest) { 741 switch (dest.kind()) { 742 case Operand::REG: 743 masm.addq_ir(imm.value, dest.reg()); 744 break; 745 case Operand::MEM_REG_DISP: 746 masm.addq_im(imm.value, dest.disp(), dest.base()); 747 break; 748 case Operand::MEM_ADDRESS32: 749 masm.addq_im(imm.value, dest.address()); 750 break; 751 default: 752 MOZ_CRASH("unexpected operand kind"); 753 } 754 } 755 void addq(Register src, Register dest) { 756 masm.addq_rr(src.encoding(), dest.encoding()); 757 } 758 void addq(const Operand& src, Register dest) { 759 switch (src.kind()) { 760 case Operand::REG: 761 masm.addq_rr(src.reg(), dest.encoding()); 762 break; 763 case Operand::MEM_REG_DISP: 764 masm.addq_mr(src.disp(), src.base(), dest.encoding()); 765 break; 766 case Operand::MEM_ADDRESS32: 767 masm.addq_mr(src.address(), dest.encoding()); 768 break; 769 case Operand::MEM_SCALE: 770 masm.addq_mr(src.disp(), src.base(), src.index(), src.scale(), 771 dest.encoding()); 772 break; 773 default: 774 MOZ_CRASH("unexpected operand kind"); 775 } 776 } 777 void addq(Register src, const Operand& dest) { 778 switch (dest.kind()) { 779 case Operand::REG: 780 masm.addq_rr(src.encoding(), dest.reg()); 781 break; 782 case Operand::MEM_REG_DISP: 783 masm.addq_rm(src.encoding(), dest.disp(), dest.base()); 784 break; 785 case Operand::MEM_SCALE: 786 masm.addq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 787 dest.scale()); 788 break; 789 default: 790 MOZ_CRASH("unexpected operand kind"); 791 } 792 } 793 794 void subq(Imm32 imm, Register dest) { 795 masm.subq_ir(imm.value, dest.encoding()); 796 } 797 void subq(Register src, Register dest) { 798 masm.subq_rr(src.encoding(), dest.encoding()); 799 } 800 void subq(const Operand& src, Register dest) { 801 switch (src.kind()) { 802 case Operand::REG: 803 masm.subq_rr(src.reg(), dest.encoding()); 804 break; 805 case Operand::MEM_REG_DISP: 806 masm.subq_mr(src.disp(), src.base(), dest.encoding()); 807 break; 808 case Operand::MEM_ADDRESS32: 809 masm.subq_mr(src.address(), dest.encoding()); 810 break; 811 default: 812 MOZ_CRASH("unexpected operand kind"); 813 } 814 } 815 void subq(Register src, const Operand& dest) { 816 switch (dest.kind()) { 817 case Operand::REG: 818 masm.subq_rr(src.encoding(), dest.reg()); 819 break; 820 case Operand::MEM_REG_DISP: 821 masm.subq_rm(src.encoding(), dest.disp(), dest.base()); 822 break; 823 case Operand::MEM_SCALE: 824 masm.subq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 825 dest.scale()); 826 break; 827 default: 828 MOZ_CRASH("unexpected operand kind"); 829 } 830 } 831 void shlq(Imm32 imm, Register dest) { 832 masm.shlq_ir(imm.value, dest.encoding()); 833 } 834 void shrq(Imm32 imm, Register dest) { 835 masm.shrq_ir(imm.value, dest.encoding()); 836 } 837 void sarq(Imm32 imm, Register dest) { 838 masm.sarq_ir(imm.value, dest.encoding()); 839 } 840 void shlq_cl(Register dest) { masm.shlq_CLr(dest.encoding()); } 841 void shrq_cl(Register dest) { masm.shrq_CLr(dest.encoding()); } 842 void sarq_cl(Register dest) { masm.sarq_CLr(dest.encoding()); } 843 void sarxq(Register src, Register shift, Register dest) { 844 MOZ_ASSERT(HasBMI2()); 845 masm.sarxq_rrr(src.encoding(), shift.encoding(), dest.encoding()); 846 } 847 void shlxq(Register src, Register shift, Register dest) { 848 MOZ_ASSERT(HasBMI2()); 849 masm.shlxq_rrr(src.encoding(), shift.encoding(), dest.encoding()); 850 } 851 void shrxq(Register src, Register shift, Register dest) { 852 MOZ_ASSERT(HasBMI2()); 853 masm.shrxq_rrr(src.encoding(), shift.encoding(), dest.encoding()); 854 } 855 void rolq(Imm32 imm, Register dest) { 856 masm.rolq_ir(imm.value, dest.encoding()); 857 } 858 void rolq_cl(Register dest) { masm.rolq_CLr(dest.encoding()); } 859 void rorq(Imm32 imm, Register dest) { 860 masm.rorq_ir(imm.value, dest.encoding()); 861 } 862 void rorq_cl(Register dest) { masm.rorq_CLr(dest.encoding()); } 863 void orq(Imm32 imm, Register dest) { 864 masm.orq_ir(imm.value, dest.encoding()); 865 } 866 void orq(Register src, Register dest) { 867 masm.orq_rr(src.encoding(), dest.encoding()); 868 } 869 void orq(const Operand& src, Register dest) { 870 switch (src.kind()) { 871 case Operand::REG: 872 masm.orq_rr(src.reg(), dest.encoding()); 873 break; 874 case Operand::MEM_REG_DISP: 875 masm.orq_mr(src.disp(), src.base(), dest.encoding()); 876 break; 877 case Operand::MEM_ADDRESS32: 878 masm.orq_mr(src.address(), dest.encoding()); 879 break; 880 default: 881 MOZ_CRASH("unexpected operand kind"); 882 } 883 } 884 void orq(Register src, const Operand& dest) { 885 switch (dest.kind()) { 886 case Operand::REG: 887 masm.orq_rr(src.encoding(), dest.reg()); 888 break; 889 case Operand::MEM_REG_DISP: 890 masm.orq_rm(src.encoding(), dest.disp(), dest.base()); 891 break; 892 case Operand::MEM_SCALE: 893 masm.orq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 894 dest.scale()); 895 break; 896 default: 897 MOZ_CRASH("unexpected operand kind"); 898 } 899 } 900 void xorq(Register src, Register dest) { 901 masm.xorq_rr(src.encoding(), dest.encoding()); 902 } 903 void xorq(Imm32 imm, Register dest) { 904 masm.xorq_ir(imm.value, dest.encoding()); 905 } 906 void xorq(const Operand& src, Register dest) { 907 switch (src.kind()) { 908 case Operand::REG: 909 masm.xorq_rr(src.reg(), dest.encoding()); 910 break; 911 case Operand::MEM_REG_DISP: 912 masm.xorq_mr(src.disp(), src.base(), dest.encoding()); 913 break; 914 case Operand::MEM_SCALE: 915 masm.xorq_mr(src.disp(), src.base(), src.index(), src.scale(), 916 dest.encoding()); 917 break; 918 case Operand::MEM_ADDRESS32: 919 masm.xorq_mr(src.address(), dest.encoding()); 920 break; 921 default: 922 MOZ_CRASH("unexpected operand kind"); 923 } 924 } 925 void xorq(Register src, const Operand& dest) { 926 switch (dest.kind()) { 927 case Operand::REG: 928 masm.xorq_rr(src.encoding(), dest.reg()); 929 break; 930 case Operand::MEM_REG_DISP: 931 masm.xorq_rm(src.encoding(), dest.disp(), dest.base()); 932 break; 933 case Operand::MEM_SCALE: 934 masm.xorq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 935 dest.scale()); 936 break; 937 default: 938 MOZ_CRASH("unexpected operand kind"); 939 } 940 } 941 942 void bsrq(const Register& src, const Register& dest) { 943 masm.bsrq_rr(src.encoding(), dest.encoding()); 944 } 945 void bsfq(const Register& src, const Register& dest) { 946 masm.bsfq_rr(src.encoding(), dest.encoding()); 947 } 948 void bswapq(const Register& reg) { masm.bswapq_r(reg.encoding()); } 949 void lzcntq(const Register& src, const Register& dest) { 950 masm.lzcntq_rr(src.encoding(), dest.encoding()); 951 } 952 void tzcntq(const Register& src, const Register& dest) { 953 masm.tzcntq_rr(src.encoding(), dest.encoding()); 954 } 955 void popcntq(const Register& src, const Register& dest) { 956 masm.popcntq_rr(src.encoding(), dest.encoding()); 957 } 958 959 void imulq(Register multiplier) { 960 // Consumes rax as the other argument and clobbers rdx, as the result is in 961 // rdx:rax. 962 masm.imulq_r(multiplier.encoding()); 963 } 964 void umulq(Register multiplier) { masm.mulq_r(multiplier.encoding()); } 965 void imulq(Imm32 imm, Register src, Register dest) { 966 masm.imulq_ir(imm.value, src.encoding(), dest.encoding()); 967 } 968 void imulq(Register src, Register dest) { 969 masm.imulq_rr(src.encoding(), dest.encoding()); 970 } 971 void imulq(const Operand& src, Register dest) { 972 switch (src.kind()) { 973 case Operand::REG: 974 masm.imulq_rr(src.reg(), dest.encoding()); 975 break; 976 case Operand::MEM_REG_DISP: 977 masm.imulq_mr(src.disp(), src.base(), dest.encoding()); 978 break; 979 case Operand::MEM_ADDRESS32: 980 MOZ_CRASH("NYI"); 981 break; 982 default: 983 MOZ_CRASH("unexpected operand kind"); 984 } 985 } 986 987 void cqo() { masm.cqo(); } 988 void idivq(Register divisor) { masm.idivq_r(divisor.encoding()); } 989 void udivq(Register divisor) { masm.divq_r(divisor.encoding()); } 990 991 void vcvtsi2sdq(Register src, FloatRegister dest) { 992 masm.vcvtsi2sdq_rr(src.encoding(), dest.encoding()); 993 } 994 995 void vpextrq(unsigned lane, FloatRegister src, Register dest) { 996 MOZ_ASSERT(HasSSE41()); 997 masm.vpextrq_irr(lane, src.encoding(), dest.encoding()); 998 } 999 1000 void vpinsrq(unsigned lane, Register src1, FloatRegister src0, 1001 FloatRegister dest) { 1002 MOZ_ASSERT(HasSSE41()); 1003 masm.vpinsrq_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); 1004 } 1005 1006 void negq(Register reg) { masm.negq_r(reg.encoding()); } 1007 1008 void notq(Register reg) { masm.notq_r(reg.encoding()); } 1009 1010 void mov(ImmWord word, Register dest) { 1011 // Use xor for setting registers to zero, as it is specially optimized 1012 // for this purpose on modern hardware. Note that it does clobber FLAGS 1013 // though. Use xorl instead of xorq since they are functionally 1014 // equivalent (32-bit instructions zero-extend their results to 64 bits) 1015 // and xorl has a smaller encoding. 1016 if (word.value == 0) { 1017 xorl(dest, dest); 1018 } else { 1019 movq(word, dest); 1020 } 1021 } 1022 void mov(ImmPtr imm, Register dest) { movq(imm, dest); } 1023 void mov(wasm::SymbolicAddress imm, Register dest) { 1024 masm.movq_i64r(-1, dest.encoding()); 1025 append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm)); 1026 } 1027 void mov(const Operand& src, Register dest) { movq(src, dest); } 1028 void mov(Register src, const Operand& dest) { movq(src, dest); } 1029 void mov(Imm32 imm32, const Operand& dest) { movq(imm32, dest); } 1030 void mov(Register src, Register dest) { movq(src, dest); } 1031 void mov(CodeLabel* label, Register dest) { 1032 masm.movq_i64r(/* placeholder */ 0, dest.encoding()); 1033 label->patchAt()->bind(masm.size()); 1034 } 1035 void xchg(Register src, Register dest) { xchgq(src, dest); } 1036 1037 void lea(const Operand& src, Register dest) { 1038 switch (src.kind()) { 1039 case Operand::MEM_REG_DISP: 1040 masm.leaq_mr(src.disp(), src.base(), dest.encoding()); 1041 break; 1042 case Operand::MEM_SCALE: 1043 masm.leaq_mr(src.disp(), src.base(), src.index(), src.scale(), 1044 dest.encoding()); 1045 break; 1046 case Operand::MEM_SCALE_NOBASE: 1047 masm.leaq_mr(src.disp(), src.index(), src.scale(), dest.encoding()); 1048 break; 1049 default: 1050 MOZ_CRASH("unexepcted operand kind"); 1051 } 1052 } 1053 1054 void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); } 1055 void cmovzPtr(const Operand& src, Register dest) { return cmovzq(src, dest); } 1056 1057 CodeOffset loadRipRelativeInt32(Register dest) { 1058 return CodeOffset(masm.movl_ripr(dest.encoding()).offset()); 1059 } 1060 CodeOffset loadRipRelativeInt64(Register dest) { 1061 return CodeOffset(masm.movq_ripr(dest.encoding()).offset()); 1062 } 1063 CodeOffset loadRipRelativeDouble(FloatRegister dest) { 1064 return CodeOffset(masm.vmovsd_ripr(dest.encoding()).offset()); 1065 } 1066 CodeOffset loadRipRelativeFloat32(FloatRegister dest) { 1067 return CodeOffset(masm.vmovss_ripr(dest.encoding()).offset()); 1068 } 1069 CodeOffset loadRipRelativeInt32x4(FloatRegister dest) { 1070 return CodeOffset(masm.vmovdqa_ripr(dest.encoding()).offset()); 1071 } 1072 CodeOffset loadRipRelativeFloat32x4(FloatRegister dest) { 1073 return CodeOffset(masm.vmovaps_ripr(dest.encoding()).offset()); 1074 } 1075 CodeOffset leaRipRelative(Register dest) { 1076 return CodeOffset(masm.leaq_rip(dest.encoding()).offset()); 1077 } 1078 1079 void cmpq(Register rhs, Register lhs) { 1080 masm.cmpq_rr(rhs.encoding(), lhs.encoding()); 1081 } 1082 void cmpq(Register rhs, const Operand& lhs) { 1083 switch (lhs.kind()) { 1084 case Operand::REG: 1085 masm.cmpq_rr(rhs.encoding(), lhs.reg()); 1086 break; 1087 case Operand::MEM_REG_DISP: 1088 masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base()); 1089 break; 1090 case Operand::MEM_SCALE: 1091 masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(), 1092 lhs.scale()); 1093 break; 1094 case Operand::MEM_ADDRESS32: 1095 masm.cmpq_rm(rhs.encoding(), lhs.address()); 1096 break; 1097 default: 1098 MOZ_CRASH("unexpected operand kind"); 1099 } 1100 } 1101 void cmpq(Imm32 rhs, Register lhs) { 1102 masm.cmpq_ir(rhs.value, lhs.encoding()); 1103 } 1104 void cmpq(Imm32 rhs, const Operand& lhs) { 1105 switch (lhs.kind()) { 1106 case Operand::REG: 1107 masm.cmpq_ir(rhs.value, lhs.reg()); 1108 break; 1109 case Operand::MEM_REG_DISP: 1110 masm.cmpq_im(rhs.value, lhs.disp(), lhs.base()); 1111 break; 1112 case Operand::MEM_SCALE: 1113 masm.cmpq_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), 1114 lhs.scale()); 1115 break; 1116 case Operand::MEM_ADDRESS32: 1117 masm.cmpq_im(rhs.value, lhs.address()); 1118 break; 1119 default: 1120 MOZ_CRASH("unexpected operand kind"); 1121 } 1122 } 1123 void cmpq(const Operand& rhs, Register lhs) { 1124 switch (rhs.kind()) { 1125 case Operand::REG: 1126 masm.cmpq_rr(rhs.reg(), lhs.encoding()); 1127 break; 1128 case Operand::MEM_REG_DISP: 1129 masm.cmpq_mr(rhs.disp(), rhs.base(), lhs.encoding()); 1130 break; 1131 default: 1132 MOZ_CRASH("unexpected operand kind"); 1133 } 1134 } 1135 1136 void testq(Imm32 rhs, Register lhs) { 1137 masm.testq_ir(rhs.value, lhs.encoding()); 1138 } 1139 void testq(Register rhs, Register lhs) { 1140 masm.testq_rr(rhs.encoding(), lhs.encoding()); 1141 } 1142 void testq(Imm32 rhs, const Operand& lhs) { 1143 switch (lhs.kind()) { 1144 case Operand::REG: 1145 masm.testq_ir(rhs.value, lhs.reg()); 1146 break; 1147 case Operand::MEM_REG_DISP: 1148 masm.testq_i32m(rhs.value, lhs.disp(), lhs.base()); 1149 break; 1150 default: 1151 MOZ_CRASH("unexpected operand kind"); 1152 break; 1153 } 1154 } 1155 1156 void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) { 1157 MOZ_ASSERT(hasCreator()); 1158 JmpSrc src = masm.jmp(); 1159 addPendingJump(src, target, reloc); 1160 } 1161 void j(Condition cond, ImmPtr target, 1162 RelocationKind reloc = RelocationKind::HARDCODED) { 1163 JmpSrc src = masm.jCC(static_cast<X86Encoding::Condition>(cond)); 1164 addPendingJump(src, target, reloc); 1165 } 1166 1167 void jmp(JitCode* target) { 1168 jmp(ImmPtr(target->raw()), RelocationKind::JITCODE); 1169 } 1170 void j(Condition cond, JitCode* target) { 1171 j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE); 1172 } 1173 void call(JitCode* target) { 1174 JmpSrc src = masm.call(); 1175 addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE); 1176 } 1177 void call(ImmWord target) { call(ImmPtr((void*)target.value)); } 1178 void call(ImmPtr target) { 1179 JmpSrc src = masm.call(); 1180 addPendingJump(src, target, RelocationKind::HARDCODED); 1181 } 1182 1183 // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch 1184 // this instruction. 1185 CodeOffset toggledCall(JitCode* target, bool enabled) { 1186 CodeOffset offset(size()); 1187 JmpSrc src = enabled ? masm.call() : masm.cmp_eax(); 1188 addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE); 1189 MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr)); 1190 return offset; 1191 } 1192 1193 static size_t ToggledCallSize(uint8_t* code) { 1194 // Size of a call instruction. 1195 return 5; 1196 } 1197 1198 // Do not mask shared implementations. 1199 using AssemblerX86Shared::call; 1200 1201 void vcvttsd2sq(FloatRegister src, Register dest) { 1202 masm.vcvttsd2sq_rr(src.encoding(), dest.encoding()); 1203 } 1204 void vcvttss2sq(FloatRegister src, Register dest) { 1205 masm.vcvttss2sq_rr(src.encoding(), dest.encoding()); 1206 } 1207 void vcvtsq2sd(Register src1, FloatRegister src0, FloatRegister dest) { 1208 masm.vcvtsq2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 1209 } 1210 void vcvtsq2ss(Register src1, FloatRegister src0, FloatRegister dest) { 1211 masm.vcvtsq2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 1212 } 1213 }; 1214 1215 static inline bool GetIntArgReg(uint32_t intArg, uint32_t floatArg, 1216 Register* out) { 1217 #if defined(_WIN64) 1218 uint32_t arg = intArg + floatArg; 1219 #else 1220 uint32_t arg = intArg; 1221 #endif 1222 if (arg >= NumIntArgRegs) { 1223 return false; 1224 } 1225 *out = IntArgRegs[arg]; 1226 return true; 1227 } 1228 1229 // Get a register in which we plan to put a quantity that will be used as an 1230 // integer argument. This differs from GetIntArgReg in that if we have no more 1231 // actual argument registers to use we will fall back on using whatever 1232 // CallTempReg* don't overlap the argument registers, and only fail once those 1233 // run out too. 1234 static inline bool GetTempRegForIntArg(uint32_t usedIntArgs, 1235 uint32_t usedFloatArgs, Register* out) { 1236 if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) { 1237 return true; 1238 } 1239 // Unfortunately, we have to assume things about the point at which 1240 // GetIntArgReg returns false, because we need to know how many registers it 1241 // can allocate. 1242 #if defined(_WIN64) 1243 uint32_t arg = usedIntArgs + usedFloatArgs; 1244 #else 1245 uint32_t arg = usedIntArgs; 1246 #endif 1247 arg -= NumIntArgRegs; 1248 if (arg >= NumCallTempNonArgRegs) { 1249 return false; 1250 } 1251 *out = CallTempNonArgRegs[arg]; 1252 return true; 1253 } 1254 1255 static inline bool GetFloatArgReg(uint32_t intArg, uint32_t floatArg, 1256 FloatRegister* out) { 1257 #if defined(_WIN64) 1258 uint32_t arg = intArg + floatArg; 1259 #else 1260 uint32_t arg = floatArg; 1261 #endif 1262 if (floatArg >= NumFloatArgRegs) { 1263 return false; 1264 } 1265 *out = FloatArgRegs[arg]; 1266 return true; 1267 } 1268 1269 } // namespace jit 1270 } // namespace js 1271 1272 #endif /* jit_x64_Assembler_x64_h */