Assembler-x86-shared.h (165967B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_x86_shared_Assembler_x86_shared_h 8 #define jit_x86_shared_Assembler_x86_shared_h 9 10 #include "mozilla/MathAlgorithms.h" 11 12 #include <cstddef> 13 14 #include "jit/shared/Assembler-shared.h" 15 #include "jit/shared/IonAssemblerBuffer.h" // jit::BufferOffset 16 17 #if defined(JS_CODEGEN_X86) 18 # include "jit/x86/BaseAssembler-x86.h" 19 #elif defined(JS_CODEGEN_X64) 20 # include "jit/x64/BaseAssembler-x64.h" 21 #else 22 # error "Unknown architecture!" 23 #endif 24 #include "jit/CompactBuffer.h" 25 #include "jit/ProcessExecutableMemory.h" 26 #include "wasm/WasmTypeDecls.h" 27 28 namespace js { 29 namespace jit { 30 31 // Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope 32 // instead. 33 struct ScratchFloat32Scope : public AutoFloatRegisterScope { 34 explicit ScratchFloat32Scope(MacroAssembler& masm) 35 : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {} 36 }; 37 38 // Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead. 39 struct ScratchDoubleScope : public AutoFloatRegisterScope { 40 explicit ScratchDoubleScope(MacroAssembler& masm) 41 : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {} 42 }; 43 44 struct ScratchSimd128Scope : public AutoFloatRegisterScope { 45 explicit ScratchSimd128Scope(MacroAssembler& masm) 46 : AutoFloatRegisterScope(masm, ScratchSimd128Reg) {} 47 }; 48 49 class Operand { 50 public: 51 enum Kind { 52 REG, 53 MEM_REG_DISP, 54 FPREG, 55 MEM_SCALE, 56 MEM_ADDRESS32, 57 MEM_SCALE_NOBASE 58 }; 59 60 private: 61 Kind kind_ : 4; 62 // Used as a Register::Encoding and a FloatRegister::Encoding. 63 uint32_t base_ : 5; 64 Scale scale_ : 3; 65 // We don't use all 8 bits, of course, but GCC complains if the size of 66 // this field is smaller than the size of Register::Encoding. 67 Register::Encoding index_ : 8; 68 int32_t disp_; 69 70 public: 71 explicit Operand(Register reg) 72 : kind_(REG), 73 base_(reg.encoding()), 74 scale_(TimesOne), 75 index_(Registers::Invalid), 76 disp_(0) {} 77 explicit Operand(FloatRegister reg) 78 : kind_(FPREG), 79 base_(reg.encoding()), 80 scale_(TimesOne), 81 index_(Registers::Invalid), 82 disp_(0) {} 83 explicit Operand(const Address& address) 84 : kind_(MEM_REG_DISP), 85 base_(address.base.encoding()), 86 scale_(TimesOne), 87 index_(Registers::Invalid), 88 disp_(address.offset) {} 89 explicit Operand(const BaseIndex& address) 90 : kind_(MEM_SCALE), 91 base_(address.base.encoding()), 92 scale_(address.scale), 93 index_(address.index.encoding()), 94 disp_(address.offset) {} 95 Operand(Register base, Register index, Scale scale, int32_t disp = 0) 96 : kind_(MEM_SCALE), 97 base_(base.encoding()), 98 scale_(scale), 99 index_(index.encoding()), 100 disp_(disp) {} 101 Operand(Register index, Scale scale, int32_t disp) 102 : kind_(MEM_SCALE_NOBASE), 103 base_(Registers::Invalid), 104 scale_(scale), 105 index_(index.encoding()), 106 disp_(disp) {} 107 Operand(Register reg, int32_t disp) 108 : kind_(MEM_REG_DISP), 109 base_(reg.encoding()), 110 scale_(TimesOne), 111 index_(Registers::Invalid), 112 disp_(disp) {} 113 explicit Operand(AbsoluteAddress address) 114 : kind_(MEM_ADDRESS32), 115 base_(Registers::Invalid), 116 scale_(TimesOne), 117 index_(Registers::Invalid), 118 disp_(X86Encoding::AddressImmediate(address.addr)) {} 119 explicit Operand(PatchedAbsoluteAddress address) 120 : kind_(MEM_ADDRESS32), 121 base_(Registers::Invalid), 122 scale_(TimesOne), 123 index_(Registers::Invalid), 124 disp_(X86Encoding::AddressImmediate(address.addr)) {} 125 126 Address toAddress() const { 127 MOZ_ASSERT(kind() == MEM_REG_DISP); 128 return Address(Register::FromCode(base()), disp()); 129 } 130 131 BaseIndex toBaseIndex() const { 132 MOZ_ASSERT(kind() == MEM_SCALE); 133 return BaseIndex(Register::FromCode(base()), Register::FromCode(index()), 134 scale(), disp()); 135 } 136 137 Kind kind() const { return kind_; } 138 Register::Encoding reg() const { 139 MOZ_ASSERT(kind() == REG); 140 return Register::Encoding(base_); 141 } 142 Register::Encoding base() const { 143 MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); 144 return Register::Encoding(base_); 145 } 146 Register::Encoding index() const { 147 MOZ_ASSERT(kind() == MEM_SCALE || kind() == MEM_SCALE_NOBASE); 148 return index_; 149 } 150 Scale scale() const { 151 MOZ_ASSERT(kind() == MEM_SCALE || kind() == MEM_SCALE_NOBASE); 152 return scale_; 153 } 154 FloatRegister::Encoding fpu() const { 155 MOZ_ASSERT(kind() == FPREG); 156 return FloatRegister::Encoding(base_); 157 } 158 int32_t disp() const { 159 MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE || 160 kind() == MEM_SCALE_NOBASE); 161 return disp_; 162 } 163 void* address() const { 164 MOZ_ASSERT(kind() == MEM_ADDRESS32); 165 return reinterpret_cast<void*>(disp_); 166 } 167 168 bool containsReg(Register r) const { 169 switch (kind()) { 170 case REG: 171 return r.encoding() == reg(); 172 case MEM_REG_DISP: 173 return r.encoding() == base(); 174 case MEM_SCALE: 175 return r.encoding() == base() || r.encoding() == index(); 176 case MEM_SCALE_NOBASE: 177 return r.encoding() == index(); 178 default: 179 return false; 180 } 181 } 182 }; 183 184 class CPUInfo { 185 public: 186 // As the SSE's were introduced in order, the presence of a later SSE implies 187 // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2 188 // support. 189 enum SSEVersion { 190 UnknownSSE = 0, 191 NoSSE = 1, 192 SSE = 2, 193 SSE2 = 3, 194 SSE3 = 4, 195 SSSE3 = 5, 196 SSE4_1 = 6, 197 SSE4_2 = 7 198 }; 199 static const int AVX_PRESENT_BIT = 8; 200 201 static SSEVersion GetSSEVersion() { 202 MOZ_ASSERT(FlagsHaveBeenComputed()); 203 MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE, 204 maxSSEVersion <= maxEnabledSSEVersion); 205 return maxSSEVersion; 206 } 207 208 static bool IsAVXPresent() { 209 MOZ_ASSERT(FlagsHaveBeenComputed()); 210 MOZ_ASSERT_IF(!avxEnabled, !avxPresent); 211 return avxPresent; 212 } 213 214 static inline uint32_t GetFingerprint() { 215 return GetSSEVersion() | (IsAVXPresent() ? AVX_PRESENT_BIT : 0); 216 } 217 218 private: 219 static SSEVersion maxSSEVersion; 220 static SSEVersion maxEnabledSSEVersion; 221 static bool avxPresent; 222 static bool avxEnabled; 223 static bool popcntPresent; 224 static bool bmi1Present; 225 static bool bmi2Present; 226 static bool lzcntPresent; 227 static bool fmaPresent; 228 static bool avx2Present; 229 static bool f16cPresent; 230 231 static void SetMaxEnabledSSEVersion(SSEVersion v) { 232 if (maxEnabledSSEVersion == UnknownSSE) { 233 maxEnabledSSEVersion = v; 234 } else { 235 maxEnabledSSEVersion = std::min(v, maxEnabledSSEVersion); 236 } 237 } 238 239 public: 240 static bool IsSSE2Present() { 241 #ifdef JS_CODEGEN_X64 242 return true; 243 #else 244 return GetSSEVersion() >= SSE2; 245 #endif 246 } 247 static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; } 248 static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; } 249 static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; } 250 static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; } 251 static bool IsPOPCNTPresent() { return popcntPresent; } 252 static bool IsBMI1Present() { return bmi1Present; } 253 static bool IsBMI2Present() { return bmi2Present; } 254 static bool IsLZCNTPresent() { return lzcntPresent; } 255 static bool IsFMAPresent() { return fmaPresent; } 256 static bool IsAVX2Present() { return avx2Present; } 257 static bool IsF16CPresent() { return f16cPresent; } 258 259 static bool FlagsHaveBeenComputed() { return maxSSEVersion != UnknownSSE; } 260 261 static void ComputeFlags(); 262 263 // The following should be called only before JS_Init (where the flags are 264 // computed). If several are called, the most restrictive setting is kept. 265 266 static void SetSSE3Disabled() { 267 MOZ_ASSERT(!FlagsHaveBeenComputed()); 268 SetMaxEnabledSSEVersion(SSE2); 269 avxEnabled = false; 270 } 271 static void SetSSSE3Disabled() { 272 MOZ_ASSERT(!FlagsHaveBeenComputed()); 273 SetMaxEnabledSSEVersion(SSE3); 274 avxEnabled = false; 275 } 276 static void SetSSE41Disabled() { 277 MOZ_ASSERT(!FlagsHaveBeenComputed()); 278 SetMaxEnabledSSEVersion(SSSE3); 279 avxEnabled = false; 280 } 281 static void SetSSE42Disabled() { 282 MOZ_ASSERT(!FlagsHaveBeenComputed()); 283 SetMaxEnabledSSEVersion(SSE4_1); 284 avxEnabled = false; 285 } 286 static void SetAVXDisabled() { 287 MOZ_ASSERT(!FlagsHaveBeenComputed()); 288 avxEnabled = false; 289 } 290 static void SetAVXEnabled() { 291 MOZ_ASSERT(!FlagsHaveBeenComputed()); 292 MOZ_ASSERT(maxEnabledSSEVersion == UnknownSSE, 293 "Can't enable AVX when SSE has been restricted"); 294 avxEnabled = true; 295 } 296 }; 297 298 class AssemblerX86Shared : public AssemblerShared { 299 protected: 300 struct RelativePatch { 301 int32_t offset; 302 void* target; 303 RelocationKind kind; 304 305 RelativePatch(int32_t offset, void* target, RelocationKind kind) 306 : offset(offset), target(target), kind(kind) {} 307 }; 308 309 CompactBufferWriter jumpRelocations_; 310 CompactBufferWriter dataRelocations_; 311 312 void writeDataRelocation(ImmGCPtr ptr) { 313 // Raw GC pointer relocations and Value relocations both end up in 314 // Assembler::TraceDataRelocations. 315 if (ptr.value) { 316 if (gc::IsInsideNursery(ptr.value)) { 317 embedsNurseryPointers_ = true; 318 } 319 dataRelocations_.writeUnsigned(masm.currentOffset()); 320 } 321 } 322 323 protected: 324 X86Encoding::BaseAssemblerSpecific masm; 325 326 using JmpSrc = X86Encoding::JmpSrc; 327 using JmpDst = X86Encoding::JmpDst; 328 329 public: 330 AssemblerX86Shared() { 331 if (!HasAVX()) { 332 masm.disableVEX(); 333 } 334 } 335 336 enum Condition { 337 Equal = X86Encoding::ConditionE, 338 NotEqual = X86Encoding::ConditionNE, 339 Above = X86Encoding::ConditionA, 340 AboveOrEqual = X86Encoding::ConditionAE, 341 Below = X86Encoding::ConditionB, 342 BelowOrEqual = X86Encoding::ConditionBE, 343 GreaterThan = X86Encoding::ConditionG, 344 GreaterThanOrEqual = X86Encoding::ConditionGE, 345 LessThan = X86Encoding::ConditionL, 346 LessThanOrEqual = X86Encoding::ConditionLE, 347 Overflow = X86Encoding::ConditionO, 348 NoOverflow = X86Encoding::ConditionNO, 349 CarrySet = X86Encoding::ConditionC, 350 CarryClear = X86Encoding::ConditionNC, 351 Signed = X86Encoding::ConditionS, 352 NotSigned = X86Encoding::ConditionNS, 353 Zero = X86Encoding::ConditionE, 354 NonZero = X86Encoding::ConditionNE, 355 Parity = X86Encoding::ConditionP, 356 NoParity = X86Encoding::ConditionNP 357 }; 358 359 enum class SSERoundingMode { 360 Nearest = int(X86Encoding::SSERoundingMode::RoundToNearest), 361 Floor = int(X86Encoding::SSERoundingMode::RoundDown), 362 Ceil = int(X86Encoding::SSERoundingMode::RoundUp), 363 Trunc = int(X86Encoding::SSERoundingMode::RoundToZero) 364 }; 365 366 // If this bit is set, the vucomisd operands have to be inverted. 367 static const int DoubleConditionBitInvert = 0x10; 368 369 // Bit set when a DoubleCondition does not map to a single x86 condition. 370 // The macro assembler has to special-case these conditions. 371 static const int DoubleConditionBitSpecial = 0x20; 372 static const int DoubleConditionBits = 373 DoubleConditionBitInvert | DoubleConditionBitSpecial; 374 375 enum DoubleCondition { 376 // These conditions will only evaluate to true if the comparison is ordered 377 // - i.e. neither operand is NaN. 378 DoubleOrdered = NoParity, 379 DoubleEqual = Equal | DoubleConditionBitSpecial, 380 DoubleNotEqual = NotEqual, 381 DoubleGreaterThan = Above, 382 DoubleGreaterThanOrEqual = AboveOrEqual, 383 DoubleLessThan = Above | DoubleConditionBitInvert, 384 DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert, 385 // If either operand is NaN, these conditions always evaluate to true. 386 DoubleUnordered = Parity, 387 DoubleEqualOrUnordered = Equal, 388 DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial, 389 DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert, 390 DoubleGreaterThanOrEqualOrUnordered = 391 BelowOrEqual | DoubleConditionBitInvert, 392 DoubleLessThanOrUnordered = Below, 393 DoubleLessThanOrEqualOrUnordered = BelowOrEqual 394 }; 395 396 enum NaNCond { NaN_HandledByCond, NaN_IsTrue, NaN_IsFalse }; 397 398 // If the primary condition returned by ConditionFromDoubleCondition doesn't 399 // handle NaNs properly, return NaN_IsFalse if the comparison should be 400 // overridden to return false on NaN, NaN_IsTrue if it should be overridden 401 // to return true on NaN, or NaN_HandledByCond if no secondary check is 402 // needed. 403 static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) { 404 switch (cond) { 405 case DoubleOrdered: 406 case DoubleNotEqual: 407 case DoubleGreaterThan: 408 case DoubleGreaterThanOrEqual: 409 case DoubleLessThan: 410 case DoubleLessThanOrEqual: 411 case DoubleUnordered: 412 case DoubleEqualOrUnordered: 413 case DoubleGreaterThanOrUnordered: 414 case DoubleGreaterThanOrEqualOrUnordered: 415 case DoubleLessThanOrUnordered: 416 case DoubleLessThanOrEqualOrUnordered: 417 return NaN_HandledByCond; 418 case DoubleEqual: 419 return NaN_IsFalse; 420 case DoubleNotEqualOrUnordered: 421 return NaN_IsTrue; 422 } 423 424 MOZ_CRASH("Unknown double condition"); 425 } 426 427 static void StaticAsserts() { 428 // DoubleConditionBits should not interfere with x86 condition codes. 429 static_assert(!((Equal | NotEqual | Above | AboveOrEqual | Below | 430 BelowOrEqual | Parity | NoParity) & 431 DoubleConditionBits)); 432 } 433 434 static Condition InvertCondition(Condition cond); 435 static Condition UnsignedCondition(Condition cond); 436 static Condition ConditionWithoutEqual(Condition cond); 437 438 static DoubleCondition InvertCondition(DoubleCondition cond); 439 440 // Return the primary condition to test. Some primary conditions may not 441 // handle NaNs properly and may therefore require a secondary condition. 442 // Use NaNCondFromDoubleCondition to determine what else is needed. 443 static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) { 444 return static_cast<Condition>(cond & ~DoubleConditionBits); 445 } 446 447 static void TraceDataRelocations(JSTracer* trc, JitCode* code, 448 CompactBufferReader& reader); 449 450 void setUnlimitedBuffer() { 451 // No-op on this platform 452 } 453 bool oom() const { 454 return AssemblerShared::oom() || masm.oom() || jumpRelocations_.oom() || 455 dataRelocations_.oom(); 456 } 457 bool reserve(size_t size) { return masm.reserve(size); } 458 bool swapBuffer(wasm::Bytes& other) { return masm.swapBuffer(other); } 459 460 void setPrinter(Sprinter* sp) { masm.setPrinter(sp); } 461 462 Register getStackPointer() const { return StackPointer; } 463 464 void executableCopy(void* buffer); 465 void processCodeLabels(uint8_t* rawCode); 466 void copyJumpRelocationTable(uint8_t* dest); 467 void copyDataRelocationTable(uint8_t* dest); 468 469 // Size of the instruction stream, in bytes. 470 size_t size() const { return masm.size(); } 471 // Size of the jump relocation table, in bytes. 472 size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); } 473 size_t dataRelocationTableBytes() const { return dataRelocations_.length(); } 474 // Size of the data table, in bytes. 475 size_t bytesNeeded() const { 476 return size() + jumpRelocationTableBytes() + dataRelocationTableBytes(); 477 } 478 479 public: 480 void haltingAlign(int alignment) { 481 MOZ_ASSERT(hasCreator()); 482 masm.haltingAlign(alignment); 483 } 484 void nopAlign(int alignment) { 485 MOZ_ASSERT(hasCreator()); 486 masm.nopAlign(alignment); 487 } 488 void writeCodePointer(CodeLabel* label) { 489 MOZ_ASSERT(hasCreator()); 490 // Use -1 as dummy value. This will be patched after codegen. 491 masm.jumpTablePointer(-1); 492 label->patchAt()->bind(masm.size()); 493 } 494 void cmovCCl(Condition cond, const Operand& src, Register dest) { 495 X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond); 496 switch (src.kind()) { 497 case Operand::REG: 498 masm.cmovCCl_rr(cc, src.reg(), dest.encoding()); 499 break; 500 case Operand::MEM_REG_DISP: 501 masm.cmovCCl_mr(cc, src.disp(), src.base(), dest.encoding()); 502 break; 503 case Operand::MEM_SCALE: 504 masm.cmovCCl_mr(cc, src.disp(), src.base(), src.index(), src.scale(), 505 dest.encoding()); 506 break; 507 default: 508 MOZ_CRASH("unexpected operand kind"); 509 } 510 } 511 void cmovCCl(Condition cond, Register src, Register dest) { 512 X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond); 513 masm.cmovCCl_rr(cc, src.encoding(), dest.encoding()); 514 } 515 void cmovzl(const Operand& src, Register dest) { 516 cmovCCl(Condition::Zero, src, dest); 517 } 518 void cmovnzl(const Operand& src, Register dest) { 519 cmovCCl(Condition::NonZero, src, dest); 520 } 521 void movl(Imm32 imm32, Register dest) { 522 MOZ_ASSERT(hasCreator()); 523 masm.movl_i32r(imm32.value, dest.encoding()); 524 } 525 void movl(Register src, Register dest) { 526 MOZ_ASSERT(hasCreator()); 527 masm.movl_rr(src.encoding(), dest.encoding()); 528 } 529 void movl(const Operand& src, Register dest) { 530 MOZ_ASSERT(hasCreator()); 531 switch (src.kind()) { 532 case Operand::REG: 533 masm.movl_rr(src.reg(), dest.encoding()); 534 break; 535 case Operand::MEM_REG_DISP: 536 masm.movl_mr(src.disp(), src.base(), dest.encoding()); 537 break; 538 case Operand::MEM_SCALE: 539 masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(), 540 dest.encoding()); 541 break; 542 case Operand::MEM_ADDRESS32: 543 masm.movl_mr(src.address(), dest.encoding()); 544 break; 545 default: 546 MOZ_CRASH("unexpected operand kind"); 547 } 548 } 549 void movl(Register src, const Operand& dest) { 550 MOZ_ASSERT(hasCreator()); 551 switch (dest.kind()) { 552 case Operand::REG: 553 masm.movl_rr(src.encoding(), dest.reg()); 554 break; 555 case Operand::MEM_REG_DISP: 556 masm.movl_rm(src.encoding(), dest.disp(), dest.base()); 557 break; 558 case Operand::MEM_SCALE: 559 masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 560 dest.scale()); 561 break; 562 case Operand::MEM_ADDRESS32: 563 masm.movl_rm(src.encoding(), dest.address()); 564 break; 565 default: 566 MOZ_CRASH("unexpected operand kind"); 567 } 568 } 569 void movl(Imm32 imm32, const Operand& dest) { 570 switch (dest.kind()) { 571 case Operand::REG: 572 masm.movl_i32r(imm32.value, dest.reg()); 573 break; 574 case Operand::MEM_REG_DISP: 575 masm.movl_i32m(imm32.value, dest.disp(), dest.base()); 576 break; 577 case Operand::MEM_SCALE: 578 masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), 579 dest.scale()); 580 break; 581 case Operand::MEM_ADDRESS32: 582 masm.movl_i32m(imm32.value, dest.address()); 583 break; 584 default: 585 MOZ_CRASH("unexpected operand kind"); 586 } 587 } 588 589 void xchgl(Register src, Register dest) { 590 masm.xchgl_rr(src.encoding(), dest.encoding()); 591 } 592 593 void vmovapd(FloatRegister src, FloatRegister dest) { 594 MOZ_ASSERT(HasSSE2()); 595 masm.vmovapd_rr(src.encoding(), dest.encoding()); 596 } 597 // Eventually vmovapd should be overloaded to support loads and 598 // stores too. 599 void vmovapd(const Operand& src, FloatRegister dest) { 600 MOZ_ASSERT(HasSSE2()); 601 switch (src.kind()) { 602 case Operand::FPREG: 603 masm.vmovapd_rr(src.fpu(), dest.encoding()); 604 break; 605 default: 606 MOZ_CRASH("unexpected operand kind"); 607 } 608 } 609 610 void vmovaps(FloatRegister src, FloatRegister dest) { 611 MOZ_ASSERT(HasSSE2()); 612 masm.vmovaps_rr(src.encoding(), dest.encoding()); 613 } 614 void vmovaps(const Operand& src, FloatRegister dest) { 615 MOZ_ASSERT(HasSSE2()); 616 switch (src.kind()) { 617 case Operand::MEM_REG_DISP: 618 masm.vmovaps_mr(src.disp(), src.base(), dest.encoding()); 619 break; 620 case Operand::MEM_SCALE: 621 masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(), 622 dest.encoding()); 623 break; 624 case Operand::FPREG: 625 masm.vmovaps_rr(src.fpu(), dest.encoding()); 626 break; 627 default: 628 MOZ_CRASH("unexpected operand kind"); 629 } 630 } 631 void vmovaps(FloatRegister src, const Operand& dest) { 632 MOZ_ASSERT(HasSSE2()); 633 switch (dest.kind()) { 634 case Operand::MEM_REG_DISP: 635 masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base()); 636 break; 637 case Operand::MEM_SCALE: 638 masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 639 dest.scale()); 640 break; 641 default: 642 MOZ_CRASH("unexpected operand kind"); 643 } 644 } 645 void vmovups(const Operand& src, FloatRegister dest) { 646 MOZ_ASSERT(HasSSE2()); 647 switch (src.kind()) { 648 case Operand::MEM_REG_DISP: 649 masm.vmovups_mr(src.disp(), src.base(), dest.encoding()); 650 break; 651 case Operand::MEM_SCALE: 652 masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(), 653 dest.encoding()); 654 break; 655 default: 656 MOZ_CRASH("unexpected operand kind"); 657 } 658 } 659 void vmovups(FloatRegister src, const Operand& dest) { 660 MOZ_ASSERT(HasSSE2()); 661 switch (dest.kind()) { 662 case Operand::MEM_REG_DISP: 663 masm.vmovups_rm(src.encoding(), dest.disp(), dest.base()); 664 break; 665 case Operand::MEM_SCALE: 666 masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 667 dest.scale()); 668 break; 669 default: 670 MOZ_CRASH("unexpected operand kind"); 671 } 672 } 673 674 void vmovsd(const Address& src, FloatRegister dest) { 675 masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding()); 676 } 677 void vmovsd(const BaseIndex& src, FloatRegister dest) { 678 masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(), 679 src.scale, dest.encoding()); 680 } 681 void vmovsd(const Operand& src, FloatRegister dest) { 682 MOZ_ASSERT(hasCreator()); 683 switch (src.kind()) { 684 case Operand::MEM_REG_DISP: 685 vmovsd(src.toAddress(), dest); 686 break; 687 case Operand::MEM_SCALE: 688 vmovsd(src.toBaseIndex(), dest); 689 break; 690 default: 691 MOZ_CRASH("Unknown operand for vmovsd"); 692 } 693 } 694 void vmovsd(FloatRegister src, const Address& dest) { 695 masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding()); 696 } 697 void vmovsd(FloatRegister src, const BaseIndex& dest) { 698 masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(), 699 dest.index.encoding(), dest.scale); 700 } 701 // Note special semantics of this - does not clobber high bits of destination. 702 void vmovsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 703 masm.vmovsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 704 } 705 void vmovss(const Address& src, FloatRegister dest) { 706 masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding()); 707 } 708 void vmovss(const BaseIndex& src, FloatRegister dest) { 709 masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(), 710 src.scale, dest.encoding()); 711 } 712 void vmovss(const Operand& src, FloatRegister dest) { 713 MOZ_ASSERT(hasCreator()); 714 switch (src.kind()) { 715 case Operand::MEM_REG_DISP: 716 vmovss(src.toAddress(), dest); 717 break; 718 case Operand::MEM_SCALE: 719 vmovss(src.toBaseIndex(), dest); 720 break; 721 default: 722 MOZ_CRASH("Unknown operand for vmovss"); 723 } 724 } 725 void vmovss(FloatRegister src, const Address& dest) { 726 masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding()); 727 } 728 void vmovss(FloatRegister src, const BaseIndex& dest) { 729 masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(), 730 dest.index.encoding(), dest.scale); 731 } 732 void vmovss(FloatRegister src, const Operand& dest) { 733 switch (dest.kind()) { 734 case Operand::MEM_REG_DISP: 735 vmovss(src, dest.toAddress()); 736 break; 737 case Operand::MEM_SCALE: 738 vmovss(src, dest.toBaseIndex()); 739 break; 740 default: 741 MOZ_CRASH("Unknown operand for vmovss"); 742 } 743 } 744 // Note special semantics of this - does not clobber high bits of destination. 745 void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 746 masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 747 } 748 void vmovdqu(const Operand& src, FloatRegister dest) { 749 MOZ_ASSERT(HasSSE2()); 750 MOZ_ASSERT(hasCreator()); 751 switch (src.kind()) { 752 case Operand::MEM_REG_DISP: 753 masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding()); 754 break; 755 case Operand::MEM_SCALE: 756 masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(), 757 dest.encoding()); 758 break; 759 default: 760 MOZ_CRASH("unexpected operand kind"); 761 } 762 } 763 void vmovdqu(FloatRegister src, const Operand& dest) { 764 MOZ_ASSERT(HasSSE2()); 765 MOZ_ASSERT(hasCreator()); 766 switch (dest.kind()) { 767 case Operand::MEM_REG_DISP: 768 masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base()); 769 break; 770 case Operand::MEM_SCALE: 771 masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 772 dest.scale()); 773 break; 774 default: 775 MOZ_CRASH("unexpected operand kind"); 776 } 777 } 778 void vmovdqa(const Operand& src, FloatRegister dest) { 779 MOZ_ASSERT(HasSSE2()); 780 switch (src.kind()) { 781 case Operand::FPREG: 782 masm.vmovdqa_rr(src.fpu(), dest.encoding()); 783 break; 784 case Operand::MEM_REG_DISP: 785 masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding()); 786 break; 787 case Operand::MEM_SCALE: 788 masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(), 789 dest.encoding()); 790 break; 791 default: 792 MOZ_CRASH("unexpected operand kind"); 793 } 794 } 795 void vmovdqa(FloatRegister src, const Operand& dest) { 796 MOZ_ASSERT(HasSSE2()); 797 switch (dest.kind()) { 798 case Operand::MEM_REG_DISP: 799 masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base()); 800 break; 801 case Operand::MEM_SCALE: 802 masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 803 dest.scale()); 804 break; 805 default: 806 MOZ_CRASH("unexpected operand kind"); 807 } 808 } 809 void vmovdqa(FloatRegister src, FloatRegister dest) { 810 MOZ_ASSERT(HasSSE2()); 811 masm.vmovdqa_rr(src.encoding(), dest.encoding()); 812 } 813 void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 814 MOZ_ASSERT(HasSSE2()); 815 masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 816 } 817 void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 818 MOZ_ASSERT(HasSSE2()); 819 masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 820 } 821 void movzbl(const Operand& src, Register dest) { 822 switch (src.kind()) { 823 case Operand::MEM_REG_DISP: 824 masm.movzbl_mr(src.disp(), src.base(), dest.encoding()); 825 break; 826 case Operand::MEM_SCALE: 827 masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(), 828 dest.encoding()); 829 break; 830 default: 831 MOZ_CRASH("unexpected operand kind"); 832 } 833 } 834 void movsbl(Register src, Register dest) { 835 masm.movsbl_rr(src.encoding(), dest.encoding()); 836 } 837 void movsbl(const Operand& src, Register dest) { 838 switch (src.kind()) { 839 case Operand::MEM_REG_DISP: 840 masm.movsbl_mr(src.disp(), src.base(), dest.encoding()); 841 break; 842 case Operand::MEM_SCALE: 843 masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(), 844 dest.encoding()); 845 break; 846 default: 847 MOZ_CRASH("unexpected operand kind"); 848 } 849 } 850 void movb(const Operand& src, Register dest) { 851 switch (src.kind()) { 852 case Operand::MEM_REG_DISP: 853 masm.movb_mr(src.disp(), src.base(), dest.encoding()); 854 break; 855 case Operand::MEM_SCALE: 856 masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(), 857 dest.encoding()); 858 break; 859 default: 860 MOZ_CRASH("unexpected operand kind"); 861 } 862 } 863 void movb(Imm32 src, Register dest) { 864 masm.movb_ir(src.value & 255, dest.encoding()); 865 } 866 void movb(Register src, const Operand& dest) { 867 switch (dest.kind()) { 868 case Operand::MEM_REG_DISP: 869 masm.movb_rm(src.encoding(), dest.disp(), dest.base()); 870 break; 871 case Operand::MEM_SCALE: 872 masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 873 dest.scale()); 874 break; 875 default: 876 MOZ_CRASH("unexpected operand kind"); 877 } 878 } 879 void movb(Imm32 src, const Operand& dest) { 880 switch (dest.kind()) { 881 case Operand::MEM_REG_DISP: 882 masm.movb_im(src.value, dest.disp(), dest.base()); 883 break; 884 case Operand::MEM_SCALE: 885 masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(), 886 dest.scale()); 887 break; 888 default: 889 MOZ_CRASH("unexpected operand kind"); 890 } 891 } 892 void movzwl(const Operand& src, Register dest) { 893 switch (src.kind()) { 894 case Operand::REG: 895 masm.movzwl_rr(src.reg(), dest.encoding()); 896 break; 897 case Operand::MEM_REG_DISP: 898 masm.movzwl_mr(src.disp(), src.base(), dest.encoding()); 899 break; 900 case Operand::MEM_SCALE: 901 masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(), 902 dest.encoding()); 903 break; 904 default: 905 MOZ_CRASH("unexpected operand kind"); 906 } 907 } 908 void movzwl(Register src, Register dest) { 909 masm.movzwl_rr(src.encoding(), dest.encoding()); 910 } 911 void movw(const Operand& src, Register dest) { 912 masm.prefix_16_for_32(); 913 movl(src, dest); 914 } 915 void movw(Imm32 src, Register dest) { 916 masm.prefix_16_for_32(); 917 movl(src, dest); 918 } 919 void movw(Register src, const Operand& dest) { 920 switch (dest.kind()) { 921 case Operand::MEM_REG_DISP: 922 masm.movw_rm(src.encoding(), dest.disp(), dest.base()); 923 break; 924 case Operand::MEM_SCALE: 925 masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 926 dest.scale()); 927 break; 928 default: 929 MOZ_CRASH("unexpected operand kind"); 930 } 931 } 932 void movw(Imm32 src, const Operand& dest) { 933 switch (dest.kind()) { 934 case Operand::MEM_REG_DISP: 935 masm.movw_im(src.value, dest.disp(), dest.base()); 936 break; 937 case Operand::MEM_SCALE: 938 masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(), 939 dest.scale()); 940 break; 941 default: 942 MOZ_CRASH("unexpected operand kind"); 943 } 944 } 945 void movswl(Register src, Register dest) { 946 masm.movswl_rr(src.encoding(), dest.encoding()); 947 } 948 void movswl(const Operand& src, Register dest) { 949 switch (src.kind()) { 950 case Operand::MEM_REG_DISP: 951 masm.movswl_mr(src.disp(), src.base(), dest.encoding()); 952 break; 953 case Operand::MEM_SCALE: 954 masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(), 955 dest.encoding()); 956 break; 957 default: 958 MOZ_CRASH("unexpected operand kind"); 959 } 960 } 961 void leal(const Operand& src, Register dest) { 962 switch (src.kind()) { 963 case Operand::MEM_REG_DISP: 964 masm.leal_mr(src.disp(), src.base(), dest.encoding()); 965 break; 966 case Operand::MEM_SCALE: 967 masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(), 968 dest.encoding()); 969 break; 970 case Operand::MEM_SCALE_NOBASE: 971 masm.leal_mr(src.disp(), src.index(), src.scale(), dest.encoding()); 972 break; 973 default: 974 MOZ_CRASH("unexpected operand kind"); 975 } 976 } 977 978 protected: 979 void jSrc(Condition cond, Label* label) { 980 if (label->bound()) { 981 // The jump can be immediately encoded to the correct destination. 982 masm.jCC_i(static_cast<X86Encoding::Condition>(cond), 983 JmpDst(label->offset())); 984 } else { 985 // Thread the jump list through the unpatched jump targets. 986 JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond)); 987 JmpSrc prev; 988 if (label->used()) { 989 prev = JmpSrc(label->offset()); 990 } 991 label->use(j.offset()); 992 masm.setNextJump(j, prev); 993 } 994 } 995 void jmpSrc(Label* label) { 996 if (label->bound()) { 997 // The jump can be immediately encoded to the correct destination. 998 masm.jmp_i(JmpDst(label->offset())); 999 } else { 1000 // Thread the jump list through the unpatched jump targets. 1001 JmpSrc j = masm.jmp(); 1002 JmpSrc prev; 1003 if (label->used()) { 1004 prev = JmpSrc(label->offset()); 1005 } 1006 label->use(j.offset()); 1007 masm.setNextJump(j, prev); 1008 } 1009 } 1010 1011 // Comparison of EAX against the address given by a Label. 1012 JmpSrc cmpSrc(Label* label) { 1013 JmpSrc j = masm.cmp_eax(); 1014 if (label->bound()) { 1015 // The jump can be immediately patched to the correct destination. 1016 masm.linkJump(j, JmpDst(label->offset())); 1017 } else { 1018 // Thread the jump list through the unpatched jump targets. 1019 JmpSrc prev; 1020 if (label->used()) { 1021 prev = JmpSrc(label->offset()); 1022 } 1023 label->use(j.offset()); 1024 masm.setNextJump(j, prev); 1025 } 1026 return j; 1027 } 1028 1029 void bind(Label* label, JmpDst dst) { 1030 if (label->used()) { 1031 bool more; 1032 JmpSrc jmp(label->offset()); 1033 do { 1034 JmpSrc next; 1035 more = masm.nextJump(jmp, &next); 1036 masm.linkJump(jmp, dst); 1037 jmp = next; 1038 } while (more); 1039 } 1040 label->bind(dst.offset()); 1041 } 1042 1043 public: 1044 void nop() { 1045 MOZ_ASSERT(hasCreator()); 1046 masm.nop(); 1047 } 1048 void nop(size_t n) { 1049 MOZ_ASSERT(hasCreator()); 1050 masm.insert_nop(n); 1051 } 1052 void j(Condition cond, Label* label) { 1053 MOZ_ASSERT(hasCreator()); 1054 jSrc(cond, label); 1055 } 1056 void jmp(Label* label) { 1057 MOZ_ASSERT(hasCreator()); 1058 jmpSrc(label); 1059 } 1060 1061 void jmp(const Operand& op) { 1062 MOZ_ASSERT(hasCreator()); 1063 switch (op.kind()) { 1064 case Operand::MEM_REG_DISP: 1065 masm.jmp_m(op.disp(), op.base()); 1066 break; 1067 case Operand::MEM_SCALE: 1068 masm.jmp_m(op.disp(), op.base(), op.index(), op.scale()); 1069 break; 1070 case Operand::REG: 1071 masm.jmp_r(op.reg()); 1072 break; 1073 default: 1074 MOZ_CRASH("unexpected operand kind"); 1075 } 1076 } 1077 void cmpEAX(Label* label) { cmpSrc(label); } 1078 void bind(Label* label) { bind(label, JmpDst(masm.label())); } 1079 void bind(CodeLabel* label) { label->target()->bind(currentOffset()); } 1080 void bind(Label* label, BufferOffset targetOffset) { 1081 bind(label, JmpDst(targetOffset.getOffset())); 1082 } 1083 uint32_t currentOffset() { return masm.label().offset(); } 1084 1085 // Re-routes pending jumps to a new label. 1086 void retarget(Label* label, Label* target) { 1087 if (!label->used()) { 1088 return; 1089 } 1090 bool more; 1091 JmpSrc jmp(label->offset()); 1092 do { 1093 JmpSrc next; 1094 more = masm.nextJump(jmp, &next); 1095 if (target->bound()) { 1096 // The jump can be immediately patched to the correct destination. 1097 masm.linkJump(jmp, JmpDst(target->offset())); 1098 } else { 1099 // Thread the jump list through the unpatched jump targets. 1100 JmpSrc prev; 1101 if (target->used()) { 1102 prev = JmpSrc(target->offset()); 1103 } 1104 target->use(jmp.offset()); 1105 masm.setNextJump(jmp, prev); 1106 } 1107 jmp = JmpSrc(next.offset()); 1108 } while (more); 1109 label->reset(); 1110 } 1111 1112 static void Bind(uint8_t* raw, const CodeLabel& label) { 1113 if (label.patchAt().bound()) { 1114 intptr_t offset = label.patchAt().offset(); 1115 intptr_t target = label.target().offset(); 1116 X86Encoding::SetPointer(raw + offset, raw + target); 1117 } 1118 } 1119 1120 BufferOffset ret() { 1121 MOZ_ASSERT(hasCreator()); 1122 BufferOffset ret(masm.currentOffset()); 1123 masm.ret(); 1124 return ret; 1125 } 1126 void retn(Imm32 n) { 1127 MOZ_ASSERT(hasCreator()); 1128 // Remove the size of the return address which is included in the frame. 1129 masm.ret_i(n.value - sizeof(void*)); 1130 } 1131 CodeOffset call(Label* label) { 1132 JmpSrc j = masm.call(); 1133 if (label->bound()) { 1134 masm.linkJump(j, JmpDst(label->offset())); 1135 } else { 1136 JmpSrc prev; 1137 if (label->used()) { 1138 prev = JmpSrc(label->offset()); 1139 } 1140 label->use(j.offset()); 1141 masm.setNextJump(j, prev); 1142 } 1143 return CodeOffset(masm.currentOffset()); 1144 } 1145 CodeOffset call(Register reg) { 1146 masm.call_r(reg.encoding()); 1147 return CodeOffset(masm.currentOffset()); 1148 } 1149 void call(const Operand& op) { 1150 switch (op.kind()) { 1151 case Operand::REG: 1152 masm.call_r(op.reg()); 1153 break; 1154 case Operand::MEM_REG_DISP: 1155 masm.call_m(op.disp(), op.base()); 1156 break; 1157 default: 1158 MOZ_CRASH("unexpected operand kind"); 1159 } 1160 } 1161 1162 CodeOffset callWithPatch() { return CodeOffset(masm.call().offset()); } 1163 1164 void patchCall(uint32_t callerOffset, uint32_t calleeOffset) { 1165 unsigned char* code = masm.data(); 1166 X86Encoding::SetRel32(code + callerOffset, code + calleeOffset); 1167 } 1168 CodeOffset farJumpWithPatch() { return CodeOffset(masm.jmp().offset()); } 1169 void patchFarJump(CodeOffset farJump, uint32_t targetOffset) { 1170 unsigned char* code = masm.data(); 1171 X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset); 1172 } 1173 static void patchFarJump(uint8_t* farJump, uint8_t* target) { 1174 MOZ_RELEASE_ASSERT(mozilla::Abs(target - farJump) <= 1175 (intptr_t)jit::MaxCodeBytesPerProcess); 1176 X86Encoding::SetRel32(farJump, target); 1177 } 1178 1179 // This is for patching during code generation, not after. 1180 void patchAddl(CodeOffset offset, int32_t n) { 1181 unsigned char* code = masm.data(); 1182 X86Encoding::SetInt32(code + offset.offset(), n); 1183 } 1184 1185 static void patchFiveByteNopToCall(uint8_t* callsite, uint8_t* target) { 1186 X86Encoding::BaseAssembler::patchFiveByteNopToCall(callsite, target); 1187 } 1188 static void patchCallToFiveByteNop(uint8_t* callsite) { 1189 X86Encoding::BaseAssembler::patchCallToFiveByteNop(callsite); 1190 } 1191 1192 void breakpoint() { masm.int3(); } 1193 CodeOffset ud2() { 1194 MOZ_ASSERT(hasCreator()); 1195 CodeOffset off(masm.currentOffset()); 1196 masm.ud2(); 1197 return off; 1198 } 1199 1200 static bool HasSSE2() { return CPUInfo::IsSSE2Present(); } 1201 static bool HasSSE3() { return CPUInfo::IsSSE3Present(); } 1202 static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); } 1203 static bool HasSSE41() { return CPUInfo::IsSSE41Present(); } 1204 static bool HasSSE42() { return CPUInfo::IsSSE42Present(); } 1205 static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); } 1206 static bool HasBMI1() { return CPUInfo::IsBMI1Present(); } 1207 static bool HasBMI2() { return CPUInfo::IsBMI2Present(); } 1208 static bool HasLZCNT() { return CPUInfo::IsLZCNTPresent(); } 1209 static bool HasF16C() { return CPUInfo::IsF16CPresent(); } 1210 static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); } 1211 static bool SupportsUnalignedAccesses() { return true; } 1212 static bool SupportsFastUnalignedFPAccesses() { return true; } 1213 static bool SupportsWasmSimd() { return CPUInfo::IsSSE41Present(); } 1214 static bool SupportsFloat64To16() { return false; } 1215 static bool SupportsFloat32To16() { return CPUInfo::IsF16CPresent(); } 1216 static bool HasAVX() { return CPUInfo::IsAVXPresent(); } 1217 static bool HasAVX2() { return CPUInfo::IsAVX2Present(); } 1218 static bool HasFMA() { return CPUInfo::IsFMAPresent(); } 1219 1220 static bool HasRoundInstruction(RoundingMode mode) { 1221 switch (mode) { 1222 case RoundingMode::Up: 1223 case RoundingMode::Down: 1224 case RoundingMode::NearestTiesToEven: 1225 case RoundingMode::TowardsZero: 1226 return CPUInfo::IsSSE41Present(); 1227 } 1228 MOZ_CRASH("unexpected mode"); 1229 } 1230 1231 void cmpl(Register rhs, Register lhs) { 1232 masm.cmpl_rr(rhs.encoding(), lhs.encoding()); 1233 } 1234 void cmpl(const Operand& rhs, Register lhs) { 1235 switch (rhs.kind()) { 1236 case Operand::REG: 1237 masm.cmpl_rr(rhs.reg(), lhs.encoding()); 1238 break; 1239 case Operand::MEM_REG_DISP: 1240 masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding()); 1241 break; 1242 case Operand::MEM_ADDRESS32: 1243 masm.cmpl_mr(rhs.address(), lhs.encoding()); 1244 break; 1245 default: 1246 MOZ_CRASH("unexpected operand kind"); 1247 } 1248 } 1249 void cmpl(Register rhs, const Operand& lhs) { 1250 switch (lhs.kind()) { 1251 case Operand::REG: 1252 masm.cmpl_rr(rhs.encoding(), lhs.reg()); 1253 break; 1254 case Operand::MEM_REG_DISP: 1255 masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base()); 1256 break; 1257 case Operand::MEM_SCALE: 1258 masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(), 1259 lhs.scale()); 1260 break; 1261 case Operand::MEM_ADDRESS32: 1262 masm.cmpl_rm(rhs.encoding(), lhs.address()); 1263 break; 1264 default: 1265 MOZ_CRASH("unexpected operand kind"); 1266 } 1267 } 1268 void cmpl(Imm32 rhs, Register lhs) { 1269 masm.cmpl_ir(rhs.value, lhs.encoding()); 1270 } 1271 void cmpl(Imm32 rhs, const Operand& lhs) { 1272 switch (lhs.kind()) { 1273 case Operand::REG: 1274 masm.cmpl_ir(rhs.value, lhs.reg()); 1275 break; 1276 case Operand::MEM_REG_DISP: 1277 masm.cmpl_im(rhs.value, lhs.disp(), lhs.base()); 1278 break; 1279 case Operand::MEM_SCALE: 1280 masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), 1281 lhs.scale()); 1282 break; 1283 case Operand::MEM_ADDRESS32: 1284 masm.cmpl_im(rhs.value, lhs.address()); 1285 break; 1286 default: 1287 MOZ_CRASH("unexpected operand kind"); 1288 } 1289 } 1290 void cmpw(Register rhs, Register lhs) { 1291 masm.cmpw_rr(rhs.encoding(), lhs.encoding()); 1292 } 1293 void cmpw(Imm32 rhs, const Operand& lhs) { 1294 switch (lhs.kind()) { 1295 case Operand::REG: 1296 masm.cmpw_ir(rhs.value, lhs.reg()); 1297 break; 1298 case Operand::MEM_REG_DISP: 1299 masm.cmpw_im(rhs.value, lhs.disp(), lhs.base()); 1300 break; 1301 case Operand::MEM_SCALE: 1302 masm.cmpw_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), 1303 lhs.scale()); 1304 break; 1305 case Operand::MEM_ADDRESS32: 1306 masm.cmpw_im(rhs.value, lhs.address()); 1307 break; 1308 default: 1309 MOZ_CRASH("unexpected operand kind"); 1310 } 1311 } 1312 void cmpb(Register rhs, const Operand& lhs) { 1313 switch (lhs.kind()) { 1314 case Operand::REG: 1315 masm.cmpb_rr(rhs.encoding(), lhs.reg()); 1316 break; 1317 case Operand::MEM_REG_DISP: 1318 masm.cmpb_rm(rhs.encoding(), lhs.disp(), lhs.base()); 1319 break; 1320 case Operand::MEM_SCALE: 1321 masm.cmpb_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(), 1322 lhs.scale()); 1323 break; 1324 case Operand::MEM_ADDRESS32: 1325 masm.cmpb_rm(rhs.encoding(), lhs.address()); 1326 break; 1327 default: 1328 MOZ_CRASH("unexpected operand kind"); 1329 } 1330 } 1331 void cmpb(Imm32 rhs, const Operand& lhs) { 1332 switch (lhs.kind()) { 1333 case Operand::REG: 1334 masm.cmpb_ir(rhs.value, lhs.reg()); 1335 break; 1336 case Operand::MEM_REG_DISP: 1337 masm.cmpb_im(rhs.value, lhs.disp(), lhs.base()); 1338 break; 1339 case Operand::MEM_SCALE: 1340 masm.cmpb_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), 1341 lhs.scale()); 1342 break; 1343 case Operand::MEM_ADDRESS32: 1344 masm.cmpb_im(rhs.value, lhs.address()); 1345 break; 1346 default: 1347 MOZ_CRASH("unexpected operand kind"); 1348 } 1349 } 1350 void setCC(Condition cond, Register r) { 1351 masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding()); 1352 } 1353 void testb(Register rhs, Register lhs) { 1354 MOZ_ASSERT( 1355 AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs)); 1356 MOZ_ASSERT( 1357 AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs)); 1358 masm.testb_rr(rhs.encoding(), lhs.encoding()); 1359 } 1360 void testw(Register rhs, Register lhs) { 1361 masm.testw_rr(lhs.encoding(), rhs.encoding()); 1362 } 1363 void testl(Register rhs, Register lhs) { 1364 masm.testl_rr(lhs.encoding(), rhs.encoding()); 1365 } 1366 void testl(Imm32 rhs, Register lhs) { 1367 masm.testl_ir(rhs.value, lhs.encoding()); 1368 } 1369 void testl(Imm32 rhs, const Operand& lhs) { 1370 switch (lhs.kind()) { 1371 case Operand::REG: 1372 masm.testl_ir(rhs.value, lhs.reg()); 1373 break; 1374 case Operand::MEM_REG_DISP: 1375 masm.testl_i32m(rhs.value, lhs.disp(), lhs.base()); 1376 break; 1377 case Operand::MEM_ADDRESS32: 1378 masm.testl_i32m(rhs.value, lhs.address()); 1379 break; 1380 default: 1381 MOZ_CRASH("unexpected operand kind"); 1382 break; 1383 } 1384 } 1385 1386 void addl(Imm32 imm, Register dest) { 1387 masm.addl_ir(imm.value, dest.encoding()); 1388 } 1389 CodeOffset addlWithPatch(Imm32 imm, Register dest) { 1390 masm.addl_i32r(imm.value, dest.encoding()); 1391 return CodeOffset(masm.currentOffset()); 1392 } 1393 void addl(Imm32 imm, const Operand& op) { 1394 switch (op.kind()) { 1395 case Operand::REG: 1396 masm.addl_ir(imm.value, op.reg()); 1397 break; 1398 case Operand::MEM_REG_DISP: 1399 masm.addl_im(imm.value, op.disp(), op.base()); 1400 break; 1401 case Operand::MEM_ADDRESS32: 1402 masm.addl_im(imm.value, op.address()); 1403 break; 1404 case Operand::MEM_SCALE: 1405 masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1406 break; 1407 default: 1408 MOZ_CRASH("unexpected operand kind"); 1409 } 1410 } 1411 void addw(Imm32 imm, const Operand& op) { 1412 switch (op.kind()) { 1413 case Operand::REG: 1414 masm.addw_ir(imm.value, op.reg()); 1415 break; 1416 case Operand::MEM_REG_DISP: 1417 masm.addw_im(imm.value, op.disp(), op.base()); 1418 break; 1419 case Operand::MEM_ADDRESS32: 1420 masm.addw_im(imm.value, op.address()); 1421 break; 1422 case Operand::MEM_SCALE: 1423 masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1424 break; 1425 default: 1426 MOZ_CRASH("unexpected operand kind"); 1427 } 1428 } 1429 void subl(Imm32 imm, Register dest) { 1430 masm.subl_ir(imm.value, dest.encoding()); 1431 } 1432 size_t subl(Imm32 imm, const Operand& op) { 1433 switch (op.kind()) { 1434 case Operand::REG: 1435 return masm.subl_ir(imm.value, op.reg()); 1436 case Operand::MEM_REG_DISP: 1437 return masm.subl_im(imm.value, op.disp(), op.base()); 1438 case Operand::MEM_SCALE: 1439 return masm.subl_im(imm.value, op.disp(), op.base(), op.index(), 1440 op.scale()); 1441 default: 1442 MOZ_CRASH("unexpected operand kind"); 1443 } 1444 } 1445 void subw(Imm32 imm, const Operand& op) { 1446 switch (op.kind()) { 1447 case Operand::REG: 1448 masm.subw_ir(imm.value, op.reg()); 1449 break; 1450 case Operand::MEM_REG_DISP: 1451 masm.subw_im(imm.value, op.disp(), op.base()); 1452 break; 1453 case Operand::MEM_SCALE: 1454 masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1455 break; 1456 default: 1457 MOZ_CRASH("unexpected operand kind"); 1458 } 1459 } 1460 void addl(Register src, Register dest) { 1461 masm.addl_rr(src.encoding(), dest.encoding()); 1462 } 1463 void addl(Register src, const Operand& dest) { 1464 switch (dest.kind()) { 1465 case Operand::REG: 1466 masm.addl_rr(src.encoding(), dest.reg()); 1467 break; 1468 case Operand::MEM_REG_DISP: 1469 masm.addl_rm(src.encoding(), dest.disp(), dest.base()); 1470 break; 1471 case Operand::MEM_SCALE: 1472 masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1473 dest.scale()); 1474 break; 1475 default: 1476 MOZ_CRASH("unexpected operand kind"); 1477 } 1478 } 1479 void addw(Register src, const Operand& dest) { 1480 switch (dest.kind()) { 1481 case Operand::REG: 1482 masm.addw_rr(src.encoding(), dest.reg()); 1483 break; 1484 case Operand::MEM_REG_DISP: 1485 masm.addw_rm(src.encoding(), dest.disp(), dest.base()); 1486 break; 1487 case Operand::MEM_SCALE: 1488 masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1489 dest.scale()); 1490 break; 1491 default: 1492 MOZ_CRASH("unexpected operand kind"); 1493 } 1494 } 1495 void sbbl(Register src, Register dest) { 1496 masm.sbbl_rr(src.encoding(), dest.encoding()); 1497 } 1498 void subl(Register src, Register dest) { 1499 masm.subl_rr(src.encoding(), dest.encoding()); 1500 } 1501 void subl(const Operand& src, Register dest) { 1502 switch (src.kind()) { 1503 case Operand::REG: 1504 masm.subl_rr(src.reg(), dest.encoding()); 1505 break; 1506 case Operand::MEM_REG_DISP: 1507 masm.subl_mr(src.disp(), src.base(), dest.encoding()); 1508 break; 1509 default: 1510 MOZ_CRASH("unexpected operand kind"); 1511 } 1512 } 1513 void subl(Register src, const Operand& dest) { 1514 switch (dest.kind()) { 1515 case Operand::REG: 1516 masm.subl_rr(src.encoding(), dest.reg()); 1517 break; 1518 case Operand::MEM_REG_DISP: 1519 masm.subl_rm(src.encoding(), dest.disp(), dest.base()); 1520 break; 1521 case Operand::MEM_SCALE: 1522 masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1523 dest.scale()); 1524 break; 1525 default: 1526 MOZ_CRASH("unexpected operand kind"); 1527 } 1528 } 1529 void subw(Register src, const Operand& dest) { 1530 switch (dest.kind()) { 1531 case Operand::REG: 1532 masm.subw_rr(src.encoding(), dest.reg()); 1533 break; 1534 case Operand::MEM_REG_DISP: 1535 masm.subw_rm(src.encoding(), dest.disp(), dest.base()); 1536 break; 1537 case Operand::MEM_SCALE: 1538 masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1539 dest.scale()); 1540 break; 1541 default: 1542 MOZ_CRASH("unexpected operand kind"); 1543 } 1544 } 1545 void orl(Register reg, Register dest) { 1546 masm.orl_rr(reg.encoding(), dest.encoding()); 1547 } 1548 void orl(Register src, const Operand& dest) { 1549 switch (dest.kind()) { 1550 case Operand::REG: 1551 masm.orl_rr(src.encoding(), dest.reg()); 1552 break; 1553 case Operand::MEM_REG_DISP: 1554 masm.orl_rm(src.encoding(), dest.disp(), dest.base()); 1555 break; 1556 case Operand::MEM_SCALE: 1557 masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1558 dest.scale()); 1559 break; 1560 default: 1561 MOZ_CRASH("unexpected operand kind"); 1562 } 1563 } 1564 void orw(Register src, const Operand& dest) { 1565 switch (dest.kind()) { 1566 case Operand::REG: 1567 masm.orw_rr(src.encoding(), dest.reg()); 1568 break; 1569 case Operand::MEM_REG_DISP: 1570 masm.orw_rm(src.encoding(), dest.disp(), dest.base()); 1571 break; 1572 case Operand::MEM_SCALE: 1573 masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1574 dest.scale()); 1575 break; 1576 default: 1577 MOZ_CRASH("unexpected operand kind"); 1578 } 1579 } 1580 void orl(Imm32 imm, Register reg) { masm.orl_ir(imm.value, reg.encoding()); } 1581 void orl(Imm32 imm, const Operand& op) { 1582 switch (op.kind()) { 1583 case Operand::REG: 1584 masm.orl_ir(imm.value, op.reg()); 1585 break; 1586 case Operand::MEM_REG_DISP: 1587 masm.orl_im(imm.value, op.disp(), op.base()); 1588 break; 1589 case Operand::MEM_SCALE: 1590 masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1591 break; 1592 default: 1593 MOZ_CRASH("unexpected operand kind"); 1594 } 1595 } 1596 void orw(Imm32 imm, const Operand& op) { 1597 switch (op.kind()) { 1598 case Operand::REG: 1599 masm.orw_ir(imm.value, op.reg()); 1600 break; 1601 case Operand::MEM_REG_DISP: 1602 masm.orw_im(imm.value, op.disp(), op.base()); 1603 break; 1604 case Operand::MEM_SCALE: 1605 masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1606 break; 1607 default: 1608 MOZ_CRASH("unexpected operand kind"); 1609 } 1610 } 1611 void xorl(Register src, Register dest) { 1612 masm.xorl_rr(src.encoding(), dest.encoding()); 1613 } 1614 void xorl(Register src, const Operand& dest) { 1615 switch (dest.kind()) { 1616 case Operand::REG: 1617 masm.xorl_rr(src.encoding(), dest.reg()); 1618 break; 1619 case Operand::MEM_REG_DISP: 1620 masm.xorl_rm(src.encoding(), dest.disp(), dest.base()); 1621 break; 1622 case Operand::MEM_SCALE: 1623 masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1624 dest.scale()); 1625 break; 1626 default: 1627 MOZ_CRASH("unexpected operand kind"); 1628 } 1629 } 1630 void xorw(Register src, const Operand& dest) { 1631 switch (dest.kind()) { 1632 case Operand::REG: 1633 masm.xorw_rr(src.encoding(), dest.reg()); 1634 break; 1635 case Operand::MEM_REG_DISP: 1636 masm.xorw_rm(src.encoding(), dest.disp(), dest.base()); 1637 break; 1638 case Operand::MEM_SCALE: 1639 masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1640 dest.scale()); 1641 break; 1642 default: 1643 MOZ_CRASH("unexpected operand kind"); 1644 } 1645 } 1646 void xorl(Imm32 imm, Register reg) { 1647 masm.xorl_ir(imm.value, reg.encoding()); 1648 } 1649 void xorl(Imm32 imm, const Operand& op) { 1650 switch (op.kind()) { 1651 case Operand::REG: 1652 masm.xorl_ir(imm.value, op.reg()); 1653 break; 1654 case Operand::MEM_REG_DISP: 1655 masm.xorl_im(imm.value, op.disp(), op.base()); 1656 break; 1657 case Operand::MEM_SCALE: 1658 masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1659 break; 1660 default: 1661 MOZ_CRASH("unexpected operand kind"); 1662 } 1663 } 1664 void xorw(Imm32 imm, const Operand& op) { 1665 switch (op.kind()) { 1666 case Operand::REG: 1667 masm.xorw_ir(imm.value, op.reg()); 1668 break; 1669 case Operand::MEM_REG_DISP: 1670 masm.xorw_im(imm.value, op.disp(), op.base()); 1671 break; 1672 case Operand::MEM_SCALE: 1673 masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1674 break; 1675 default: 1676 MOZ_CRASH("unexpected operand kind"); 1677 } 1678 } 1679 void andl(Register src, Register dest) { 1680 masm.andl_rr(src.encoding(), dest.encoding()); 1681 } 1682 void andl(Register src, const Operand& dest) { 1683 switch (dest.kind()) { 1684 case Operand::REG: 1685 masm.andl_rr(src.encoding(), dest.reg()); 1686 break; 1687 case Operand::MEM_REG_DISP: 1688 masm.andl_rm(src.encoding(), dest.disp(), dest.base()); 1689 break; 1690 case Operand::MEM_SCALE: 1691 masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1692 dest.scale()); 1693 break; 1694 default: 1695 MOZ_CRASH("unexpected operand kind"); 1696 } 1697 } 1698 void andw(Register src, const Operand& dest) { 1699 switch (dest.kind()) { 1700 case Operand::REG: 1701 masm.andw_rr(src.encoding(), dest.reg()); 1702 break; 1703 case Operand::MEM_REG_DISP: 1704 masm.andw_rm(src.encoding(), dest.disp(), dest.base()); 1705 break; 1706 case Operand::MEM_SCALE: 1707 masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 1708 dest.scale()); 1709 break; 1710 default: 1711 MOZ_CRASH("unexpected operand kind"); 1712 } 1713 } 1714 void andl(Imm32 imm, Register dest) { 1715 masm.andl_ir(imm.value, dest.encoding()); 1716 } 1717 void andl(Imm32 imm, const Operand& op) { 1718 switch (op.kind()) { 1719 case Operand::REG: 1720 masm.andl_ir(imm.value, op.reg()); 1721 break; 1722 case Operand::MEM_REG_DISP: 1723 masm.andl_im(imm.value, op.disp(), op.base()); 1724 break; 1725 case Operand::MEM_SCALE: 1726 masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1727 break; 1728 default: 1729 MOZ_CRASH("unexpected operand kind"); 1730 } 1731 } 1732 void andw(Imm32 imm, const Operand& op) { 1733 switch (op.kind()) { 1734 case Operand::REG: 1735 masm.andw_ir(imm.value, op.reg()); 1736 break; 1737 case Operand::MEM_REG_DISP: 1738 masm.andw_im(imm.value, op.disp(), op.base()); 1739 break; 1740 case Operand::MEM_SCALE: 1741 masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1742 break; 1743 default: 1744 MOZ_CRASH("unexpected operand kind"); 1745 } 1746 } 1747 void addl(const Operand& src, Register dest) { 1748 switch (src.kind()) { 1749 case Operand::REG: 1750 masm.addl_rr(src.reg(), dest.encoding()); 1751 break; 1752 case Operand::MEM_REG_DISP: 1753 masm.addl_mr(src.disp(), src.base(), dest.encoding()); 1754 break; 1755 default: 1756 MOZ_CRASH("unexpected operand kind"); 1757 } 1758 } 1759 void andnl(Register src1, Register src2, Register dest) { 1760 MOZ_ASSERT(HasBMI1()); 1761 masm.andnl_rrr(src1.encoding(), src2.encoding(), dest.encoding()); 1762 } 1763 void orl(const Operand& src, Register dest) { 1764 switch (src.kind()) { 1765 case Operand::REG: 1766 masm.orl_rr(src.reg(), dest.encoding()); 1767 break; 1768 case Operand::MEM_REG_DISP: 1769 masm.orl_mr(src.disp(), src.base(), dest.encoding()); 1770 break; 1771 default: 1772 MOZ_CRASH("unexpected operand kind"); 1773 } 1774 } 1775 void xorl(const Operand& src, Register dest) { 1776 switch (src.kind()) { 1777 case Operand::REG: 1778 masm.xorl_rr(src.reg(), dest.encoding()); 1779 break; 1780 case Operand::MEM_REG_DISP: 1781 masm.xorl_mr(src.disp(), src.base(), dest.encoding()); 1782 break; 1783 default: 1784 MOZ_CRASH("unexpected operand kind"); 1785 } 1786 } 1787 void andl(const Operand& src, Register dest) { 1788 switch (src.kind()) { 1789 case Operand::REG: 1790 masm.andl_rr(src.reg(), dest.encoding()); 1791 break; 1792 case Operand::MEM_REG_DISP: 1793 masm.andl_mr(src.disp(), src.base(), dest.encoding()); 1794 break; 1795 case Operand::MEM_SCALE: 1796 masm.andl_mr(src.disp(), src.base(), src.index(), src.scale(), 1797 dest.encoding()); 1798 break; 1799 default: 1800 MOZ_CRASH("unexpected operand kind"); 1801 } 1802 } 1803 void bsrl(const Register& src, const Register& dest) { 1804 masm.bsrl_rr(src.encoding(), dest.encoding()); 1805 } 1806 void bsfl(const Register& src, const Register& dest) { 1807 masm.bsfl_rr(src.encoding(), dest.encoding()); 1808 } 1809 void bswapl(Register reg) { masm.bswapl_r(reg.encoding()); } 1810 void lzcntl(const Register& src, const Register& dest) { 1811 masm.lzcntl_rr(src.encoding(), dest.encoding()); 1812 } 1813 void tzcntl(const Register& src, const Register& dest) { 1814 masm.tzcntl_rr(src.encoding(), dest.encoding()); 1815 } 1816 void popcntl(const Register& src, const Register& dest) { 1817 masm.popcntl_rr(src.encoding(), dest.encoding()); 1818 } 1819 void imull(Register multiplier) { 1820 // Consumes eax as the other argument 1821 // and clobbers edx, as result is in edx:eax 1822 masm.imull_r(multiplier.encoding()); 1823 } 1824 void umull(Register multiplier) { masm.mull_r(multiplier.encoding()); } 1825 void imull(Imm32 imm, Register dest) { 1826 masm.imull_ir(imm.value, dest.encoding(), dest.encoding()); 1827 } 1828 void imull(Register src, Register dest) { 1829 masm.imull_rr(src.encoding(), dest.encoding()); 1830 } 1831 void imull(Imm32 imm, Register src, Register dest) { 1832 masm.imull_ir(imm.value, src.encoding(), dest.encoding()); 1833 } 1834 void imull(const Operand& src, Register dest) { 1835 switch (src.kind()) { 1836 case Operand::REG: 1837 masm.imull_rr(src.reg(), dest.encoding()); 1838 break; 1839 case Operand::MEM_REG_DISP: 1840 masm.imull_mr(src.disp(), src.base(), dest.encoding()); 1841 break; 1842 default: 1843 MOZ_CRASH("unexpected operand kind"); 1844 } 1845 } 1846 void negl(const Operand& src) { 1847 switch (src.kind()) { 1848 case Operand::REG: 1849 masm.negl_r(src.reg()); 1850 break; 1851 case Operand::MEM_REG_DISP: 1852 masm.negl_m(src.disp(), src.base()); 1853 break; 1854 default: 1855 MOZ_CRASH("unexpected operand kind"); 1856 } 1857 } 1858 void negl(Register reg) { masm.negl_r(reg.encoding()); } 1859 void notl(const Operand& src) { 1860 switch (src.kind()) { 1861 case Operand::REG: 1862 masm.notl_r(src.reg()); 1863 break; 1864 case Operand::MEM_REG_DISP: 1865 masm.notl_m(src.disp(), src.base()); 1866 break; 1867 default: 1868 MOZ_CRASH("unexpected operand kind"); 1869 } 1870 } 1871 void notl(Register reg) { masm.notl_r(reg.encoding()); } 1872 void shrl(const Imm32 imm, Register dest) { 1873 masm.shrl_ir(imm.value, dest.encoding()); 1874 } 1875 void shll(const Imm32 imm, Register dest) { 1876 masm.shll_ir(imm.value, dest.encoding()); 1877 } 1878 void sarl(const Imm32 imm, Register dest) { 1879 masm.sarl_ir(imm.value, dest.encoding()); 1880 } 1881 void shrl_cl(Register dest) { masm.shrl_CLr(dest.encoding()); } 1882 void shll_cl(Register dest) { masm.shll_CLr(dest.encoding()); } 1883 void sarl_cl(Register dest) { masm.sarl_CLr(dest.encoding()); } 1884 void shrdl_cl(Register src, Register dest) { 1885 masm.shrdl_CLr(src.encoding(), dest.encoding()); 1886 } 1887 void shldl_cl(Register src, Register dest) { 1888 masm.shldl_CLr(src.encoding(), dest.encoding()); 1889 } 1890 1891 void sarxl(Register src, Register shift, Register dest) { 1892 MOZ_ASSERT(HasBMI2()); 1893 masm.sarxl_rrr(src.encoding(), shift.encoding(), dest.encoding()); 1894 } 1895 void shlxl(Register src, Register shift, Register dest) { 1896 MOZ_ASSERT(HasBMI2()); 1897 masm.shlxl_rrr(src.encoding(), shift.encoding(), dest.encoding()); 1898 } 1899 void shrxl(Register src, Register shift, Register dest) { 1900 MOZ_ASSERT(HasBMI2()); 1901 masm.shrxl_rrr(src.encoding(), shift.encoding(), dest.encoding()); 1902 } 1903 1904 void roll(const Imm32 imm, Register dest) { 1905 masm.roll_ir(imm.value, dest.encoding()); 1906 } 1907 void roll_cl(Register dest) { masm.roll_CLr(dest.encoding()); } 1908 void rolw(const Imm32 imm, Register dest) { 1909 masm.rolw_ir(imm.value, dest.encoding()); 1910 } 1911 void rorl(const Imm32 imm, Register dest) { 1912 masm.rorl_ir(imm.value, dest.encoding()); 1913 } 1914 void rorl_cl(Register dest) { masm.rorl_CLr(dest.encoding()); } 1915 1916 void incl(const Operand& op) { 1917 switch (op.kind()) { 1918 case Operand::MEM_REG_DISP: 1919 masm.incl_m32(op.disp(), op.base()); 1920 break; 1921 default: 1922 MOZ_CRASH("unexpected operand kind"); 1923 } 1924 } 1925 void lock_incl(const Operand& op) { 1926 masm.prefix_lock(); 1927 incl(op); 1928 } 1929 1930 void decl(const Operand& op) { 1931 switch (op.kind()) { 1932 case Operand::MEM_REG_DISP: 1933 masm.decl_m32(op.disp(), op.base()); 1934 break; 1935 default: 1936 MOZ_CRASH("unexpected operand kind"); 1937 } 1938 } 1939 void lock_decl(const Operand& op) { 1940 masm.prefix_lock(); 1941 decl(op); 1942 } 1943 1944 void addb(Imm32 imm, const Operand& op) { 1945 switch (op.kind()) { 1946 case Operand::MEM_REG_DISP: 1947 masm.addb_im(imm.value, op.disp(), op.base()); 1948 break; 1949 case Operand::MEM_SCALE: 1950 masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1951 break; 1952 default: 1953 MOZ_CRASH("unexpected operand kind"); 1954 break; 1955 } 1956 } 1957 void addb(Register src, const Operand& op) { 1958 switch (op.kind()) { 1959 case Operand::MEM_REG_DISP: 1960 masm.addb_rm(src.encoding(), op.disp(), op.base()); 1961 break; 1962 case Operand::MEM_SCALE: 1963 masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(), 1964 op.scale()); 1965 break; 1966 default: 1967 MOZ_CRASH("unexpected operand kind"); 1968 break; 1969 } 1970 } 1971 1972 void subb(Imm32 imm, const Operand& op) { 1973 switch (op.kind()) { 1974 case Operand::MEM_REG_DISP: 1975 masm.subb_im(imm.value, op.disp(), op.base()); 1976 break; 1977 case Operand::MEM_SCALE: 1978 masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1979 break; 1980 default: 1981 MOZ_CRASH("unexpected operand kind"); 1982 break; 1983 } 1984 } 1985 void subb(Register src, const Operand& op) { 1986 switch (op.kind()) { 1987 case Operand::MEM_REG_DISP: 1988 masm.subb_rm(src.encoding(), op.disp(), op.base()); 1989 break; 1990 case Operand::MEM_SCALE: 1991 masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(), 1992 op.scale()); 1993 break; 1994 default: 1995 MOZ_CRASH("unexpected operand kind"); 1996 break; 1997 } 1998 } 1999 2000 void andb(Imm32 imm, const Operand& op) { 2001 switch (op.kind()) { 2002 case Operand::MEM_REG_DISP: 2003 masm.andb_im(imm.value, op.disp(), op.base()); 2004 break; 2005 case Operand::MEM_SCALE: 2006 masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 2007 break; 2008 default: 2009 MOZ_CRASH("unexpected operand kind"); 2010 break; 2011 } 2012 } 2013 void andb(Register src, const Operand& op) { 2014 switch (op.kind()) { 2015 case Operand::MEM_REG_DISP: 2016 masm.andb_rm(src.encoding(), op.disp(), op.base()); 2017 break; 2018 case Operand::MEM_SCALE: 2019 masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(), 2020 op.scale()); 2021 break; 2022 default: 2023 MOZ_CRASH("unexpected operand kind"); 2024 break; 2025 } 2026 } 2027 2028 void orb(Imm32 imm, const Operand& op) { 2029 switch (op.kind()) { 2030 case Operand::MEM_REG_DISP: 2031 masm.orb_im(imm.value, op.disp(), op.base()); 2032 break; 2033 case Operand::MEM_SCALE: 2034 masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 2035 break; 2036 default: 2037 MOZ_CRASH("unexpected operand kind"); 2038 break; 2039 } 2040 } 2041 void orb(Register src, const Operand& op) { 2042 switch (op.kind()) { 2043 case Operand::MEM_REG_DISP: 2044 masm.orb_rm(src.encoding(), op.disp(), op.base()); 2045 break; 2046 case Operand::MEM_SCALE: 2047 masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(), 2048 op.scale()); 2049 break; 2050 default: 2051 MOZ_CRASH("unexpected operand kind"); 2052 break; 2053 } 2054 } 2055 2056 void xorb(Imm32 imm, const Operand& op) { 2057 switch (op.kind()) { 2058 case Operand::MEM_REG_DISP: 2059 masm.xorb_im(imm.value, op.disp(), op.base()); 2060 break; 2061 case Operand::MEM_SCALE: 2062 masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 2063 break; 2064 default: 2065 MOZ_CRASH("unexpected operand kind"); 2066 break; 2067 } 2068 } 2069 void xorb(Register src, const Operand& op) { 2070 switch (op.kind()) { 2071 case Operand::MEM_REG_DISP: 2072 masm.xorb_rm(src.encoding(), op.disp(), op.base()); 2073 break; 2074 case Operand::MEM_SCALE: 2075 masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(), 2076 op.scale()); 2077 break; 2078 default: 2079 MOZ_CRASH("unexpected operand kind"); 2080 break; 2081 } 2082 } 2083 2084 template <typename T> 2085 void lock_addb(T src, const Operand& op) { 2086 masm.prefix_lock(); 2087 addb(src, op); 2088 } 2089 template <typename T> 2090 void lock_subb(T src, const Operand& op) { 2091 masm.prefix_lock(); 2092 subb(src, op); 2093 } 2094 template <typename T> 2095 void lock_andb(T src, const Operand& op) { 2096 masm.prefix_lock(); 2097 andb(src, op); 2098 } 2099 template <typename T> 2100 void lock_orb(T src, const Operand& op) { 2101 masm.prefix_lock(); 2102 orb(src, op); 2103 } 2104 template <typename T> 2105 void lock_xorb(T src, const Operand& op) { 2106 masm.prefix_lock(); 2107 xorb(src, op); 2108 } 2109 2110 template <typename T> 2111 void lock_addw(T src, const Operand& op) { 2112 masm.prefix_lock(); 2113 addw(src, op); 2114 } 2115 template <typename T> 2116 void lock_subw(T src, const Operand& op) { 2117 masm.prefix_lock(); 2118 subw(src, op); 2119 } 2120 template <typename T> 2121 void lock_andw(T src, const Operand& op) { 2122 masm.prefix_lock(); 2123 andw(src, op); 2124 } 2125 template <typename T> 2126 void lock_orw(T src, const Operand& op) { 2127 masm.prefix_lock(); 2128 orw(src, op); 2129 } 2130 template <typename T> 2131 void lock_xorw(T src, const Operand& op) { 2132 masm.prefix_lock(); 2133 xorw(src, op); 2134 } 2135 2136 // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems, 2137 // among other things. Do not optimize, replace by XADDL, or similar. 2138 template <typename T> 2139 void lock_addl(T src, const Operand& op) { 2140 masm.prefix_lock(); 2141 addl(src, op); 2142 } 2143 template <typename T> 2144 void lock_subl(T src, const Operand& op) { 2145 masm.prefix_lock(); 2146 subl(src, op); 2147 } 2148 template <typename T> 2149 void lock_andl(T src, const Operand& op) { 2150 masm.prefix_lock(); 2151 andl(src, op); 2152 } 2153 template <typename T> 2154 void lock_orl(T src, const Operand& op) { 2155 masm.prefix_lock(); 2156 orl(src, op); 2157 } 2158 template <typename T> 2159 void lock_xorl(T src, const Operand& op) { 2160 masm.prefix_lock(); 2161 xorl(src, op); 2162 } 2163 2164 void lock_cmpxchgb(Register src, const Operand& mem) { 2165 masm.prefix_lock(); 2166 switch (mem.kind()) { 2167 case Operand::MEM_REG_DISP: 2168 masm.cmpxchgb(src.encoding(), mem.disp(), mem.base()); 2169 break; 2170 case Operand::MEM_SCALE: 2171 masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(), 2172 mem.scale()); 2173 break; 2174 default: 2175 MOZ_CRASH("unexpected operand kind"); 2176 } 2177 } 2178 void lock_cmpxchgw(Register src, const Operand& mem) { 2179 masm.prefix_lock(); 2180 switch (mem.kind()) { 2181 case Operand::MEM_REG_DISP: 2182 masm.cmpxchgw(src.encoding(), mem.disp(), mem.base()); 2183 break; 2184 case Operand::MEM_SCALE: 2185 masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(), 2186 mem.scale()); 2187 break; 2188 default: 2189 MOZ_CRASH("unexpected operand kind"); 2190 } 2191 } 2192 void lock_cmpxchgl(Register src, const Operand& mem) { 2193 masm.prefix_lock(); 2194 switch (mem.kind()) { 2195 case Operand::MEM_REG_DISP: 2196 masm.cmpxchgl(src.encoding(), mem.disp(), mem.base()); 2197 break; 2198 case Operand::MEM_SCALE: 2199 masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(), 2200 mem.scale()); 2201 break; 2202 default: 2203 MOZ_CRASH("unexpected operand kind"); 2204 } 2205 } 2206 void lock_cmpxchg8b(Register srcHi, Register srcLo, Register newHi, 2207 Register newLo, const Operand& mem) { 2208 masm.prefix_lock(); 2209 switch (mem.kind()) { 2210 case Operand::MEM_REG_DISP: 2211 masm.cmpxchg8b(srcHi.encoding(), srcLo.encoding(), newHi.encoding(), 2212 newLo.encoding(), mem.disp(), mem.base()); 2213 break; 2214 case Operand::MEM_SCALE: 2215 masm.cmpxchg8b(srcHi.encoding(), srcLo.encoding(), newHi.encoding(), 2216 newLo.encoding(), mem.disp(), mem.base(), mem.index(), 2217 mem.scale()); 2218 break; 2219 default: 2220 MOZ_CRASH("unexpected operand kind"); 2221 } 2222 } 2223 2224 void xchgb(Register src, const Operand& mem) { 2225 switch (mem.kind()) { 2226 case Operand::MEM_REG_DISP: 2227 masm.xchgb_rm(src.encoding(), mem.disp(), mem.base()); 2228 break; 2229 case Operand::MEM_SCALE: 2230 masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), 2231 mem.scale()); 2232 break; 2233 default: 2234 MOZ_CRASH("unexpected operand kind"); 2235 } 2236 } 2237 void xchgw(Register src, const Operand& mem) { 2238 switch (mem.kind()) { 2239 case Operand::MEM_REG_DISP: 2240 masm.xchgw_rm(src.encoding(), mem.disp(), mem.base()); 2241 break; 2242 case Operand::MEM_SCALE: 2243 masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), 2244 mem.scale()); 2245 break; 2246 default: 2247 MOZ_CRASH("unexpected operand kind"); 2248 } 2249 } 2250 void xchgl(Register src, const Operand& mem) { 2251 switch (mem.kind()) { 2252 case Operand::MEM_REG_DISP: 2253 masm.xchgl_rm(src.encoding(), mem.disp(), mem.base()); 2254 break; 2255 case Operand::MEM_SCALE: 2256 masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), 2257 mem.scale()); 2258 break; 2259 default: 2260 MOZ_CRASH("unexpected operand kind"); 2261 } 2262 } 2263 2264 void lock_xaddb(Register srcdest, const Operand& mem) { 2265 switch (mem.kind()) { 2266 case Operand::MEM_REG_DISP: 2267 masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base()); 2268 break; 2269 case Operand::MEM_SCALE: 2270 masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(), 2271 mem.index(), mem.scale()); 2272 break; 2273 default: 2274 MOZ_CRASH("unexpected operand kind"); 2275 } 2276 } 2277 void lock_xaddw(Register srcdest, const Operand& mem) { 2278 masm.prefix_16_for_32(); 2279 lock_xaddl(srcdest, mem); 2280 } 2281 void lock_xaddl(Register srcdest, const Operand& mem) { 2282 switch (mem.kind()) { 2283 case Operand::MEM_REG_DISP: 2284 masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base()); 2285 break; 2286 case Operand::MEM_SCALE: 2287 masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(), 2288 mem.index(), mem.scale()); 2289 break; 2290 default: 2291 MOZ_CRASH("unexpected operand kind"); 2292 } 2293 } 2294 2295 void push(const Imm32 imm) { masm.push_i(imm.value); } 2296 2297 void push(const Operand& src) { 2298 MOZ_ASSERT(hasCreator()); 2299 switch (src.kind()) { 2300 case Operand::REG: 2301 masm.push_r(src.reg()); 2302 break; 2303 case Operand::MEM_REG_DISP: 2304 masm.push_m(src.disp(), src.base()); 2305 break; 2306 case Operand::MEM_SCALE: 2307 masm.push_m(src.disp(), src.base(), src.index(), src.scale()); 2308 break; 2309 default: 2310 MOZ_CRASH("unexpected operand kind"); 2311 } 2312 } 2313 void push(Register src) { 2314 MOZ_ASSERT(hasCreator()); 2315 masm.push_r(src.encoding()); 2316 } 2317 void push(const Address& src) { 2318 masm.push_m(src.offset, src.base.encoding()); 2319 } 2320 2321 void pop(const Operand& src) { 2322 MOZ_ASSERT(hasCreator()); 2323 switch (src.kind()) { 2324 case Operand::REG: 2325 masm.pop_r(src.reg()); 2326 break; 2327 case Operand::MEM_REG_DISP: 2328 masm.pop_m(src.disp(), src.base()); 2329 break; 2330 default: 2331 MOZ_CRASH("unexpected operand kind"); 2332 } 2333 } 2334 void pop(Register src) { 2335 MOZ_ASSERT(hasCreator()); 2336 masm.pop_r(src.encoding()); 2337 } 2338 void pop(const Address& src) { masm.pop_m(src.offset, src.base.encoding()); } 2339 2340 void pushFlags() { masm.push_flags(); } 2341 void popFlags() { masm.pop_flags(); } 2342 2343 #ifdef JS_CODEGEN_X86 2344 void pushAllRegs() { masm.pusha(); } 2345 void popAllRegs() { masm.popa(); } 2346 #endif 2347 2348 // Zero-extend byte to 32-bit integer. 2349 void movzbl(Register src, Register dest) { 2350 masm.movzbl_rr(src.encoding(), dest.encoding()); 2351 } 2352 2353 void cdq() { masm.cdq(); } 2354 void idiv(Register divisor) { masm.idivl_r(divisor.encoding()); } 2355 void udiv(Register divisor) { masm.divl_r(divisor.encoding()); } 2356 2357 void vpblendw(uint32_t mask, FloatRegister src1, FloatRegister src0, 2358 FloatRegister dest) { 2359 MOZ_ASSERT(HasSSE41()); 2360 masm.vpblendw_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 2361 } 2362 2363 void vpblendvb(FloatRegister mask, FloatRegister src1, FloatRegister src0, 2364 FloatRegister dest) { 2365 MOZ_ASSERT(HasSSE41()); 2366 masm.vpblendvb_rr(mask.encoding(), src1.encoding(), src0.encoding(), 2367 dest.encoding()); 2368 } 2369 2370 void vpinsrb(unsigned lane, const Operand& src1, FloatRegister src0, 2371 FloatRegister dest) { 2372 MOZ_ASSERT(HasSSE41()); 2373 switch (src1.kind()) { 2374 case Operand::REG: 2375 masm.vpinsrb_irr(lane, src1.reg(), src0.encoding(), dest.encoding()); 2376 break; 2377 case Operand::MEM_REG_DISP: 2378 masm.vpinsrb_imr(lane, src1.disp(), src1.base(), src0.encoding(), 2379 dest.encoding()); 2380 break; 2381 case Operand::MEM_SCALE: 2382 masm.vpinsrb_imr(lane, src1.disp(), src1.base(), src1.index(), 2383 src1.scale(), src0.encoding(), dest.encoding()); 2384 break; 2385 default: 2386 MOZ_CRASH("unexpected operand kind"); 2387 } 2388 } 2389 void vpinsrw(unsigned lane, const Operand& src1, FloatRegister src0, 2390 FloatRegister dest) { 2391 switch (src1.kind()) { 2392 case Operand::REG: 2393 masm.vpinsrw_irr(lane, src1.reg(), src0.encoding(), dest.encoding()); 2394 break; 2395 case Operand::MEM_REG_DISP: 2396 masm.vpinsrw_imr(lane, src1.disp(), src1.base(), src0.encoding(), 2397 dest.encoding()); 2398 break; 2399 case Operand::MEM_SCALE: 2400 masm.vpinsrw_imr(lane, src1.disp(), src1.base(), src1.index(), 2401 src1.scale(), src0.encoding(), dest.encoding()); 2402 break; 2403 default: 2404 MOZ_CRASH("unexpected operand kind"); 2405 } 2406 } 2407 2408 void vpinsrd(unsigned lane, Register src1, FloatRegister src0, 2409 FloatRegister dest) { 2410 MOZ_ASSERT(HasSSE41()); 2411 masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); 2412 } 2413 2414 void vpextrb(unsigned lane, FloatRegister src, const Operand& dest) { 2415 MOZ_ASSERT(HasSSE41()); 2416 switch (dest.kind()) { 2417 case Operand::REG: 2418 masm.vpextrb_irr(lane, src.encoding(), dest.reg()); 2419 break; 2420 case Operand::MEM_REG_DISP: 2421 masm.vpextrb_irm(lane, src.encoding(), dest.disp(), dest.base()); 2422 break; 2423 case Operand::MEM_SCALE: 2424 masm.vpextrb_irm(lane, src.encoding(), dest.disp(), dest.base(), 2425 dest.index(), dest.scale()); 2426 break; 2427 default: 2428 MOZ_CRASH("unexpected operand kind"); 2429 } 2430 } 2431 void vpextrw(unsigned lane, FloatRegister src, const Operand& dest) { 2432 MOZ_ASSERT(HasSSE41()); 2433 switch (dest.kind()) { 2434 case Operand::REG: 2435 masm.vpextrw_irr(lane, src.encoding(), dest.reg()); 2436 break; 2437 case Operand::MEM_REG_DISP: 2438 masm.vpextrw_irm(lane, src.encoding(), dest.disp(), dest.base()); 2439 break; 2440 case Operand::MEM_SCALE: 2441 masm.vpextrw_irm(lane, src.encoding(), dest.disp(), dest.base(), 2442 dest.index(), dest.scale()); 2443 break; 2444 default: 2445 MOZ_CRASH("unexpected operand kind"); 2446 } 2447 } 2448 void vpextrd(unsigned lane, FloatRegister src, Register dest) { 2449 MOZ_ASSERT(HasSSE41()); 2450 masm.vpextrd_irr(lane, src.encoding(), dest.encoding()); 2451 } 2452 void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) { 2453 MOZ_ASSERT(HasSSE2()); 2454 masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding()); 2455 } 2456 void vpslldq(Imm32 shift, FloatRegister src, FloatRegister dest) { 2457 MOZ_ASSERT(HasSSE2()); 2458 masm.vpslldq_ir(shift.value, src.encoding(), dest.encoding()); 2459 } 2460 void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) { 2461 MOZ_ASSERT(HasSSE2()); 2462 masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding()); 2463 } 2464 void vpsllq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2465 MOZ_ASSERT(HasSSE2()); 2466 masm.vpsllq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2467 } 2468 void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) { 2469 MOZ_ASSERT(HasSSE2()); 2470 masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding()); 2471 } 2472 void vpsrlq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2473 MOZ_ASSERT(HasSSE2()); 2474 masm.vpsrlq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2475 } 2476 void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2477 MOZ_ASSERT(HasSSE2()); 2478 masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2479 } 2480 void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) { 2481 MOZ_ASSERT(HasSSE2()); 2482 masm.vpslld_ir(count.value, src0.encoding(), dest.encoding()); 2483 } 2484 void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2485 MOZ_ASSERT(HasSSE2()); 2486 masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2487 } 2488 void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) { 2489 MOZ_ASSERT(HasSSE2()); 2490 masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding()); 2491 } 2492 void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2493 MOZ_ASSERT(HasSSE2()); 2494 masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2495 } 2496 void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) { 2497 MOZ_ASSERT(HasSSE2()); 2498 masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding()); 2499 } 2500 2501 void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2502 MOZ_ASSERT(HasSSE2()); 2503 masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2504 } 2505 void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) { 2506 MOZ_ASSERT(HasSSE2()); 2507 masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding()); 2508 } 2509 void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2510 MOZ_ASSERT(HasSSE2()); 2511 masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2512 } 2513 void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) { 2514 MOZ_ASSERT(HasSSE2()); 2515 masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding()); 2516 } 2517 void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2518 MOZ_ASSERT(HasSSE2()); 2519 masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2520 } 2521 void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) { 2522 MOZ_ASSERT(HasSSE2()); 2523 masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding()); 2524 } 2525 2526 void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2527 MOZ_ASSERT(HasSSE2()); 2528 switch (src1.kind()) { 2529 case Operand::REG: 2530 masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding()); 2531 break; 2532 case Operand::MEM_REG_DISP: 2533 masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(), 2534 dest.encoding()); 2535 break; 2536 case Operand::MEM_SCALE: 2537 masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), 2538 src0.encoding(), dest.encoding()); 2539 break; 2540 default: 2541 MOZ_CRASH("unexpected operand kind"); 2542 } 2543 } 2544 void vcvttsd2si(FloatRegister src, Register dest) { 2545 MOZ_ASSERT(HasSSE2()); 2546 masm.vcvttsd2si_rr(src.encoding(), dest.encoding()); 2547 } 2548 void vcvttss2si(FloatRegister src, Register dest) { 2549 MOZ_ASSERT(HasSSE2()); 2550 masm.vcvttss2si_rr(src.encoding(), dest.encoding()); 2551 } 2552 void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2553 MOZ_ASSERT(HasSSE2()); 2554 switch (src1.kind()) { 2555 case Operand::REG: 2556 masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding()); 2557 break; 2558 case Operand::MEM_REG_DISP: 2559 masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(), 2560 dest.encoding()); 2561 break; 2562 case Operand::MEM_SCALE: 2563 masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), 2564 src0.encoding(), dest.encoding()); 2565 break; 2566 default: 2567 MOZ_CRASH("unexpected operand kind"); 2568 } 2569 } 2570 void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) { 2571 MOZ_ASSERT(HasSSE2()); 2572 masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2573 } 2574 void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) { 2575 MOZ_ASSERT(HasSSE2()); 2576 masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2577 } 2578 void vcvttps2dq(FloatRegister src, FloatRegister dest) { 2579 MOZ_ASSERT(HasSSE2()); 2580 masm.vcvttps2dq_rr(src.encoding(), dest.encoding()); 2581 } 2582 void vcvttpd2dq(FloatRegister src, FloatRegister dest) { 2583 masm.vcvttpd2dq_rr(src.encoding(), dest.encoding()); 2584 } 2585 void vcvtdq2ps(FloatRegister src, FloatRegister dest) { 2586 MOZ_ASSERT(HasSSE2()); 2587 masm.vcvtdq2ps_rr(src.encoding(), dest.encoding()); 2588 } 2589 void vcvtdq2pd(FloatRegister src, FloatRegister dest) { 2590 masm.vcvtdq2pd_rr(src.encoding(), dest.encoding()); 2591 } 2592 void vcvtps2pd(FloatRegister src, FloatRegister dest) { 2593 masm.vcvtps2pd_rr(src.encoding(), dest.encoding()); 2594 } 2595 void vcvtpd2ps(FloatRegister src, FloatRegister dest) { 2596 masm.vcvtpd2ps_rr(src.encoding(), dest.encoding()); 2597 } 2598 void vcvtph2ps(FloatRegister src, FloatRegister dest) { 2599 MOZ_ASSERT(HasF16C()); 2600 masm.vcvtph2ps_rr(src.encoding(), dest.encoding()); 2601 } 2602 void vcvtps2ph(FloatRegister src, FloatRegister dest) { 2603 MOZ_ASSERT(HasF16C()); 2604 masm.vcvtps2ph_rr(src.encoding(), dest.encoding()); 2605 } 2606 void vmovmskpd(FloatRegister src, Register dest) { 2607 MOZ_ASSERT(HasSSE2()); 2608 masm.vmovmskpd_rr(src.encoding(), dest.encoding()); 2609 } 2610 void vmovmskps(FloatRegister src, Register dest) { 2611 MOZ_ASSERT(HasSSE2()); 2612 masm.vmovmskps_rr(src.encoding(), dest.encoding()); 2613 } 2614 void vpmovmskb(FloatRegister src, Register dest) { 2615 MOZ_ASSERT(HasSSE2()); 2616 masm.vpmovmskb_rr(src.encoding(), dest.encoding()); 2617 } 2618 void vptest(FloatRegister rhs, FloatRegister lhs) { 2619 MOZ_ASSERT(HasSSE41()); 2620 masm.vptest_rr(rhs.encoding(), lhs.encoding()); 2621 } 2622 void vucomisd(FloatRegister rhs, FloatRegister lhs) { 2623 MOZ_ASSERT(HasSSE2()); 2624 masm.vucomisd_rr(rhs.encoding(), lhs.encoding()); 2625 } 2626 void vucomiss(FloatRegister rhs, FloatRegister lhs) { 2627 MOZ_ASSERT(HasSSE2()); 2628 masm.vucomiss_rr(rhs.encoding(), lhs.encoding()); 2629 } 2630 2631 void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2632 MOZ_ASSERT(HasSSE2()); 2633 switch (rhs.kind()) { 2634 case Operand::FPREG: 2635 masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2636 break; 2637 case Operand::MEM_REG_DISP: 2638 masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2639 dest.encoding()); 2640 break; 2641 case Operand::MEM_ADDRESS32: 2642 masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2643 break; 2644 default: 2645 MOZ_CRASH("unexpected operand kind"); 2646 } 2647 } 2648 void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2649 MOZ_ASSERT(HasSSE2()); 2650 switch (rhs.kind()) { 2651 case Operand::FPREG: 2652 masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2653 break; 2654 case Operand::MEM_REG_DISP: 2655 masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2656 dest.encoding()); 2657 break; 2658 case Operand::MEM_ADDRESS32: 2659 masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2660 break; 2661 default: 2662 MOZ_CRASH("unexpected operand kind"); 2663 } 2664 } 2665 2666 void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2667 MOZ_ASSERT(HasSSE2()); 2668 switch (rhs.kind()) { 2669 case Operand::FPREG: 2670 masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2671 break; 2672 case Operand::MEM_REG_DISP: 2673 masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2674 dest.encoding()); 2675 break; 2676 case Operand::MEM_ADDRESS32: 2677 masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2678 break; 2679 default: 2680 MOZ_CRASH("unexpected operand kind"); 2681 } 2682 } 2683 void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2684 MOZ_ASSERT(HasSSE2()); 2685 switch (rhs.kind()) { 2686 case Operand::FPREG: 2687 masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2688 break; 2689 case Operand::MEM_REG_DISP: 2690 masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2691 dest.encoding()); 2692 break; 2693 case Operand::MEM_ADDRESS32: 2694 masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2695 break; 2696 default: 2697 MOZ_CRASH("unexpected operand kind"); 2698 } 2699 } 2700 2701 void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2702 MOZ_ASSERT(HasSSE2()); 2703 switch (rhs.kind()) { 2704 case Operand::FPREG: 2705 masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2706 break; 2707 case Operand::MEM_REG_DISP: 2708 masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2709 dest.encoding()); 2710 break; 2711 case Operand::MEM_ADDRESS32: 2712 masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2713 break; 2714 default: 2715 MOZ_CRASH("unexpected operand kind"); 2716 } 2717 } 2718 void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2719 MOZ_ASSERT(HasSSE2()); 2720 switch (rhs.kind()) { 2721 case Operand::FPREG: 2722 masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2723 break; 2724 case Operand::MEM_REG_DISP: 2725 masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2726 dest.encoding()); 2727 break; 2728 case Operand::MEM_ADDRESS32: 2729 masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2730 break; 2731 default: 2732 MOZ_CRASH("unexpected operand kind"); 2733 } 2734 } 2735 void vpcmpgtq(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2736 MOZ_ASSERT(HasSSE42()); 2737 switch (rhs.kind()) { 2738 case Operand::FPREG: 2739 masm.vpcmpgtq_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2740 break; 2741 default: 2742 MOZ_CRASH("unexpected operand kind"); 2743 } 2744 } 2745 2746 void vpcmpeqq(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2747 MOZ_ASSERT(HasSSE41()); 2748 switch (rhs.kind()) { 2749 case Operand::FPREG: 2750 masm.vpcmpeqq_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2751 break; 2752 case Operand::MEM_REG_DISP: 2753 masm.vpcmpeqq_mr(rhs.disp(), rhs.base(), lhs.encoding(), 2754 dest.encoding()); 2755 break; 2756 case Operand::MEM_ADDRESS32: 2757 masm.vpcmpeqq_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2758 break; 2759 default: 2760 MOZ_CRASH("unexpected operand kind"); 2761 } 2762 } 2763 2764 void vcmpps(uint8_t order, Operand rhs, FloatRegister lhs, 2765 FloatRegister dest) { 2766 MOZ_ASSERT(HasSSE2()); 2767 switch (rhs.kind()) { 2768 case Operand::FPREG: 2769 masm.vcmpps_rr(order, rhs.fpu(), lhs.encoding(), dest.encoding()); 2770 break; 2771 case Operand::MEM_REG_DISP: 2772 masm.vcmpps_mr(order, rhs.disp(), rhs.base(), lhs.encoding(), 2773 dest.encoding()); 2774 break; 2775 case Operand::MEM_ADDRESS32: 2776 masm.vcmpps_mr(order, rhs.address(), lhs.encoding(), dest.encoding()); 2777 break; 2778 default: 2779 MOZ_CRASH("unexpected operand kind"); 2780 } 2781 } 2782 void vcmpeqps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2783 vcmpps(X86Encoding::ConditionCmp_EQ, rhs, lhs, dest); 2784 } 2785 void vcmpltps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2786 vcmpps(X86Encoding::ConditionCmp_LT, rhs, lhs, dest); 2787 } 2788 void vcmpleps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2789 vcmpps(X86Encoding::ConditionCmp_LE, rhs, lhs, dest); 2790 } 2791 void vcmpunordps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2792 vcmpps(X86Encoding::ConditionCmp_UNORD, rhs, lhs, dest); 2793 } 2794 void vcmpneqps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2795 vcmpps(X86Encoding::ConditionCmp_NEQ, rhs, lhs, dest); 2796 } 2797 void vcmpordps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2798 vcmpps(X86Encoding::ConditionCmp_ORD, rhs, lhs, dest); 2799 } 2800 void vcmppd(uint8_t order, Operand rhs, FloatRegister lhs, 2801 FloatRegister dest) { 2802 switch (rhs.kind()) { 2803 case Operand::FPREG: 2804 masm.vcmppd_rr(order, rhs.fpu(), lhs.encoding(), dest.encoding()); 2805 break; 2806 default: 2807 MOZ_CRASH("NYI"); 2808 } 2809 } 2810 void vcmpeqpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2811 vcmppd(X86Encoding::ConditionCmp_EQ, rhs, lhs, dest); 2812 } 2813 void vcmpltpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2814 vcmppd(X86Encoding::ConditionCmp_LT, rhs, lhs, dest); 2815 } 2816 void vcmplepd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2817 vcmppd(X86Encoding::ConditionCmp_LE, rhs, lhs, dest); 2818 } 2819 void vcmpneqpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2820 vcmppd(X86Encoding::ConditionCmp_NEQ, rhs, lhs, dest); 2821 } 2822 void vcmpordpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2823 vcmppd(X86Encoding::ConditionCmp_ORD, rhs, lhs, dest); 2824 } 2825 void vcmpunordpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2826 vcmppd(X86Encoding::ConditionCmp_UNORD, rhs, lhs, dest); 2827 } 2828 void vrcpps(const Operand& src, FloatRegister dest) { 2829 MOZ_ASSERT(HasSSE2()); 2830 switch (src.kind()) { 2831 case Operand::FPREG: 2832 masm.vrcpps_rr(src.fpu(), dest.encoding()); 2833 break; 2834 case Operand::MEM_REG_DISP: 2835 masm.vrcpps_mr(src.disp(), src.base(), dest.encoding()); 2836 break; 2837 case Operand::MEM_ADDRESS32: 2838 masm.vrcpps_mr(src.address(), dest.encoding()); 2839 break; 2840 default: 2841 MOZ_CRASH("unexpected operand kind"); 2842 } 2843 } 2844 void vsqrtps(const Operand& src, FloatRegister dest) { 2845 MOZ_ASSERT(HasSSE2()); 2846 switch (src.kind()) { 2847 case Operand::FPREG: 2848 masm.vsqrtps_rr(src.fpu(), dest.encoding()); 2849 break; 2850 case Operand::MEM_REG_DISP: 2851 masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding()); 2852 break; 2853 case Operand::MEM_ADDRESS32: 2854 masm.vsqrtps_mr(src.address(), dest.encoding()); 2855 break; 2856 default: 2857 MOZ_CRASH("unexpected operand kind"); 2858 } 2859 } 2860 void vrsqrtps(const Operand& src, FloatRegister dest) { 2861 MOZ_ASSERT(HasSSE2()); 2862 switch (src.kind()) { 2863 case Operand::FPREG: 2864 masm.vrsqrtps_rr(src.fpu(), dest.encoding()); 2865 break; 2866 case Operand::MEM_REG_DISP: 2867 masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding()); 2868 break; 2869 case Operand::MEM_ADDRESS32: 2870 masm.vrsqrtps_mr(src.address(), dest.encoding()); 2871 break; 2872 default: 2873 MOZ_CRASH("unexpected operand kind"); 2874 } 2875 } 2876 void vsqrtpd(const Operand& src, FloatRegister dest) { 2877 MOZ_ASSERT(HasSSE2()); 2878 switch (src.kind()) { 2879 case Operand::FPREG: 2880 masm.vsqrtpd_rr(src.fpu(), dest.encoding()); 2881 break; 2882 default: 2883 MOZ_CRASH("unexpected operand kind"); 2884 } 2885 } 2886 void vmovd(Register src, FloatRegister dest) { 2887 MOZ_ASSERT(HasSSE2()); 2888 masm.vmovd_rr(src.encoding(), dest.encoding()); 2889 } 2890 void vmovd(FloatRegister src, Register dest) { 2891 MOZ_ASSERT(HasSSE2()); 2892 masm.vmovd_rr(src.encoding(), dest.encoding()); 2893 } 2894 void vmovd(const Operand& src, FloatRegister dest) { 2895 MOZ_ASSERT(HasSSE2()); 2896 switch (src.kind()) { 2897 case Operand::MEM_REG_DISP: 2898 masm.vmovd_mr(src.disp(), src.base(), dest.encoding()); 2899 break; 2900 case Operand::MEM_SCALE: 2901 masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(), 2902 dest.encoding()); 2903 break; 2904 default: 2905 MOZ_CRASH("unexpected operand kind"); 2906 } 2907 } 2908 void vmovd(FloatRegister src, const Operand& dest) { 2909 MOZ_ASSERT(HasSSE2()); 2910 switch (dest.kind()) { 2911 case Operand::MEM_REG_DISP: 2912 masm.vmovd_rm(src.encoding(), dest.disp(), dest.base()); 2913 break; 2914 case Operand::MEM_SCALE: 2915 masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 2916 dest.scale()); 2917 break; 2918 case Operand::MEM_ADDRESS32: 2919 masm.vmovq_rm(src.encoding(), dest.address()); 2920 break; 2921 default: 2922 MOZ_CRASH("unexpected operand kind"); 2923 } 2924 } 2925 void vmovq(const Operand& src, FloatRegister dest) { 2926 MOZ_ASSERT(HasSSE2()); 2927 switch (src.kind()) { 2928 case Operand::MEM_REG_DISP: 2929 masm.vmovq_mr(src.disp(), src.base(), dest.encoding()); 2930 break; 2931 case Operand::MEM_SCALE: 2932 masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(), 2933 dest.encoding()); 2934 break; 2935 case Operand::MEM_ADDRESS32: 2936 masm.vmovq_mr(src.address(), dest.encoding()); 2937 break; 2938 default: 2939 MOZ_CRASH("unexpected operand kind"); 2940 } 2941 } 2942 void vmovq(FloatRegister src, const Operand& dest) { 2943 MOZ_ASSERT(HasSSE2()); 2944 switch (dest.kind()) { 2945 case Operand::MEM_REG_DISP: 2946 masm.vmovq_rm(src.encoding(), dest.disp(), dest.base()); 2947 break; 2948 case Operand::MEM_SCALE: 2949 masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 2950 dest.scale()); 2951 break; 2952 default: 2953 MOZ_CRASH("unexpected operand kind"); 2954 } 2955 } 2956 void vpmaddubsw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2957 MOZ_ASSERT(HasSSSE3()); 2958 masm.vpmaddubsw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2959 } 2960 void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2961 MOZ_ASSERT(HasSSE2()); 2962 switch (src1.kind()) { 2963 case Operand::FPREG: 2964 masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2965 break; 2966 case Operand::MEM_REG_DISP: 2967 masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(), 2968 dest.encoding()); 2969 break; 2970 case Operand::MEM_ADDRESS32: 2971 masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding()); 2972 break; 2973 default: 2974 MOZ_CRASH("unexpected operand kind"); 2975 } 2976 } 2977 void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2978 MOZ_ASSERT(HasSSE2()); 2979 switch (src1.kind()) { 2980 case Operand::FPREG: 2981 masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2982 break; 2983 case Operand::MEM_REG_DISP: 2984 masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(), 2985 dest.encoding()); 2986 break; 2987 case Operand::MEM_ADDRESS32: 2988 masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding()); 2989 break; 2990 default: 2991 MOZ_CRASH("unexpected operand kind"); 2992 } 2993 } 2994 void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2995 MOZ_ASSERT(HasSSE2()); 2996 switch (src1.kind()) { 2997 case Operand::FPREG: 2998 masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2999 break; 3000 case Operand::MEM_REG_DISP: 3001 masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(), 3002 dest.encoding()); 3003 break; 3004 case Operand::MEM_ADDRESS32: 3005 masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding()); 3006 break; 3007 default: 3008 MOZ_CRASH("unexpected operand kind"); 3009 } 3010 } 3011 void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3012 MOZ_ASSERT(HasSSE2()); 3013 switch (src1.kind()) { 3014 case Operand::FPREG: 3015 masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3016 break; 3017 case Operand::MEM_REG_DISP: 3018 masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(), 3019 dest.encoding()); 3020 break; 3021 case Operand::MEM_ADDRESS32: 3022 masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding()); 3023 break; 3024 default: 3025 MOZ_CRASH("unexpected operand kind"); 3026 } 3027 } 3028 void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3029 MOZ_ASSERT(HasSSE2()); 3030 switch (src1.kind()) { 3031 case Operand::FPREG: 3032 masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3033 break; 3034 case Operand::MEM_REG_DISP: 3035 masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(), 3036 dest.encoding()); 3037 break; 3038 case Operand::MEM_ADDRESS32: 3039 masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding()); 3040 break; 3041 default: 3042 MOZ_CRASH("unexpected operand kind"); 3043 } 3044 } 3045 void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3046 MOZ_ASSERT(HasSSE2()); 3047 switch (src1.kind()) { 3048 case Operand::FPREG: 3049 masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3050 break; 3051 case Operand::MEM_REG_DISP: 3052 masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(), 3053 dest.encoding()); 3054 break; 3055 case Operand::MEM_ADDRESS32: 3056 masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding()); 3057 break; 3058 default: 3059 MOZ_CRASH("unexpected operand kind"); 3060 } 3061 } 3062 void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3063 MOZ_ASSERT(HasSSE2()); 3064 switch (src1.kind()) { 3065 case Operand::FPREG: 3066 masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3067 break; 3068 case Operand::MEM_REG_DISP: 3069 masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(), 3070 dest.encoding()); 3071 break; 3072 case Operand::MEM_ADDRESS32: 3073 masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding()); 3074 break; 3075 default: 3076 MOZ_CRASH("unexpected operand kind"); 3077 } 3078 } 3079 void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3080 MOZ_ASSERT(HasSSE2()); 3081 switch (src1.kind()) { 3082 case Operand::FPREG: 3083 masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3084 break; 3085 case Operand::MEM_REG_DISP: 3086 masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(), 3087 dest.encoding()); 3088 break; 3089 case Operand::MEM_ADDRESS32: 3090 masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding()); 3091 break; 3092 default: 3093 MOZ_CRASH("unexpected operand kind"); 3094 } 3095 } 3096 void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3097 MOZ_ASSERT(HasSSE2()); 3098 switch (src1.kind()) { 3099 case Operand::FPREG: 3100 masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3101 break; 3102 case Operand::MEM_REG_DISP: 3103 masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(), 3104 dest.encoding()); 3105 break; 3106 case Operand::MEM_ADDRESS32: 3107 masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding()); 3108 break; 3109 default: 3110 MOZ_CRASH("unexpected operand kind"); 3111 } 3112 } 3113 void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3114 MOZ_ASSERT(HasSSE2()); 3115 switch (src1.kind()) { 3116 case Operand::FPREG: 3117 masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3118 break; 3119 case Operand::MEM_REG_DISP: 3120 masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(), 3121 dest.encoding()); 3122 break; 3123 case Operand::MEM_ADDRESS32: 3124 masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding()); 3125 break; 3126 default: 3127 MOZ_CRASH("unexpected operand kind"); 3128 } 3129 } 3130 void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3131 MOZ_ASSERT(HasSSE2()); 3132 switch (src1.kind()) { 3133 case Operand::FPREG: 3134 masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3135 break; 3136 case Operand::MEM_REG_DISP: 3137 masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(), 3138 dest.encoding()); 3139 break; 3140 case Operand::MEM_ADDRESS32: 3141 masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding()); 3142 break; 3143 default: 3144 MOZ_CRASH("unexpected operand kind"); 3145 } 3146 } 3147 void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3148 MOZ_ASSERT(HasSSE2()); 3149 switch (src1.kind()) { 3150 case Operand::FPREG: 3151 masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3152 break; 3153 case Operand::MEM_REG_DISP: 3154 masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(), 3155 dest.encoding()); 3156 break; 3157 case Operand::MEM_ADDRESS32: 3158 masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding()); 3159 break; 3160 default: 3161 MOZ_CRASH("unexpected operand kind"); 3162 } 3163 } 3164 void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3165 MOZ_ASSERT(HasSSE2()); 3166 switch (src1.kind()) { 3167 case Operand::FPREG: 3168 masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3169 break; 3170 case Operand::MEM_REG_DISP: 3171 masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(), 3172 dest.encoding()); 3173 break; 3174 case Operand::MEM_ADDRESS32: 3175 masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding()); 3176 break; 3177 default: 3178 MOZ_CRASH("unexpected operand kind"); 3179 } 3180 } 3181 void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3182 MOZ_ASSERT(HasSSE2()); 3183 switch (src1.kind()) { 3184 case Operand::FPREG: 3185 masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3186 break; 3187 case Operand::MEM_REG_DISP: 3188 masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(), 3189 dest.encoding()); 3190 break; 3191 case Operand::MEM_ADDRESS32: 3192 masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding()); 3193 break; 3194 default: 3195 MOZ_CRASH("unexpected operand kind"); 3196 } 3197 } 3198 void vpmuldq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3199 MOZ_ASSERT(HasSSE41()); 3200 masm.vpmuldq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3201 } 3202 void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3203 MOZ_ASSERT(HasSSE2()); 3204 masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3205 } 3206 void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3207 MOZ_ASSERT(HasSSE2()); 3208 switch (src1.kind()) { 3209 case Operand::FPREG: 3210 masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3211 break; 3212 case Operand::MEM_REG_DISP: 3213 masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(), 3214 dest.encoding()); 3215 break; 3216 default: 3217 MOZ_CRASH("unexpected operand kind"); 3218 } 3219 } 3220 void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3221 MOZ_ASSERT(HasSSE2()); 3222 switch (src1.kind()) { 3223 case Operand::FPREG: 3224 masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3225 break; 3226 case Operand::MEM_REG_DISP: 3227 masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(), 3228 dest.encoding()); 3229 break; 3230 default: 3231 MOZ_CRASH("unexpected operand kind"); 3232 } 3233 } 3234 void vpmulhw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3235 MOZ_ASSERT(HasSSE2()); 3236 switch (src1.kind()) { 3237 case Operand::FPREG: 3238 masm.vpmulhw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3239 break; 3240 case Operand::MEM_REG_DISP: 3241 masm.vpmulhw_mr(src1.disp(), src1.base(), src0.encoding(), 3242 dest.encoding()); 3243 break; 3244 default: 3245 MOZ_CRASH("unexpected operand kind"); 3246 } 3247 } 3248 void vpmulhuw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3249 MOZ_ASSERT(HasSSE2()); 3250 switch (src1.kind()) { 3251 case Operand::FPREG: 3252 masm.vpmulhuw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3253 break; 3254 case Operand::MEM_REG_DISP: 3255 masm.vpmulhuw_mr(src1.disp(), src1.base(), src0.encoding(), 3256 dest.encoding()); 3257 break; 3258 default: 3259 MOZ_CRASH("unexpected operand kind"); 3260 } 3261 } 3262 void vpmulhrsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3263 MOZ_ASSERT(HasSSE2()); 3264 switch (src1.kind()) { 3265 case Operand::FPREG: 3266 masm.vpmulhrsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3267 break; 3268 case Operand::MEM_REG_DISP: 3269 masm.vpmulhrsw_mr(src1.disp(), src1.base(), src0.encoding(), 3270 dest.encoding()); 3271 break; 3272 default: 3273 MOZ_CRASH("unexpected operand kind"); 3274 } 3275 } 3276 void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3277 MOZ_ASSERT(HasSSE41()); 3278 switch (src1.kind()) { 3279 case Operand::FPREG: 3280 masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3281 break; 3282 case Operand::MEM_REG_DISP: 3283 masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(), 3284 dest.encoding()); 3285 break; 3286 case Operand::MEM_ADDRESS32: 3287 masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding()); 3288 break; 3289 default: 3290 MOZ_CRASH("unexpected operand kind"); 3291 } 3292 } 3293 void vpmaddwd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3294 MOZ_ASSERT(HasSSE2()); 3295 switch (src1.kind()) { 3296 case Operand::FPREG: 3297 masm.vpmaddwd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3298 break; 3299 default: 3300 MOZ_CRASH("unexpected operand kind"); 3301 } 3302 } 3303 void vpaddq(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3304 MOZ_ASSERT(HasSSE2()); 3305 switch (src1.kind()) { 3306 case Operand::FPREG: 3307 masm.vpaddq_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3308 break; 3309 default: 3310 MOZ_CRASH("unexpected operand kind"); 3311 } 3312 } 3313 void vpsubq(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3314 MOZ_ASSERT(HasSSE2()); 3315 switch (src1.kind()) { 3316 case Operand::FPREG: 3317 masm.vpsubq_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3318 break; 3319 default: 3320 MOZ_CRASH("unexpected operand kind"); 3321 } 3322 } 3323 void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3324 MOZ_ASSERT(HasSSE2()); 3325 switch (src1.kind()) { 3326 case Operand::FPREG: 3327 masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3328 break; 3329 case Operand::MEM_REG_DISP: 3330 masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(), 3331 dest.encoding()); 3332 break; 3333 case Operand::MEM_ADDRESS32: 3334 masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding()); 3335 break; 3336 default: 3337 MOZ_CRASH("unexpected operand kind"); 3338 } 3339 } 3340 void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3341 MOZ_ASSERT(HasSSE2()); 3342 switch (src1.kind()) { 3343 case Operand::FPREG: 3344 masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3345 break; 3346 case Operand::MEM_REG_DISP: 3347 masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(), 3348 dest.encoding()); 3349 break; 3350 case Operand::MEM_ADDRESS32: 3351 masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding()); 3352 break; 3353 default: 3354 MOZ_CRASH("unexpected operand kind"); 3355 } 3356 } 3357 void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3358 MOZ_ASSERT(HasSSE2()); 3359 switch (src1.kind()) { 3360 case Operand::FPREG: 3361 masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3362 break; 3363 case Operand::MEM_REG_DISP: 3364 masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(), 3365 dest.encoding()); 3366 break; 3367 case Operand::MEM_ADDRESS32: 3368 masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding()); 3369 break; 3370 default: 3371 MOZ_CRASH("unexpected operand kind"); 3372 } 3373 } 3374 void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3375 MOZ_ASSERT(HasSSE2()); 3376 switch (src1.kind()) { 3377 case Operand::FPREG: 3378 masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3379 break; 3380 case Operand::MEM_REG_DISP: 3381 masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(), 3382 dest.encoding()); 3383 break; 3384 case Operand::MEM_ADDRESS32: 3385 masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding()); 3386 break; 3387 default: 3388 MOZ_CRASH("unexpected operand kind"); 3389 } 3390 } 3391 void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3392 MOZ_ASSERT(HasSSE2()); 3393 switch (src1.kind()) { 3394 case Operand::FPREG: 3395 masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3396 break; 3397 case Operand::MEM_REG_DISP: 3398 masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(), 3399 dest.encoding()); 3400 break; 3401 case Operand::MEM_ADDRESS32: 3402 masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding()); 3403 break; 3404 default: 3405 MOZ_CRASH("unexpected operand kind"); 3406 } 3407 } 3408 void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3409 MOZ_ASSERT(HasSSE2()); 3410 switch (src1.kind()) { 3411 case Operand::FPREG: 3412 masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3413 break; 3414 case Operand::MEM_REG_DISP: 3415 masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(), 3416 dest.encoding()); 3417 break; 3418 case Operand::MEM_ADDRESS32: 3419 masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding()); 3420 break; 3421 default: 3422 MOZ_CRASH("unexpected operand kind"); 3423 } 3424 } 3425 void vminpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3426 MOZ_ASSERT(HasSSE2()); 3427 switch (src1.kind()) { 3428 case Operand::FPREG: 3429 masm.vminpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3430 break; 3431 default: 3432 MOZ_CRASH("unexpected operand kind"); 3433 } 3434 } 3435 void vmaxpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3436 MOZ_ASSERT(HasSSE2()); 3437 switch (src1.kind()) { 3438 case Operand::FPREG: 3439 masm.vmaxpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3440 break; 3441 default: 3442 MOZ_CRASH("unexpected operand kind"); 3443 } 3444 } 3445 void vaddpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3446 MOZ_ASSERT(HasSSE2()); 3447 switch (src1.kind()) { 3448 case Operand::FPREG: 3449 masm.vaddpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3450 break; 3451 default: 3452 MOZ_CRASH("unexpected operand kind"); 3453 } 3454 } 3455 void vsubpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3456 MOZ_ASSERT(HasSSE2()); 3457 switch (src1.kind()) { 3458 case Operand::FPREG: 3459 masm.vsubpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3460 break; 3461 default: 3462 MOZ_CRASH("unexpected operand kind"); 3463 } 3464 } 3465 void vmulpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3466 MOZ_ASSERT(HasSSE2()); 3467 switch (src1.kind()) { 3468 case Operand::FPREG: 3469 masm.vmulpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3470 break; 3471 default: 3472 MOZ_CRASH("unexpected operand kind"); 3473 } 3474 } 3475 void vdivpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3476 MOZ_ASSERT(HasSSE2()); 3477 switch (src1.kind()) { 3478 case Operand::FPREG: 3479 masm.vdivpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3480 break; 3481 default: 3482 MOZ_CRASH("unexpected operand kind"); 3483 } 3484 } 3485 void vpavgb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3486 MOZ_ASSERT(HasSSE2()); 3487 switch (src1.kind()) { 3488 case Operand::FPREG: 3489 masm.vpavgb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3490 break; 3491 default: 3492 MOZ_CRASH("unexpected operand kind"); 3493 } 3494 } 3495 void vpavgw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3496 MOZ_ASSERT(HasSSE2()); 3497 switch (src1.kind()) { 3498 case Operand::FPREG: 3499 masm.vpavgw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3500 break; 3501 default: 3502 MOZ_CRASH("unexpected operand kind"); 3503 } 3504 } 3505 void vpminsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3506 MOZ_ASSERT(HasSSE41()); 3507 switch (src1.kind()) { 3508 case Operand::FPREG: 3509 masm.vpminsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3510 break; 3511 default: 3512 MOZ_CRASH("unexpected operand kind"); 3513 } 3514 } 3515 void vpminub(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3516 MOZ_ASSERT(HasSSE2()); 3517 switch (src1.kind()) { 3518 case Operand::FPREG: 3519 masm.vpminub_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3520 break; 3521 default: 3522 MOZ_CRASH("unexpected operand kind"); 3523 } 3524 } 3525 void vpmaxsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3526 MOZ_ASSERT(HasSSE41()); 3527 switch (src1.kind()) { 3528 case Operand::FPREG: 3529 masm.vpmaxsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3530 break; 3531 default: 3532 MOZ_CRASH("unexpected operand kind"); 3533 } 3534 } 3535 void vpmaxub(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3536 MOZ_ASSERT(HasSSE2()); 3537 switch (src1.kind()) { 3538 case Operand::FPREG: 3539 masm.vpmaxub_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3540 break; 3541 default: 3542 MOZ_CRASH("unexpected operand kind"); 3543 } 3544 } 3545 void vpminsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3546 MOZ_ASSERT(HasSSE2()); 3547 switch (src1.kind()) { 3548 case Operand::FPREG: 3549 masm.vpminsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3550 break; 3551 default: 3552 MOZ_CRASH("unexpected operand kind"); 3553 } 3554 } 3555 void vpminuw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3556 MOZ_ASSERT(HasSSE41()); 3557 switch (src1.kind()) { 3558 case Operand::FPREG: 3559 masm.vpminuw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3560 break; 3561 default: 3562 MOZ_CRASH("unexpected operand kind"); 3563 } 3564 } 3565 void vpmaxsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3566 MOZ_ASSERT(HasSSE2()); 3567 switch (src1.kind()) { 3568 case Operand::FPREG: 3569 masm.vpmaxsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3570 break; 3571 default: 3572 MOZ_CRASH("unexpected operand kind"); 3573 } 3574 } 3575 void vpmaxuw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3576 MOZ_ASSERT(HasSSE41()); 3577 switch (src1.kind()) { 3578 case Operand::FPREG: 3579 masm.vpmaxuw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3580 break; 3581 default: 3582 MOZ_CRASH("unexpected operand kind"); 3583 } 3584 } 3585 void vpminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3586 MOZ_ASSERT(HasSSE41()); 3587 switch (src1.kind()) { 3588 case Operand::FPREG: 3589 masm.vpminsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3590 break; 3591 default: 3592 MOZ_CRASH("unexpected operand kind"); 3593 } 3594 } 3595 void vpminud(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3596 MOZ_ASSERT(HasSSE41()); 3597 switch (src1.kind()) { 3598 case Operand::FPREG: 3599 masm.vpminud_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3600 break; 3601 default: 3602 MOZ_CRASH("unexpected operand kind"); 3603 } 3604 } 3605 void vpmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3606 MOZ_ASSERT(HasSSE41()); 3607 switch (src1.kind()) { 3608 case Operand::FPREG: 3609 masm.vpmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3610 break; 3611 default: 3612 MOZ_CRASH("unexpected operand kind"); 3613 } 3614 } 3615 void vpmaxud(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3616 MOZ_ASSERT(HasSSE41()); 3617 switch (src1.kind()) { 3618 case Operand::FPREG: 3619 masm.vpmaxud_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3620 break; 3621 default: 3622 MOZ_CRASH("unexpected operand kind"); 3623 } 3624 } 3625 void vpacksswb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3626 MOZ_ASSERT(HasSSE2()); 3627 switch (src1.kind()) { 3628 case Operand::FPREG: 3629 masm.vpacksswb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3630 break; 3631 default: 3632 MOZ_CRASH("unexpected operand kind"); 3633 } 3634 } 3635 void vpackuswb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3636 MOZ_ASSERT(HasSSE2()); 3637 switch (src1.kind()) { 3638 case Operand::FPREG: 3639 masm.vpackuswb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3640 break; 3641 default: 3642 MOZ_CRASH("unexpected operand kind"); 3643 } 3644 } 3645 void vpackssdw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3646 MOZ_ASSERT(HasSSE2()); 3647 switch (src1.kind()) { 3648 case Operand::FPREG: 3649 masm.vpackssdw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3650 break; 3651 default: 3652 MOZ_CRASH("unexpected operand kind"); 3653 } 3654 } 3655 void vpackusdw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3656 MOZ_ASSERT(HasSSE41()); 3657 switch (src1.kind()) { 3658 case Operand::FPREG: 3659 masm.vpackusdw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3660 break; 3661 default: 3662 MOZ_CRASH("unexpected operand kind"); 3663 } 3664 } 3665 void vpabsb(const Operand& src, FloatRegister dest) { 3666 MOZ_ASSERT(HasSSE3()); 3667 switch (src.kind()) { 3668 case Operand::FPREG: 3669 masm.vpabsb_rr(src.fpu(), dest.encoding()); 3670 break; 3671 default: 3672 MOZ_CRASH("unexpected operand kind"); 3673 } 3674 } 3675 void vpabsw(const Operand& src, FloatRegister dest) { 3676 MOZ_ASSERT(HasSSE3()); 3677 switch (src.kind()) { 3678 case Operand::FPREG: 3679 masm.vpabsw_rr(src.fpu(), dest.encoding()); 3680 break; 3681 default: 3682 MOZ_CRASH("unexpected operand kind"); 3683 } 3684 } 3685 void vpabsd(const Operand& src, FloatRegister dest) { 3686 MOZ_ASSERT(HasSSE3()); 3687 switch (src.kind()) { 3688 case Operand::FPREG: 3689 masm.vpabsd_rr(src.fpu(), dest.encoding()); 3690 break; 3691 default: 3692 MOZ_CRASH("unexpected operand kind"); 3693 } 3694 } 3695 void vpmovsxbw(const Operand& src, FloatRegister dest) { 3696 MOZ_ASSERT(HasSSE41()); 3697 switch (src.kind()) { 3698 case Operand::FPREG: 3699 masm.vpmovsxbw_rr(src.fpu(), dest.encoding()); 3700 break; 3701 case Operand::MEM_REG_DISP: 3702 masm.vpmovsxbw_mr(src.disp(), src.base(), dest.encoding()); 3703 break; 3704 case Operand::MEM_SCALE: 3705 masm.vpmovsxbw_mr(src.disp(), src.base(), src.index(), src.scale(), 3706 dest.encoding()); 3707 break; 3708 default: 3709 MOZ_CRASH("unexpected operand kind"); 3710 } 3711 } 3712 void vpmovzxbw(const Operand& src, FloatRegister dest) { 3713 MOZ_ASSERT(HasSSE41()); 3714 switch (src.kind()) { 3715 case Operand::FPREG: 3716 masm.vpmovzxbw_rr(src.fpu(), dest.encoding()); 3717 break; 3718 case Operand::MEM_REG_DISP: 3719 masm.vpmovzxbw_mr(src.disp(), src.base(), dest.encoding()); 3720 break; 3721 case Operand::MEM_SCALE: 3722 masm.vpmovzxbw_mr(src.disp(), src.base(), src.index(), src.scale(), 3723 dest.encoding()); 3724 break; 3725 default: 3726 MOZ_CRASH("unexpected operand kind"); 3727 } 3728 } 3729 void vpmovzxbd(const Operand& src, FloatRegister dest) { 3730 MOZ_ASSERT(HasSSE41()); 3731 switch (src.kind()) { 3732 case Operand::FPREG: 3733 masm.vpmovzxbd_rr(src.fpu(), dest.encoding()); 3734 break; 3735 default: 3736 MOZ_CRASH("unexpected operand kind"); 3737 } 3738 } 3739 void vpmovzxbq(const Operand& src, FloatRegister dest) { 3740 MOZ_ASSERT(HasSSE41()); 3741 switch (src.kind()) { 3742 case Operand::FPREG: 3743 masm.vpmovzxbq_rr(src.fpu(), dest.encoding()); 3744 break; 3745 default: 3746 MOZ_CRASH("unexpected operand kind"); 3747 } 3748 } 3749 void vpmovsxwd(const Operand& src, FloatRegister dest) { 3750 MOZ_ASSERT(HasSSE41()); 3751 switch (src.kind()) { 3752 case Operand::FPREG: 3753 masm.vpmovsxwd_rr(src.fpu(), dest.encoding()); 3754 break; 3755 case Operand::MEM_REG_DISP: 3756 masm.vpmovsxwd_mr(src.disp(), src.base(), dest.encoding()); 3757 break; 3758 case Operand::MEM_SCALE: 3759 masm.vpmovsxwd_mr(src.disp(), src.base(), src.index(), src.scale(), 3760 dest.encoding()); 3761 break; 3762 default: 3763 MOZ_CRASH("unexpected operand kind"); 3764 } 3765 } 3766 void vpmovzxwd(const Operand& src, FloatRegister dest) { 3767 MOZ_ASSERT(HasSSE41()); 3768 switch (src.kind()) { 3769 case Operand::FPREG: 3770 masm.vpmovzxwd_rr(src.fpu(), dest.encoding()); 3771 break; 3772 case Operand::MEM_REG_DISP: 3773 masm.vpmovzxwd_mr(src.disp(), src.base(), dest.encoding()); 3774 break; 3775 case Operand::MEM_SCALE: 3776 masm.vpmovzxwd_mr(src.disp(), src.base(), src.index(), src.scale(), 3777 dest.encoding()); 3778 break; 3779 default: 3780 MOZ_CRASH("unexpected operand kind"); 3781 } 3782 } 3783 void vpmovzxwq(const Operand& src, FloatRegister dest) { 3784 MOZ_ASSERT(HasSSE41()); 3785 switch (src.kind()) { 3786 case Operand::FPREG: 3787 masm.vpmovzxwq_rr(src.fpu(), dest.encoding()); 3788 break; 3789 default: 3790 MOZ_CRASH("unexpected operand kind"); 3791 } 3792 } 3793 void vpmovsxdq(const Operand& src, FloatRegister dest) { 3794 MOZ_ASSERT(HasSSE41()); 3795 switch (src.kind()) { 3796 case Operand::FPREG: 3797 masm.vpmovsxdq_rr(src.fpu(), dest.encoding()); 3798 break; 3799 case Operand::MEM_REG_DISP: 3800 masm.vpmovsxdq_mr(src.disp(), src.base(), dest.encoding()); 3801 break; 3802 case Operand::MEM_SCALE: 3803 masm.vpmovsxdq_mr(src.disp(), src.base(), src.index(), src.scale(), 3804 dest.encoding()); 3805 break; 3806 default: 3807 MOZ_CRASH("unexpected operand kind"); 3808 } 3809 } 3810 void vpmovzxdq(const Operand& src, FloatRegister dest) { 3811 MOZ_ASSERT(HasSSE41()); 3812 switch (src.kind()) { 3813 case Operand::FPREG: 3814 masm.vpmovzxdq_rr(src.fpu(), dest.encoding()); 3815 break; 3816 case Operand::MEM_REG_DISP: 3817 masm.vpmovzxdq_mr(src.disp(), src.base(), dest.encoding()); 3818 break; 3819 case Operand::MEM_SCALE: 3820 masm.vpmovzxdq_mr(src.disp(), src.base(), src.index(), src.scale(), 3821 dest.encoding()); 3822 break; 3823 default: 3824 MOZ_CRASH("unexpected operand kind"); 3825 } 3826 } 3827 void vphaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3828 MOZ_ASSERT(HasSSE41()); 3829 switch (src1.kind()) { 3830 case Operand::FPREG: 3831 masm.vphaddd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3832 break; 3833 default: 3834 MOZ_CRASH("unexpected operand kind"); 3835 } 3836 } 3837 void vpalignr(const Operand& src1, FloatRegister src0, FloatRegister dest, 3838 uint8_t shift) { 3839 MOZ_ASSERT(HasSSE3()); 3840 switch (src1.kind()) { 3841 case Operand::FPREG: 3842 masm.vpalignr_irr(shift, src1.fpu(), src0.encoding(), dest.encoding()); 3843 break; 3844 default: 3845 MOZ_CRASH("unexpected operand kind"); 3846 } 3847 } 3848 void vpunpcklbw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3849 MOZ_ASSERT(HasSSE2()); 3850 MOZ_ASSERT(src0.size() == 16); 3851 MOZ_ASSERT(src1.size() == 16); 3852 MOZ_ASSERT(dest.size() == 16); 3853 masm.vpunpcklbw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3854 } 3855 void vpunpckhbw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3856 MOZ_ASSERT(HasSSE2()); 3857 MOZ_ASSERT(src0.size() == 16); 3858 MOZ_ASSERT(src1.size() == 16); 3859 MOZ_ASSERT(dest.size() == 16); 3860 masm.vpunpckhbw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3861 } 3862 void vpunpcklbw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3863 MOZ_ASSERT(HasSSE2()); 3864 switch (src1.kind()) { 3865 case Operand::FPREG: 3866 masm.vpunpcklbw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3867 break; 3868 default: 3869 MOZ_CRASH("unexpected operand kind"); 3870 } 3871 } 3872 void vpunpckldq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3873 MOZ_ASSERT(HasSSE2()); 3874 MOZ_ASSERT(src0.size() == 16); 3875 MOZ_ASSERT(src1.size() == 16); 3876 MOZ_ASSERT(dest.size() == 16); 3877 masm.vpunpckldq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3878 } 3879 void vpunpckldq(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3880 MOZ_ASSERT(HasSSE2()); 3881 MOZ_ASSERT(src0.size() == 16); 3882 MOZ_ASSERT(dest.size() == 16); 3883 switch (src1.kind()) { 3884 case Operand::MEM_REG_DISP: 3885 masm.vpunpckldq_mr(src1.disp(), src1.base(), src0.encoding(), 3886 dest.encoding()); 3887 break; 3888 case Operand::MEM_ADDRESS32: 3889 masm.vpunpckldq_mr(src1.address(), src0.encoding(), dest.encoding()); 3890 break; 3891 default: 3892 MOZ_CRASH("unexpected operand kind"); 3893 } 3894 } 3895 void vpunpcklqdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3896 MOZ_ASSERT(HasSSE2()); 3897 MOZ_ASSERT(src0.size() == 16); 3898 MOZ_ASSERT(src1.size() == 16); 3899 MOZ_ASSERT(dest.size() == 16); 3900 masm.vpunpcklqdq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3901 } 3902 void vpunpcklqdq(const Operand& src1, FloatRegister src0, 3903 FloatRegister dest) { 3904 MOZ_ASSERT(HasSSE2()); 3905 MOZ_ASSERT(src0.size() == 16); 3906 MOZ_ASSERT(dest.size() == 16); 3907 switch (src1.kind()) { 3908 case Operand::MEM_REG_DISP: 3909 masm.vpunpcklqdq_mr(src1.disp(), src1.base(), src0.encoding(), 3910 dest.encoding()); 3911 break; 3912 case Operand::MEM_ADDRESS32: 3913 masm.vpunpcklqdq_mr(src1.address(), src0.encoding(), dest.encoding()); 3914 break; 3915 default: 3916 MOZ_CRASH("unexpected operand kind"); 3917 } 3918 } 3919 void vpunpckhdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3920 MOZ_ASSERT(HasSSE2()); 3921 MOZ_ASSERT(src0.size() == 16); 3922 MOZ_ASSERT(src1.size() == 16); 3923 MOZ_ASSERT(dest.size() == 16); 3924 masm.vpunpckhdq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3925 } 3926 void vpunpckhqdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3927 MOZ_ASSERT(HasSSE2()); 3928 MOZ_ASSERT(src0.size() == 16); 3929 MOZ_ASSERT(src1.size() == 16); 3930 MOZ_ASSERT(dest.size() == 16); 3931 masm.vpunpckhqdq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3932 } 3933 void vpunpcklwd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3934 MOZ_ASSERT(HasSSE2()); 3935 MOZ_ASSERT(src0.size() == 16); 3936 MOZ_ASSERT(src1.size() == 16); 3937 MOZ_ASSERT(dest.size() == 16); 3938 masm.vpunpcklwd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3939 } 3940 void vpunpckhwd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3941 MOZ_ASSERT(HasSSE2()); 3942 MOZ_ASSERT(src0.size() == 16); 3943 MOZ_ASSERT(src1.size() == 16); 3944 MOZ_ASSERT(dest.size() == 16); 3945 masm.vpunpckhwd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3946 } 3947 3948 void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3949 MOZ_ASSERT(HasSSE2()); 3950 switch (src1.kind()) { 3951 case Operand::FPREG: 3952 masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3953 break; 3954 case Operand::MEM_REG_DISP: 3955 masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(), 3956 dest.encoding()); 3957 break; 3958 case Operand::MEM_ADDRESS32: 3959 masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding()); 3960 break; 3961 default: 3962 MOZ_CRASH("unexpected operand kind"); 3963 } 3964 } 3965 void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3966 // Negates bits of dest and then applies AND 3967 MOZ_ASSERT(HasSSE2()); 3968 switch (src1.kind()) { 3969 case Operand::FPREG: 3970 masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3971 break; 3972 case Operand::MEM_REG_DISP: 3973 masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(), 3974 dest.encoding()); 3975 break; 3976 case Operand::MEM_ADDRESS32: 3977 masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding()); 3978 break; 3979 default: 3980 MOZ_CRASH("unexpected operand kind"); 3981 } 3982 } 3983 void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3984 MOZ_ASSERT(HasSSE2()); 3985 switch (src1.kind()) { 3986 case Operand::FPREG: 3987 masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3988 break; 3989 case Operand::MEM_REG_DISP: 3990 masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(), 3991 dest.encoding()); 3992 break; 3993 case Operand::MEM_ADDRESS32: 3994 masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding()); 3995 break; 3996 default: 3997 MOZ_CRASH("unexpected operand kind"); 3998 } 3999 } 4000 void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4001 MOZ_ASSERT(HasSSE2()); 4002 switch (src1.kind()) { 4003 case Operand::FPREG: 4004 masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4005 break; 4006 case Operand::MEM_REG_DISP: 4007 masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(), 4008 dest.encoding()); 4009 break; 4010 case Operand::MEM_ADDRESS32: 4011 masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding()); 4012 break; 4013 default: 4014 MOZ_CRASH("unexpected operand kind"); 4015 } 4016 } 4017 void vandpd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4018 MOZ_ASSERT(HasSSE2()); 4019 switch (src1.kind()) { 4020 case Operand::FPREG: 4021 masm.vandpd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4022 break; 4023 default: 4024 MOZ_CRASH("unexpected operand kind"); 4025 } 4026 } 4027 4028 void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4029 MOZ_ASSERT(HasSSE2()); 4030 masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4031 } 4032 void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4033 MOZ_ASSERT(HasSSE2()); 4034 switch (src1.kind()) { 4035 case Operand::FPREG: 4036 masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4037 break; 4038 case Operand::MEM_REG_DISP: 4039 masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(), 4040 dest.encoding()); 4041 break; 4042 case Operand::MEM_ADDRESS32: 4043 masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding()); 4044 break; 4045 default: 4046 MOZ_CRASH("unexpected operand kind"); 4047 } 4048 } 4049 void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4050 MOZ_ASSERT(HasSSE2()); 4051 masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4052 } 4053 void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4054 MOZ_ASSERT(HasSSE2()); 4055 switch (src1.kind()) { 4056 case Operand::FPREG: 4057 masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4058 break; 4059 case Operand::MEM_REG_DISP: 4060 masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(), 4061 dest.encoding()); 4062 break; 4063 case Operand::MEM_ADDRESS32: 4064 masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding()); 4065 break; 4066 default: 4067 MOZ_CRASH("unexpected operand kind"); 4068 } 4069 } 4070 void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4071 MOZ_ASSERT(HasSSE2()); 4072 masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4073 } 4074 void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4075 MOZ_ASSERT(HasSSE2()); 4076 switch (src1.kind()) { 4077 case Operand::FPREG: 4078 masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4079 break; 4080 case Operand::MEM_REG_DISP: 4081 masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(), 4082 dest.encoding()); 4083 break; 4084 case Operand::MEM_ADDRESS32: 4085 masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding()); 4086 break; 4087 default: 4088 MOZ_CRASH("unexpected operand kind"); 4089 } 4090 } 4091 void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4092 MOZ_ASSERT(HasSSE2()); 4093 masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4094 } 4095 void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4096 MOZ_ASSERT(HasSSE2()); 4097 switch (src1.kind()) { 4098 case Operand::FPREG: 4099 masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4100 break; 4101 case Operand::MEM_REG_DISP: 4102 masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(), 4103 dest.encoding()); 4104 break; 4105 case Operand::MEM_ADDRESS32: 4106 masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding()); 4107 break; 4108 default: 4109 MOZ_CRASH("unexpected operand kind"); 4110 } 4111 } 4112 4113 void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) { 4114 MOZ_ASSERT(HasSSE2()); 4115 masm.vpshufd_irr(mask, src.encoding(), dest.encoding()); 4116 } 4117 void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) { 4118 MOZ_ASSERT(HasSSE2()); 4119 switch (src1.kind()) { 4120 case Operand::FPREG: 4121 masm.vpshufd_irr(mask, src1.fpu(), dest.encoding()); 4122 break; 4123 case Operand::MEM_REG_DISP: 4124 masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding()); 4125 break; 4126 case Operand::MEM_ADDRESS32: 4127 masm.vpshufd_imr(mask, src1.address(), dest.encoding()); 4128 break; 4129 default: 4130 MOZ_CRASH("unexpected operand kind"); 4131 } 4132 } 4133 4134 void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) { 4135 MOZ_ASSERT(HasSSE2()); 4136 masm.vpshuflw_irr(mask, src.encoding(), dest.encoding()); 4137 } 4138 void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) { 4139 MOZ_ASSERT(HasSSE2()); 4140 masm.vpshufhw_irr(mask, src.encoding(), dest.encoding()); 4141 } 4142 void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) { 4143 MOZ_ASSERT(HasSSSE3()); 4144 masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding()); 4145 } 4146 void vmovddup(const Operand& src, FloatRegister dest) { 4147 MOZ_ASSERT(HasSSE3()); 4148 switch (src.kind()) { 4149 case Operand::FPREG: 4150 masm.vmovddup_rr(src.fpu(), dest.encoding()); 4151 break; 4152 case Operand::MEM_REG_DISP: 4153 masm.vmovddup_mr(src.disp(), src.base(), dest.encoding()); 4154 break; 4155 case Operand::MEM_SCALE: 4156 masm.vmovddup_mr(src.disp(), src.base(), src.index(), src.scale(), 4157 dest.encoding()); 4158 break; 4159 default: 4160 MOZ_CRASH("unexpected operand kind"); 4161 } 4162 } 4163 void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4164 MOZ_ASSERT(HasSSE2()); 4165 masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4166 } 4167 void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4168 MOZ_ASSERT(HasSSE2()); 4169 masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4170 } 4171 void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4172 MOZ_ASSERT(HasSSE2()); 4173 masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4174 } 4175 void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4176 MOZ_ASSERT(HasSSE2()); 4177 switch (src1.kind()) { 4178 case Operand::FPREG: 4179 masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4180 break; 4181 case Operand::MEM_REG_DISP: 4182 masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(), 4183 dest.encoding()); 4184 break; 4185 case Operand::MEM_ADDRESS32: 4186 masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding()); 4187 break; 4188 default: 4189 MOZ_CRASH("unexpected operand kind"); 4190 } 4191 } 4192 void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4193 MOZ_ASSERT(HasSSE2()); 4194 masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4195 } 4196 void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4197 MOZ_ASSERT(HasSSE2()); 4198 switch (src1.kind()) { 4199 case Operand::FPREG: 4200 masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4201 break; 4202 case Operand::MEM_REG_DISP: 4203 masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(), 4204 dest.encoding()); 4205 break; 4206 case Operand::MEM_ADDRESS32: 4207 masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding()); 4208 break; 4209 default: 4210 MOZ_CRASH("unexpected operand kind"); 4211 } 4212 } 4213 void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0, 4214 FloatRegister dest) { 4215 MOZ_ASSERT(HasSSE2()); 4216 masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 4217 } 4218 void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0, 4219 FloatRegister dest) { 4220 MOZ_ASSERT(HasSSE2()); 4221 switch (src1.kind()) { 4222 case Operand::FPREG: 4223 masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); 4224 break; 4225 case Operand::MEM_REG_DISP: 4226 masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(), 4227 dest.encoding()); 4228 break; 4229 case Operand::MEM_ADDRESS32: 4230 masm.vshufps_imr(mask, src1.address(), src0.encoding(), 4231 dest.encoding()); 4232 break; 4233 default: 4234 MOZ_CRASH("unexpected operand kind"); 4235 } 4236 } 4237 void vshufpd(uint32_t mask, FloatRegister src1, FloatRegister src0, 4238 FloatRegister dest) { 4239 MOZ_ASSERT(HasSSE2()); 4240 masm.vshufpd_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 4241 } 4242 void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4243 MOZ_ASSERT(HasSSE2()); 4244 masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4245 } 4246 void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4247 MOZ_ASSERT(HasSSE2()); 4248 masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4249 } 4250 void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4251 MOZ_ASSERT(HasSSE2()); 4252 switch (src1.kind()) { 4253 case Operand::FPREG: 4254 masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4255 break; 4256 case Operand::MEM_REG_DISP: 4257 masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(), 4258 dest.encoding()); 4259 break; 4260 case Operand::MEM_ADDRESS32: 4261 masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding()); 4262 break; 4263 default: 4264 MOZ_CRASH("unexpected operand kind"); 4265 } 4266 } 4267 void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4268 MOZ_ASSERT(HasSSE2()); 4269 switch (src1.kind()) { 4270 case Operand::FPREG: 4271 masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4272 break; 4273 case Operand::MEM_REG_DISP: 4274 masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(), 4275 dest.encoding()); 4276 break; 4277 case Operand::MEM_ADDRESS32: 4278 masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding()); 4279 break; 4280 default: 4281 MOZ_CRASH("unexpected operand kind"); 4282 } 4283 } 4284 void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4285 MOZ_ASSERT(HasSSE2()); 4286 masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4287 } 4288 void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4289 MOZ_ASSERT(HasSSE2()); 4290 masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4291 } 4292 void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4293 MOZ_ASSERT(HasSSE2()); 4294 switch (src1.kind()) { 4295 case Operand::FPREG: 4296 masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4297 break; 4298 case Operand::MEM_REG_DISP: 4299 masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(), 4300 dest.encoding()); 4301 break; 4302 default: 4303 MOZ_CRASH("unexpected operand kind"); 4304 } 4305 } 4306 void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4307 MOZ_ASSERT(HasSSE2()); 4308 switch (src1.kind()) { 4309 case Operand::FPREG: 4310 masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4311 break; 4312 case Operand::MEM_REG_DISP: 4313 masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(), 4314 dest.encoding()); 4315 break; 4316 default: 4317 MOZ_CRASH("unexpected operand kind"); 4318 } 4319 } 4320 void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4321 MOZ_ASSERT(HasSSE2()); 4322 masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4323 } 4324 void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4325 MOZ_ASSERT(HasSSE2()); 4326 switch (src1.kind()) { 4327 case Operand::FPREG: 4328 masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4329 break; 4330 case Operand::MEM_REG_DISP: 4331 masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(), 4332 dest.encoding()); 4333 break; 4334 default: 4335 MOZ_CRASH("unexpected operand kind"); 4336 } 4337 } 4338 void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4339 MOZ_ASSERT(HasSSE2()); 4340 switch (src1.kind()) { 4341 case Operand::FPREG: 4342 masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4343 break; 4344 case Operand::MEM_REG_DISP: 4345 masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(), 4346 dest.encoding()); 4347 break; 4348 default: 4349 MOZ_CRASH("unexpected operand kind"); 4350 } 4351 } 4352 void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4353 MOZ_ASSERT(HasSSE2()); 4354 masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4355 } 4356 void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4357 MOZ_ASSERT(HasSSE2()); 4358 masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4359 } 4360 void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4361 MOZ_ASSERT(HasSSE2()); 4362 masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4363 } 4364 void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4365 MOZ_ASSERT(HasSSE2()); 4366 switch (src1.kind()) { 4367 case Operand::FPREG: 4368 masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4369 break; 4370 case Operand::MEM_REG_DISP: 4371 masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(), 4372 dest.encoding()); 4373 break; 4374 default: 4375 MOZ_CRASH("unexpected operand kind"); 4376 } 4377 } 4378 void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4379 MOZ_ASSERT(HasSSE2()); 4380 switch (src1.kind()) { 4381 case Operand::FPREG: 4382 masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4383 break; 4384 case Operand::MEM_REG_DISP: 4385 masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(), 4386 dest.encoding()); 4387 break; 4388 default: 4389 MOZ_CRASH("unexpected operand kind"); 4390 } 4391 } 4392 void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4393 MOZ_ASSERT(HasSSE2()); 4394 masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4395 } 4396 void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4397 MOZ_ASSERT(HasSSE2()); 4398 masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4399 } 4400 void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4401 MOZ_ASSERT(HasSSE2()); 4402 masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4403 } 4404 void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4405 MOZ_ASSERT(HasSSE2()); 4406 masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4407 } 4408 void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4409 MOZ_ASSERT(HasSSE2()); 4410 masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4411 } 4412 void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4413 MOZ_ASSERT(HasSSE2()); 4414 masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4415 } 4416 void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4417 MOZ_ASSERT(HasSSE2()); 4418 masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4419 } 4420 void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4421 MOZ_ASSERT(HasSSE2()); 4422 masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4423 } 4424 void vroundps(SSERoundingMode mode, const Operand& src, FloatRegister dest) { 4425 MOZ_ASSERT(HasSSE41()); 4426 switch (src.kind()) { 4427 case Operand::FPREG: 4428 masm.vroundps_irr((X86Encoding::SSERoundingMode)mode, src.fpu(), 4429 dest.encoding()); 4430 break; 4431 default: 4432 MOZ_CRASH("unexpected operand kind"); 4433 } 4434 } 4435 void vroundpd(SSERoundingMode mode, const Operand& src, FloatRegister dest) { 4436 MOZ_ASSERT(HasSSE41()); 4437 switch (src.kind()) { 4438 case Operand::FPREG: 4439 masm.vroundpd_irr((X86Encoding::SSERoundingMode)mode, src.fpu(), 4440 dest.encoding()); 4441 break; 4442 default: 4443 MOZ_CRASH("unexpected operand kind"); 4444 } 4445 } 4446 4447 static X86Encoding::RoundingMode ToX86RoundingMode(RoundingMode mode) { 4448 switch (mode) { 4449 case RoundingMode::Up: 4450 return X86Encoding::RoundUp; 4451 case RoundingMode::Down: 4452 return X86Encoding::RoundDown; 4453 case RoundingMode::NearestTiesToEven: 4454 return X86Encoding::RoundToNearest; 4455 case RoundingMode::TowardsZero: 4456 return X86Encoding::RoundToZero; 4457 } 4458 MOZ_CRASH("unexpected mode"); 4459 } 4460 void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src, 4461 FloatRegister dest) { 4462 MOZ_ASSERT(HasSSE41()); 4463 masm.vroundsd_irr(mode, src.encoding(), dest.encoding()); 4464 } 4465 void vroundss(X86Encoding::RoundingMode mode, FloatRegister src, 4466 FloatRegister dest) { 4467 MOZ_ASSERT(HasSSE41()); 4468 masm.vroundss_irr(mode, src.encoding(), dest.encoding()); 4469 } 4470 4471 unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane, 4472 unsigned zeroMask = 0) { 4473 // Note that the sourceLane bits are ignored in the case of a source 4474 // memory operand, and the source is the given 32-bits memory location. 4475 MOZ_ASSERT(zeroMask < 16); 4476 unsigned ret = zeroMask; 4477 ret |= destLane << 4; 4478 ret |= sourceLane << 6; 4479 MOZ_ASSERT(ret < 256); 4480 return ret; 4481 } 4482 void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0, 4483 FloatRegister dest) { 4484 MOZ_ASSERT(HasSSE41()); 4485 masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 4486 } 4487 void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0, 4488 FloatRegister dest) { 4489 MOZ_ASSERT(HasSSE41()); 4490 switch (src1.kind()) { 4491 case Operand::FPREG: 4492 masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); 4493 break; 4494 case Operand::MEM_REG_DISP: 4495 masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(), 4496 dest.encoding()); 4497 break; 4498 case Operand::MEM_SCALE: 4499 masm.vinsertps_imr(mask, src1.disp(), src1.base(), src1.index(), 4500 src1.scale(), src0.encoding(), dest.encoding()); 4501 break; 4502 default: 4503 MOZ_CRASH("unexpected operand kind"); 4504 } 4505 } 4506 void vmovlps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4507 switch (src1.kind()) { 4508 case Operand::MEM_REG_DISP: 4509 masm.vmovlps_mr(src1.disp(), src1.base(), src0.encoding(), 4510 dest.encoding()); 4511 break; 4512 case Operand::MEM_SCALE: 4513 masm.vmovlps_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), 4514 src0.encoding(), dest.encoding()); 4515 break; 4516 default: 4517 MOZ_CRASH("unexpected operand kind"); 4518 } 4519 } 4520 void vmovlps(FloatRegister src, const Operand& dest) { 4521 switch (dest.kind()) { 4522 case Operand::MEM_REG_DISP: 4523 masm.vmovlps_rm(src.encoding(), dest.disp(), dest.base()); 4524 break; 4525 case Operand::MEM_SCALE: 4526 masm.vmovlps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 4527 dest.scale()); 4528 break; 4529 default: 4530 MOZ_CRASH("unexpected operand kind"); 4531 } 4532 } 4533 void vmovhps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4534 switch (src1.kind()) { 4535 case Operand::MEM_REG_DISP: 4536 masm.vmovhps_mr(src1.disp(), src1.base(), src0.encoding(), 4537 dest.encoding()); 4538 break; 4539 case Operand::MEM_SCALE: 4540 masm.vmovhps_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), 4541 src0.encoding(), dest.encoding()); 4542 break; 4543 default: 4544 MOZ_CRASH("unexpected operand kind"); 4545 } 4546 } 4547 void vmovhps(FloatRegister src, const Operand& dest) { 4548 switch (dest.kind()) { 4549 case Operand::MEM_REG_DISP: 4550 masm.vmovhps_rm(src.encoding(), dest.disp(), dest.base()); 4551 break; 4552 case Operand::MEM_SCALE: 4553 masm.vmovhps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), 4554 dest.scale()); 4555 break; 4556 default: 4557 MOZ_CRASH("unexpected operand kind"); 4558 } 4559 } 4560 void vextractps(unsigned lane, FloatRegister src, const Operand& dest) { 4561 MOZ_ASSERT(HasSSE41()); 4562 MOZ_ASSERT(lane < 4); 4563 switch (dest.kind()) { 4564 case Operand::MEM_REG_DISP: 4565 masm.vextractps_rm(lane, src.encoding(), dest.disp(), dest.base()); 4566 break; 4567 case Operand::MEM_SCALE: 4568 masm.vextractps_rm(lane, src.encoding(), dest.disp(), dest.base(), 4569 dest.index(), dest.scale()); 4570 break; 4571 default: 4572 MOZ_CRASH("unexpected operand kind"); 4573 } 4574 } 4575 unsigned blendpsMask(bool x, bool y, bool z, bool w) { 4576 return (x << 0) | (y << 1) | (z << 2) | (w << 3); 4577 } 4578 void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0, 4579 FloatRegister dest) { 4580 MOZ_ASSERT(HasSSE41()); 4581 masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 4582 } 4583 void vblendps(unsigned mask, const Operand& src1, FloatRegister src0, 4584 FloatRegister dest) { 4585 MOZ_ASSERT(HasSSE41()); 4586 switch (src1.kind()) { 4587 case Operand::FPREG: 4588 masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); 4589 break; 4590 case Operand::MEM_REG_DISP: 4591 masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(), 4592 dest.encoding()); 4593 break; 4594 default: 4595 MOZ_CRASH("unexpected operand kind"); 4596 } 4597 } 4598 void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0, 4599 FloatRegister dest) { 4600 MOZ_ASSERT(HasSSE41()); 4601 masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(), 4602 dest.encoding()); 4603 } 4604 void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0, 4605 FloatRegister dest) { 4606 MOZ_ASSERT(HasSSE41()); 4607 switch (src1.kind()) { 4608 case Operand::FPREG: 4609 masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(), 4610 dest.encoding()); 4611 break; 4612 case Operand::MEM_REG_DISP: 4613 masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(), 4614 src0.encoding(), dest.encoding()); 4615 break; 4616 default: 4617 MOZ_CRASH("unexpected operand kind"); 4618 } 4619 } 4620 void vblendvpd(FloatRegister mask, FloatRegister src1, FloatRegister src0, 4621 FloatRegister dest) { 4622 MOZ_ASSERT(HasSSE41()); 4623 masm.vblendvpd_rr(mask.encoding(), src1.encoding(), src0.encoding(), 4624 dest.encoding()); 4625 } 4626 void vmovsldup(FloatRegister src, FloatRegister dest) { 4627 MOZ_ASSERT(HasSSE3()); 4628 masm.vmovsldup_rr(src.encoding(), dest.encoding()); 4629 } 4630 void vmovsldup(const Operand& src, FloatRegister dest) { 4631 MOZ_ASSERT(HasSSE3()); 4632 switch (src.kind()) { 4633 case Operand::FPREG: 4634 masm.vmovsldup_rr(src.fpu(), dest.encoding()); 4635 break; 4636 case Operand::MEM_REG_DISP: 4637 masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding()); 4638 break; 4639 default: 4640 MOZ_CRASH("unexpected operand kind"); 4641 } 4642 } 4643 void vmovshdup(FloatRegister src, FloatRegister dest) { 4644 MOZ_ASSERT(HasSSE3()); 4645 masm.vmovshdup_rr(src.encoding(), dest.encoding()); 4646 } 4647 void vmovshdup(const Operand& src, FloatRegister dest) { 4648 MOZ_ASSERT(HasSSE3()); 4649 switch (src.kind()) { 4650 case Operand::FPREG: 4651 masm.vmovshdup_rr(src.fpu(), dest.encoding()); 4652 break; 4653 case Operand::MEM_REG_DISP: 4654 masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding()); 4655 break; 4656 default: 4657 MOZ_CRASH("unexpected operand kind"); 4658 } 4659 } 4660 void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4661 MOZ_ASSERT(HasSSE2()); 4662 masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4663 } 4664 void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4665 MOZ_ASSERT(HasSSE2()); 4666 switch (src1.kind()) { 4667 case Operand::FPREG: 4668 masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4669 break; 4670 case Operand::MEM_REG_DISP: 4671 masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(), 4672 dest.encoding()); 4673 break; 4674 default: 4675 MOZ_CRASH("unexpected operand kind"); 4676 } 4677 } 4678 void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4679 MOZ_ASSERT(HasSSE2()); 4680 masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4681 } 4682 void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4683 MOZ_ASSERT(HasSSE2()); 4684 masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4685 } 4686 void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4687 MOZ_ASSERT(HasSSE2()); 4688 switch (src1.kind()) { 4689 case Operand::FPREG: 4690 masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4691 break; 4692 case Operand::MEM_REG_DISP: 4693 masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(), 4694 dest.encoding()); 4695 break; 4696 default: 4697 MOZ_CRASH("unexpected operand kind"); 4698 } 4699 } 4700 void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4701 MOZ_ASSERT(HasSSE2()); 4702 masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 4703 } 4704 void fisttp(const Operand& dest) { 4705 MOZ_ASSERT(HasSSE3()); 4706 switch (dest.kind()) { 4707 case Operand::MEM_REG_DISP: 4708 masm.fisttp_m(dest.disp(), dest.base()); 4709 break; 4710 default: 4711 MOZ_CRASH("unexpected operand kind"); 4712 } 4713 } 4714 void fistp(const Operand& dest) { 4715 switch (dest.kind()) { 4716 case Operand::MEM_REG_DISP: 4717 masm.fistp_m(dest.disp(), dest.base()); 4718 break; 4719 default: 4720 MOZ_CRASH("unexpected operand kind"); 4721 } 4722 } 4723 void fnstcw(const Operand& dest) { 4724 switch (dest.kind()) { 4725 case Operand::MEM_REG_DISP: 4726 masm.fnstcw_m(dest.disp(), dest.base()); 4727 break; 4728 default: 4729 MOZ_CRASH("unexpected operand kind"); 4730 } 4731 } 4732 void fldcw(const Operand& dest) { 4733 switch (dest.kind()) { 4734 case Operand::MEM_REG_DISP: 4735 masm.fldcw_m(dest.disp(), dest.base()); 4736 break; 4737 default: 4738 MOZ_CRASH("unexpected operand kind"); 4739 } 4740 } 4741 void fnstsw(const Operand& dest) { 4742 switch (dest.kind()) { 4743 case Operand::MEM_REG_DISP: 4744 masm.fnstsw_m(dest.disp(), dest.base()); 4745 break; 4746 default: 4747 MOZ_CRASH("unexpected operand kind"); 4748 } 4749 } 4750 void fld(const Operand& dest) { 4751 switch (dest.kind()) { 4752 case Operand::MEM_REG_DISP: 4753 masm.fld_m(dest.disp(), dest.base()); 4754 break; 4755 default: 4756 MOZ_CRASH("unexpected operand kind"); 4757 } 4758 } 4759 void fld32(const Operand& dest) { 4760 switch (dest.kind()) { 4761 case Operand::MEM_REG_DISP: 4762 masm.fld32_m(dest.disp(), dest.base()); 4763 break; 4764 default: 4765 MOZ_CRASH("unexpected operand kind"); 4766 } 4767 } 4768 void fstp(const Operand& src) { 4769 switch (src.kind()) { 4770 case Operand::MEM_REG_DISP: 4771 masm.fstp_m(src.disp(), src.base()); 4772 break; 4773 default: 4774 MOZ_CRASH("unexpected operand kind"); 4775 } 4776 } 4777 void fstp32(const Operand& src) { 4778 switch (src.kind()) { 4779 case Operand::MEM_REG_DISP: 4780 masm.fstp32_m(src.disp(), src.base()); 4781 break; 4782 default: 4783 MOZ_CRASH("unexpected operand kind"); 4784 } 4785 } 4786 4787 void vbroadcastb(const Operand& src, FloatRegister dest) { 4788 MOZ_ASSERT(HasAVX2()); 4789 switch (src.kind()) { 4790 case Operand::FPREG: 4791 masm.vbroadcastb_rr(src.fpu(), dest.encoding()); 4792 break; 4793 case Operand::MEM_REG_DISP: 4794 masm.vbroadcastb_mr(src.disp(), src.base(), dest.encoding()); 4795 break; 4796 case Operand::MEM_SCALE: 4797 masm.vbroadcastb_mr(src.disp(), src.base(), src.index(), src.scale(), 4798 dest.encoding()); 4799 break; 4800 default: 4801 MOZ_CRASH("unexpected operand kind"); 4802 } 4803 } 4804 void vbroadcastw(const Operand& src, FloatRegister dest) { 4805 MOZ_ASSERT(HasAVX2()); 4806 switch (src.kind()) { 4807 case Operand::FPREG: 4808 masm.vbroadcastw_rr(src.fpu(), dest.encoding()); 4809 break; 4810 case Operand::MEM_REG_DISP: 4811 masm.vbroadcastw_mr(src.disp(), src.base(), dest.encoding()); 4812 break; 4813 case Operand::MEM_SCALE: 4814 masm.vbroadcastw_mr(src.disp(), src.base(), src.index(), src.scale(), 4815 dest.encoding()); 4816 break; 4817 default: 4818 MOZ_CRASH("unexpected operand kind"); 4819 } 4820 } 4821 void vbroadcastd(const Operand& src, FloatRegister dest) { 4822 MOZ_ASSERT(HasAVX2()); 4823 switch (src.kind()) { 4824 case Operand::FPREG: 4825 masm.vbroadcastd_rr(src.fpu(), dest.encoding()); 4826 break; 4827 case Operand::MEM_REG_DISP: 4828 masm.vbroadcastd_mr(src.disp(), src.base(), dest.encoding()); 4829 break; 4830 case Operand::MEM_SCALE: 4831 masm.vbroadcastd_mr(src.disp(), src.base(), src.index(), src.scale(), 4832 dest.encoding()); 4833 break; 4834 default: 4835 MOZ_CRASH("unexpected operand kind"); 4836 } 4837 } 4838 void vbroadcastq(const Operand& src, FloatRegister dest) { 4839 MOZ_ASSERT(HasAVX2()); 4840 switch (src.kind()) { 4841 case Operand::FPREG: 4842 masm.vbroadcastq_rr(src.fpu(), dest.encoding()); 4843 break; 4844 case Operand::MEM_REG_DISP: 4845 masm.vbroadcastq_mr(src.disp(), src.base(), dest.encoding()); 4846 break; 4847 case Operand::MEM_SCALE: 4848 masm.vbroadcastq_mr(src.disp(), src.base(), src.index(), src.scale(), 4849 dest.encoding()); 4850 break; 4851 default: 4852 MOZ_CRASH("unexpected operand kind"); 4853 } 4854 } 4855 void vbroadcastss(const Operand& src, FloatRegister dest) { 4856 MOZ_ASSERT(HasAVX2()); 4857 switch (src.kind()) { 4858 case Operand::FPREG: 4859 masm.vbroadcastss_rr(src.fpu(), dest.encoding()); 4860 break; 4861 case Operand::MEM_REG_DISP: 4862 masm.vbroadcastss_mr(src.disp(), src.base(), dest.encoding()); 4863 break; 4864 case Operand::MEM_SCALE: 4865 masm.vbroadcastss_mr(src.disp(), src.base(), src.index(), src.scale(), 4866 dest.encoding()); 4867 break; 4868 default: 4869 MOZ_CRASH("unexpected operand kind"); 4870 } 4871 } 4872 4873 void vpsignd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 4874 MOZ_ASSERT(HasSSSE3()); 4875 switch (src1.kind()) { 4876 case Operand::FPREG: 4877 masm.vpsignd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 4878 break; 4879 default: 4880 MOZ_CRASH("unexpected operand kind"); 4881 } 4882 } 4883 4884 void vfmadd231ps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4885 MOZ_ASSERT(HasFMA()); 4886 masm.vfmadd231ps_rrr(src1.encoding(), src0.encoding(), dest.encoding()); 4887 } 4888 void vfnmadd231ps(FloatRegister src1, FloatRegister src0, 4889 FloatRegister dest) { 4890 MOZ_ASSERT(HasFMA()); 4891 masm.vfnmadd231ps_rrr(src1.encoding(), src0.encoding(), dest.encoding()); 4892 } 4893 void vfmadd231pd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 4894 MOZ_ASSERT(HasFMA()); 4895 masm.vfmadd231pd_rrr(src1.encoding(), src0.encoding(), dest.encoding()); 4896 } 4897 void vfnmadd231pd(FloatRegister src1, FloatRegister src0, 4898 FloatRegister dest) { 4899 MOZ_ASSERT(HasFMA()); 4900 masm.vfnmadd231pd_rrr(src1.encoding(), src0.encoding(), dest.encoding()); 4901 } 4902 4903 void flushBuffer() {} 4904 4905 // Patching. 4906 4907 static size_t PatchWrite_NearCallSize() { return 5; } 4908 static uintptr_t GetPointer(uint8_t* instPtr) { 4909 uint8_t* ptr = instPtr - sizeof(uintptr_t); 4910 return mozilla::LittleEndian::readUintptr(ptr); 4911 } 4912 // Write a relative call at the start location |dataLabel|. 4913 // Note that this DOES NOT patch data that comes before |label|. 4914 static void PatchWrite_NearCall(CodeLocationLabel startLabel, 4915 CodeLocationLabel target) { 4916 uint8_t* start = startLabel.raw(); 4917 *start = 0xE8; // <CALL> rel32 4918 ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize(); 4919 MOZ_ASSERT(int32_t(offset) == offset); 4920 mozilla::LittleEndian::writeInt32(start + 1, offset); // CALL <rel32> 4921 } 4922 4923 static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) { 4924 // dataLabel is a code location which targets the end of an instruction 4925 // which has a 32 bits immediate. Thus writting a value requires shifting 4926 // back to the address of the 32 bits immediate within the instruction. 4927 uint8_t* ptr = dataLabel.raw(); 4928 mozilla::LittleEndian::writeInt32(ptr - sizeof(int32_t), toWrite.value); 4929 } 4930 4931 static void PatchDataWithValueCheck(CodeLocationLabel data, 4932 PatchedImmPtr newData, 4933 PatchedImmPtr expectedData) { 4934 // The pointer given is a pointer to *after* the data. 4935 uint8_t* ptr = data.raw() - sizeof(uintptr_t); 4936 MOZ_ASSERT(mozilla::LittleEndian::readUintptr(ptr) == 4937 uintptr_t(expectedData.value)); 4938 mozilla::LittleEndian::writeUintptr(ptr, uintptr_t(newData.value)); 4939 } 4940 static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData, 4941 ImmPtr expectedData) { 4942 PatchDataWithValueCheck(data, PatchedImmPtr(newData.value), 4943 PatchedImmPtr(expectedData.value)); 4944 } 4945 4946 static uint32_t NopSize() { return 1; } 4947 static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) { 4948 MOZ_CRASH("nextInstruction NYI on x86"); 4949 } 4950 4951 // Toggle a jmp or cmp emitted by toggledJump(). 4952 static void ToggleToJmp(CodeLocationLabel inst) { 4953 uint8_t* ptr = (uint8_t*)inst.raw(); 4954 MOZ_ASSERT(*ptr == 0x3D); // <CMP> eax, imm32 4955 *ptr = 0xE9; // <JMP> rel32 4956 } 4957 static void ToggleToCmp(CodeLocationLabel inst) { 4958 uint8_t* ptr = (uint8_t*)inst.raw(); 4959 MOZ_ASSERT(*ptr == 0xE9); // <JMP> rel32 4960 *ptr = 0x3D; // <CMP> eax, imm32 4961 } 4962 static void ToggleCall(CodeLocationLabel inst, bool enabled) { 4963 uint8_t* ptr = (uint8_t*)inst.raw(); 4964 MOZ_ASSERT(*ptr == 0x3D || // <CMP> eax, imm32 4965 *ptr == 0xE8); // <CALL> rel32 4966 *ptr = enabled ? 0xE8 : 0x3D; 4967 } 4968 4969 MOZ_COLD void verifyHeapAccessDisassembly( 4970 uint32_t begin, uint32_t end, const Disassembler::HeapAccess& heapAccess); 4971 }; 4972 4973 } // namespace jit 4974 } // namespace js 4975 4976 #endif /* jit_x86_shared_Assembler_x86_shared_h */