[ tor-browser ].git.dasho

MacroAssembler-x64.h (45590B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #ifndef jit_x64_MacroAssembler_x64_h
      8 #define jit_x64_MacroAssembler_x64_h
      9 
     10 #include "jit/x86-shared/MacroAssembler-x86-shared.h"
     11 #include "js/HeapAPI.h"
     12 #include "wasm/WasmBuiltins.h"
     13 
     14 namespace js {
     15 namespace jit {
     16 
     17 struct ImmShiftedTag : public ImmWord {
     18  explicit ImmShiftedTag(JSValueType type)
     19      : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {}
     20 };
     21 
     22 struct ImmTag : public Imm32 {
     23  explicit ImmTag(JSValueTag tag) : Imm32(tag) {}
     24 };
     25 
     26 // ScratchTagScope and ScratchTagScopeRelease are used to manage the tag
     27 // register for splitTagForTest(), which has different register management on
     28 // different platforms.  On 64-bit platforms it requires a scratch register that
     29 // does not interfere with other operations; on 32-bit platforms it uses a
     30 // register that is already part of the Value.
     31 //
     32 // The ScratchTagScope RAII type acquires the appropriate register; a reference
     33 // to a variable of this type is then passed to splitTagForTest().
     34 //
     35 // On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register
     36 // available in a dynamic scope during compilation.  However it is important to
     37 // remember that that does not preserve the register value in any way, so this
     38 // RAII type should only be used along paths that eventually branch past further
     39 // uses of the extracted tag value.
     40 //
     41 // On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not
     42 // manage a register, it only aliases a register in the ValueOperand.
     43 
     44 class ScratchTagScope : public ScratchRegisterScope {
     45 public:
     46  ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
     47      : ScratchRegisterScope(masm) {}
     48 };
     49 
     50 class ScratchTagScopeRelease {
     51  ScratchTagScope* ts_;
     52 
     53 public:
     54  explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
     55    ts_->release();
     56  }
     57  ~ScratchTagScopeRelease() { ts_->reacquire(); }
     58 };
     59 
     60 class MacroAssemblerX64 : public MacroAssemblerX86Shared {
     61 private:
     62  // Perform a downcast. Should be removed by Bug 996602.
     63  MacroAssembler& asMasm();
     64  const MacroAssembler& asMasm() const;
     65 
     66  void bindOffsets(const MacroAssemblerX86Shared::UsesVector&);
     67 
     68  void vpRiprOpSimd128(const SimdConstant& v, FloatRegister reg,
     69                       JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
     70                           X86Encoding::XMMRegisterID id));
     71 
     72  void vpRiprOpSimd128(const SimdConstant& v, FloatRegister lhs,
     73                       FloatRegister dest,
     74                       JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
     75                           X86Encoding::XMMRegisterID srcId,
     76                           X86Encoding::XMMRegisterID destId));
     77 
     78 protected:
     79  void flexibleDivMod64(Register lhs, Register rhs, Register output,
     80                        bool isUnsigned, bool isDiv);
     81 
     82 public:
     83  using MacroAssemblerX86Shared::load32;
     84  using MacroAssemblerX86Shared::store16;
     85  using MacroAssemblerX86Shared::store32;
     86 
     87  MacroAssemblerX64() = default;
     88 
     89  // The buffer is about to be linked, make sure any constant pools or excess
     90  // bookkeeping has been flushed to the instruction stream.
     91  void finish();
     92 
     93  /////////////////////////////////////////////////////////////////
     94  // X64 helpers.
     95  /////////////////////////////////////////////////////////////////
     96  void writeDataRelocation(const Value& val) {
     97    MOZ_ASSERT(val.isGCThing(), "only called for gc-things");
     98 
     99    // Raw GC pointer relocations and Value relocations both end up in
    100    // Assembler::TraceDataRelocations.
    101    gc::Cell* cell = val.toGCThing();
    102    if (cell && gc::IsInsideNursery(cell)) {
    103      embedsNurseryPointers_ = true;
    104    }
    105    dataRelocations_.writeUnsigned(masm.currentOffset());
    106  }
    107 
    108  // Refers to the upper 32 bits of a 64-bit Value operand.
    109  // On x86_64, the upper 32 bits do not necessarily only contain the type.
    110  Operand ToUpper32(Operand base) {
    111    switch (base.kind()) {
    112      case Operand::MEM_REG_DISP:
    113        return Operand(Register::FromCode(base.base()), base.disp() + 4);
    114 
    115      case Operand::MEM_SCALE:
    116        return Operand(Register::FromCode(base.base()),
    117                       Register::FromCode(base.index()), base.scale(),
    118                       base.disp() + 4);
    119 
    120      default:
    121        MOZ_CRASH("unexpected operand kind");
    122    }
    123  }
    124  static inline Operand ToUpper32(const Address& address) {
    125    return Operand(address.base, address.offset + 4);
    126  }
    127  static inline Operand ToUpper32(const BaseIndex& address) {
    128    return Operand(address.base, address.index, address.scale,
    129                   address.offset + 4);
    130  }
    131 
    132  uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); }
    133 
    134  JSValueShiftedTag GetShiftedTag(JSValueType type) {
    135    return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
    136  }
    137 
    138  /////////////////////////////////////////////////////////////////
    139  // X86/X64-common interface.
    140  /////////////////////////////////////////////////////////////////
    141 
    142  void storeValue(ValueOperand val, Operand dest) {
    143    movq(val.valueReg(), dest);
    144  }
    145  void storeValue(ValueOperand val, const Address& dest) {
    146    storeValue(val, Operand(dest));
    147  }
    148  template <typename T>
    149  void storeValue(JSValueType type, Register reg, const T& dest) {
    150    ScratchRegisterScope scratch(asMasm());
    151    boxValue(type, reg, scratch);
    152    movq(scratch, Operand(dest));
    153  }
    154  template <typename T>
    155  void storeValue(const Value& val, const T& dest) {
    156    if (val.isGCThing()) {
    157      ScratchRegisterScope scratch(asMasm());
    158      movWithPatch(ImmWord(val.asRawBits()), scratch);
    159      writeDataRelocation(val);
    160      movq(scratch, Operand(dest));
    161    } else {
    162      storePtr(ImmWord(val.asRawBits()), dest);
    163    }
    164  }
    165  void storeValue(ValueOperand val, BaseIndex dest) {
    166    storeValue(val, Operand(dest));
    167  }
    168  void storeValue(const Address& src, const Address& dest, Register temp) {
    169    loadPtr(src, temp);
    170    storePtr(temp, dest);
    171  }
    172  void storePrivateValue(Register src, const Address& dest) {
    173    storePtr(src, dest);
    174  }
    175  void storePrivateValue(ImmGCPtr imm, const Address& dest) {
    176    storePtr(imm, dest);
    177  }
    178  void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); }
    179  void loadValue(Address src, ValueOperand val) {
    180    loadValue(Operand(src), val);
    181  }
    182  void loadValue(const BaseIndex& src, ValueOperand val) {
    183    loadValue(Operand(src), val);
    184  }
    185  void loadUnalignedValue(const Address& src, ValueOperand dest) {
    186    loadValue(src, dest);
    187  }
    188  void tagValue(JSValueType type, Register payload, ValueOperand dest);
    189  void pushValue(ValueOperand val) { push(val.valueReg()); }
    190  void popValue(ValueOperand val) { pop(val.valueReg()); }
    191  void pushValue(const Value& val) {
    192    if (val.isGCThing()) {
    193      ScratchRegisterScope scratch(asMasm());
    194      movWithPatch(ImmWord(val.asRawBits()), scratch);
    195      writeDataRelocation(val);
    196      push(scratch);
    197    } else {
    198      push(ImmWord(val.asRawBits()));
    199    }
    200  }
    201  void pushValue(JSValueType type, Register reg) {
    202    ScratchRegisterScope scratch(asMasm());
    203    boxValue(type, reg, scratch);
    204    push(scratch);
    205  }
    206  void pushValue(const Address& addr) { push(Operand(addr)); }
    207 
    208  void pushValue(const BaseIndex& addr, Register scratch) {
    209    push(Operand(addr));
    210  }
    211 
    212  void boxValue(JSValueType type, Register src, Register dest);
    213  void boxValue(Register type, Register src, Register dest);
    214 
    215  Condition testUndefined(Condition cond, Register tag) {
    216    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    217    cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED));
    218    return cond;
    219  }
    220  Condition testInt32(Condition cond, Register tag) {
    221    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    222    cmp32(tag, ImmTag(JSVAL_TAG_INT32));
    223    return cond;
    224  }
    225  Condition testBoolean(Condition cond, Register tag) {
    226    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    227    cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN));
    228    return cond;
    229  }
    230  Condition testNull(Condition cond, Register tag) {
    231    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    232    cmp32(tag, ImmTag(JSVAL_TAG_NULL));
    233    return cond;
    234  }
    235  Condition testString(Condition cond, Register tag) {
    236    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    237    cmp32(tag, ImmTag(JSVAL_TAG_STRING));
    238    return cond;
    239  }
    240  Condition testSymbol(Condition cond, Register tag) {
    241    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    242    cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL));
    243    return cond;
    244  }
    245  Condition testBigInt(Condition cond, Register tag) {
    246    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    247    cmp32(tag, ImmTag(JSVAL_TAG_BIGINT));
    248    return cond;
    249  }
    250  Condition testObject(Condition cond, Register tag) {
    251    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    252    cmp32(tag, ImmTag(JSVAL_TAG_OBJECT));
    253    return cond;
    254  }
    255  Condition testDouble(Condition cond, Register tag) {
    256    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    257    cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE));
    258    return cond == Equal ? BelowOrEqual : Above;
    259  }
    260  Condition testNumber(Condition cond, Register tag) {
    261    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    262    cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag));
    263    return cond == Equal ? BelowOrEqual : Above;
    264  }
    265  Condition testGCThing(Condition cond, Register tag) {
    266    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    267    cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag));
    268    return cond == Equal ? AboveOrEqual : Below;
    269  }
    270 
    271  Condition testMagic(Condition cond, Register tag) {
    272    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    273    cmp32(tag, ImmTag(JSVAL_TAG_MAGIC));
    274    return cond;
    275  }
    276  Condition testError(Condition cond, Register tag) {
    277    return testMagic(cond, tag);
    278  }
    279  Condition testPrimitive(Condition cond, Register tag) {
    280    MOZ_ASSERT(cond == Equal || cond == NotEqual);
    281    cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
    282    return cond == Equal ? Below : AboveOrEqual;
    283  }
    284 
    285  Condition testUndefined(Condition cond, const ValueOperand& src) {
    286    ScratchRegisterScope scratch(asMasm());
    287    splitTag(src, scratch);
    288    return testUndefined(cond, scratch);
    289  }
    290  Condition testInt32(Condition cond, const ValueOperand& src) {
    291    ScratchRegisterScope scratch(asMasm());
    292    splitTag(src, scratch);
    293    return testInt32(cond, scratch);
    294  }
    295  Condition testBoolean(Condition cond, const ValueOperand& src) {
    296    ScratchRegisterScope scratch(asMasm());
    297    splitTag(src, scratch);
    298    return testBoolean(cond, scratch);
    299  }
    300  Condition testDouble(Condition cond, const ValueOperand& src) {
    301    ScratchRegisterScope scratch(asMasm());
    302    splitTag(src, scratch);
    303    return testDouble(cond, scratch);
    304  }
    305  Condition testNumber(Condition cond, const ValueOperand& src) {
    306    ScratchRegisterScope scratch(asMasm());
    307    splitTag(src, scratch);
    308    return testNumber(cond, scratch);
    309  }
    310  Condition testNull(Condition cond, const ValueOperand& src) {
    311    ScratchRegisterScope scratch(asMasm());
    312    splitTag(src, scratch);
    313    return testNull(cond, scratch);
    314  }
    315  Condition testString(Condition cond, const ValueOperand& src) {
    316    ScratchRegisterScope scratch(asMasm());
    317    splitTag(src, scratch);
    318    return testString(cond, scratch);
    319  }
    320  Condition testSymbol(Condition cond, const ValueOperand& src) {
    321    ScratchRegisterScope scratch(asMasm());
    322    splitTag(src, scratch);
    323    return testSymbol(cond, scratch);
    324  }
    325  Condition testBigInt(Condition cond, const ValueOperand& src) {
    326    ScratchRegisterScope scratch(asMasm());
    327    splitTag(src, scratch);
    328    return testBigInt(cond, scratch);
    329  }
    330  Condition testObject(Condition cond, const ValueOperand& src) {
    331    ScratchRegisterScope scratch(asMasm());
    332    splitTag(src, scratch);
    333    return testObject(cond, scratch);
    334  }
    335  Condition testGCThing(Condition cond, const ValueOperand& src) {
    336    ScratchRegisterScope scratch(asMasm());
    337    splitTag(src, scratch);
    338    return testGCThing(cond, scratch);
    339  }
    340  Condition testPrimitive(Condition cond, const ValueOperand& src) {
    341    ScratchRegisterScope scratch(asMasm());
    342    splitTag(src, scratch);
    343    return testPrimitive(cond, scratch);
    344  }
    345 
    346  Condition testUndefined(Condition cond, const Address& src) {
    347    cmp32(ToUpper32(src),
    348          Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED))));
    349    return cond;
    350  }
    351  Condition testInt32(Condition cond, const Address& src) {
    352    cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32))));
    353    return cond;
    354  }
    355  Condition testBoolean(Condition cond, const Address& src) {
    356    cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN))));
    357    return cond;
    358  }
    359  Condition testDouble(Condition cond, const Address& src) {
    360    ScratchRegisterScope scratch(asMasm());
    361    splitTag(src, scratch);
    362    return testDouble(cond, scratch);
    363  }
    364  Condition testNumber(Condition cond, const Address& src) {
    365    ScratchRegisterScope scratch(asMasm());
    366    splitTag(src, scratch);
    367    return testNumber(cond, scratch);
    368  }
    369  Condition testNull(Condition cond, const Address& src) {
    370    cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL))));
    371    return cond;
    372  }
    373  Condition testString(Condition cond, const Address& src) {
    374    ScratchRegisterScope scratch(asMasm());
    375    splitTag(src, scratch);
    376    return testString(cond, scratch);
    377  }
    378  Condition testSymbol(Condition cond, const Address& src) {
    379    ScratchRegisterScope scratch(asMasm());
    380    splitTag(src, scratch);
    381    return testSymbol(cond, scratch);
    382  }
    383  Condition testBigInt(Condition cond, const Address& src) {
    384    ScratchRegisterScope scratch(asMasm());
    385    splitTag(src, scratch);
    386    return testBigInt(cond, scratch);
    387  }
    388  Condition testObject(Condition cond, const Address& src) {
    389    ScratchRegisterScope scratch(asMasm());
    390    splitTag(src, scratch);
    391    return testObject(cond, scratch);
    392  }
    393  Condition testPrimitive(Condition cond, const Address& src) {
    394    ScratchRegisterScope scratch(asMasm());
    395    splitTag(src, scratch);
    396    return testPrimitive(cond, scratch);
    397  }
    398  Condition testGCThing(Condition cond, const Address& src) {
    399    ScratchRegisterScope scratch(asMasm());
    400    splitTag(src, scratch);
    401    return testGCThing(cond, scratch);
    402  }
    403  Condition testMagic(Condition cond, const Address& src) {
    404    ScratchRegisterScope scratch(asMasm());
    405    splitTag(src, scratch);
    406    return testMagic(cond, scratch);
    407  }
    408 
    409  Condition testUndefined(Condition cond, const BaseIndex& src) {
    410    ScratchRegisterScope scratch(asMasm());
    411    splitTag(src, scratch);
    412    return testUndefined(cond, scratch);
    413  }
    414  Condition testNull(Condition cond, const BaseIndex& src) {
    415    ScratchRegisterScope scratch(asMasm());
    416    splitTag(src, scratch);
    417    return testNull(cond, scratch);
    418  }
    419  Condition testBoolean(Condition cond, const BaseIndex& src) {
    420    ScratchRegisterScope scratch(asMasm());
    421    splitTag(src, scratch);
    422    return testBoolean(cond, scratch);
    423  }
    424  Condition testString(Condition cond, const BaseIndex& src) {
    425    ScratchRegisterScope scratch(asMasm());
    426    splitTag(src, scratch);
    427    return testString(cond, scratch);
    428  }
    429  Condition testSymbol(Condition cond, const BaseIndex& src) {
    430    ScratchRegisterScope scratch(asMasm());
    431    splitTag(src, scratch);
    432    return testSymbol(cond, scratch);
    433  }
    434  Condition testBigInt(Condition cond, const BaseIndex& src) {
    435    ScratchRegisterScope scratch(asMasm());
    436    splitTag(src, scratch);
    437    return testBigInt(cond, scratch);
    438  }
    439  Condition testInt32(Condition cond, const BaseIndex& src) {
    440    ScratchRegisterScope scratch(asMasm());
    441    splitTag(src, scratch);
    442    return testInt32(cond, scratch);
    443  }
    444  Condition testObject(Condition cond, const BaseIndex& src) {
    445    ScratchRegisterScope scratch(asMasm());
    446    splitTag(src, scratch);
    447    return testObject(cond, scratch);
    448  }
    449  Condition testDouble(Condition cond, const BaseIndex& src) {
    450    ScratchRegisterScope scratch(asMasm());
    451    splitTag(src, scratch);
    452    return testDouble(cond, scratch);
    453  }
    454  Condition testMagic(Condition cond, const BaseIndex& src) {
    455    ScratchRegisterScope scratch(asMasm());
    456    splitTag(src, scratch);
    457    return testMagic(cond, scratch);
    458  }
    459  Condition testGCThing(Condition cond, const BaseIndex& src) {
    460    ScratchRegisterScope scratch(asMasm());
    461    splitTag(src, scratch);
    462    return testGCThing(cond, scratch);
    463  }
    464 
    465  Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) {
    466    uint64_t magic = MagicValue(why).asRawBits();
    467    cmpPtr(src.valueReg(), ImmWord(magic));
    468    return cond;
    469  }
    470 
    471  void cmpPtr(Register lhs, const ImmWord rhs) {
    472    ScratchRegisterScope scratch(asMasm());
    473    MOZ_ASSERT(lhs != scratch);
    474    if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) {
    475      cmpPtr(lhs, Imm32(int32_t(rhs.value)));
    476    } else {
    477      movePtr(rhs, scratch);
    478      cmpPtr(lhs, scratch);
    479    }
    480  }
    481  void cmpPtr(Register lhs, const ImmPtr rhs) {
    482    cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
    483  }
    484  void cmpPtr(Register lhs, const ImmGCPtr rhs) {
    485    ScratchRegisterScope scratch(asMasm());
    486    MOZ_ASSERT(lhs != scratch);
    487    movePtr(rhs, scratch);
    488    cmpPtr(lhs, scratch);
    489  }
    490  void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
    491  void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) {
    492    ScratchRegisterScope scratch(asMasm());
    493    MOZ_ASSERT(!lhs.containsReg(scratch));
    494    movePtr(rhs, scratch);
    495    cmpPtr(lhs, scratch);
    496  }
    497  void cmpPtr(const Operand& lhs, const ImmWord rhs) {
    498    if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
    499      cmpPtr(lhs, Imm32((int32_t)rhs.value));
    500    } else {
    501      ScratchRegisterScope scratch(asMasm());
    502      movePtr(rhs, scratch);
    503      cmpPtr(lhs, scratch);
    504    }
    505  }
    506  void cmpPtr(const Operand& lhs, const ImmPtr rhs) {
    507    cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
    508  }
    509  void cmpPtr(const Address& lhs, const ImmGCPtr rhs) {
    510    cmpPtr(Operand(lhs), rhs);
    511  }
    512  void cmpPtr(const Address& lhs, const ImmWord rhs) {
    513    cmpPtr(Operand(lhs), rhs);
    514  }
    515  void cmpPtr(const Address& lhs, const ImmPtr rhs) {
    516    cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
    517  }
    518  void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); }
    519  void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); }
    520  void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
    521  void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); }
    522  void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); }
    523  void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); }
    524  void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); }
    525  void testPtr(Register lhs, ImmWord rhs) { test64(lhs, Imm64(rhs.value)); }
    526  void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); }
    527  void test64(Register lhs, Register rhs) { testq(rhs, lhs); }
    528  void test64(Register lhs, const Imm64 rhs) {
    529    if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
    530      testq(Imm32((int32_t)rhs.value), lhs);
    531    } else {
    532      ScratchRegisterScope scratch(asMasm());
    533      movq(ImmWord(rhs.value), scratch);
    534      testq(scratch, lhs);
    535    }
    536  }
    537 
    538  // Compare-then-conditionally-move/load, for integer types
    539  template <size_t CmpSize, size_t MoveSize>
    540  void cmpMove(Condition cond, Register lhs, Register rhs, Register falseVal,
    541               Register trueValAndDest);
    542 
    543  template <size_t CmpSize, size_t MoveSize>
    544  void cmpMove(Condition cond, Register lhs, const Address& rhs,
    545               Register falseVal, Register trueValAndDest);
    546 
    547  template <size_t CmpSize, size_t LoadSize>
    548  void cmpLoad(Condition cond, Register lhs, Register rhs,
    549               const Address& falseVal, Register trueValAndDest);
    550 
    551  template <size_t CmpSize, size_t LoadSize>
    552  void cmpLoad(Condition cond, Register lhs, const Address& rhs,
    553               const Address& falseVal, Register trueValAndDest);
    554 
    555  /////////////////////////////////////////////////////////////////
    556  // Common interface.
    557  /////////////////////////////////////////////////////////////////
    558 
    559  void movePtr(Register src, Register dest) { movq(src, dest); }
    560  void movePtr(Register src, const Operand& dest) { movq(src, dest); }
    561  void movePtr(ImmWord imm, Register dest) { mov(imm, dest); }
    562  void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); }
    563  void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); }
    564  void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); }
    565  void loadPtr(AbsoluteAddress address, Register dest) {
    566    if (X86Encoding::IsAddressImmediate(address.addr)) {
    567      movq(Operand(address), dest);
    568    } else {
    569      ScratchRegisterScope scratch(asMasm());
    570      mov(ImmPtr(address.addr), scratch);
    571      loadPtr(Address(scratch, 0x0), dest);
    572    }
    573  }
    574  FaultingCodeOffset loadPtr(const Address& address, Register dest) {
    575    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    576    movq(Operand(address), dest);
    577    return fco;
    578  }
    579  void load64(const Address& address, Register dest) {
    580    movq(Operand(address), dest);
    581  }
    582  void loadPtr(const Operand& src, Register dest) { movq(src, dest); }
    583  FaultingCodeOffset loadPtr(const BaseIndex& src, Register dest) {
    584    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    585    movq(Operand(src), dest);
    586    return fco;
    587  }
    588  void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); }
    589  void load32(AbsoluteAddress address, Register dest) {
    590    if (X86Encoding::IsAddressImmediate(address.addr)) {
    591      movl(Operand(address), dest);
    592    } else {
    593      ScratchRegisterScope scratch(asMasm());
    594      mov(ImmPtr(address.addr), scratch);
    595      load32(Address(scratch, 0x0), dest);
    596    }
    597  }
    598  void load64(const Operand& address, Register64 dest) {
    599    movq(address, dest.reg);
    600  }
    601  FaultingCodeOffset load64(const Address& address, Register64 dest) {
    602    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    603    movq(Operand(address), dest.reg);
    604    return fco;
    605  }
    606  FaultingCodeOffset load64(const BaseIndex& address, Register64 dest) {
    607    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    608    movq(Operand(address), dest.reg);
    609    return fco;
    610  }
    611  template <typename S>
    612  void load64Unaligned(const S& src, Register64 dest) {
    613    load64(src, dest);
    614  }
    615  template <typename T>
    616  void storePtr(ImmWord imm, T address) {
    617    if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
    618      movq(Imm32((int32_t)imm.value), Operand(address));
    619    } else {
    620      ScratchRegisterScope scratch(asMasm());
    621      mov(imm, scratch);
    622      movq(scratch, Operand(address));
    623    }
    624  }
    625  template <typename T>
    626  void storePtr(ImmPtr imm, T address) {
    627    storePtr(ImmWord(uintptr_t(imm.value)), address);
    628  }
    629  template <typename T>
    630  void storePtr(ImmGCPtr imm, T address) {
    631    ScratchRegisterScope scratch(asMasm());
    632    movq(imm, scratch);
    633    movq(scratch, Operand(address));
    634  }
    635  FaultingCodeOffset storePtr(Register src, const Address& address) {
    636    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    637    movq(src, Operand(address));
    638    return fco;
    639  }
    640  void store64(Register src, const Address& address) {
    641    movq(src, Operand(address));
    642  }
    643  void store64(Register64 src, const Operand& address) {
    644    movq(src.reg, address);
    645  }
    646  FaultingCodeOffset storePtr(Register src, const BaseIndex& address) {
    647    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    648    movq(src, Operand(address));
    649    return fco;
    650  }
    651  void storePtr(Register src, const Operand& dest) { movq(src, dest); }
    652  void storePtr(Register src, AbsoluteAddress address) {
    653    if (X86Encoding::IsAddressImmediate(address.addr)) {
    654      movq(src, Operand(address));
    655    } else {
    656      ScratchRegisterScope scratch(asMasm());
    657      mov(ImmPtr(address.addr), scratch);
    658      storePtr(src, Address(scratch, 0x0));
    659    }
    660  }
    661  void store32(Register src, AbsoluteAddress address) {
    662    if (X86Encoding::IsAddressImmediate(address.addr)) {
    663      movl(src, Operand(address));
    664    } else {
    665      ScratchRegisterScope scratch(asMasm());
    666      mov(ImmPtr(address.addr), scratch);
    667      store32(src, Address(scratch, 0x0));
    668    }
    669  }
    670  void store16(Register src, AbsoluteAddress address) {
    671    if (X86Encoding::IsAddressImmediate(address.addr)) {
    672      movw(src, Operand(address));
    673    } else {
    674      ScratchRegisterScope scratch(asMasm());
    675      mov(ImmPtr(address.addr), scratch);
    676      store16(src, Address(scratch, 0x0));
    677    }
    678  }
    679  FaultingCodeOffset store64(Register64 src, Address address) {
    680    FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
    681    storePtr(src.reg, address);
    682    return fco;
    683  }
    684  FaultingCodeOffset store64(Register64 src, const BaseIndex& address) {
    685    return storePtr(src.reg, address);
    686  }
    687  void store64(Imm64 imm, Address address) {
    688    storePtr(ImmWord(imm.value), address);
    689  }
    690  void store64(Imm64 imm, const BaseIndex& address) {
    691    storePtr(ImmWord(imm.value), address);
    692  }
    693  template <typename S, typename T>
    694  void store64Unaligned(const S& src, const T& dest) {
    695    store64(src, dest);
    696  }
    697 
    698  void splitTag(Register src, Register dest) {
    699    if (src != dest) {
    700      movq(src, dest);
    701    }
    702    shrq(Imm32(JSVAL_TAG_SHIFT), dest);
    703  }
    704  void splitTag(const ValueOperand& operand, Register dest) {
    705    splitTag(operand.valueReg(), dest);
    706  }
    707  void splitTag(const Operand& operand, Register dest) {
    708    movq(operand, dest);
    709    shrq(Imm32(JSVAL_TAG_SHIFT), dest);
    710  }
    711  void splitTag(const Address& operand, Register dest) {
    712    splitTag(Operand(operand), dest);
    713  }
    714  void splitTag(const BaseIndex& operand, Register dest) {
    715    splitTag(Operand(operand), dest);
    716  }
    717 
    718  // Extracts the tag of a value and places it in tag.
    719  void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
    720    splitTag(value, tag);
    721  }
    722  void cmpTag(const ValueOperand& operand, ImmTag tag) {
    723    ScratchTagScope reg(asMasm(), operand);
    724    splitTagForTest(operand, reg);
    725    cmp32(reg, tag);
    726  }
    727 
    728  Condition testMagic(Condition cond, const ValueOperand& src) {
    729    ScratchTagScope scratch(asMasm(), src);
    730    splitTagForTest(src, scratch);
    731    return testMagic(cond, scratch);
    732  }
    733  Condition testError(Condition cond, const ValueOperand& src) {
    734    return testMagic(cond, src);
    735  }
    736 
    737  void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
    738    bool destIsZero = maybeEmitSetZeroByteRegister(value, dest);
    739    cond = testNull(cond, value);
    740    emitSet(cond, dest, destIsZero);
    741  }
    742 
    743  void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
    744    bool destIsZero = maybeEmitSetZeroByteRegister(value, dest);
    745    cond = testObject(cond, value);
    746    emitSet(cond, dest, destIsZero);
    747  }
    748 
    749  void testUndefinedSet(Condition cond, const ValueOperand& value,
    750                        Register dest) {
    751    bool destIsZero = maybeEmitSetZeroByteRegister(value, dest);
    752    cond = testUndefined(cond, value);
    753    emitSet(cond, dest, destIsZero);
    754  }
    755 
    756  void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) {
    757    vmovq(src, dest.valueReg());
    758  }
    759  void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
    760    boxValue(type, src, dest.valueReg());
    761  }
    762  void boxNonDouble(Register type, Register src, const ValueOperand& dest) {
    763    boxValue(type, src, dest.valueReg());
    764  }
    765 
    766  // Note that the |dest| register here may be ScratchReg, so we shouldn't
    767  // use it.
    768  void unboxInt32(const ValueOperand& src, Register dest) {
    769    movl(src.valueReg(), dest);
    770  }
    771  void unboxInt32(const Operand& src, Register dest) { movl(src, dest); }
    772  void unboxInt32(const Address& src, Register dest) {
    773    unboxInt32(Operand(src), dest);
    774  }
    775  void unboxInt32(const BaseIndex& src, Register dest) {
    776    unboxInt32(Operand(src), dest);
    777  }
    778  template <typename T>
    779  void unboxDouble(const T& src, FloatRegister dest) {
    780    loadDouble(Operand(src), dest);
    781  }
    782 
    783  void unboxBoolean(const ValueOperand& src, Register dest) {
    784    movl(src.valueReg(), dest);
    785  }
    786  void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); }
    787  void unboxBoolean(const Address& src, Register dest) {
    788    unboxBoolean(Operand(src), dest);
    789  }
    790  void unboxBoolean(const BaseIndex& src, Register dest) {
    791    unboxBoolean(Operand(src), dest);
    792  }
    793 
    794  void unboxMagic(const ValueOperand& src, Register dest) {
    795    movl(src.valueReg(), dest);
    796  }
    797 
    798  void unboxDouble(const ValueOperand& src, FloatRegister dest) {
    799    vmovq(src.valueReg(), dest);
    800  }
    801 
    802  void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); }
    803 
    804  void unboxNonDouble(const ValueOperand& src, Register dest,
    805                      JSValueType type) {
    806    MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
    807    if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
    808      movl(src.valueReg(), dest);
    809      return;
    810    }
    811    if (src.valueReg() == dest) {
    812      ScratchRegisterScope scratch(asMasm());
    813      mov(ImmShiftedTag(type), scratch);
    814      xorq(scratch, dest);
    815    } else {
    816      mov(ImmShiftedTag(type), dest);
    817      xorq(src.valueReg(), dest);
    818    }
    819  }
    820  void unboxNonDouble(const Operand& src, Register dest, JSValueType type) {
    821    MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
    822    if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
    823      movl(src, dest);
    824      return;
    825    }
    826    // Explicitly permits |dest| to be used in |src|.
    827    ScratchRegisterScope scratch(asMasm());
    828    MOZ_ASSERT(dest != scratch);
    829    if (src.containsReg(dest)) {
    830      mov(ImmShiftedTag(type), scratch);
    831      // If src is already a register, then src and dest are the same
    832      // thing and we don't need to move anything into dest.
    833      if (src.kind() != Operand::REG) {
    834        movq(src, dest);
    835      }
    836      xorq(scratch, dest);
    837    } else {
    838      mov(ImmShiftedTag(type), dest);
    839      xorq(src, dest);
    840    }
    841  }
    842  void unboxNonDouble(const Address& src, Register dest, JSValueType type) {
    843    unboxNonDouble(Operand(src), dest, type);
    844  }
    845  void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) {
    846    unboxNonDouble(Operand(src), dest, type);
    847  }
    848 
    849  void unboxString(const ValueOperand& src, Register dest) {
    850    unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
    851  }
    852  void unboxString(const Operand& src, Register dest) {
    853    unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
    854  }
    855  void unboxString(const Address& src, Register dest) {
    856    unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
    857  }
    858 
    859  void unboxSymbol(const ValueOperand& src, Register dest) {
    860    unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
    861  }
    862  void unboxSymbol(const Operand& src, Register dest) {
    863    unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
    864  }
    865 
    866  void unboxBigInt(const ValueOperand& src, Register dest) {
    867    unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
    868  }
    869  void unboxBigInt(const Operand& src, Register dest) {
    870    unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
    871  }
    872  void unboxBigInt(const Address& src, Register dest) {
    873    unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
    874  }
    875 
    876  void unboxObject(const ValueOperand& src, Register dest) {
    877    unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
    878  }
    879  void unboxObject(const Operand& src, Register dest) {
    880    unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
    881  }
    882  void unboxObject(const Address& src, Register dest) {
    883    unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
    884  }
    885  void unboxObject(const BaseIndex& src, Register dest) {
    886    unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
    887  }
    888 
    889  // This should only be used for GC barrier code, to unbox a GC thing Value.
    890  // It's fine there because we don't depend on the actual Value type (all Cells
    891  // are treated the same way). In almost all other cases this would be
    892  // Spectre-unsafe - use unboxNonDouble and friends instead.
    893  void unboxGCThingForGCBarrier(const Address& src, Register dest) {
    894    movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
    895    andq(Operand(src), dest);
    896  }
    897  void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
    898    MOZ_ASSERT(src.valueReg() != dest);
    899    movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
    900    andq(src.valueReg(), dest);
    901  }
    902 
    903  void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
    904    movq(ImmWord(wasm::AnyRef::GCThingMask), dest);
    905    andq(Operand(src), dest);
    906  }
    907 
    908  // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
    909  void getGCThingValueChunk(const Address& src, Register dest) {
    910    movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
    911    andq(Operand(src), dest);
    912  }
    913  void getGCThingValueChunk(const ValueOperand& src, Register dest) {
    914    MOZ_ASSERT(src.valueReg() != dest);
    915    movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
    916    andq(src.valueReg(), dest);
    917  }
    918 
    919  void getWasmAnyRefGCThingChunk(Register src, Register dest) {
    920    MOZ_ASSERT(src != dest);
    921    movq(ImmWord(wasm::AnyRef::GCThingChunkMask), dest);
    922    andq(src, dest);
    923  }
    924 
    925  inline void fallibleUnboxPtrImpl(const Operand& src, Register dest,
    926                                   JSValueType type, Label* fail);
    927 
    928  // Extended unboxing API. If the payload is already in a register, returns
    929  // that register. Otherwise, provides a move to the given scratch register,
    930  // and returns that.
    931  [[nodiscard]] Register extractObject(const Address& address,
    932                                       Register scratch) {
    933    MOZ_ASSERT(scratch != ScratchReg);
    934    unboxObject(address, scratch);
    935    return scratch;
    936  }
    937  [[nodiscard]] Register extractObject(const ValueOperand& value,
    938                                       Register scratch) {
    939    MOZ_ASSERT(scratch != ScratchReg);
    940    unboxObject(value, scratch);
    941    return scratch;
    942  }
    943  [[nodiscard]] Register extractSymbol(const ValueOperand& value,
    944                                       Register scratch) {
    945    MOZ_ASSERT(scratch != ScratchReg);
    946    unboxSymbol(value, scratch);
    947    return scratch;
    948  }
    949  [[nodiscard]] Register extractInt32(const ValueOperand& value,
    950                                      Register scratch) {
    951    MOZ_ASSERT(scratch != ScratchReg);
    952    unboxInt32(value, scratch);
    953    return scratch;
    954  }
    955  [[nodiscard]] Register extractBoolean(const ValueOperand& value,
    956                                        Register scratch) {
    957    MOZ_ASSERT(scratch != ScratchReg);
    958    unboxBoolean(value, scratch);
    959    return scratch;
    960  }
    961  [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
    962    MOZ_ASSERT(scratch != ScratchReg);
    963    loadPtr(address, scratch);
    964    splitTag(scratch, scratch);
    965    return scratch;
    966  }
    967  [[nodiscard]] Register extractTag(const ValueOperand& value,
    968                                    Register scratch) {
    969    MOZ_ASSERT(scratch != ScratchReg);
    970    splitTag(value, scratch);
    971    return scratch;
    972  }
    973 
    974  inline void unboxValue(const ValueOperand& src, AnyRegister dest,
    975                         JSValueType type);
    976 
    977  void loadConstantDouble(double d, FloatRegister dest);
    978  void loadConstantFloat32(float f, FloatRegister dest);
    979 
    980  void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest);
    981  void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest);
    982  void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
    983                     FloatRegister dest);
    984  void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
    985                     FloatRegister dest);
    986  void vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
    987                     FloatRegister dest);
    988  void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
    989                     FloatRegister dest);
    990  void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
    991                     FloatRegister dest);
    992  void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
    993                     FloatRegister dest);
    994  void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
    995                     FloatRegister dest);
    996  void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
    997                     FloatRegister dest);
    998  void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
    999                      FloatRegister dest);
   1000  void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
   1001                      FloatRegister dest);
   1002  void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
   1003                      FloatRegister dest);
   1004  void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs,
   1005                       FloatRegister dest);
   1006  void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
   1007                      FloatRegister dest);
   1008  void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs,
   1009                       FloatRegister dest);
   1010  void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
   1011                      FloatRegister dest);
   1012  void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs,
   1013                       FloatRegister dest);
   1014  void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
   1015                      FloatRegister dest);
   1016  void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs,
   1017                       FloatRegister dest);
   1018  void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
   1019                      FloatRegister dest);
   1020  void vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
   1021                      FloatRegister dest);
   1022  void vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
   1023                      FloatRegister dest);
   1024  void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
   1025                      FloatRegister dest);
   1026  void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
   1027                      FloatRegister dest);
   1028  void vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
   1029                      FloatRegister dest);
   1030  void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
   1031                      FloatRegister dest);
   1032  void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
   1033                      FloatRegister dest);
   1034  void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
   1035                      FloatRegister dest);
   1036  void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
   1037                      FloatRegister dest);
   1038  void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
   1039                      FloatRegister dest);
   1040  void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
   1041                      FloatRegister dest);
   1042  void vpandSimd128(const SimdConstant& v, FloatRegister lhs,
   1043                    FloatRegister dest);
   1044  void vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
   1045                    FloatRegister dest);
   1046  void vporSimd128(const SimdConstant& v, FloatRegister lhs,
   1047                   FloatRegister dest);
   1048  void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1049                     FloatRegister dest);
   1050  void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1051                     FloatRegister dest);
   1052  void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1053                     FloatRegister dest);
   1054  void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1055                     FloatRegister dest);
   1056  void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1057                     FloatRegister dest);
   1058  void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1059                     FloatRegister dest);
   1060  void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1061                     FloatRegister dest);
   1062  void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1063                     FloatRegister dest);
   1064  void vandpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1065                     FloatRegister dest);
   1066  void vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1067                     FloatRegister dest);
   1068  void vxorpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1069                     FloatRegister dest);
   1070  void vxorpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1071                     FloatRegister dest);
   1072  void vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1073                     FloatRegister dest);
   1074  void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs,
   1075                        FloatRegister dest);
   1076  void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs,
   1077                        FloatRegister dest);
   1078  void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs,
   1079                        FloatRegister dest);
   1080  void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs,
   1081                        FloatRegister dest);
   1082  void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs,
   1083                         FloatRegister dest);
   1084  void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1085                        FloatRegister dest);
   1086  void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
   1087                      FloatRegister dest);
   1088  void vptestSimd128(const SimdConstant& v, FloatRegister lhs);
   1089  void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs,
   1090                       FloatRegister dest);
   1091  void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs,
   1092                       FloatRegister dest);
   1093  void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs,
   1094                       FloatRegister dest);
   1095  void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs,
   1096                       FloatRegister dest);
   1097  void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs,
   1098                       FloatRegister dest);
   1099  void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs,
   1100                       FloatRegister dest);
   1101  void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs,
   1102                       FloatRegister dest);
   1103  void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1104                       FloatRegister dest);
   1105  void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1106                        FloatRegister dest);
   1107  void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs,
   1108                       FloatRegister dest);
   1109  void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs,
   1110                       FloatRegister dest);
   1111  void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs,
   1112                       FloatRegister dest);
   1113  void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1114                       FloatRegister dest);
   1115  void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1116                        FloatRegister dest);
   1117  void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs,
   1118                       FloatRegister dest);
   1119  void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs,
   1120                       FloatRegister dest);
   1121  void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs,
   1122                         FloatRegister dest);
   1123  void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs,
   1124                       FloatRegister dest);
   1125 
   1126 public:
   1127  Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
   1128    test32(operand.valueReg(), operand.valueReg());
   1129    return truthy ? NonZero : Zero;
   1130  }
   1131  Condition testStringTruthy(bool truthy, const ValueOperand& value);
   1132  Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
   1133 
   1134  template <typename T>
   1135  inline void loadInt32OrDouble(const T& src, FloatRegister dest);
   1136 
   1137  template <typename T>
   1138  void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) {
   1139    if (dest.isFloat()) {
   1140      loadInt32OrDouble(src, dest.fpu());
   1141    } else {
   1142      unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type));
   1143    }
   1144  }
   1145 
   1146  // Checks whether a double is representable as a 64-bit integer. If so, the
   1147  // integer is written to the output register. Otherwise, a bailout is taken to
   1148  // the given snapshot. This function overwrites the scratch float register.
   1149  void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
   1150                          bool negativeZeroCheck = true);
   1151 
   1152  void convertUInt32ToDouble(Register src, FloatRegister dest) {
   1153    // Zero the output register to break dependencies, see convertInt32ToDouble.
   1154    zeroDouble(dest);
   1155 
   1156    vcvtsq2sd(src, dest, dest);
   1157  }
   1158 
   1159  void convertUInt32ToFloat32(Register src, FloatRegister dest) {
   1160    // Zero the output register to break dependencies, see convertInt32ToDouble.
   1161    zeroDouble(dest);
   1162 
   1163    vcvtsq2ss(src, dest, dest);
   1164  }
   1165 
   1166  void truncateFloat32ModUint32(FloatRegister src, Register dest) {
   1167    // vcvttss2sq returns 0x8000000000000000 on failure. It fails if
   1168    // 1. The input is non-finite (NaN or ±Infinity).
   1169    // 2. The input's exponent is at least 63.
   1170    //
   1171    // In both cases the input is too large for an int32 and the truncated
   1172    // result is zero. So unconditionally zeroing the upper 32-bits gives the
   1173    // correct result for all inputs.
   1174 
   1175    vcvttss2sq(src, dest);
   1176    movl(dest, dest);  // Zero upper 32-bits.
   1177  }
   1178 
   1179  inline void incrementInt32Value(const Address& addr);
   1180 
   1181  void minMax32(Register lhs, Register rhs, Register dest, bool isMax);
   1182  void minMax32(Register lhs, Imm32 rhs, Register dest, bool isMax);
   1183 
   1184  void minMaxPtr(Register lhs, Register rhs, Register dest, bool isMax);
   1185  void minMaxPtr(Register lhs, ImmWord rhs, Register dest, bool isMax);
   1186 
   1187 public:
   1188  void handleFailureWithHandlerTail(Label* profilerExitTail, Label* bailoutTail,
   1189                                    uint32_t* returnValueCheckOffset);
   1190 
   1191  // Instrumentation for entering and leaving the profiler.
   1192  void profilerEnterFrame(Register framePtr, Register scratch);
   1193  void profilerExitFrame();
   1194 };
   1195 
   1196 using MacroAssemblerSpecific = MacroAssemblerX64;
   1197 
   1198 }  // namespace jit
   1199 }  // namespace js
   1200 
   1201 #endif /* jit_x64_MacroAssembler_x64_h */
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE