[ tor-browser ].git.dasho

CodeGenerator-x64.cpp (47830B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "jit/x64/CodeGenerator-x64.h"
      8 
      9 #include "mozilla/CheckedInt.h"
     10 #include "mozilla/FloatingPoint.h"
     11 
     12 #include "jit/CodeGenerator.h"
     13 #include "jit/MIR-wasm.h"
     14 #include "jit/MIR.h"
     15 #include "jit/ReciprocalMulConstants.h"
     16 #include "js/ScalarType.h"  // js::Scalar::Type
     17 
     18 #include "jit/MacroAssembler-inl.h"
     19 #include "jit/shared/CodeGenerator-shared-inl.h"
     20 
     21 using namespace js;
     22 using namespace js::jit;
     23 
     24 CodeGeneratorX64::CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph,
     25                                   MacroAssembler* masm,
     26                                   const wasm::CodeMetadata* wasmCodeMeta)
     27    : CodeGeneratorX86Shared(gen, graph, masm, wasmCodeMeta) {}
     28 
     29 Operand CodeGeneratorX64::ToOperand64(const LInt64Allocation& a64) {
     30  const LAllocation& a = a64.value();
     31  MOZ_ASSERT(!a.isFloatReg());
     32  if (a.isGeneralReg()) {
     33    return Operand(a.toGeneralReg()->reg());
     34  }
     35  return Operand(ToAddress(a));
     36 }
     37 
     38 void CodeGenerator::visitBox(LBox* box) {
     39  const LAllocation* in = box->payload();
     40  ValueOperand result = ToOutValue(box);
     41 
     42  masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
     43 
     44  if (JitOptions.spectreValueMasking && IsFloatingPointType(box->type())) {
     45    ScratchRegisterScope scratch(masm);
     46    masm.movePtr(ImmWord(JSVAL_SHIFTED_TAG_MAX_DOUBLE), scratch);
     47    masm.cmpPtrMovePtr(Assembler::Below, scratch, result.valueReg(), scratch,
     48                       result.valueReg());
     49  }
     50 }
     51 
     52 void CodeGenerator::visitUnbox(LUnbox* unbox) {
     53  MUnbox* mir = unbox->mir();
     54 
     55  Register result = ToRegister(unbox->output());
     56 
     57  if (mir->fallible()) {
     58    Label bail;
     59    auto fallibleUnboxImpl = [&](auto value) {
     60      switch (mir->type()) {
     61        case MIRType::Int32:
     62          masm.fallibleUnboxInt32(value, result, &bail);
     63          break;
     64        case MIRType::Boolean:
     65          masm.fallibleUnboxBoolean(value, result, &bail);
     66          break;
     67        case MIRType::Object:
     68          masm.fallibleUnboxObject(value, result, &bail);
     69          break;
     70        case MIRType::String:
     71          masm.fallibleUnboxString(value, result, &bail);
     72          break;
     73        case MIRType::Symbol:
     74          masm.fallibleUnboxSymbol(value, result, &bail);
     75          break;
     76        case MIRType::BigInt:
     77          masm.fallibleUnboxBigInt(value, result, &bail);
     78          break;
     79        default:
     80          MOZ_CRASH("Given MIRType cannot be unboxed.");
     81      }
     82    };
     83    LAllocation* input = unbox->getOperand(LUnbox::Input);
     84    if (input->isGeneralReg()) {
     85      fallibleUnboxImpl(ValueOperand(ToRegister(input)));
     86    } else {
     87      fallibleUnboxImpl(ToAddress(input));
     88    }
     89    bailoutFrom(&bail, unbox->snapshot());
     90    return;
     91  }
     92 
     93  // Infallible unbox.
     94 
     95  Operand input = ToOperand(unbox->getOperand(LUnbox::Input));
     96 
     97 #ifdef DEBUG
     98  // Assert the types match.
     99  JSValueTag tag = MIRTypeToTag(mir->type());
    100  Label ok;
    101  masm.splitTag(input, ScratchReg);
    102  masm.branch32(Assembler::Equal, ScratchReg, Imm32(tag), &ok);
    103  masm.assumeUnreachable("Infallible unbox type mismatch");
    104  masm.bind(&ok);
    105 #endif
    106 
    107  switch (mir->type()) {
    108    case MIRType::Int32:
    109      masm.unboxInt32(input, result);
    110      break;
    111    case MIRType::Boolean:
    112      masm.unboxBoolean(input, result);
    113      break;
    114    case MIRType::Object:
    115      masm.unboxObject(input, result);
    116      break;
    117    case MIRType::String:
    118      masm.unboxString(input, result);
    119      break;
    120    case MIRType::Symbol:
    121      masm.unboxSymbol(input, result);
    122      break;
    123    case MIRType::BigInt:
    124      masm.unboxBigInt(input, result);
    125      break;
    126    default:
    127      MOZ_CRASH("Given MIRType cannot be unboxed.");
    128  }
    129 }
    130 
    131 void CodeGenerator::visitMulI64(LMulI64* lir) {
    132  Register lhs = ToRegister64(lir->lhs()).reg;
    133  LInt64Allocation rhs = lir->rhs();
    134  Register out = ToOutRegister64(lir).reg;
    135 
    136  if (IsConstant(rhs)) {
    137    int64_t constant = ToInt64(rhs);
    138    switch (constant) {
    139      case -1:
    140        if (lhs != out) {
    141          masm.movq(lhs, out);
    142        }
    143        masm.negq(out);
    144        break;
    145      case 0:
    146        masm.xorq(out, out);
    147        break;
    148      case 1:
    149        if (lhs != out) {
    150          masm.movq(lhs, out);
    151        }
    152        break;
    153      case 2:
    154        if (lhs == out) {
    155          masm.addq(lhs, lhs);
    156        } else {
    157          masm.lea(Operand(lhs, lhs, TimesOne), out);
    158        }
    159        break;
    160      case 3:
    161        masm.lea(Operand(lhs, lhs, TimesTwo), out);
    162        break;
    163      case 4:
    164        if (lhs == out) {
    165          masm.shlq(Imm32(2), lhs);
    166        } else {
    167          masm.lea(Operand(lhs, TimesFour, 0), out);
    168        }
    169        break;
    170      case 5:
    171        masm.lea(Operand(lhs, lhs, TimesFour), out);
    172        break;
    173      case 8:
    174        if (lhs == out) {
    175          masm.shlq(Imm32(3), lhs);
    176        } else {
    177          masm.lea(Operand(lhs, TimesEight, 0), out);
    178        }
    179        break;
    180      case 9:
    181        masm.lea(Operand(lhs, lhs, TimesEight), out);
    182        break;
    183      default: {
    184        // Use shift if constant is power of 2.
    185        int32_t shift = mozilla::FloorLog2(constant);
    186        if (constant > 0 && (1 << shift) == constant) {
    187          if (lhs != out) {
    188            masm.movq(lhs, out);
    189          }
    190          masm.shlq(Imm32(shift), out);
    191        } else if (int32_t(constant) == constant) {
    192          masm.imulq(Imm32(constant), lhs, out);
    193        } else {
    194          MOZ_ASSERT(out == lhs);
    195          masm.mul64(Imm64(constant), Register64(lhs));
    196        }
    197        break;
    198      }
    199    }
    200  } else {
    201    MOZ_ASSERT(out == lhs);
    202    masm.imulq(ToOperandOrRegister64(rhs), lhs);
    203  }
    204 }
    205 
    206 template <class LIR>
    207 static void TrapIfDivideByZero(MacroAssembler& masm, LIR* lir, Register rhs) {
    208  auto* mir = lir->mir();
    209  MOZ_ASSERT(mir->trapOnError());
    210 
    211  if (mir->canBeDivideByZero()) {
    212    Label nonZero;
    213    masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
    214    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc());
    215    masm.bind(&nonZero);
    216  }
    217 }
    218 
    219 void CodeGenerator::visitDivI64(LDivI64* lir) {
    220  Register lhs = ToRegister(lir->lhs());
    221  Register rhs = ToRegister(lir->rhs());
    222 
    223  MOZ_ASSERT(lhs == rax);
    224  MOZ_ASSERT(rhs != rax);
    225  MOZ_ASSERT(rhs != rdx);
    226  MOZ_ASSERT(ToRegister(lir->output()) == rax);
    227  MOZ_ASSERT(ToRegister(lir->temp0()) == rdx);
    228 
    229  MDiv* mir = lir->mir();
    230 
    231  // Handle divide by zero.
    232  TrapIfDivideByZero(masm, lir, rhs);
    233 
    234  // Handle an integer overflow exception from INT64_MIN / -1.
    235  if (mir->canBeNegativeOverflow()) {
    236    Label notOverflow;
    237    masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
    238    masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
    239    masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->trapSiteDesc());
    240    masm.bind(&notOverflow);
    241  }
    242 
    243  // Sign extend the lhs into rdx to make rdx:rax.
    244  masm.cqo();
    245  masm.idivq(rhs);
    246 }
    247 
    248 void CodeGenerator::visitModI64(LModI64* lir) {
    249  Register lhs = ToRegister(lir->lhs());
    250  Register rhs = ToRegister(lir->rhs());
    251  Register output = ToRegister(lir->output());
    252 
    253  MOZ_ASSERT(lhs == rax);
    254  MOZ_ASSERT(rhs != rax);
    255  MOZ_ASSERT(rhs != rdx);
    256  MOZ_ASSERT(ToRegister(lir->output()) == rdx);
    257  MOZ_ASSERT(ToRegister(lir->temp0()) == rax);
    258 
    259  MMod* mir = lir->mir();
    260 
    261  Label done;
    262 
    263  // Handle divide by zero.
    264  TrapIfDivideByZero(masm, lir, rhs);
    265 
    266  // Handle an integer overflow exception from INT64_MIN / -1.
    267  if (mir->canBeNegativeDividend()) {
    268    Label notOverflow;
    269    masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
    270    masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
    271    {
    272      masm.xorl(output, output);
    273      masm.jump(&done);
    274    }
    275    masm.bind(&notOverflow);
    276  }
    277 
    278  // Sign extend the lhs into rdx to make rdx:rax.
    279  masm.cqo();
    280  masm.idivq(rhs);
    281 
    282  masm.bind(&done);
    283 }
    284 
    285 void CodeGenerator::visitUDivI64(LUDivI64* lir) {
    286  Register rhs = ToRegister(lir->rhs());
    287 
    288  MOZ_ASSERT(ToRegister(lir->lhs()) == rax);
    289  MOZ_ASSERT(rhs != rax);
    290  MOZ_ASSERT(rhs != rdx);
    291  MOZ_ASSERT(ToRegister(lir->output()) == rax);
    292  MOZ_ASSERT(ToRegister(lir->temp0()) == rdx);
    293 
    294  // Prevent divide by zero.
    295  TrapIfDivideByZero(masm, lir, rhs);
    296 
    297  // Zero extend the lhs into rdx to make (rdx:rax).
    298  masm.xorl(rdx, rdx);
    299  masm.udivq(rhs);
    300 }
    301 
    302 void CodeGenerator::visitUModI64(LUModI64* lir) {
    303  Register rhs = ToRegister(lir->rhs());
    304 
    305  MOZ_ASSERT(ToRegister(lir->lhs()) == rax);
    306  MOZ_ASSERT(rhs != rax);
    307  MOZ_ASSERT(rhs != rdx);
    308  MOZ_ASSERT(ToRegister(lir->output()) == rdx);
    309  MOZ_ASSERT(ToRegister(lir->temp0()) == rax);
    310 
    311  // Prevent divide by zero.
    312  TrapIfDivideByZero(masm, lir, rhs);
    313 
    314  // Zero extend the lhs into rdx to make (rdx:rax).
    315  masm.xorl(rdx, rdx);
    316  masm.udivq(rhs);
    317 }
    318 
    319 void CodeGenerator::visitDivPowTwoI64(LDivPowTwoI64* ins) {
    320  Register lhs = ToRegister(ins->numerator());
    321 
    322  int32_t shift = ins->shift();
    323  bool negativeDivisor = ins->negativeDivisor();
    324  MDiv* mir = ins->mir();
    325 
    326  // We use defineReuseInput so these should always be the same, which is
    327  // convenient since all of our instructions here are two-address.
    328  MOZ_ASSERT(lhs == ToRegister(ins->output()));
    329 
    330  // Unsigned division is just a right-shift.
    331  if (mir->isUnsigned()) {
    332    if (shift != 0) {
    333      masm.shrq(Imm32(shift), lhs);
    334    }
    335    return;
    336  }
    337 
    338  if (shift != 0) {
    339    // Adjust the value so that shifting produces a correctly rounded result
    340    // when the numerator is negative.
    341    // See 10-1 "Signed Division by a Known Power of 2" in Henry S. Warren,
    342    // Jr.'s Hacker's Delight.
    343    if (mir->canBeNegativeDividend()) {
    344      Register lhsCopy = ToRegister(ins->numeratorCopy());
    345      MOZ_ASSERT(lhsCopy != lhs);
    346      if (shift > 1) {
    347        // Copy the sign bit of the numerator. (= (2^63 - 1) or 0)
    348        masm.sarq(Imm32(63), lhs);
    349      }
    350      // Divide by 2^(64 - shift)
    351      // i.e. (= (2^64 - 1) / 2^(64 - shift) or 0)
    352      // i.e. (= (2^shift - 1) or 0)
    353      masm.shrq(Imm32(64 - shift), lhs);
    354      // If signed, make any 1 bit below the shifted bits to bubble up, such
    355      // that once shifted the value would be rounded towards 0.
    356      masm.addq(lhsCopy, lhs);
    357    }
    358    masm.sarq(Imm32(shift), lhs);
    359  }
    360 
    361  if (negativeDivisor) {
    362    masm.negq(lhs);
    363  }
    364 
    365  if (shift == 0 && negativeDivisor) {
    366    // INT64_MIN / -1 overflows.
    367    Label ok;
    368    masm.j(Assembler::NoOverflow, &ok);
    369    masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->trapSiteDesc());
    370    masm.bind(&ok);
    371  }
    372 }
    373 
    374 void CodeGenerator::visitModPowTwoI64(LModPowTwoI64* ins) {
    375  Register64 lhs = Register64(ToRegister(ins->input()));
    376  int32_t shift = ins->shift();
    377  bool canBeNegative =
    378      !ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend();
    379 
    380  MOZ_ASSERT(lhs.reg == ToRegister(ins->output()));
    381 
    382  if (shift == 0) {
    383    masm.xorl(lhs.reg, lhs.reg);
    384    return;
    385  }
    386 
    387  auto clearHighBits = [&]() {
    388    switch (shift) {
    389      case 8:
    390        masm.movzbl(lhs.reg, lhs.reg);
    391        break;
    392      case 16:
    393        masm.movzwl(lhs.reg, lhs.reg);
    394        break;
    395      case 32:
    396        masm.movl(lhs.reg, lhs.reg);
    397        break;
    398      default:
    399        masm.and64(Imm64((uint64_t(1) << shift) - 1), lhs);
    400        break;
    401    }
    402  };
    403 
    404  Label negative;
    405 
    406  if (canBeNegative) {
    407    // Switch based on sign of the lhs.
    408    // Positive numbers are just a bitmask
    409    masm.branchTest64(Assembler::Signed, lhs, lhs, &negative);
    410  }
    411 
    412  clearHighBits();
    413 
    414  if (canBeNegative) {
    415    Label done;
    416    masm.jump(&done);
    417 
    418    // Negative numbers need a negate, bitmask, negate
    419    masm.bind(&negative);
    420 
    421    // Unlike in the visitModI64 case, we are not computing the mod by means of
    422    // a division. Therefore, the divisor = -1 case isn't problematic (the andq
    423    // always returns 0, which is what we expect).
    424    //
    425    // The negq instruction overflows if lhs == INT64_MIN, but this is also not
    426    // a problem: shift is at most 63, and so the andq also always returns 0.
    427    masm.neg64(lhs);
    428    clearHighBits();
    429    masm.neg64(lhs);
    430 
    431    masm.bind(&done);
    432  }
    433 }
    434 
    435 template <class LDivOrMod>
    436 static void Divide64WithConstant(MacroAssembler& masm, LDivOrMod* ins) {
    437  Register lhs = ToRegister(ins->numerator());
    438  [[maybe_unused]] Register output = ToRegister(ins->output());
    439  [[maybe_unused]] Register temp = ToRegister(ins->temp0());
    440  int64_t d = ins->denominator();
    441 
    442  MOZ_ASSERT(lhs != rax && lhs != rdx);
    443  MOZ_ASSERT((output == rax && temp == rdx) || (output == rdx && temp == rax));
    444 
    445  // The absolute value of the denominator isn't a power of 2 (see LDivPowTwoI64
    446  // and LModPowTwoI64).
    447  MOZ_ASSERT(!mozilla::IsPowerOfTwo(mozilla::Abs(d)));
    448 
    449  auto* mir = ins->mir();
    450 
    451  // We will first divide by Abs(d), and negate the answer if d is negative.
    452  // If desired, this can be avoided by generalizing computeDivisionConstants.
    453  auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(d);
    454 
    455  // We first compute (M * n) >> 64, where M = rmc.multiplier.
    456  masm.movq(ImmWord(uint64_t(rmc.multiplier)), rax);
    457  masm.imulq(lhs);
    458  if (rmc.multiplier > Int128(INT64_MAX)) {
    459    MOZ_ASSERT(rmc.multiplier < (Int128(1) << 64));
    460 
    461    // We actually computed rdx = ((int64_t(M) * n) >> 64) instead. Since
    462    // (M * n) >> 64 is the same as (rdx + n), we can correct for the overflow.
    463    // (rdx + n) can't overflow, as n and rdx have opposite signs because
    464    // int64_t(M) is negative.
    465    masm.addq(lhs, rdx);
    466  }
    467  // (M * n) >> (64 + shift) is the truncated division answer if n is
    468  // non-negative, as proved in the comments of computeDivisionConstants. We
    469  // must add 1 later if n is negative to get the right answer in all cases.
    470  if (rmc.shiftAmount > 0) {
    471    masm.sarq(Imm32(rmc.shiftAmount), rdx);
    472  }
    473 
    474  // We'll subtract -1 instead of adding 1, because (n < 0 ? -1 : 0) can be
    475  // computed with just a sign-extending shift of 63 bits.
    476  if (mir->canBeNegativeDividend()) {
    477    masm.movq(lhs, rax);
    478    masm.sarq(Imm32(63), rax);
    479    masm.subq(rax, rdx);
    480  }
    481 
    482  // After this, rdx contains the correct truncated division result.
    483  if (d < 0) {
    484    masm.negq(rdx);
    485  }
    486 }
    487 
    488 void CodeGenerator::visitDivConstantI64(LDivConstantI64* ins) {
    489  int32_t d = ins->denominator();
    490 
    491  // This emits the division answer into rdx.
    492  MOZ_ASSERT(ToRegister(ins->output()) == rdx);
    493  MOZ_ASSERT(ToRegister(ins->temp0()) == rax);
    494 
    495  if (d == 0) {
    496    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->mir()->trapSiteDesc());
    497    return;
    498  }
    499 
    500  // Compute the truncated division result in rdx.
    501  Divide64WithConstant(masm, ins);
    502 }
    503 
    504 void CodeGenerator::visitModConstantI64(LModConstantI64* ins) {
    505  Register lhs = ToRegister(ins->numerator());
    506  int64_t d = ins->denominator();
    507 
    508  // This emits the modulus answer into rax.
    509  MOZ_ASSERT(ToRegister(ins->output()) == rax);
    510  MOZ_ASSERT(ToRegister(ins->temp0()) == rdx);
    511 
    512  if (d == 0) {
    513    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->mir()->trapSiteDesc());
    514    return;
    515  }
    516 
    517  // Compute the truncated division result in rdx.
    518  Divide64WithConstant(masm, ins);
    519 
    520  // Compute the remainder in |rax|: rax = lhs - d * rdx
    521  masm.mul64(Imm64(d), Register64(rdx));
    522  masm.movq(lhs, rax);
    523  masm.subq(rdx, rax);
    524 }
    525 
    526 template <class LUDivOrUMod>
    527 static void UnsignedDivide64WithConstant(MacroAssembler& masm,
    528                                         LUDivOrUMod* ins) {
    529  Register lhs = ToRegister(ins->numerator());
    530  [[maybe_unused]] Register output = ToRegister(ins->output());
    531  [[maybe_unused]] Register temp = ToRegister(ins->temp0());
    532  uint64_t d = ins->denominator();
    533 
    534  MOZ_ASSERT(lhs != rax && lhs != rdx);
    535  MOZ_ASSERT((output == rax && temp == rdx) || (output == rdx && temp == rax));
    536 
    537  // The denominator isn't a power of 2 (see LDivPowTwoI and LModPowTwoI).
    538  MOZ_ASSERT(!mozilla::IsPowerOfTwo(d));
    539 
    540  auto rmc = ReciprocalMulConstants::computeUnsignedDivisionConstants(d);
    541 
    542  // We first compute (M * n) >> 64, where M = rmc.multiplier.
    543  masm.movq(ImmWord(uint64_t(rmc.multiplier)), rax);
    544  masm.umulq(lhs);
    545  if (rmc.multiplier > Int128(UINT64_MAX)) {
    546    // M >= 2^64 and shift == 0 is impossible, as d >= 2 implies that
    547    // ((M * n) >> (64 + shift)) >= n > floor(n/d) whenever n >= d,
    548    // contradicting the proof of correctness in computeDivisionConstants.
    549    MOZ_ASSERT(rmc.shiftAmount > 0);
    550    MOZ_ASSERT(rmc.multiplier < (Int128(1) << 65));
    551 
    552    // We actually computed rdx = ((uint64_t(M) * n) >> 64) instead. Since
    553    // (M * n) >> (64 + shift) is the same as (rdx + n) >> shift, we can correct
    554    // for the overflow. This case is a bit trickier than the signed case,
    555    // though, as the (rdx + n) addition itself can overflow; however, note that
    556    // (rdx + n) >> shift == (((n - rdx) >> 1) + rdx) >> (shift - 1),
    557    // which is overflow-free. See Hacker's Delight, section 10-8 for details.
    558 
    559    // Compute (n - rdx) >> 1 into temp.
    560    masm.movq(lhs, rax);
    561    masm.subq(rdx, rax);
    562    masm.shrq(Imm32(1), rax);
    563 
    564    // Finish the computation.
    565    masm.addq(rax, rdx);
    566    masm.shrq(Imm32(rmc.shiftAmount - 1), rdx);
    567  } else {
    568    if (rmc.shiftAmount > 0) {
    569      masm.shrq(Imm32(rmc.shiftAmount), rdx);
    570    }
    571  }
    572 }
    573 
    574 void CodeGenerator::visitUDivConstantI64(LUDivConstantI64* ins) {
    575  uint64_t d = ins->denominator();
    576 
    577  // This emits the division answer into rdx.
    578  MOZ_ASSERT(ToRegister(ins->output()) == rdx);
    579  MOZ_ASSERT(ToRegister(ins->temp0()) == rax);
    580 
    581  if (d == 0) {
    582    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->mir()->trapSiteDesc());
    583    return;
    584  }
    585 
    586  // Compute the truncated division result in rdx.
    587  UnsignedDivide64WithConstant(masm, ins);
    588 }
    589 
    590 void CodeGenerator::visitUModConstantI64(LUModConstantI64* ins) {
    591  Register lhs = ToRegister(ins->numerator());
    592  uint64_t d = ins->denominator();
    593 
    594  // This emits the modulus answer into rax.
    595  MOZ_ASSERT(ToRegister(ins->output()) == rax);
    596  MOZ_ASSERT(ToRegister(ins->temp0()) == rdx);
    597 
    598  if (d == 0) {
    599    masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->mir()->trapSiteDesc());
    600    return;
    601  }
    602 
    603  // Compute the truncated division result in rdx.
    604  UnsignedDivide64WithConstant(masm, ins);
    605 
    606  // Compute the remainder in |rax|: rax = lhs - d * rdx
    607  masm.mul64(Imm64(d), Register64(rdx));
    608  masm.movq(lhs, rax);
    609  masm.subq(rdx, rax);
    610 }
    611 
    612 void CodeGeneratorX64::emitBigIntPtrDiv(LBigIntPtrDiv* ins, Register dividend,
    613                                        Register divisor, Register output) {
    614  // Callers handle division by zero and integer overflow.
    615 
    616  MOZ_ASSERT(ToRegister(ins->temp0()) == rdx);
    617  MOZ_ASSERT(output == rax);
    618 
    619  if (dividend != rax) {
    620    masm.movePtr(dividend, rax);
    621  }
    622 
    623  // Sign extend the lhs into rdx to make rdx:rax.
    624  masm.cqo();
    625 
    626  masm.idivq(divisor);
    627 }
    628 
    629 void CodeGeneratorX64::emitBigIntPtrMod(LBigIntPtrMod* ins, Register dividend,
    630                                        Register divisor, Register output) {
    631  // Callers handle division by zero and integer overflow.
    632 
    633  MOZ_ASSERT(dividend == rax);
    634  MOZ_ASSERT(output == rdx);
    635 
    636  // Sign extend the lhs into rdx to make rdx:rax.
    637  masm.cqo();
    638 
    639  masm.idivq(divisor);
    640 }
    641 
    642 void CodeGenerator::visitShiftIntPtr(LShiftIntPtr* ins) {
    643  Register lhs = ToRegister(ins->lhs());
    644  const LAllocation* rhs = ins->rhs();
    645  Register out = ToRegister(ins->output());
    646 
    647  if (rhs->isConstant()) {
    648    MOZ_ASSERT(out == lhs);
    649 
    650    int32_t shift = ToIntPtr(rhs) & 0x3f;
    651    switch (ins->bitop()) {
    652      case JSOp::Lsh:
    653        if (shift) {
    654          masm.lshiftPtr(Imm32(shift), lhs);
    655        }
    656        break;
    657      case JSOp::Rsh:
    658        if (shift) {
    659          masm.rshiftPtrArithmetic(Imm32(shift), lhs);
    660        }
    661        break;
    662      case JSOp::Ursh:
    663        if (shift) {
    664          masm.rshiftPtr(Imm32(shift), lhs);
    665        }
    666        break;
    667      default:
    668        MOZ_CRASH("Unexpected shift op");
    669    }
    670  } else {
    671    Register shift = ToRegister(rhs);
    672    MOZ_ASSERT_IF(out != lhs, Assembler::HasBMI2());
    673 
    674    switch (ins->bitop()) {
    675      case JSOp::Lsh:
    676        if (out != lhs) {
    677          masm.shlxq(lhs, shift, out);
    678        } else {
    679          masm.lshiftPtr(shift, lhs);
    680        }
    681        break;
    682      case JSOp::Rsh:
    683        if (out != lhs) {
    684          masm.sarxq(lhs, shift, out);
    685        } else {
    686          masm.rshiftPtrArithmetic(shift, lhs);
    687        }
    688        break;
    689      case JSOp::Ursh:
    690        if (out != lhs) {
    691          masm.shrxq(lhs, shift, out);
    692        } else {
    693          masm.rshiftPtr(shift, lhs);
    694        }
    695        break;
    696      default:
    697        MOZ_CRASH("Unexpected shift op");
    698    }
    699  }
    700 }
    701 
    702 void CodeGenerator::visitShiftI64(LShiftI64* lir) {
    703  Register lhs = ToRegister64(lir->lhs()).reg;
    704  const LAllocation* rhs = lir->rhs();
    705  Register out = ToOutRegister64(lir).reg;
    706 
    707  if (rhs->isConstant()) {
    708    MOZ_ASSERT(out == lhs);
    709 
    710    int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
    711    switch (lir->bitop()) {
    712      case JSOp::Lsh:
    713        if (shift) {
    714          masm.lshiftPtr(Imm32(shift), lhs);
    715        }
    716        break;
    717      case JSOp::Rsh:
    718        if (shift) {
    719          masm.rshiftPtrArithmetic(Imm32(shift), lhs);
    720        }
    721        break;
    722      case JSOp::Ursh:
    723        if (shift) {
    724          masm.rshiftPtr(Imm32(shift), lhs);
    725        }
    726        break;
    727      default:
    728        MOZ_CRASH("Unexpected shift op");
    729    }
    730    return;
    731  }
    732 
    733  Register shift = ToRegister(rhs);
    734  MOZ_ASSERT_IF(out != lhs, Assembler::HasBMI2());
    735 
    736  switch (lir->bitop()) {
    737    case JSOp::Lsh:
    738      if (out != lhs) {
    739        masm.shlxq(lhs, shift, out);
    740      } else {
    741        masm.lshiftPtr(shift, lhs);
    742      }
    743      break;
    744    case JSOp::Rsh:
    745      if (out != lhs) {
    746        masm.sarxq(lhs, shift, out);
    747      } else {
    748        masm.rshiftPtrArithmetic(shift, lhs);
    749      }
    750      break;
    751    case JSOp::Ursh:
    752      if (out != lhs) {
    753        masm.shrxq(lhs, shift, out);
    754      } else {
    755        masm.rshiftPtr(shift, lhs);
    756      }
    757      break;
    758    default:
    759      MOZ_CRASH("Unexpected shift op");
    760  }
    761 }
    762 
    763 void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
    764  Register elements = ToRegister(lir->elements());
    765  Register64 out = ToOutRegister64(lir);
    766 
    767  Scalar::Type storageType = lir->mir()->storageType();
    768 
    769  auto source = ToAddressOrBaseIndex(elements, lir->index(), storageType);
    770 
    771  // NOTE: the generated code must match the assembly code in gen_load in
    772  // GenerateAtomicOperations.py
    773  auto sync = Synchronization::Load();
    774 
    775  masm.memoryBarrierBefore(sync);
    776  source.match([&](const auto& source) { masm.load64(source, out); });
    777  masm.memoryBarrierAfter(sync);
    778 }
    779 
    780 void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
    781  Register elements = ToRegister(lir->elements());
    782  Register64 value = ToRegister64(lir->value());
    783 
    784  Scalar::Type writeType = lir->mir()->writeType();
    785 
    786  auto dest = ToAddressOrBaseIndex(elements, lir->index(), writeType);
    787 
    788  // NOTE: the generated code must match the assembly code in gen_store in
    789  // GenerateAtomicOperations.py
    790  auto sync = Synchronization::Store();
    791 
    792  masm.memoryBarrierBefore(sync);
    793  dest.match([&](const auto& dest) { masm.store64(value, dest); });
    794  masm.memoryBarrierAfter(sync);
    795 }
    796 
    797 void CodeGenerator::visitCompareExchangeTypedArrayElement64(
    798    LCompareExchangeTypedArrayElement64* lir) {
    799  Register elements = ToRegister(lir->elements());
    800  Register64 oldval = ToRegister64(lir->oldval());
    801  Register64 newval = ToRegister64(lir->newval());
    802  Register64 out = ToOutRegister64(lir);
    803 
    804  MOZ_ASSERT(out.reg == rax);
    805 
    806  Scalar::Type arrayType = lir->mir()->arrayType();
    807 
    808  auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);
    809 
    810  dest.match([&](const auto& dest) {
    811    masm.compareExchange64(Synchronization::Full(), dest, oldval, newval, out);
    812  });
    813 }
    814 
    815 void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
    816    LAtomicExchangeTypedArrayElement64* lir) {
    817  Register elements = ToRegister(lir->elements());
    818  Register64 value = ToRegister64(lir->value());
    819  Register64 out = ToOutRegister64(lir);
    820 
    821  Scalar::Type arrayType = lir->mir()->arrayType();
    822 
    823  auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);
    824 
    825  dest.match([&](const auto& dest) {
    826    masm.atomicExchange64(Synchronization::Full(), dest, value, out);
    827  });
    828 }
    829 
    830 void CodeGenerator::visitAtomicTypedArrayElementBinop64(
    831    LAtomicTypedArrayElementBinop64* lir) {
    832  MOZ_ASSERT(!lir->mir()->isForEffect());
    833 
    834  Register elements = ToRegister(lir->elements());
    835  Register64 value = ToRegister64(lir->value());
    836  Register64 temp = ToTempRegister64OrInvalid(lir->temp0());
    837  Register64 out = ToOutRegister64(lir);
    838 
    839  Scalar::Type arrayType = lir->mir()->arrayType();
    840  AtomicOp atomicOp = lir->mir()->operation();
    841 
    842  // Add and Sub don't need |temp| and can save a `mov` when the value and
    843  // output register are equal to each other.
    844  if (atomicOp == AtomicOp::Add || atomicOp == AtomicOp::Sub) {
    845    MOZ_ASSERT(temp == Register64::Invalid());
    846    MOZ_ASSERT(value == out);
    847  } else {
    848    MOZ_ASSERT(out.reg == rax);
    849  }
    850 
    851  auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);
    852 
    853  dest.match([&](const auto& dest) {
    854    masm.atomicFetchOp64(Synchronization::Full(), atomicOp, value, dest, temp,
    855                         out);
    856  });
    857 }
    858 
    859 void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
    860    LAtomicTypedArrayElementBinopForEffect64* lir) {
    861  MOZ_ASSERT(lir->mir()->isForEffect());
    862 
    863  Register elements = ToRegister(lir->elements());
    864  Register64 value = ToRegister64(lir->value());
    865 
    866  Scalar::Type arrayType = lir->mir()->arrayType();
    867  AtomicOp atomicOp = lir->mir()->operation();
    868 
    869  auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType);
    870 
    871  dest.match([&](const auto& dest) {
    872    masm.atomicEffectOp64(Synchronization::Full(), atomicOp, value, dest);
    873  });
    874 }
    875 
    876 void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
    877  MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
    878 
    879  Register cond = ToRegister(lir->condExpr());
    880 
    881  Operand falseExpr = ToOperandOrRegister64(lir->falseExpr());
    882 
    883  Register64 out = ToOutRegister64(lir);
    884  MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
    885             "true expr is reused for input");
    886 
    887  masm.test32(cond, cond);
    888  masm.cmovzq(falseExpr, out.reg);
    889 }
    890 
    891 // We expect to handle only the cases: compare is {U,}Int{32,64}, and select
    892 // is {U,}Int{32,64}, independently.  Some values may be stack allocated, and
    893 // the "true" input is reused for the output.
    894 void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
    895  bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
    896                    ins->compareType() == MCompare::Compare_UInt32;
    897  bool cmpIs64bit = ins->compareType() == MCompare::Compare_Int64 ||
    898                    ins->compareType() == MCompare::Compare_UInt64;
    899  bool selIs32bit = ins->mir()->type() == MIRType::Int32;
    900  bool selIs64bit = ins->mir()->type() == MIRType::Int64;
    901 
    902  // Throw out unhandled cases
    903  MOZ_RELEASE_ASSERT(
    904      cmpIs32bit != cmpIs64bit && selIs32bit != selIs64bit,
    905      "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
    906 
    907  using C = Assembler::Condition;
    908  using R = Register;
    909  using A = const Address&;
    910 
    911  // Identify macroassembler methods to generate instructions, based on the
    912  // type of the comparison and the select.  This avoids having to duplicate
    913  // the code-generation tree below 4 times.  These assignments to
    914  // `cmpMove_CRRRR` et al are unambiguous as a result of the combination of
    915  // the template parameters and the 5 argument types ((C, R, R, R, R) etc).
    916  void (MacroAssembler::*cmpMove_CRRRR)(C, R, R, R, R) = nullptr;
    917  void (MacroAssembler::*cmpMove_CRARR)(C, R, A, R, R) = nullptr;
    918  void (MacroAssembler::*cmpLoad_CRRAR)(C, R, R, A, R) = nullptr;
    919  void (MacroAssembler::*cmpLoad_CRAAR)(C, R, A, A, R) = nullptr;
    920 
    921  if (cmpIs32bit) {
    922    if (selIs32bit) {
    923      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 32>;
    924      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 32>;
    925      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 32>;
    926      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 32>;
    927    } else {
    928      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 64>;
    929      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 64>;
    930      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 64>;
    931      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 64>;
    932    }
    933  } else {
    934    if (selIs32bit) {
    935      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 32>;
    936      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 32>;
    937      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 32>;
    938      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 32>;
    939    } else {
    940      cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 64>;
    941      cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 64>;
    942      cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 64>;
    943      cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 64>;
    944    }
    945  }
    946 
    947  Register trueExprAndDest = ToRegister(ins->output());
    948  MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
    949             "true expr input is reused for output");
    950 
    951  Assembler::Condition cond = Assembler::InvertCondition(
    952      JSOpToCondition(ins->compareType(), ins->jsop()));
    953  const LAllocation* rhs = ins->rightExpr();
    954  const LAllocation* falseExpr = ins->ifFalseExpr();
    955  Register lhs = ToRegister(ins->leftExpr());
    956 
    957  // We generate one of four cmp+cmov pairings, depending on whether one of
    958  // the cmp args and one of the cmov args is in memory or a register.
    959  if (rhs->isGeneralReg()) {
    960    if (falseExpr->isGeneralReg()) {
    961      (masm.*cmpMove_CRRRR)(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
    962                            trueExprAndDest);
    963    } else {
    964      (masm.*cmpLoad_CRRAR)(cond, lhs, ToRegister(rhs), ToAddress(falseExpr),
    965                            trueExprAndDest);
    966    }
    967  } else {
    968    if (falseExpr->isGeneralReg()) {
    969      (masm.*cmpMove_CRARR)(cond, lhs, ToAddress(rhs), ToRegister(falseExpr),
    970                            trueExprAndDest);
    971    } else {
    972      (masm.*cmpLoad_CRAAR)(cond, lhs, ToAddress(rhs), ToAddress(falseExpr),
    973                            trueExprAndDest);
    974    }
    975  }
    976 }
    977 
    978 void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
    979  masm.convertUInt32ToDouble(ToRegister(lir->input()),
    980                             ToFloatRegister(lir->output()));
    981 }
    982 
    983 void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
    984  masm.convertUInt32ToFloat32(ToRegister(lir->input()),
    985                              ToFloatRegister(lir->output()));
    986 }
    987 
    988 void CodeGeneratorX64::wasmStore(const wasm::MemoryAccessDesc& access,
    989                                 const LAllocation* value, Operand dstAddr) {
    990  if (value->isConstant()) {
    991    masm.memoryBarrierBefore(access.sync());
    992 
    993    const MConstant* mir = value->toConstant();
    994    Imm32 cst =
    995        Imm32(mir->type() == MIRType::Int32 ? mir->toInt32() : mir->toInt64());
    996 
    997    switch (access.type()) {
    998      case Scalar::Int8:
    999      case Scalar::Uint8:
   1000        masm.append(access, wasm::TrapMachineInsn::Store8,
   1001                    FaultingCodeOffset(masm.currentOffset()));
   1002        masm.movb(cst, dstAddr);
   1003        break;
   1004      case Scalar::Int16:
   1005      case Scalar::Uint16:
   1006        masm.append(access, wasm::TrapMachineInsn::Store16,
   1007                    FaultingCodeOffset(masm.currentOffset()));
   1008        masm.movw(cst, dstAddr);
   1009        break;
   1010      case Scalar::Int32:
   1011      case Scalar::Uint32:
   1012        masm.append(access, wasm::TrapMachineInsn::Store32,
   1013                    FaultingCodeOffset(masm.currentOffset()));
   1014        masm.movl(cst, dstAddr);
   1015        break;
   1016      case Scalar::Int64:
   1017        MOZ_ASSERT_IF(mir->type() == MIRType::Int64,
   1018                      mozilla::CheckedInt32(mir->toInt64()).isValid());
   1019        masm.append(access, wasm::TrapMachineInsn::Store64,
   1020                    FaultingCodeOffset(masm.currentOffset()));
   1021        masm.movq(cst, dstAddr);
   1022        break;
   1023      case Scalar::Simd128:
   1024      case Scalar::Float16:
   1025      case Scalar::Float32:
   1026      case Scalar::Float64:
   1027      case Scalar::Uint8Clamped:
   1028      case Scalar::BigInt64:
   1029      case Scalar::BigUint64:
   1030      case Scalar::MaxTypedArrayViewType:
   1031        MOZ_CRASH("unexpected array type");
   1032    }
   1033 
   1034    masm.memoryBarrierAfter(access.sync());
   1035  } else {
   1036    masm.wasmStore(access, ToAnyRegister(value), dstAddr);
   1037  }
   1038 }
   1039 
   1040 template <typename T>
   1041 void CodeGeneratorX64::emitWasmLoad(T* ins) {
   1042  const MWasmLoad* mir = ins->mir();
   1043 
   1044  mir->access().assertOffsetInGuardPages();
   1045  uint32_t offset = mir->access().offset32();
   1046 
   1047  // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
   1048  // true 64-bit value.
   1049  const LAllocation* ptr = ins->ptr();
   1050  Register memoryBase = ToRegister(ins->memoryBase());
   1051  Operand srcAddr =
   1052      ptr->isBogus() ? Operand(memoryBase, offset)
   1053                     : Operand(memoryBase, ToRegister(ptr), TimesOne, offset);
   1054 
   1055  if (mir->type() == MIRType::Int64) {
   1056    masm.wasmLoadI64(mir->access(), srcAddr, ToOutRegister64(ins));
   1057  } else {
   1058    masm.wasmLoad(mir->access(), srcAddr, ToAnyRegister(ins->output()));
   1059  }
   1060 }
   1061 
   1062 void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { emitWasmLoad(ins); }
   1063 
   1064 void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { emitWasmLoad(ins); }
   1065 
   1066 template <typename T>
   1067 void CodeGeneratorX64::emitWasmStore(T* ins) {
   1068  const MWasmStore* mir = ins->mir();
   1069  const wasm::MemoryAccessDesc& access = mir->access();
   1070 
   1071  mir->access().assertOffsetInGuardPages();
   1072  uint32_t offset = access.offset32();
   1073 
   1074  const LAllocation* value = ins->value();
   1075  const LAllocation* ptr = ins->ptr();
   1076  Register memoryBase = ToRegister(ins->memoryBase());
   1077  Operand dstAddr =
   1078      ptr->isBogus() ? Operand(memoryBase, offset)
   1079                     : Operand(memoryBase, ToRegister(ptr), TimesOne, offset);
   1080 
   1081  wasmStore(access, value, dstAddr);
   1082 }
   1083 
   1084 void CodeGenerator::visitWasmStore(LWasmStore* ins) { emitWasmStore(ins); }
   1085 
   1086 void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) {
   1087  MOZ_CRASH("Unused on this platform");
   1088 }
   1089 
   1090 void CodeGenerator::visitWasmCompareExchangeHeap(
   1091    LWasmCompareExchangeHeap* ins) {
   1092  MWasmCompareExchangeHeap* mir = ins->mir();
   1093 
   1094  Register ptr = ToRegister(ins->ptr());
   1095  Register oldval = ToRegister(ins->oldValue());
   1096  Register newval = ToRegister(ins->newValue());
   1097  Register memoryBase = ToRegister(ins->memoryBase());
   1098 
   1099  Scalar::Type accessType = mir->access().type();
   1100  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset32());
   1101 
   1102  if (accessType == Scalar::Int64) {
   1103    masm.wasmCompareExchange64(mir->access(), srcAddr, Register64(oldval),
   1104                               Register64(newval), ToOutRegister64(ins));
   1105  } else {
   1106    masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval,
   1107                             ToRegister(ins->output()));
   1108  }
   1109 }
   1110 
   1111 void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
   1112  MWasmAtomicExchangeHeap* mir = ins->mir();
   1113 
   1114  Register ptr = ToRegister(ins->ptr());
   1115  Register value = ToRegister(ins->value());
   1116  Register memoryBase = ToRegister(ins->memoryBase());
   1117 
   1118  Scalar::Type accessType = mir->access().type();
   1119 
   1120  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset32());
   1121 
   1122  if (accessType == Scalar::Int64) {
   1123    masm.wasmAtomicExchange64(mir->access(), srcAddr, Register64(value),
   1124                              ToOutRegister64(ins));
   1125  } else {
   1126    masm.wasmAtomicExchange(mir->access(), srcAddr, value,
   1127                            ToRegister(ins->output()));
   1128  }
   1129 }
   1130 
   1131 void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
   1132  MWasmAtomicBinopHeap* mir = ins->mir();
   1133  MOZ_ASSERT(mir->hasUses());
   1134 
   1135  Register ptr = ToRegister(ins->ptr());
   1136  Register memoryBase = ToRegister(ins->memoryBase());
   1137  const LAllocation* value = ins->value();
   1138  Register temp = ToTempRegisterOrInvalid(ins->temp0());
   1139  Register output = ToRegister(ins->output());
   1140 
   1141  Scalar::Type accessType = mir->access().type();
   1142  if (accessType == Scalar::Uint32) {
   1143    accessType = Scalar::Int32;
   1144  }
   1145 
   1146  AtomicOp op = mir->operation();
   1147  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset32());
   1148 
   1149  if (accessType == Scalar::Int64) {
   1150    Register64 val = Register64(ToRegister(value));
   1151    Register64 out = Register64(output);
   1152    Register64 tmp = Register64(temp);
   1153    masm.wasmAtomicFetchOp64(mir->access(), op, val, srcAddr, tmp, out);
   1154  } else if (value->isConstant()) {
   1155    masm.wasmAtomicFetchOp(mir->access(), op, Imm32(ToInt32(value)), srcAddr,
   1156                           temp, output);
   1157  } else {
   1158    masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), srcAddr, temp,
   1159                           output);
   1160  }
   1161 }
   1162 
   1163 void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
   1164    LWasmAtomicBinopHeapForEffect* ins) {
   1165  MWasmAtomicBinopHeap* mir = ins->mir();
   1166  MOZ_ASSERT(!mir->hasUses());
   1167 
   1168  Register ptr = ToRegister(ins->ptr());
   1169  Register memoryBase = ToRegister(ins->memoryBase());
   1170  const LAllocation* value = ins->value();
   1171 
   1172  Scalar::Type accessType = mir->access().type();
   1173  AtomicOp op = mir->operation();
   1174 
   1175  BaseIndex srcAddr(memoryBase, ptr, TimesOne, mir->access().offset32());
   1176 
   1177  if (accessType == Scalar::Int64) {
   1178    Register64 val = Register64(ToRegister(value));
   1179    masm.wasmAtomicEffectOp64(mir->access(), op, val, srcAddr);
   1180  } else if (value->isConstant()) {
   1181    Imm32 c(0);
   1182    if (value->toConstant()->type() == MIRType::Int64) {
   1183      c = Imm32(ToInt64(value));
   1184    } else {
   1185      c = Imm32(ToInt32(value));
   1186    }
   1187    masm.wasmAtomicEffectOp(mir->access(), op, c, srcAddr, InvalidReg);
   1188  } else {
   1189    masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), srcAddr,
   1190                            InvalidReg);
   1191  }
   1192 }
   1193 
   1194 class js::jit::OutOfLineTruncate : public OutOfLineCodeBase<CodeGeneratorX64> {
   1195  FloatRegister input_;
   1196  Register output_;
   1197  Register temp_;
   1198 
   1199 public:
   1200  OutOfLineTruncate(FloatRegister input, Register output, Register temp)
   1201      : input_(input), output_(output), temp_(temp) {}
   1202 
   1203  void accept(CodeGeneratorX64* codegen) override {
   1204    codegen->visitOutOfLineTruncate(this);
   1205  }
   1206 
   1207  FloatRegister input() const { return input_; }
   1208  Register output() const { return output_; }
   1209  Register temp() const { return temp_; }
   1210 };
   1211 
   1212 void CodeGeneratorX64::visitOutOfLineTruncate(OutOfLineTruncate* ool) {
   1213  FloatRegister input = ool->input();
   1214  Register output = ool->output();
   1215  Register temp = ool->temp();
   1216 
   1217  // Inline implementation of `JS::ToInt32(double)` for double values whose
   1218  // exponent is ≥63.
   1219 
   1220 #ifdef DEBUG
   1221  Label ok;
   1222  masm.branchTruncateDoubleMaybeModUint32(input, output, &ok);
   1223  masm.assumeUnreachable("OOL path only used when vcvttsd2sq failed");
   1224  masm.bind(&ok);
   1225 #endif
   1226 
   1227  constexpr uint32_t ShiftedExponentBits =
   1228      mozilla::FloatingPoint<double>::kExponentBits >>
   1229      mozilla::FloatingPoint<double>::kExponentShift;
   1230  static_assert(ShiftedExponentBits == 0x7ff);
   1231 
   1232  constexpr uint32_t ExponentBiasAndShift =
   1233      mozilla::FloatingPoint<double>::kExponentBias +
   1234      mozilla::FloatingPoint<double>::kExponentShift;
   1235  static_assert(ExponentBiasAndShift == (1023 + 52));
   1236 
   1237  constexpr size_t ResultWidth = CHAR_BIT * sizeof(int32_t);
   1238 
   1239  // Extract the bit representation of |input|.
   1240  masm.moveDoubleToGPR64(input, Register64(output));
   1241 
   1242  // Extract the exponent.
   1243  masm.rshiftPtr(Imm32(mozilla::FloatingPoint<double>::kExponentShift), output,
   1244                 temp);
   1245  masm.and32(Imm32(ShiftedExponentBits), temp);
   1246 #ifdef DEBUG
   1247  // The biased exponent must be at least `1023 + 63`, because otherwise
   1248  // vcvttsd2sq wouldn't have failed.
   1249  constexpr uint32_t MinBiasedExponent =
   1250      mozilla::FloatingPoint<double>::kExponentBias + 63;
   1251 
   1252  Label exponentOk;
   1253  masm.branch32(Assembler::GreaterThanOrEqual, temp, Imm32(MinBiasedExponent),
   1254                &exponentOk);
   1255  masm.assumeUnreachable("exponent is greater-than-or-equals to 63");
   1256  masm.bind(&exponentOk);
   1257 #endif
   1258  masm.sub32(Imm32(ExponentBiasAndShift), temp);
   1259 
   1260  // If the exponent is greater than or equal to |ResultWidth|, the number is
   1261  // either infinite, NaN, or too large to have lower-order bits. We have to
   1262  // return zero in this case.
   1263  {
   1264    ScratchRegisterScope scratch(masm);
   1265    masm.movePtr(ImmWord(0), scratch);
   1266    masm.cmp32MovePtr(Assembler::AboveOrEqual, temp, Imm32(ResultWidth),
   1267                      scratch, output);
   1268  }
   1269 
   1270  // Negate if the sign bit is set.
   1271  {
   1272    ScratchRegisterScope scratch(masm);
   1273    masm.movePtr(output, scratch);
   1274    masm.negPtr(scratch);
   1275    masm.testPtr(output, output);
   1276    masm.cmovCCq(Assembler::Signed, scratch, output);
   1277  }
   1278 
   1279  // The significand contains the bits that will determine the final result.
   1280  // Shift those bits left by the exponent value in |temp|.
   1281  masm.lshift32(temp, output);
   1282 
   1283  // Return from OOL path.
   1284  masm.jump(ool->rejoin());
   1285 }
   1286 
   1287 void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
   1288  FloatRegister input = ToFloatRegister(ins->input());
   1289  Register output = ToRegister(ins->output());
   1290  Register temp = ToRegister(ins->temp0());
   1291 
   1292  auto* ool = new (alloc()) OutOfLineTruncate(input, output, temp);
   1293  addOutOfLineCode(ool, ins->mir());
   1294 
   1295  masm.branchTruncateDoubleMaybeModUint32(input, output, ool->entry());
   1296  masm.bind(ool->rejoin());
   1297 }
   1298 
   1299 void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
   1300    LWasmBuiltinTruncateDToInt32* lir) {
   1301  FloatRegister input = ToFloatRegister(lir->input());
   1302  Register output = ToRegister(lir->output());
   1303  Register temp = ToRegister(lir->temp0());
   1304  MOZ_ASSERT(lir->instance()->isBogus(), "instance not used for x64");
   1305 
   1306  auto* ool = new (alloc()) OutOfLineTruncate(input, output, temp);
   1307  addOutOfLineCode(ool, lir->mir());
   1308 
   1309  masm.branchTruncateDoubleMaybeModUint32(input, output, ool->entry());
   1310  masm.bind(ool->rejoin());
   1311 }
   1312 
   1313 void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
   1314    LWasmBuiltinTruncateFToInt32* lir) {
   1315  FloatRegister input = ToFloatRegister(lir->input());
   1316  Register output = ToRegister(lir->output());
   1317  MOZ_ASSERT(lir->instance()->isBogus(), "instance not used for x64");
   1318 
   1319  masm.truncateFloat32ModUint32(input, output);
   1320 }
   1321 
   1322 void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
   1323  FloatRegister input = ToFloatRegister(ins->input());
   1324  Register output = ToRegister(ins->output());
   1325 
   1326  masm.truncateFloat32ModUint32(input, output);
   1327 }
   1328 
   1329 void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
   1330  LInt64Allocation input = lir->input();
   1331  Register output = ToRegister(lir->output());
   1332 
   1333  if (lir->mir()->bottomHalf()) {
   1334    if (input.value().isMemory()) {
   1335      masm.load32(ToAddress(input), output);
   1336    } else {
   1337      masm.move64To32(ToRegister64(input), output);
   1338    }
   1339  } else {
   1340    MOZ_CRASH("Not implemented.");
   1341  }
   1342 }
   1343 
   1344 void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
   1345  const LAllocation* input = lir->input();
   1346  Register output = ToRegister(lir->output());
   1347 
   1348  if (lir->mir()->isUnsigned()) {
   1349    masm.movl(ToOperand(input), output);
   1350  } else {
   1351    masm.movslq(ToOperand(input), output);
   1352  }
   1353 }
   1354 
   1355 void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
   1356  // Generates no code on this platform because the input is assumed to have
   1357  // canonical form.
   1358  Register output = ToRegister(lir->output());
   1359  MOZ_ASSERT(ToRegister(lir->input()) == output);
   1360  masm.debugAssertCanonicalInt32(output);
   1361 }
   1362 
   1363 void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
   1364  // Generates no code on this platform because the input is assumed to have
   1365  // canonical form.
   1366  Register output = ToRegister(lir->output());
   1367  MOZ_ASSERT(ToRegister(lir->input()) == output);
   1368  masm.debugAssertCanonicalInt32(output);
   1369 }
   1370 
   1371 void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* ins) {
   1372  Register64 input = ToRegister64(ins->input());
   1373  Register64 output = ToOutRegister64(ins);
   1374  switch (ins->mir()->mode()) {
   1375    case MSignExtendInt64::Byte:
   1376      masm.movsbq(Operand(input.reg), output.reg);
   1377      break;
   1378    case MSignExtendInt64::Half:
   1379      masm.movswq(Operand(input.reg), output.reg);
   1380      break;
   1381    case MSignExtendInt64::Word:
   1382      masm.movslq(Operand(input.reg), output.reg);
   1383      break;
   1384  }
   1385 }
   1386 
   1387 void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
   1388  FloatRegister input = ToFloatRegister(lir->input());
   1389  Register64 output = ToOutRegister64(lir);
   1390 
   1391  MWasmTruncateToInt64* mir = lir->mir();
   1392  MIRType inputType = mir->input()->type();
   1393 
   1394  MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
   1395 
   1396  auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
   1397  addOutOfLineCode(ool, mir);
   1398 
   1399  FloatRegister temp =
   1400      mir->isUnsigned() ? ToFloatRegister(lir->temp0()) : InvalidFloatReg;
   1401 
   1402  Label* oolEntry = ool->entry();
   1403  Label* oolRejoin = ool->rejoin();
   1404  bool isSaturating = mir->isSaturating();
   1405  if (inputType == MIRType::Double) {
   1406    if (mir->isUnsigned()) {
   1407      masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
   1408                                      oolRejoin, temp);
   1409    } else {
   1410      masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
   1411                                     oolRejoin, temp);
   1412    }
   1413  } else {
   1414    if (mir->isUnsigned()) {
   1415      masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
   1416                                       oolRejoin, temp);
   1417    } else {
   1418      masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
   1419                                      oolRejoin, temp);
   1420    }
   1421  }
   1422 }
   1423 
   1424 void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
   1425  Register64 input = ToRegister64(lir->input());
   1426  FloatRegister output = ToFloatRegister(lir->output());
   1427  Register temp = ToTempRegisterOrInvalid(lir->temp0());
   1428 
   1429  MInt64ToFloatingPoint* mir = lir->mir();
   1430  bool isUnsigned = mir->isUnsigned();
   1431 
   1432  MIRType outputType = mir->type();
   1433  MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
   1434  MOZ_ASSERT(isUnsigned == (temp != InvalidReg));
   1435 
   1436  if (outputType == MIRType::Double) {
   1437    if (isUnsigned) {
   1438      masm.convertUInt64ToDouble(input, output, temp);
   1439    } else {
   1440      masm.convertInt64ToDouble(input, output);
   1441    }
   1442  } else {
   1443    if (isUnsigned) {
   1444      masm.convertUInt64ToFloat32(input, output, temp);
   1445    } else {
   1446      masm.convertInt64ToFloat32(input, output);
   1447    }
   1448  }
   1449 }
   1450 
   1451 void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
   1452  LInt64Allocation input = ins->input();
   1453  MOZ_ASSERT(!IsConstant(input));
   1454  Register64 inputR = ToRegister64(input);
   1455  MOZ_ASSERT(inputR == ToOutRegister64(ins));
   1456  masm.notq(inputR.reg);
   1457 }
   1458 
   1459 void CodeGenerator::visitAddIntPtr(LAddIntPtr* ins) {
   1460  Register lhs = ToRegister(ins->lhs());
   1461  MOZ_ASSERT(ToRegister(ins->output()) == lhs);
   1462 
   1463  if (ins->rhs()->isConstant()) {
   1464    masm.addPtr(ImmWord(ToIntPtr(ins->rhs())), lhs);
   1465  } else {
   1466    masm.addq(ToOperand(ins->rhs()), lhs);
   1467  }
   1468 }
   1469 
   1470 void CodeGenerator::visitSubIntPtr(LSubIntPtr* ins) {
   1471  Register lhs = ToRegister(ins->lhs());
   1472  MOZ_ASSERT(ToRegister(ins->output()) == lhs);
   1473 
   1474  if (ins->rhs()->isConstant()) {
   1475    masm.subPtr(ImmWord(ToIntPtr(ins->rhs())), lhs);
   1476  } else {
   1477    masm.subq(ToOperand(ins->rhs()), lhs);
   1478  }
   1479 }
   1480 
   1481 void CodeGenerator::visitMulIntPtr(LMulIntPtr* ins) {
   1482  Register lhs = ToRegister(ins->lhs());
   1483  MOZ_ASSERT(ToRegister(ins->output()) == lhs);
   1484  const LAllocation* rhs = ins->rhs();
   1485 
   1486  if (rhs->isConstant()) {
   1487    intptr_t constant = ToIntPtr(rhs);
   1488 
   1489    switch (constant) {
   1490      case -1:
   1491        masm.negPtr(lhs);
   1492        return;
   1493      case 0:
   1494        masm.xorPtr(lhs, lhs);
   1495        return;
   1496      case 1:
   1497        return;
   1498      case 2:
   1499        masm.addPtr(lhs, lhs);
   1500        return;
   1501    }
   1502 
   1503    // Use shift if constant is a power of 2.
   1504    if (constant > 0 && mozilla::IsPowerOfTwo(uintptr_t(constant))) {
   1505      uint32_t shift = mozilla::FloorLog2(constant);
   1506      masm.lshiftPtr(Imm32(shift), lhs);
   1507      return;
   1508    }
   1509 
   1510    masm.mulPtr(ImmWord(constant), lhs);
   1511  } else {
   1512    masm.imulq(ToOperand(rhs), lhs);
   1513  }
   1514 }
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE