[ tor-browser ].git.dasho

Lowering-arm64.cpp (48829B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "jit/arm64/Lowering-arm64.h"
      8 
      9 #include "mozilla/MathAlgorithms.h"
     10 
     11 #include "jit/arm64/Assembler-arm64.h"
     12 #include "jit/Lowering.h"
     13 #include "jit/MIR-wasm.h"
     14 #include "jit/MIR.h"
     15 #include "wasm/WasmFeatures.h"  // for wasm::ReportSimdAnalysis
     16 
     17 #include "jit/shared/Lowering-shared-inl.h"
     18 
     19 using namespace js;
     20 using namespace js::jit;
     21 
     22 LBoxAllocation LIRGeneratorARM64::useBoxFixed(MDefinition* mir, Register reg1,
     23                                              Register, bool useAtStart) {
     24  MOZ_ASSERT(mir->type() == MIRType::Value);
     25 
     26  ensureDefined(mir);
     27  return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
     28 }
     29 
     30 LAllocation LIRGeneratorARM64::useByteOpRegister(MDefinition* mir) {
     31  return useRegister(mir);
     32 }
     33 
     34 LAllocation LIRGeneratorARM64::useByteOpRegisterAtStart(MDefinition* mir) {
     35  return useRegisterAtStart(mir);
     36 }
     37 
     38 LAllocation LIRGeneratorARM64::useByteOpRegisterOrNonDoubleConstant(
     39    MDefinition* mir) {
     40  return useRegisterOrNonDoubleConstant(mir);
     41 }
     42 
     43 LDefinition LIRGeneratorARM64::tempByteOpRegister() { return temp(); }
     44 
     45 LDefinition LIRGeneratorARM64::tempToUnbox() { return temp(); }
     46 
     47 void LIRGenerator::visitBox(MBox* box) {
     48  MDefinition* opd = box->getOperand(0);
     49 
     50  // If the operand is a constant, emit near its uses.
     51  if (opd->isConstant() && box->canEmitAtUses()) {
     52    emitAtUses(box);
     53    return;
     54  }
     55 
     56  if (opd->isConstant()) {
     57    define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
     58           LDefinition(LDefinition::BOX));
     59  } else {
     60    LBox* ins = new (alloc()) LBox(useRegisterAtStart(opd), opd->type());
     61    define(ins, box, LDefinition(LDefinition::BOX));
     62  }
     63 }
     64 
     65 void LIRGenerator::visitUnbox(MUnbox* unbox) {
     66  MDefinition* box = unbox->getOperand(0);
     67  MOZ_ASSERT(box->type() == MIRType::Value);
     68 
     69  LInstructionHelper<1, BOX_PIECES, 0>* lir;
     70  if (IsFloatingPointType(unbox->type())) {
     71    MOZ_ASSERT(unbox->type() == MIRType::Double);
     72    lir = new (alloc()) LUnboxFloatingPoint(useBoxAtStart(box));
     73  } else if (unbox->fallible()) {
     74    // If the unbox is fallible, load the Value in a register first to
     75    // avoid multiple loads.
     76    lir = new (alloc()) LUnbox(useRegisterAtStart(box));
     77  } else {
     78    // FIXME: It should be possible to useAtStart() here, but the DEBUG
     79    // code in CodeGenerator::visitUnbox() needs to handle non-Register
     80    // cases. ARM64 doesn't have an Operand type.
     81    lir = new (alloc()) LUnbox(useRegisterAtStart(box));
     82  }
     83 
     84  if (unbox->fallible()) {
     85    assignSnapshot(lir, unbox->bailoutKind());
     86  }
     87 
     88  define(lir, unbox);
     89 }
     90 
     91 void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
     92  MOZ_ASSERT(opd->type() == MIRType::Value);
     93 
     94  LReturn* ins = new (alloc()) LReturn(isGenerator);
     95  ins->setOperand(0, useFixed(opd, JSReturnReg));
     96  add(ins);
     97 }
     98 
     99 // x = !y
    100 void LIRGeneratorARM64::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
    101                                    MDefinition* mir, MDefinition* input) {
    102  // Unary ALU operations don't read the input after writing to the output, even
    103  // for fallible operations, so we can use at-start allocations.
    104  ins->setOperand(0, useRegisterAtStart(input));
    105  define(ins, mir);
    106 }
    107 
    108 // z = x+y
    109 void LIRGeneratorARM64::lowerForALU(LInstructionHelper<1, 2, 0>* ins,
    110                                    MDefinition* mir, MDefinition* lhs,
    111                                    MDefinition* rhs) {
    112  // Binary ALU operations don't read any input after writing to the output,
    113  // even for fallible operations, so we can use at-start allocations.
    114  ins->setOperand(0, useRegisterAtStart(lhs));
    115  ins->setOperand(1, useRegisterOrConstantAtStart(rhs));
    116  define(ins, mir);
    117 }
    118 
    119 void LIRGeneratorARM64::lowerForFPU(LInstructionHelper<1, 1, 0>* ins,
    120                                    MDefinition* mir, MDefinition* input) {
    121  ins->setOperand(0, useRegisterAtStart(input));
    122  define(ins, mir);
    123 }
    124 
    125 void LIRGeneratorARM64::lowerForFPU(LInstructionHelper<1, 2, 0>* ins,
    126                                    MDefinition* mir, MDefinition* lhs,
    127                                    MDefinition* rhs) {
    128  ins->setOperand(0, useRegisterAtStart(lhs));
    129  ins->setOperand(1, useRegisterAtStart(rhs));
    130  define(ins, mir);
    131 }
    132 
    133 void LIRGeneratorARM64::lowerForALUInt64(
    134    LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
    135    MDefinition* input) {
    136  ins->setInt64Operand(0, useInt64RegisterAtStart(input));
    137  defineInt64(ins, mir);
    138 }
    139 
    140 void LIRGeneratorARM64::lowerForALUInt64(
    141    LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
    142    MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
    143  ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
    144  ins->setInt64Operand(INT64_PIECES, useInt64RegisterOrConstantAtStart(rhs));
    145  defineInt64(ins, mir);
    146 }
    147 
    148 void LIRGeneratorARM64::lowerForMulInt64(LMulI64* ins, MMul* mir,
    149                                         MDefinition* lhs, MDefinition* rhs) {
    150  lowerForALUInt64(ins, mir, lhs, rhs);
    151 }
    152 
    153 template <class LInstr>
    154 void LIRGeneratorARM64::lowerForShiftInt64(LInstr* ins, MDefinition* mir,
    155                                           MDefinition* lhs, MDefinition* rhs) {
    156  if constexpr (std::is_same_v<LInstr, LShiftI64>) {
    157    ins->setLhs(useInt64RegisterAtStart(lhs));
    158    ins->setRhs(useRegisterOrConstantAtStart(rhs));
    159  } else {
    160    ins->setInput(useInt64RegisterAtStart(lhs));
    161    ins->setCount(useRegisterOrConstantAtStart(rhs));
    162  }
    163  defineInt64(ins, mir);
    164 }
    165 
    166 template void LIRGeneratorARM64::lowerForShiftInt64(LShiftI64* ins,
    167                                                    MDefinition* mir,
    168                                                    MDefinition* lhs,
    169                                                    MDefinition* rhs);
    170 template void LIRGeneratorARM64::lowerForShiftInt64(LRotateI64* ins,
    171                                                    MDefinition* mir,
    172                                                    MDefinition* lhs,
    173                                                    MDefinition* rhs);
    174 
    175 void LIRGeneratorARM64::lowerWasmBuiltinTruncateToInt32(
    176    MWasmBuiltinTruncateToInt32* ins) {
    177  MDefinition* opd = ins->input();
    178  MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
    179 
    180  if (opd->type() == MIRType::Double) {
    181    define(new (alloc()) LWasmBuiltinTruncateDToInt32(
    182               useRegister(opd), useFixed(ins->instance(), InstanceReg),
    183               LDefinition::BogusTemp()),
    184           ins);
    185    return;
    186  }
    187 
    188  define(new (alloc()) LWasmBuiltinTruncateFToInt32(
    189             useRegister(opd), LAllocation(), LDefinition::BogusTemp()),
    190         ins);
    191 }
    192 
    193 void LIRGeneratorARM64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
    194                                             LBlock* block, size_t lirIndex) {
    195  lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
    196 }
    197 
    198 void LIRGeneratorARM64::lowerForShift(LInstructionHelper<1, 2, 0>* ins,
    199                                      MDefinition* mir, MDefinition* lhs,
    200                                      MDefinition* rhs) {
    201  lowerForALU(ins, mir, lhs, rhs);
    202 }
    203 
    204 void LIRGeneratorARM64::lowerDivI(MDiv* div) {
    205  if (div->rhs()->isConstant()) {
    206    LAllocation lhs = useRegister(div->lhs());
    207 
    208    int32_t rhs = div->rhs()->toConstant()->toInt32();
    209    int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs));
    210 
    211    if (rhs != 0 && uint32_t(1) << shift == mozilla::Abs(rhs)) {
    212      auto* lir = new (alloc()) LDivPowTwoI(lhs, shift, rhs < 0);
    213      if (div->fallible()) {
    214        assignSnapshot(lir, div->bailoutKind());
    215      }
    216      define(lir, div);
    217      return;
    218    }
    219 
    220    auto* lir = new (alloc()) LDivConstantI(lhs, rhs);
    221    if (div->fallible()) {
    222      assignSnapshot(lir, div->bailoutKind());
    223    }
    224    define(lir, div);
    225    return;
    226  }
    227 
    228  LAllocation lhs, rhs;
    229  if (div->canTruncateRemainder()) {
    230    lhs = useRegisterAtStart(div->lhs());
    231    rhs = useRegisterAtStart(div->rhs());
    232  } else {
    233    lhs = useRegister(div->lhs());
    234    rhs = useRegister(div->rhs());
    235  }
    236 
    237  // ARM64 has plenty of scratch registers, so we don't need to request an
    238  // additonal temp register from the register allocator.
    239  auto* lir = new (alloc()) LDivI(lhs, rhs, LDefinition::BogusTemp());
    240  if (div->fallible()) {
    241    assignSnapshot(lir, div->bailoutKind());
    242  }
    243  define(lir, div);
    244 }
    245 
    246 void LIRGeneratorARM64::lowerMulI(MMul* mul, MDefinition* lhs,
    247                                  MDefinition* rhs) {
    248  LMulI* lir = new (alloc()) LMulI;
    249  if (mul->fallible()) {
    250    assignSnapshot(lir, mul->bailoutKind());
    251  }
    252 
    253  // Negative zero check reads |lhs| and |rhs| after writing to the output, so
    254  // we can't use at-start allocations.
    255  if (mul->canBeNegativeZero() && !rhs->isConstant()) {
    256    lir->setOperand(0, useRegister(lhs));
    257    lir->setOperand(1, useRegister(rhs));
    258    define(lir, mul);
    259    return;
    260  }
    261 
    262  lowerForALU(lir, mul, lhs, rhs);
    263 }
    264 
    265 void LIRGeneratorARM64::lowerModI(MMod* mod) {
    266  LAllocation lhs = useRegister(mod->lhs());
    267 
    268  if (mod->rhs()->isConstant()) {
    269    int32_t rhs = mod->rhs()->toConstant()->toInt32();
    270    int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs));
    271 
    272    if (rhs != 0 && uint32_t(1) << shift == mozilla::Abs(rhs)) {
    273      auto* lir = new (alloc()) LModPowTwoI(lhs, shift);
    274      if (mod->fallible()) {
    275        assignSnapshot(lir, mod->bailoutKind());
    276      }
    277      define(lir, mod);
    278      return;
    279    }
    280 
    281    auto* lir = new (alloc()) LModConstantI(lhs, rhs);
    282    if (mod->fallible()) {
    283      assignSnapshot(lir, mod->bailoutKind());
    284    }
    285    define(lir, mod);
    286    return;
    287  }
    288 
    289  auto* lir = new (alloc()) LModI(lhs, useRegister(mod->rhs()));
    290  if (mod->fallible()) {
    291    assignSnapshot(lir, mod->bailoutKind());
    292  }
    293  define(lir, mod);
    294 }
    295 
    296 void LIRGeneratorARM64::lowerDivI64(MDiv* div) {
    297  if (div->rhs()->isConstant()) {
    298    LAllocation lhs = useRegister(div->lhs());
    299    int64_t rhs = div->rhs()->toConstant()->toInt64();
    300 
    301    if (mozilla::IsPowerOfTwo(mozilla::Abs(rhs))) {
    302      int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs));
    303 
    304      auto* lir = new (alloc()) LDivPowTwoI64(lhs, shift, rhs < 0);
    305      define(lir, div);
    306      return;
    307    }
    308 
    309    auto* lir = new (alloc()) LDivConstantI64(lhs, rhs);
    310    define(lir, div);
    311    return;
    312  }
    313 
    314  auto* lir = new (alloc())
    315      LDivI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs()));
    316  define(lir, div);
    317 }
    318 
    319 void LIRGeneratorARM64::lowerModI64(MMod* mod) {
    320  LAllocation lhs = useRegister(mod->lhs());
    321 
    322  if (mod->rhs()->isConstant()) {
    323    int64_t rhs = mod->rhs()->toConstant()->toInt64();
    324 
    325    if (mozilla::IsPowerOfTwo(mozilla::Abs(rhs))) {
    326      int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs));
    327 
    328      auto* lir = new (alloc()) LModPowTwoI64(lhs, shift);
    329      define(lir, mod);
    330      return;
    331    }
    332 
    333    auto* lir = new (alloc()) LModConstantI64(lhs, rhs);
    334    define(lir, mod);
    335    return;
    336  }
    337 
    338  auto* lir = new (alloc()) LModI64(lhs, useRegister(mod->rhs()));
    339  define(lir, mod);
    340 }
    341 
    342 void LIRGeneratorARM64::lowerUDivI64(MDiv* div) {
    343  if (div->rhs()->isConstant()) {
    344    LAllocation lhs = useRegister(div->lhs());
    345 
    346    // NOTE: the result of toInt64 is coerced to uint64_t.
    347    uint64_t rhs = div->rhs()->toConstant()->toInt64();
    348 
    349    if (mozilla::IsPowerOfTwo(rhs)) {
    350      int32_t shift = mozilla::FloorLog2(rhs);
    351 
    352      auto* lir = new (alloc()) LDivPowTwoI64(lhs, shift, false);
    353      define(lir, div);
    354      return;
    355    }
    356 
    357    auto* lir = new (alloc()) LUDivConstantI64(lhs, rhs);
    358    define(lir, div);
    359    return;
    360  }
    361 
    362  auto* lir = new (alloc())
    363      LUDivI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs()));
    364  define(lir, div);
    365 }
    366 
    367 void LIRGeneratorARM64::lowerUModI64(MMod* mod) {
    368  LAllocation lhs = useRegister(mod->lhs());
    369 
    370  if (mod->rhs()->isConstant()) {
    371    // NOTE: the result of toInt64 is coerced to uint64_t.
    372    uint64_t rhs = mod->rhs()->toConstant()->toInt64();
    373 
    374    if (mozilla::IsPowerOfTwo(rhs)) {
    375      int32_t shift = mozilla::FloorLog2(rhs);
    376 
    377      auto* lir = new (alloc()) LModPowTwoI64(lhs, shift);
    378      define(lir, mod);
    379      return;
    380    }
    381 
    382    auto* lir = new (alloc()) LUModConstantI64(lhs, rhs);
    383    define(lir, mod);
    384    return;
    385  }
    386 
    387  auto* lir = new (alloc()) LUModI64(lhs, useRegister(mod->rhs()));
    388  define(lir, mod);
    389 }
    390 
    391 void LIRGeneratorARM64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
    392  MOZ_CRASH("We don't use runtime div for this architecture");
    393 }
    394 
    395 void LIRGeneratorARM64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
    396  MOZ_CRASH("We don't use runtime mod for this architecture");
    397 }
    398 
    399 void LIRGeneratorARM64::lowerWasmSelectI(MWasmSelect* select) {
    400  if (select->type() == MIRType::Simd128) {
    401    LAllocation t = useRegisterAtStart(select->trueExpr());
    402    LAllocation f = useRegister(select->falseExpr());
    403    LAllocation c = useRegister(select->condExpr());
    404    auto* lir = new (alloc()) LWasmSelect(t, f, c);
    405    defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
    406  } else {
    407    LAllocation t = useRegisterAtStart(select->trueExpr());
    408    LAllocation f = useRegisterAtStart(select->falseExpr());
    409    LAllocation c = useRegisterAtStart(select->condExpr());
    410    define(new (alloc()) LWasmSelect(t, f, c), select);
    411  }
    412 }
    413 
    414 void LIRGeneratorARM64::lowerWasmSelectI64(MWasmSelect* select) {
    415  LInt64Allocation t = useInt64RegisterAtStart(select->trueExpr());
    416  LInt64Allocation f = useInt64RegisterAtStart(select->falseExpr());
    417  LAllocation c = useRegisterAtStart(select->condExpr());
    418  defineInt64(new (alloc()) LWasmSelectI64(t, f, c), select);
    419 }
    420 
    421 // On arm64 we specialize the cases: compare is {{U,}Int32, {U,}Int64},
    422 // Float32, Double}, and select is {{U,}Int32, {U,}Int64}, Float32, Double},
    423 // independently.
    424 bool LIRGeneratorARM64::canSpecializeWasmCompareAndSelect(
    425    MCompare::CompareType compTy, MIRType insTy) {
    426  return (insTy == MIRType::Int32 || insTy == MIRType::Int64 ||
    427          insTy == MIRType::Float32 || insTy == MIRType::Double) &&
    428         (compTy == MCompare::Compare_Int32 ||
    429          compTy == MCompare::Compare_UInt32 ||
    430          compTy == MCompare::Compare_Int64 ||
    431          compTy == MCompare::Compare_UInt64 ||
    432          compTy == MCompare::Compare_Float32 ||
    433          compTy == MCompare::Compare_Double);
    434 }
    435 
    436 void LIRGeneratorARM64::lowerWasmCompareAndSelect(MWasmSelect* ins,
    437                                                  MDefinition* lhs,
    438                                                  MDefinition* rhs,
    439                                                  MCompare::CompareType compTy,
    440                                                  JSOp jsop) {
    441  MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
    442  LAllocation rhsAlloc;
    443  if (compTy == MCompare::Compare_Float32 ||
    444      compTy == MCompare::Compare_Double) {
    445    rhsAlloc = useRegisterAtStart(rhs);
    446  } else if (compTy == MCompare::Compare_Int32 ||
    447             compTy == MCompare::Compare_UInt32 ||
    448             compTy == MCompare::Compare_Int64 ||
    449             compTy == MCompare::Compare_UInt64) {
    450    rhsAlloc = useRegisterOrConstantAtStart(rhs);
    451  } else {
    452    MOZ_CRASH("Unexpected type");
    453  }
    454  auto* lir = new (alloc()) LWasmCompareAndSelect(
    455      useRegisterAtStart(lhs), rhsAlloc, useRegisterAtStart(ins->trueExpr()),
    456      useRegisterAtStart(ins->falseExpr()), compTy, jsop);
    457  define(lir, ins);
    458 }
    459 
    460 LTableSwitch* LIRGeneratorARM64::newLTableSwitch(const LAllocation& in,
    461                                                 const LDefinition& inputCopy) {
    462  return new (alloc()) LTableSwitch(in, inputCopy, temp());
    463 }
    464 
    465 LTableSwitchV* LIRGeneratorARM64::newLTableSwitchV(const LBoxAllocation& in) {
    466  return new (alloc()) LTableSwitchV(in, temp(), tempDouble(), temp());
    467 }
    468 
    469 void LIRGeneratorARM64::lowerUrshD(MUrsh* mir) {
    470  MDefinition* lhs = mir->lhs();
    471  MDefinition* rhs = mir->rhs();
    472 
    473  MOZ_ASSERT(lhs->type() == MIRType::Int32);
    474  MOZ_ASSERT(rhs->type() == MIRType::Int32);
    475 
    476  LUrshD* lir = new (alloc())
    477      LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp());
    478  define(lir, mir);
    479 }
    480 
    481 void LIRGeneratorARM64::lowerPowOfTwoI(MPow* mir) {
    482  int32_t base = mir->input()->toConstant()->toInt32();
    483  MDefinition* power = mir->power();
    484 
    485  auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base);
    486  assignSnapshot(lir, mir->bailoutKind());
    487  define(lir, mir);
    488 }
    489 
    490 void LIRGeneratorARM64::lowerBigIntPtrLsh(MBigIntPtrLsh* ins) {
    491  auto* lir = new (alloc()) LBigIntPtrLsh(
    492      useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp());
    493  assignSnapshot(lir, ins->bailoutKind());
    494  define(lir, ins);
    495 }
    496 
    497 void LIRGeneratorARM64::lowerBigIntPtrRsh(MBigIntPtrRsh* ins) {
    498  auto* lir = new (alloc()) LBigIntPtrRsh(
    499      useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp());
    500  assignSnapshot(lir, ins->bailoutKind());
    501  define(lir, ins);
    502 }
    503 
    504 void LIRGeneratorARM64::lowerBigIntPtrDiv(MBigIntPtrDiv* ins) {
    505  auto* lir = new (alloc())
    506      LBigIntPtrDiv(useRegister(ins->lhs()), useRegister(ins->rhs()),
    507                    LDefinition::BogusTemp(), LDefinition::BogusTemp());
    508  assignSnapshot(lir, ins->bailoutKind());
    509  define(lir, ins);
    510 }
    511 
    512 void LIRGeneratorARM64::lowerBigIntPtrMod(MBigIntPtrMod* ins) {
    513  auto* lir = new (alloc())
    514      LBigIntPtrMod(useRegister(ins->lhs()), useRegister(ins->rhs()), temp(),
    515                    LDefinition::BogusTemp());
    516  if (ins->canBeDivideByZero()) {
    517    assignSnapshot(lir, ins->bailoutKind());
    518  }
    519  define(lir, ins);
    520 }
    521 
    522 #ifdef ENABLE_WASM_SIMD
    523 
    524 bool LIRGeneratorARM64::canFoldReduceSimd128AndBranch(wasm::SimdOp op) {
    525  switch (op) {
    526    case wasm::SimdOp::V128AnyTrue:
    527    case wasm::SimdOp::I8x16AllTrue:
    528    case wasm::SimdOp::I16x8AllTrue:
    529    case wasm::SimdOp::I32x4AllTrue:
    530    case wasm::SimdOp::I64x2AllTrue:
    531      return true;
    532    default:
    533      return false;
    534  }
    535 }
    536 
    537 bool LIRGeneratorARM64::canEmitWasmReduceSimd128AtUses(
    538    MWasmReduceSimd128* ins) {
    539  if (!ins->canEmitAtUses()) {
    540    return false;
    541  }
    542  // Only specific ops generating int32.
    543  if (ins->type() != MIRType::Int32) {
    544    return false;
    545  }
    546  if (!canFoldReduceSimd128AndBranch(ins->simdOp())) {
    547    return false;
    548  }
    549  // If never used then defer (it will be removed).
    550  MUseIterator iter(ins->usesBegin());
    551  if (iter == ins->usesEnd()) {
    552    return true;
    553  }
    554  // We require an MTest consumer.
    555  MNode* node = iter->consumer();
    556  if (!node->isDefinition() || !node->toDefinition()->isTest()) {
    557    return false;
    558  }
    559  // Defer only if there's only one use.
    560  iter++;
    561  return iter == ins->usesEnd();
    562 }
    563 
    564 #endif
    565 
    566 void LIRGeneratorARM64::lowerUDiv(MDiv* div) {
    567  if (div->rhs()->isConstant()) {
    568    LAllocation lhs = useRegister(div->lhs());
    569 
    570    // NOTE: the result of toInt32 is coerced to uint32_t.
    571    uint32_t rhs = div->rhs()->toConstant()->toInt32();
    572    int32_t shift = mozilla::FloorLog2(rhs);
    573 
    574    if (rhs != 0 && uint32_t(1) << shift == rhs) {
    575      auto* lir = new (alloc()) LDivPowTwoI(lhs, shift, false);
    576      if (div->fallible()) {
    577        assignSnapshot(lir, div->bailoutKind());
    578      }
    579      define(lir, div);
    580      return;
    581    }
    582 
    583    auto* lir = new (alloc()) LUDivConstant(lhs, rhs);
    584    if (div->fallible()) {
    585      assignSnapshot(lir, div->bailoutKind());
    586    }
    587    define(lir, div);
    588    return;
    589  }
    590 
    591  // Generate UDiv
    592  LAllocation lhs, rhs;
    593  if (div->canTruncateRemainder()) {
    594    lhs = useRegisterAtStart(div->lhs());
    595    rhs = useRegisterAtStart(div->rhs());
    596  } else {
    597    lhs = useRegister(div->lhs());
    598    rhs = useRegister(div->rhs());
    599  }
    600 
    601  auto* lir = new (alloc()) LUDiv(lhs, rhs);
    602  if (div->fallible()) {
    603    assignSnapshot(lir, div->bailoutKind());
    604  }
    605  define(lir, div);
    606 }
    607 
    608 void LIRGeneratorARM64::lowerUMod(MMod* mod) {
    609  LAllocation lhs = useRegister(mod->lhs());
    610 
    611  if (mod->rhs()->isConstant()) {
    612    // NOTE: the result of toInt32 is coerced to uint32_t.
    613    uint32_t rhs = mod->rhs()->toConstant()->toInt32();
    614    int32_t shift = mozilla::FloorLog2(rhs);
    615 
    616    if (rhs != 0 && uint32_t(1) << shift == rhs) {
    617      auto* lir = new (alloc()) LModPowTwoI(lhs, shift);
    618      if (mod->fallible()) {
    619        assignSnapshot(lir, mod->bailoutKind());
    620      }
    621      define(lir, mod);
    622      return;
    623    }
    624 
    625    auto* lir = new (alloc()) LUModConstant(lhs, rhs);
    626    if (mod->fallible()) {
    627      assignSnapshot(lir, mod->bailoutKind());
    628    }
    629    define(lir, mod);
    630    return;
    631  }
    632 
    633  auto* lir = new (alloc()) LUMod(lhs, useRegister(mod->rhs()));
    634  if (mod->fallible()) {
    635    assignSnapshot(lir, mod->bailoutKind());
    636  }
    637  define(lir, mod);
    638 }
    639 
    640 void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
    641  MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
    642  LWasmUint32ToDouble* lir =
    643      new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
    644  define(lir, ins);
    645 }
    646 
    647 void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
    648  MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
    649  LWasmUint32ToFloat32* lir =
    650      new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
    651  define(lir, ins);
    652 }
    653 
    654 void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) {
    655  MDefinition* base = ins->base();
    656  MOZ_ASSERT(base->type() == MIRType::Int32);
    657 
    658  MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
    659  MOZ_ASSERT_IF(ins->needsBoundsCheck(),
    660                boundsCheckLimit->type() == MIRType::Int32);
    661 
    662  LAllocation baseAlloc = useRegisterAtStart(base);
    663 
    664  LAllocation limitAlloc = ins->needsBoundsCheck()
    665                               ? useRegisterAtStart(boundsCheckLimit)
    666                               : LAllocation();
    667 
    668  // We have no memory-base value, meaning that HeapReg is to be used as the
    669  // memory base.  This follows from the definition of
    670  // FunctionCompiler::maybeLoadMemoryBase() in WasmIonCompile.cpp.
    671  MOZ_ASSERT(!ins->hasMemoryBase());
    672  auto* lir =
    673      new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation());
    674  define(lir, ins);
    675 }
    676 
    677 void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) {
    678  MDefinition* base = ins->base();
    679  MOZ_ASSERT(base->type() == MIRType::Int32);
    680 
    681  MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
    682  MOZ_ASSERT_IF(ins->needsBoundsCheck(),
    683                boundsCheckLimit->type() == MIRType::Int32);
    684 
    685  LAllocation baseAlloc = useRegisterAtStart(base);
    686 
    687  LAllocation limitAlloc = ins->needsBoundsCheck()
    688                               ? useRegisterAtStart(boundsCheckLimit)
    689                               : LAllocation();
    690 
    691  // See comment in LIRGenerator::visitAsmJSStoreHeap just above.
    692  MOZ_ASSERT(!ins->hasMemoryBase());
    693  add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()),
    694                                    limitAlloc, LAllocation()),
    695      ins);
    696 }
    697 
    698 void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
    699  MDefinition* base = ins->base();
    700  // See comment in visitWasmLoad re the type of 'base'.
    701  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
    702 
    703  LAllocation memoryBase = ins->hasMemoryBase()
    704                               ? LAllocation(useRegister(ins->memoryBase()))
    705                               : LGeneralReg(HeapReg);
    706 
    707  // Note, the access type may be Int64 here.
    708 
    709  LWasmCompareExchangeHeap* lir = new (alloc())
    710      LWasmCompareExchangeHeap(useRegister(base), useRegister(ins->oldValue()),
    711                               useRegister(ins->newValue()), memoryBase);
    712 
    713  define(lir, ins);
    714 }
    715 
    716 void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
    717  MDefinition* base = ins->base();
    718  // See comment in visitWasmLoad re the type of 'base'.
    719  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
    720 
    721  LAllocation memoryBase = ins->hasMemoryBase()
    722                               ? LAllocation(useRegister(ins->memoryBase()))
    723                               : LGeneralReg(HeapReg);
    724 
    725  // Note, the access type may be Int64 here.
    726 
    727  LWasmAtomicExchangeHeap* lir = new (alloc()) LWasmAtomicExchangeHeap(
    728      useRegister(base), useRegister(ins->value()), memoryBase);
    729  define(lir, ins);
    730 }
    731 
    732 void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
    733  MDefinition* base = ins->base();
    734  // See comment in visitWasmLoad re the type of 'base'.
    735  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
    736 
    737  LAllocation memoryBase = ins->hasMemoryBase()
    738                               ? LAllocation(useRegister(ins->memoryBase()))
    739                               : LGeneralReg(HeapReg);
    740 
    741  // Note, the access type may be Int64 here.
    742 
    743  if (!ins->hasUses()) {
    744    auto* lir = new (alloc()) LWasmAtomicBinopHeapForEffect(
    745        useRegister(base), useRegister(ins->value()), memoryBase, temp());
    746    add(lir, ins);
    747    return;
    748  }
    749 
    750  auto* lir = new (alloc()) LWasmAtomicBinopHeap(
    751      useRegister(base), useRegister(ins->value()), memoryBase, temp());
    752  define(lir, ins);
    753 }
    754 
    755 void LIRGeneratorARM64::lowerTruncateDToInt32(MTruncateToInt32* ins) {
    756  MDefinition* opd = ins->input();
    757  MOZ_ASSERT(opd->type() == MIRType::Double);
    758  define(new (alloc())
    759             LTruncateDToInt32(useRegister(opd), LDefinition::BogusTemp()),
    760         ins);
    761 }
    762 
    763 void LIRGeneratorARM64::lowerTruncateFToInt32(MTruncateToInt32* ins) {
    764  MDefinition* opd = ins->input();
    765  MOZ_ASSERT(opd->type() == MIRType::Float32);
    766  define(new (alloc())
    767             LTruncateFToInt32(useRegister(opd), LDefinition::BogusTemp()),
    768         ins);
    769 }
    770 
    771 void LIRGenerator::visitAtomicTypedArrayElementBinop(
    772    MAtomicTypedArrayElementBinop* ins) {
    773  MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
    774  MOZ_ASSERT(!Scalar::isFloatingType(ins->arrayType()));
    775  MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
    776  MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
    777 
    778  const LUse elements = useRegister(ins->elements());
    779  const LAllocation index =
    780      useRegisterOrIndexConstant(ins->index(), ins->arrayType());
    781 
    782  if (Scalar::isBigIntType(ins->arrayType())) {
    783    LInt64Allocation value = useInt64Register(ins->value());
    784    LInt64Definition temp = tempInt64();
    785 
    786    // Case 1: the result of the operation is not used.
    787 
    788    if (ins->isForEffect()) {
    789      auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
    790          elements, index, value, temp);
    791      add(lir, ins);
    792      return;
    793    }
    794 
    795    // Case 2: the result of the operation is used.
    796 
    797    auto* lir = new (alloc())
    798        LAtomicTypedArrayElementBinop64(elements, index, value, temp);
    799    defineInt64(lir, ins);
    800    return;
    801  }
    802 
    803  LAllocation value = useRegister(ins->value());
    804 
    805  if (ins->isForEffect()) {
    806    auto* lir = new (alloc())
    807        LAtomicTypedArrayElementBinopForEffect(elements, index, value, temp());
    808    add(lir, ins);
    809    return;
    810  }
    811 
    812  LDefinition tempDef1 = temp();
    813  LDefinition tempDef2 = LDefinition::BogusTemp();
    814  if (ins->arrayType() == Scalar::Uint32) {
    815    tempDef2 = temp();
    816  }
    817 
    818  LAtomicTypedArrayElementBinop* lir = new (alloc())
    819      LAtomicTypedArrayElementBinop(elements, index, value, tempDef1, tempDef2);
    820 
    821  define(lir, ins);
    822 }
    823 
    824 void LIRGenerator::visitCompareExchangeTypedArrayElement(
    825    MCompareExchangeTypedArrayElement* ins) {
    826  MOZ_ASSERT(!Scalar::isFloatingType(ins->arrayType()));
    827  MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
    828  MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
    829 
    830  const LUse elements = useRegister(ins->elements());
    831  const LAllocation index =
    832      useRegisterOrIndexConstant(ins->index(), ins->arrayType());
    833 
    834  if (Scalar::isBigIntType(ins->arrayType())) {
    835    LInt64Allocation oldval = useInt64Register(ins->oldval());
    836    LInt64Allocation newval = useInt64Register(ins->newval());
    837 
    838    auto* lir = new (alloc())
    839        LCompareExchangeTypedArrayElement64(elements, index, oldval, newval);
    840    defineInt64(lir, ins);
    841    return;
    842  }
    843 
    844  const LAllocation oldval = useRegister(ins->oldval());
    845  const LAllocation newval = useRegister(ins->newval());
    846 
    847  // If the target is an FPReg then we need a temporary at the CodeGenerator
    848  // level for creating the result.
    849 
    850  LDefinition outTemp = LDefinition::BogusTemp();
    851  if (ins->arrayType() == Scalar::Uint32) {
    852    outTemp = temp();
    853  }
    854 
    855  LCompareExchangeTypedArrayElement* lir =
    856      new (alloc()) LCompareExchangeTypedArrayElement(elements, index, oldval,
    857                                                      newval, outTemp);
    858 
    859  define(lir, ins);
    860 }
    861 
    862 void LIRGenerator::visitAtomicExchangeTypedArrayElement(
    863    MAtomicExchangeTypedArrayElement* ins) {
    864  MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
    865  MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
    866 
    867  const LUse elements = useRegister(ins->elements());
    868  const LAllocation index =
    869      useRegisterOrIndexConstant(ins->index(), ins->arrayType());
    870 
    871  if (Scalar::isBigIntType(ins->arrayType())) {
    872    LInt64Allocation value = useInt64Register(ins->value());
    873 
    874    auto* lir = new (alloc())
    875        LAtomicExchangeTypedArrayElement64(elements, index, value);
    876    defineInt64(lir, ins);
    877    return;
    878  }
    879 
    880  MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
    881 
    882  const LAllocation value = useRegister(ins->value());
    883 
    884  LDefinition tempDef = LDefinition::BogusTemp();
    885  if (ins->arrayType() == Scalar::Uint32) {
    886    tempDef = temp();
    887  }
    888 
    889  LAtomicExchangeTypedArrayElement* lir = new (alloc())
    890      LAtomicExchangeTypedArrayElement(elements, index, value, tempDef);
    891 
    892  define(lir, ins);
    893 }
    894 
    895 void LIRGeneratorARM64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
    896  const LUse elements = useRegister(ins->elements());
    897  const LAllocation index =
    898      useRegisterOrIndexConstant(ins->index(), ins->storageType());
    899 
    900  auto* lir = new (alloc()) LAtomicLoad64(elements, index);
    901  defineInt64(lir, ins);
    902 }
    903 
    904 void LIRGeneratorARM64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
    905  LUse elements = useRegister(ins->elements());
    906  LAllocation index =
    907      useRegisterOrIndexConstant(ins->index(), ins->writeType());
    908  LInt64Allocation value = useInt64Register(ins->value());
    909 
    910  add(new (alloc()) LAtomicStore64(elements, index, value), ins);
    911 }
    912 
    913 void LIRGenerator::visitSubstr(MSubstr* ins) {
    914  LSubstr* lir = new (alloc())
    915      LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
    916              useRegister(ins->length()), temp(), temp(), temp());
    917  define(lir, ins);
    918  assignSafepoint(lir, ins);
    919 }
    920 
    921 void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
    922  MDefinition* opd = ins->input();
    923  MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
    924 
    925  defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd)), ins);
    926 }
    927 
    928 void LIRGeneratorARM64::lowerWasmBuiltinTruncateToInt64(
    929    MWasmBuiltinTruncateToInt64* ins) {
    930  MOZ_CRASH("We don't use WasmBuiltinTruncateToInt64 for arm64");
    931 }
    932 
    933 void LIRGeneratorARM64::lowerBuiltinInt64ToFloatingPoint(
    934    MBuiltinInt64ToFloatingPoint* ins) {
    935  MOZ_CRASH("We don't use it for this architecture");
    936 }
    937 
    938 void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
    939  MDefinition* base = ins->base();
    940  // 'base' is a GPR but may be of either type.  If it is 32-bit it is
    941  // zero-extended and can act as 64-bit.
    942  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
    943 
    944  LAllocation memoryBase =
    945      ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase()))
    946                           : LGeneralReg(HeapReg);
    947  LAllocation ptr = useRegisterOrConstantAtStart(base);
    948 
    949  if (ins->type() == MIRType::Int64) {
    950    auto* lir = new (alloc()) LWasmLoadI64(ptr, memoryBase);
    951    defineInt64(lir, ins);
    952  } else {
    953    auto* lir = new (alloc()) LWasmLoad(ptr, memoryBase);
    954    define(lir, ins);
    955  }
    956 }
    957 
    958 void LIRGenerator::visitWasmStore(MWasmStore* ins) {
    959  MDefinition* base = ins->base();
    960  // See comment in visitWasmLoad re the type of 'base'.
    961  MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
    962 
    963  MDefinition* value = ins->value();
    964 
    965  LAllocation memoryBase =
    966      ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase()))
    967                           : LGeneralReg(HeapReg);
    968  LAllocation baseAlloc = useRegisterOrConstantAtStart(base);
    969 
    970  if (ins->access().type() == Scalar::Int64) {
    971    LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
    972    auto* lir = new (alloc()) LWasmStoreI64(baseAlloc, valueAlloc, memoryBase);
    973    add(lir, ins);
    974    return;
    975  }
    976 
    977  LAllocation valueAlloc = useRegisterAtStart(value);
    978  auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc, memoryBase);
    979  add(lir, ins);
    980 }
    981 
    982 void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
    983  MDefinition* opd = ins->input();
    984  MOZ_ASSERT(opd->type() == MIRType::Int64);
    985  MOZ_ASSERT(IsFloatingPointType(ins->type()));
    986 
    987  define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins);
    988 }
    989 
    990 void LIRGenerator::visitCopySign(MCopySign* ins) {
    991  MDefinition* lhs = ins->lhs();
    992  MDefinition* rhs = ins->rhs();
    993 
    994  MOZ_ASSERT(IsFloatingPointType(lhs->type()));
    995  MOZ_ASSERT(lhs->type() == rhs->type());
    996  MOZ_ASSERT(lhs->type() == ins->type());
    997 
    998  LInstructionHelper<1, 2, 0>* lir;
    999  if (lhs->type() == MIRType::Double) {
   1000    lir = new (alloc()) LCopySignD();
   1001  } else {
   1002    lir = new (alloc()) LCopySignF();
   1003  }
   1004 
   1005  lir->setOperand(0, useRegisterAtStart(lhs));
   1006  lir->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs)
   1007                         ? useRegister(rhs)
   1008                         : useRegisterAtStart(rhs));
   1009  // The copySignDouble and copySignFloat32 are optimized for lhs == output.
   1010  // It also prevents rhs == output when lhs != output, avoids clobbering.
   1011  defineReuseInput(lir, ins, 0);
   1012 }
   1013 
   1014 void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
   1015  defineInt64(
   1016      new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins);
   1017 }
   1018 
   1019 void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
   1020  defineInt64(new (alloc())
   1021                  LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
   1022              ins);
   1023 }
   1024 
   1025 void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
   1026 #ifdef ENABLE_WASM_SIMD
   1027  MOZ_ASSERT(ins->v0()->type() == MIRType::Simd128);
   1028  MOZ_ASSERT(ins->v1()->type() == MIRType::Simd128);
   1029  MOZ_ASSERT(ins->v2()->type() == MIRType::Simd128);
   1030  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1031 
   1032  switch (ins->simdOp()) {
   1033    case wasm::SimdOp::V128Bitselect: {
   1034      auto* lir = new (alloc())
   1035          LWasmTernarySimd128(useRegister(ins->v0()), useRegister(ins->v1()),
   1036                              useRegisterAtStart(ins->v2()),
   1037                              LDefinition::BogusTemp(), ins->simdOp());
   1038      // On ARM64, control register is used as output at machine instruction.
   1039      defineReuseInput(lir, ins, LWasmTernarySimd128::V2Index);
   1040      break;
   1041    }
   1042    case wasm::SimdOp::F32x4RelaxedMadd:
   1043    case wasm::SimdOp::F32x4RelaxedNmadd:
   1044    case wasm::SimdOp::F64x2RelaxedMadd:
   1045    case wasm::SimdOp::F64x2RelaxedNmadd: {
   1046      auto* lir = new (alloc())
   1047          LWasmTernarySimd128(useRegister(ins->v0()), useRegister(ins->v1()),
   1048                              useRegisterAtStart(ins->v2()),
   1049                              LDefinition::BogusTemp(), ins->simdOp());
   1050      defineReuseInput(lir, ins, LWasmTernarySimd128::V2Index);
   1051      break;
   1052    }
   1053    case wasm::SimdOp::I32x4RelaxedDotI8x16I7x16AddS: {
   1054      auto* lir = new (alloc()) LWasmTernarySimd128(
   1055          useRegister(ins->v0()), useRegister(ins->v1()),
   1056          useRegisterAtStart(ins->v2()), tempSimd128(), ins->simdOp());
   1057      defineReuseInput(lir, ins, LWasmTernarySimd128::V2Index);
   1058      break;
   1059    }
   1060    case wasm::SimdOp::I8x16RelaxedLaneSelect:
   1061    case wasm::SimdOp::I16x8RelaxedLaneSelect:
   1062    case wasm::SimdOp::I32x4RelaxedLaneSelect:
   1063    case wasm::SimdOp::I64x2RelaxedLaneSelect: {
   1064      auto* lir = new (alloc())
   1065          LWasmTernarySimd128(useRegister(ins->v0()), useRegister(ins->v1()),
   1066                              useRegisterAtStart(ins->v2()),
   1067                              LDefinition::BogusTemp(), ins->simdOp());
   1068      defineReuseInput(lir, ins, LWasmTernarySimd128::V2Index);
   1069      break;
   1070    }
   1071    default:
   1072      MOZ_CRASH("NYI");
   1073  }
   1074 #else
   1075  MOZ_CRASH("No SIMD");
   1076 #endif
   1077 }
   1078 
   1079 void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
   1080 #ifdef ENABLE_WASM_SIMD
   1081  MDefinition* lhs = ins->lhs();
   1082  MDefinition* rhs = ins->rhs();
   1083  wasm::SimdOp op = ins->simdOp();
   1084 
   1085  MOZ_ASSERT(lhs->type() == MIRType::Simd128);
   1086  MOZ_ASSERT(rhs->type() == MIRType::Simd128);
   1087  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1088 
   1089  LAllocation lhsAlloc = useRegisterAtStart(lhs);
   1090  LAllocation rhsAlloc = useRegisterAtStart(rhs);
   1091  LDefinition tempReg0 = LDefinition::BogusTemp();
   1092  LDefinition tempReg1 = LDefinition::BogusTemp();
   1093  if (op == wasm::SimdOp::I64x2Mul) {
   1094    tempReg0 = tempSimd128();
   1095    tempReg1 = tempSimd128();
   1096  }
   1097  auto* lir = new (alloc())
   1098      LWasmBinarySimd128(lhsAlloc, rhsAlloc, tempReg0, tempReg1, op);
   1099  define(lir, ins);
   1100 #else
   1101  MOZ_CRASH("No SIMD");
   1102 #endif
   1103 }
   1104 
   1105 #ifdef ENABLE_WASM_SIMD
   1106 bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle(
   1107    int8_t shuffle[16]) {
   1108  return false;
   1109 }
   1110 bool MWasmTernarySimd128::canRelaxBitselect() { return false; }
   1111 
   1112 bool MWasmBinarySimd128::canPmaddubsw() { return false; }
   1113 #endif
   1114 
   1115 bool MWasmBinarySimd128::specializeForConstantRhs() {
   1116  // Probably many we want to do here
   1117  return false;
   1118 }
   1119 
   1120 void LIRGenerator::visitWasmBinarySimd128WithConstant(
   1121    MWasmBinarySimd128WithConstant* ins) {
   1122  MOZ_CRASH("binary SIMD with constant NYI");
   1123 }
   1124 
   1125 void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
   1126 #ifdef ENABLE_WASM_SIMD
   1127  MDefinition* lhs = ins->lhs();
   1128  MDefinition* rhs = ins->rhs();
   1129 
   1130  MOZ_ASSERT(lhs->type() == MIRType::Simd128);
   1131  MOZ_ASSERT(rhs->type() == MIRType::Int32);
   1132  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1133 
   1134  if (rhs->isConstant()) {
   1135    int32_t shiftCount = rhs->toConstant()->toInt32();
   1136    switch (ins->simdOp()) {
   1137      case wasm::SimdOp::I8x16Shl:
   1138      case wasm::SimdOp::I8x16ShrU:
   1139      case wasm::SimdOp::I8x16ShrS:
   1140        shiftCount &= 7;
   1141        break;
   1142      case wasm::SimdOp::I16x8Shl:
   1143      case wasm::SimdOp::I16x8ShrU:
   1144      case wasm::SimdOp::I16x8ShrS:
   1145        shiftCount &= 15;
   1146        break;
   1147      case wasm::SimdOp::I32x4Shl:
   1148      case wasm::SimdOp::I32x4ShrU:
   1149      case wasm::SimdOp::I32x4ShrS:
   1150        shiftCount &= 31;
   1151        break;
   1152      case wasm::SimdOp::I64x2Shl:
   1153      case wasm::SimdOp::I64x2ShrU:
   1154      case wasm::SimdOp::I64x2ShrS:
   1155        shiftCount &= 63;
   1156        break;
   1157      default:
   1158        MOZ_CRASH("Unexpected shift operation");
   1159    }
   1160 #  ifdef DEBUG
   1161    js::wasm::ReportSimdAnalysis("shift -> constant shift");
   1162 #  endif
   1163    auto* lir = new (alloc())
   1164        LWasmConstantShiftSimd128(useRegisterAtStart(lhs), shiftCount);
   1165    define(lir, ins);
   1166    return;
   1167  }
   1168 
   1169 #  ifdef DEBUG
   1170  js::wasm::ReportSimdAnalysis("shift -> variable shift");
   1171 #  endif
   1172 
   1173  LAllocation lhsDestAlloc = useRegisterAtStart(lhs);
   1174  LAllocation rhsAlloc = useRegisterAtStart(rhs);
   1175  auto* lir = new (alloc()) LWasmVariableShiftSimd128(lhsDestAlloc, rhsAlloc);
   1176  define(lir, ins);
   1177 #else
   1178  MOZ_CRASH("No SIMD");
   1179 #endif
   1180 }
   1181 
   1182 void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
   1183 #ifdef ENABLE_WASM_SIMD
   1184  MOZ_ASSERT(ins->lhs()->type() == MIRType::Simd128);
   1185  MOZ_ASSERT(ins->rhs()->type() == MIRType::Simd128);
   1186  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1187 
   1188  SimdShuffle s = ins->shuffle();
   1189  switch (s.opd) {
   1190    case SimdShuffle::Operand::LEFT:
   1191    case SimdShuffle::Operand::RIGHT: {
   1192      LAllocation src;
   1193      switch (*s.permuteOp) {
   1194        case SimdPermuteOp::MOVE:
   1195        case SimdPermuteOp::BROADCAST_8x16:
   1196        case SimdPermuteOp::BROADCAST_16x8:
   1197        case SimdPermuteOp::PERMUTE_8x16:
   1198        case SimdPermuteOp::PERMUTE_16x8:
   1199        case SimdPermuteOp::PERMUTE_32x4:
   1200        case SimdPermuteOp::ROTATE_RIGHT_8x16:
   1201        case SimdPermuteOp::SHIFT_LEFT_8x16:
   1202        case SimdPermuteOp::SHIFT_RIGHT_8x16:
   1203        case SimdPermuteOp::REVERSE_16x8:
   1204        case SimdPermuteOp::REVERSE_32x4:
   1205        case SimdPermuteOp::REVERSE_64x2:
   1206        case SimdPermuteOp::ZERO_EXTEND_8x16_TO_16x8:
   1207        case SimdPermuteOp::ZERO_EXTEND_8x16_TO_32x4:
   1208        case SimdPermuteOp::ZERO_EXTEND_8x16_TO_64x2:
   1209        case SimdPermuteOp::ZERO_EXTEND_16x8_TO_32x4:
   1210        case SimdPermuteOp::ZERO_EXTEND_16x8_TO_64x2:
   1211        case SimdPermuteOp::ZERO_EXTEND_32x4_TO_64x2:
   1212          break;
   1213        default:
   1214          MOZ_CRASH("Unexpected operator");
   1215      }
   1216      if (s.opd == SimdShuffle::Operand::LEFT) {
   1217        src = useRegisterAtStart(ins->lhs());
   1218      } else {
   1219        src = useRegisterAtStart(ins->rhs());
   1220      }
   1221      auto* lir =
   1222          new (alloc()) LWasmPermuteSimd128(src, *s.permuteOp, s.control);
   1223      define(lir, ins);
   1224      break;
   1225    }
   1226    case SimdShuffle::Operand::BOTH:
   1227    case SimdShuffle::Operand::BOTH_SWAPPED: {
   1228      LAllocation lhs;
   1229      LAllocation rhs;
   1230      if (s.opd == SimdShuffle::Operand::BOTH) {
   1231        lhs = useRegisterAtStart(ins->lhs());
   1232        rhs = useRegisterAtStart(ins->rhs());
   1233      } else {
   1234        lhs = useRegisterAtStart(ins->rhs());
   1235        rhs = useRegisterAtStart(ins->lhs());
   1236      }
   1237      auto* lir =
   1238          new (alloc()) LWasmShuffleSimd128(lhs, rhs, *s.shuffleOp, s.control);
   1239      define(lir, ins);
   1240      break;
   1241    }
   1242  }
   1243 #else
   1244  MOZ_CRASH("No SIMD");
   1245 #endif
   1246 }
   1247 
   1248 void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
   1249 #ifdef ENABLE_WASM_SIMD
   1250  MOZ_ASSERT(ins->lhs()->type() == MIRType::Simd128);
   1251  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1252 
   1253  // Optimal code generation reuses the lhs register because the rhs scalar is
   1254  // merged into a vector lhs.
   1255  LAllocation lhs = useRegisterAtStart(ins->lhs());
   1256  if (ins->rhs()->type() == MIRType::Int64) {
   1257    auto* lir = new (alloc())
   1258        LWasmReplaceInt64LaneSimd128(lhs, useInt64Register(ins->rhs()));
   1259    defineReuseInput(lir, ins, 0);
   1260  } else {
   1261    auto* lir =
   1262        new (alloc()) LWasmReplaceLaneSimd128(lhs, useRegister(ins->rhs()));
   1263    defineReuseInput(lir, ins, 0);
   1264  }
   1265 #else
   1266  MOZ_CRASH("No SIMD");
   1267 #endif
   1268 }
   1269 
   1270 void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
   1271 #ifdef ENABLE_WASM_SIMD
   1272  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1273 
   1274  switch (ins->input()->type()) {
   1275    case MIRType::Int64: {
   1276      // 64-bit integer splats.
   1277      // Load-and-(sign|zero)extend.
   1278      auto* lir = new (alloc())
   1279          LWasmInt64ToSimd128(useInt64RegisterAtStart(ins->input()));
   1280      define(lir, ins);
   1281      break;
   1282    }
   1283    case MIRType::Float32:
   1284    case MIRType::Double: {
   1285      // Floating-point splats.
   1286      auto* lir =
   1287          new (alloc()) LWasmScalarToSimd128(useRegisterAtStart(ins->input()));
   1288      define(lir, ins);
   1289      break;
   1290    }
   1291    default: {
   1292      // 32-bit integer splats.
   1293      auto* lir =
   1294          new (alloc()) LWasmScalarToSimd128(useRegisterAtStart(ins->input()));
   1295      define(lir, ins);
   1296      break;
   1297    }
   1298  }
   1299 #else
   1300  MOZ_CRASH("No SIMD");
   1301 #endif
   1302 }
   1303 
   1304 void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
   1305 #ifdef ENABLE_WASM_SIMD
   1306  MOZ_ASSERT(ins->input()->type() == MIRType::Simd128);
   1307  MOZ_ASSERT(ins->type() == MIRType::Simd128);
   1308 
   1309  LDefinition tempReg = LDefinition::BogusTemp();
   1310  switch (ins->simdOp()) {
   1311    case wasm::SimdOp::I8x16Neg:
   1312    case wasm::SimdOp::I16x8Neg:
   1313    case wasm::SimdOp::I32x4Neg:
   1314    case wasm::SimdOp::I64x2Neg:
   1315    case wasm::SimdOp::F32x4Neg:
   1316    case wasm::SimdOp::F64x2Neg:
   1317    case wasm::SimdOp::F32x4Abs:
   1318    case wasm::SimdOp::F64x2Abs:
   1319    case wasm::SimdOp::V128Not:
   1320    case wasm::SimdOp::F32x4Sqrt:
   1321    case wasm::SimdOp::F64x2Sqrt:
   1322    case wasm::SimdOp::I8x16Abs:
   1323    case wasm::SimdOp::I16x8Abs:
   1324    case wasm::SimdOp::I32x4Abs:
   1325    case wasm::SimdOp::I64x2Abs:
   1326    case wasm::SimdOp::I32x4TruncSatF32x4S:
   1327    case wasm::SimdOp::F32x4ConvertI32x4U:
   1328    case wasm::SimdOp::I32x4TruncSatF32x4U:
   1329    case wasm::SimdOp::I16x8ExtendLowI8x16S:
   1330    case wasm::SimdOp::I16x8ExtendHighI8x16S:
   1331    case wasm::SimdOp::I16x8ExtendLowI8x16U:
   1332    case wasm::SimdOp::I16x8ExtendHighI8x16U:
   1333    case wasm::SimdOp::I32x4ExtendLowI16x8S:
   1334    case wasm::SimdOp::I32x4ExtendHighI16x8S:
   1335    case wasm::SimdOp::I32x4ExtendLowI16x8U:
   1336    case wasm::SimdOp::I32x4ExtendHighI16x8U:
   1337    case wasm::SimdOp::I64x2ExtendLowI32x4S:
   1338    case wasm::SimdOp::I64x2ExtendHighI32x4S:
   1339    case wasm::SimdOp::I64x2ExtendLowI32x4U:
   1340    case wasm::SimdOp::I64x2ExtendHighI32x4U:
   1341    case wasm::SimdOp::F32x4ConvertI32x4S:
   1342    case wasm::SimdOp::F32x4Ceil:
   1343    case wasm::SimdOp::F32x4Floor:
   1344    case wasm::SimdOp::F32x4Trunc:
   1345    case wasm::SimdOp::F32x4Nearest:
   1346    case wasm::SimdOp::F64x2Ceil:
   1347    case wasm::SimdOp::F64x2Floor:
   1348    case wasm::SimdOp::F64x2Trunc:
   1349    case wasm::SimdOp::F64x2Nearest:
   1350    case wasm::SimdOp::F32x4DemoteF64x2Zero:
   1351    case wasm::SimdOp::F64x2PromoteLowF32x4:
   1352    case wasm::SimdOp::F64x2ConvertLowI32x4S:
   1353    case wasm::SimdOp::F64x2ConvertLowI32x4U:
   1354    case wasm::SimdOp::I16x8ExtaddPairwiseI8x16S:
   1355    case wasm::SimdOp::I16x8ExtaddPairwiseI8x16U:
   1356    case wasm::SimdOp::I32x4ExtaddPairwiseI16x8S:
   1357    case wasm::SimdOp::I32x4ExtaddPairwiseI16x8U:
   1358    case wasm::SimdOp::I8x16Popcnt:
   1359    case wasm::SimdOp::I32x4RelaxedTruncF32x4S:
   1360    case wasm::SimdOp::I32x4RelaxedTruncF32x4U:
   1361    case wasm::SimdOp::I32x4RelaxedTruncF64x2SZero:
   1362    case wasm::SimdOp::I32x4RelaxedTruncF64x2UZero:
   1363      break;
   1364    case wasm::SimdOp::I32x4TruncSatF64x2SZero:
   1365    case wasm::SimdOp::I32x4TruncSatF64x2UZero:
   1366      tempReg = tempSimd128();
   1367      break;
   1368    default:
   1369      MOZ_CRASH("Unary SimdOp not implemented");
   1370  }
   1371 
   1372  LUse input = useRegisterAtStart(ins->input());
   1373  LWasmUnarySimd128* lir = new (alloc()) LWasmUnarySimd128(input, tempReg);
   1374  define(lir, ins);
   1375 #else
   1376  MOZ_CRASH("No SIMD");
   1377 #endif
   1378 }
   1379 
   1380 void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
   1381 #ifdef ENABLE_WASM_SIMD
   1382  if (canEmitWasmReduceSimd128AtUses(ins)) {
   1383    emitAtUses(ins);
   1384    return;
   1385  }
   1386 
   1387  // Reductions (any_true, all_true, bitmask, extract_lane) uniformly prefer
   1388  // useRegisterAtStart:
   1389  //
   1390  // - In most cases, the input type differs from the output type, so there's no
   1391  //   conflict and it doesn't really matter.
   1392  //
   1393  // - For extract_lane(0) on F32x4 and F64x2, input == output results in zero
   1394  //   code being generated.
   1395  //
   1396  // - For extract_lane(k > 0) on F32x4 and F64x2, allowing the input register
   1397  //   to be targeted lowers register pressure if it's the last use of the
   1398  //   input.
   1399 
   1400  if (ins->type() == MIRType::Int64) {
   1401    auto* lir = new (alloc())
   1402        LWasmReduceSimd128ToInt64(useRegisterAtStart(ins->input()));
   1403    defineInt64(lir, ins);
   1404  } else {
   1405    LDefinition tempReg = LDefinition::BogusTemp();
   1406    switch (ins->simdOp()) {
   1407      case wasm::SimdOp::I8x16Bitmask:
   1408      case wasm::SimdOp::I16x8Bitmask:
   1409      case wasm::SimdOp::I32x4Bitmask:
   1410      case wasm::SimdOp::I64x2Bitmask:
   1411        tempReg = tempSimd128();
   1412        break;
   1413      default:
   1414        break;
   1415    }
   1416 
   1417    // Ideally we would reuse the input register for floating extract_lane if
   1418    // the lane is zero, but constraints in the register allocator require the
   1419    // input and output register types to be the same.
   1420    auto* lir = new (alloc())
   1421        LWasmReduceSimd128(useRegisterAtStart(ins->input()), tempReg);
   1422    define(lir, ins);
   1423  }
   1424 #else
   1425  MOZ_CRASH("No SIMD");
   1426 #endif
   1427 }
   1428 
   1429 void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
   1430 #ifdef ENABLE_WASM_SIMD
   1431  // On 64-bit systems, the base pointer can be 32 bits or 64 bits.  Either way,
   1432  // it fits in a GPR so we can ignore the Register/Register64 distinction here.
   1433 
   1434  // Optimal allocation here reuses the value input for the output register
   1435  // because codegen otherwise has to copy the input to the output; this is
   1436  // because load-lane is implemented as load + replace-lane.  Bug 1706106 may
   1437  // change all of that, so leave it alone for now.
   1438  LUse base = useRegisterAtStart(ins->base());
   1439  LUse inputUse = useRegisterAtStart(ins->value());
   1440  LAllocation memoryBase =
   1441      ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase()))
   1442                           : LGeneralReg(HeapReg);
   1443  auto* lir =
   1444      new (alloc()) LWasmLoadLaneSimd128(base, inputUse, memoryBase, temp());
   1445  define(lir, ins);
   1446 #else
   1447  MOZ_CRASH("No SIMD");
   1448 #endif
   1449 }
   1450 
   1451 void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
   1452 #ifdef ENABLE_WASM_SIMD
   1453  // See comment above about the base pointer.
   1454 
   1455  LUse base = useRegisterAtStart(ins->base());
   1456  LUse input = useRegisterAtStart(ins->value());
   1457  LAllocation memoryBase =
   1458      ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase()))
   1459                           : LGeneralReg(HeapReg);
   1460  auto* lir =
   1461      new (alloc()) LWasmStoreLaneSimd128(base, input, memoryBase, temp());
   1462  add(lir, ins);
   1463 #else
   1464  MOZ_CRASH("No SIMD");
   1465 #endif
   1466 }
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE