[ tor-browser ].git.dasho

MacroAssembler-riscv64.cpp (231499B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 // Copyright 2021 the V8 project authors. All rights reserved.
      8 // Use of this source code is governed by a BSD-style license that can be
      9 // found in the LICENSE file.
     10 #include "jit/riscv64/MacroAssembler-riscv64.h"
     11 
     12 #include "jsmath.h"
     13 #include "jit/Bailouts.h"
     14 #include "jit/BaselineFrame.h"
     15 #include "jit/JitFrames.h"
     16 #include "jit/JitRuntime.h"
     17 #include "jit/MacroAssembler.h"
     18 #include "jit/MoveEmitter.h"
     19 #include "jit/riscv64/SharedICRegisters-riscv64.h"
     20 #include "util/Memory.h"
     21 #include "vm/JitActivation.h"  // jit::JitActivation
     22 #include "vm/JSContext.h"
     23 #include "wasm/WasmStubs.h"
     24 
     25 #include "jit/MacroAssembler-inl.h"
     26 
     27 namespace js {
     28 namespace jit {
     29 
     30 MacroAssembler& MacroAssemblerRiscv64::asMasm() {
     31  return *static_cast<MacroAssembler*>(this);
     32 }
     33 
     34 const MacroAssembler& MacroAssemblerRiscv64::asMasm() const {
     35  return *static_cast<const MacroAssembler*>(this);
     36 }
     37 
     38 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Register rj, ImmWord imm,
     39                                       Condition c) {
     40  if (imm.value <= INT32_MAX) {
     41    ma_cmp_set(rd, rj, Imm32(uint32_t(imm.value)), c);
     42  } else {
     43    UseScratchRegisterScope temps(this);
     44    Register scratch = temps.Acquire();
     45    ma_li(scratch, imm);
     46    ma_cmp_set(rd, rj, scratch, c);
     47  }
     48 }
     49 
     50 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Register rj, ImmPtr imm,
     51                                       Condition c) {
     52  ma_cmp_set(rd, rj, ImmWord(uintptr_t(imm.value)), c);
     53 }
     54 
     55 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Register rj, ImmGCPtr imm,
     56                                       Condition c) {
     57  UseScratchRegisterScope temps(this);
     58  Register scratch = temps.Acquire();
     59  ma_li(scratch, imm);
     60  ma_cmp_set(rd, rj, scratch, c);
     61 }
     62 
     63 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Address address,
     64                                       Register rhs, Condition c) {
     65  UseScratchRegisterScope temps(this);
     66  Register scratch2 = temps.Acquire();
     67  ma_load(scratch2, address, SizeDouble);
     68  ma_cmp_set(rd, Register(scratch2), rhs, c);
     69 }
     70 
     71 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Address address, Imm32 imm,
     72                                       Condition c) {
     73  // TODO(riscv): 32-bit ma_cmp_set?
     74  UseScratchRegisterScope temps(this);
     75  Register scratch2 = temps.Acquire();
     76  ma_load(scratch2, address, SizeWord);
     77  ma_cmp_set(rd, Register(scratch2), imm, c);
     78 }
     79 
     80 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Address address,
     81                                       ImmWord imm, Condition c) {
     82  UseScratchRegisterScope temps(this);
     83  Register scratch2 = temps.Acquire();
     84  ma_load(scratch2, address, SizeDouble);
     85  ma_cmp_set(rd, Register(scratch2), imm, c);
     86 }
     87 
     88 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Register rj, Imm32 imm,
     89                                       Condition c) {
     90  if (imm.value == 0) {
     91    switch (c) {
     92      case Equal:
     93      case BelowOrEqual:
     94        ma_sltu(rd, rj, Operand(1));
     95        break;
     96      case NotEqual:
     97      case Above:
     98        sltu(rd, zero, rj);
     99        break;
    100      case AboveOrEqual:
    101      case Below:
    102        ori(rd, zero, c == AboveOrEqual ? 1 : 0);
    103        break;
    104      case GreaterThan:
    105      case LessThanOrEqual:
    106        slt(rd, zero, rj);
    107        if (c == LessThanOrEqual) {
    108          xori(rd, rd, 1);
    109        }
    110        break;
    111      case LessThan:
    112      case GreaterThanOrEqual:
    113        slt(rd, rj, zero);
    114        if (c == GreaterThanOrEqual) {
    115          xori(rd, rd, 1);
    116        }
    117        break;
    118      case Zero:
    119        ma_sltu(rd, rj, Operand(1));
    120        break;
    121      case NonZero:
    122        sltu(rd, zero, rj);
    123        break;
    124      case Signed:
    125        slt(rd, rj, zero);
    126        break;
    127      case NotSigned:
    128        slt(rd, rj, zero);
    129        xori(rd, rd, 1);
    130        break;
    131      default:
    132        MOZ_CRASH("Invalid condition.");
    133    }
    134    return;
    135  }
    136 
    137  switch (c) {
    138    case Equal:
    139    case NotEqual:
    140      ma_xor(rd, rj, imm);
    141      if (c == Equal) {
    142        ma_sltu(rd, rd, Operand(1));
    143      } else {
    144        sltu(rd, zero, rd);
    145      }
    146      break;
    147    case Zero:
    148    case NonZero:
    149    case Signed:
    150    case NotSigned:
    151      MOZ_CRASH("Invalid condition.");
    152    default:
    153      Condition cond = ma_cmp(rd, rj, imm, c);
    154      MOZ_ASSERT(cond == Equal || cond == NotEqual);
    155 
    156      if (cond == Equal) xori(rd, rd, 1);
    157  }
    158 }
    159 
    160 Assembler::Condition MacroAssemblerRiscv64::ma_cmp(Register dest, Register lhs,
    161                                                   Register rhs, Condition c) {
    162  switch (c) {
    163    case Above:
    164      // bgtu s,t,label =>
    165      //   sltu at,t,s
    166      //   bne at,$zero,offs
    167      sltu(dest, rhs, lhs);
    168      return NotEqual;
    169    case AboveOrEqual:
    170      // bgeu s,t,label =>
    171      //   sltu at,s,t
    172      //   beq at,$zero,offs
    173      sltu(dest, lhs, rhs);
    174      return Equal;
    175    case Below:
    176      // bltu s,t,label =>
    177      //   sltu at,s,t
    178      //   bne at,$zero,offs
    179      sltu(dest, lhs, rhs);
    180      return NotEqual;
    181    case BelowOrEqual:
    182      // bleu s,t,label =>
    183      //   sltu at,t,s
    184      //   beq at,$zero,offs
    185      sltu(dest, rhs, lhs);
    186      return Equal;
    187    case GreaterThan:
    188      // bgt s,t,label =>
    189      //   slt at,t,s
    190      //   bne at,$zero,offs
    191      slt(dest, rhs, lhs);
    192      return NotEqual;
    193    case GreaterThanOrEqual:
    194      // bge s,t,label =>
    195      //   slt at,s,t
    196      //   beq at,$zero,offs
    197      slt(dest, lhs, rhs);
    198      return Equal;
    199    case LessThan:
    200      // blt s,t,label =>
    201      //   slt at,s,t
    202      //   bne at,$zero,offs
    203      slt(dest, lhs, rhs);
    204      return NotEqual;
    205    case LessThanOrEqual:
    206      // ble s,t,label =>
    207      //   slt at,t,s
    208      //   beq at,$zero,offs
    209      slt(dest, rhs, lhs);
    210      return Equal;
    211    default:
    212      MOZ_CRASH("Invalid condition.");
    213  }
    214  return Always;
    215 }
    216 
    217 Assembler::Condition MacroAssemblerRiscv64::ma_cmp(Register dest, Register lhs,
    218                                                   Imm32 imm, Condition c) {
    219  UseScratchRegisterScope temps(this);
    220  Register scratch = temps.Acquire();
    221  MOZ_RELEASE_ASSERT(lhs != scratch);
    222 
    223  switch (c) {
    224    case Above:
    225    case BelowOrEqual:
    226      if (imm.value != 0x7fffffff && is_intn(imm.value + 1, 12) &&
    227          imm.value != -1) {
    228        // lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow
    229        ma_sltu(dest, lhs, Operand(imm.value + 1));
    230 
    231        return (c == BelowOrEqual ? NotEqual : Equal);
    232      } else {
    233        ma_li(scratch, imm);
    234        sltu(dest, scratch, lhs);
    235        return (c == BelowOrEqual ? Equal : NotEqual);
    236      }
    237    case AboveOrEqual:
    238    case Below:
    239      if (is_intn(imm.value, 12)) {
    240        ma_sltu(dest, lhs, Operand(imm.value));
    241      } else {
    242        ma_li(scratch, imm);
    243        sltu(dest, lhs, scratch);
    244      }
    245      return (c == AboveOrEqual ? Equal : NotEqual);
    246    case GreaterThan:
    247    case LessThanOrEqual:
    248      if (imm.value != 0x7fffffff && is_intn(imm.value + 1, 12)) {
    249        // lhs <= rhs via lhs < rhs + 1.
    250        ma_slt(dest, lhs, Operand(imm.value + 1));
    251        return (c == LessThanOrEqual ? NotEqual : Equal);
    252      } else {
    253        ma_li(scratch, imm);
    254        slt(dest, scratch, lhs);
    255        return (c == LessThanOrEqual ? Equal : NotEqual);
    256      }
    257    case GreaterThanOrEqual:
    258    case LessThan:
    259      if (is_intn(imm.value, 12)) {
    260        ma_slt(dest, lhs, imm);
    261      } else {
    262        ma_li(scratch, imm);
    263        slt(dest, lhs, scratch);
    264      }
    265      return (c == GreaterThanOrEqual ? Equal : NotEqual);
    266    default:
    267      MOZ_CRASH("Invalid condition.");
    268  }
    269  return Always;
    270 }
    271 
    272 void MacroAssemblerRiscv64::ma_cmp_set(Register rd, Register rj, Register rk,
    273                                       Condition c) {
    274  switch (c) {
    275    case Equal:
    276      // seq d,s,t =>
    277      //   xor d,s,t
    278      //   sltiu d,d,1
    279      xor_(rd, rj, rk);
    280      ma_sltu(rd, rd, Operand(1));
    281      break;
    282    case NotEqual:
    283      // sne d,s,t =>
    284      //   xor d,s,t
    285      //   sltu d,$zero,d
    286      xor_(rd, rj, rk);
    287      sltu(rd, zero, rd);
    288      break;
    289    case Above:
    290      // sgtu d,s,t =>
    291      //   sltu d,t,s
    292      sltu(rd, rk, rj);
    293      break;
    294    case AboveOrEqual:
    295      // sgeu d,s,t =>
    296      //   sltu d,s,t
    297      //   xori d,d,1
    298      sltu(rd, rj, rk);
    299      xori(rd, rd, 1);
    300      break;
    301    case Below:
    302      // sltu d,s,t
    303      sltu(rd, rj, rk);
    304      break;
    305    case BelowOrEqual:
    306      // sleu d,s,t =>
    307      //   sltu d,t,s
    308      //   xori d,d,1
    309      sltu(rd, rk, rj);
    310      xori(rd, rd, 1);
    311      break;
    312    case GreaterThan:
    313      // sgt d,s,t =>
    314      //   slt d,t,s
    315      slt(rd, rk, rj);
    316      break;
    317    case GreaterThanOrEqual:
    318      // sge d,s,t =>
    319      //   slt d,s,t
    320      //   xori d,d,1
    321      slt(rd, rj, rk);
    322      xori(rd, rd, 1);
    323      break;
    324    case LessThan:
    325      // slt d,s,t
    326      slt(rd, rj, rk);
    327      break;
    328    case LessThanOrEqual:
    329      // sle d,s,t =>
    330      //   slt d,t,s
    331      //   xori d,d,1
    332      slt(rd, rk, rj);
    333      xori(rd, rd, 1);
    334      break;
    335    case Zero:
    336      MOZ_ASSERT(rj == rk);
    337      // seq d,s,$zero =>
    338      //   sltiu d,s,1
    339      ma_sltu(rd, rj, Operand(1));
    340      break;
    341    case NonZero:
    342      MOZ_ASSERT(rj == rk);
    343      // sne d,s,$zero =>
    344      //   sltu d,$zero,s
    345      sltu(rd, zero, rj);
    346      break;
    347    case Signed:
    348      MOZ_ASSERT(rj == rk);
    349      slt(rd, rj, zero);
    350      break;
    351    case NotSigned:
    352      MOZ_ASSERT(rj == rk);
    353      // sge d,s,$zero =>
    354      //   slt d,s,$zero
    355      //   xori d,d,1
    356      slt(rd, rj, zero);
    357      xori(rd, rd, 1);
    358      break;
    359    default:
    360      MOZ_CRASH("Invalid condition.");
    361  }
    362 }
    363 
    364 void MacroAssemblerRiscv64::ma_compareF32(Register rd, DoubleCondition cc,
    365                                          FloatRegister cmp1,
    366                                          FloatRegister cmp2) {
    367  switch (cc) {
    368    case DoubleEqual:
    369      feq_s(rd, cmp1, cmp2);
    370      return;
    371    case DoubleEqualOrUnordered: {
    372      UseScratchRegisterScope temps(this);
    373      Register scratch = temps.Acquire();
    374      flt_s(rd, cmp1, cmp2);
    375      flt_s(scratch, cmp2, cmp1);
    376      or_(rd, rd, scratch);
    377      NegateBool(rd, rd);
    378      return;
    379    }
    380    case DoubleNotEqual: {
    381      UseScratchRegisterScope temps(this);
    382      Register scratch = temps.Acquire();
    383      flt_s(rd, cmp1, cmp2);
    384      flt_s(scratch, cmp2, cmp1);
    385      or_(rd, rd, scratch);
    386      return;
    387    }
    388    case DoubleNotEqualOrUnordered:
    389      feq_s(rd, cmp1, cmp2);
    390      NegateBool(rd, rd);
    391      return;
    392    case DoubleLessThan:
    393      flt_s(rd, cmp1, cmp2);
    394      return;
    395    case DoubleLessThanOrUnordered:
    396      fle_s(rd, cmp2, cmp1);
    397      NegateBool(rd, rd);
    398      return;
    399    case DoubleGreaterThanOrEqual:
    400      fle_s(rd, cmp2, cmp1);
    401      return;
    402    case DoubleGreaterThanOrEqualOrUnordered:
    403      flt_s(rd, cmp1, cmp2);
    404      NegateBool(rd, rd);
    405      return;
    406    case DoubleLessThanOrEqual:
    407      fle_s(rd, cmp1, cmp2);
    408      return;
    409    case DoubleLessThanOrEqualOrUnordered:
    410      flt_s(rd, cmp2, cmp1);
    411      NegateBool(rd, rd);
    412      return;
    413    case DoubleGreaterThan:
    414      flt_s(rd, cmp2, cmp1);
    415      return;
    416    case DoubleGreaterThanOrUnordered:
    417      fle_s(rd, cmp1, cmp2);
    418      NegateBool(rd, rd);
    419      return;
    420    case DoubleOrdered:
    421      CompareIsNotNanF32(rd, cmp1, cmp2);
    422      return;
    423    case DoubleUnordered:
    424      CompareIsNanF32(rd, cmp1, cmp2);
    425      return;
    426  }
    427 }
    428 
    429 void MacroAssemblerRiscv64::ma_compareF64(Register rd, DoubleCondition cc,
    430                                          FloatRegister cmp1,
    431                                          FloatRegister cmp2) {
    432  switch (cc) {
    433    case DoubleEqual:
    434      feq_d(rd, cmp1, cmp2);
    435      return;
    436    case DoubleEqualOrUnordered: {
    437      UseScratchRegisterScope temps(this);
    438      Register scratch = temps.Acquire();
    439      flt_d(rd, cmp1, cmp2);
    440      flt_d(scratch, cmp2, cmp1);
    441      or_(rd, rd, scratch);
    442      NegateBool(rd, rd);
    443      return;
    444    }
    445    case DoubleNotEqual: {
    446      UseScratchRegisterScope temps(this);
    447      Register scratch = temps.Acquire();
    448      flt_d(rd, cmp1, cmp2);
    449      flt_d(scratch, cmp2, cmp1);
    450      or_(rd, rd, scratch);
    451      return;
    452    }
    453    case DoubleNotEqualOrUnordered:
    454      feq_d(rd, cmp1, cmp2);
    455      NegateBool(rd, rd);
    456      return;
    457    case DoubleLessThan:
    458      flt_d(rd, cmp1, cmp2);
    459      return;
    460    case DoubleLessThanOrUnordered:
    461      fle_d(rd, cmp2, cmp1);
    462      NegateBool(rd, rd);
    463      return;
    464    case DoubleGreaterThanOrEqual:
    465      fle_d(rd, cmp2, cmp1);
    466      return;
    467    case DoubleGreaterThanOrEqualOrUnordered:
    468      flt_d(rd, cmp1, cmp2);
    469      NegateBool(rd, rd);
    470      return;
    471    case DoubleLessThanOrEqual:
    472      fle_d(rd, cmp1, cmp2);
    473      return;
    474    case DoubleLessThanOrEqualOrUnordered:
    475      flt_d(rd, cmp2, cmp1);
    476      NegateBool(rd, rd);
    477      return;
    478    case DoubleGreaterThan:
    479      flt_d(rd, cmp2, cmp1);
    480      return;
    481    case DoubleGreaterThanOrUnordered:
    482      fle_d(rd, cmp1, cmp2);
    483      NegateBool(rd, rd);
    484      return;
    485    case DoubleOrdered:
    486      CompareIsNotNanF64(rd, cmp1, cmp2);
    487      return;
    488    case DoubleUnordered:
    489      CompareIsNanF64(rd, cmp1, cmp2);
    490      return;
    491  }
    492 }
    493 
    494 void MacroAssemblerRiscv64Compat::movePtr(Register src, Register dest) {
    495  mv(dest, src);
    496 }
    497 void MacroAssemblerRiscv64Compat::movePtr(ImmWord imm, Register dest) {
    498  ma_li(dest, imm);
    499 }
    500 
    501 void MacroAssemblerRiscv64Compat::movePtr(ImmGCPtr imm, Register dest) {
    502  ma_li(dest, imm);
    503 }
    504 
    505 void MacroAssemblerRiscv64Compat::movePtr(ImmPtr imm, Register dest) {
    506  movePtr(ImmWord(uintptr_t(imm.value)), dest);
    507 }
    508 void MacroAssemblerRiscv64Compat::movePtr(wasm::SymbolicAddress imm,
    509                                          Register dest) {
    510  DEBUG_PRINTF("[ %s\n", __FUNCTION__);
    511  BlockTrampolinePoolScope block_trampoline_pool(this, 8);
    512  append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm));
    513  ma_liPatchable(dest, ImmWord(-1), Li64);
    514  DEBUG_PRINTF("]\n");
    515 }
    516 
    517 bool MacroAssemblerRiscv64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
    518  asMasm().Push(FrameDescriptor(FrameType::IonJS));  // descriptor_
    519  asMasm().Push(ImmPtr(fakeReturnAddr));
    520  asMasm().Push(FramePointer);
    521  return true;
    522 }
    523 
    524 void MacroAssemblerRiscv64Compat::convertUInt32ToDouble(Register src,
    525                                                        FloatRegister dest) {
    526  fcvt_d_wu(dest, src);
    527 }
    528 
    529 void MacroAssemblerRiscv64Compat::convertUInt64ToDouble(Register src,
    530                                                        FloatRegister dest) {
    531  fcvt_d_lu(dest, src);
    532 }
    533 
    534 void MacroAssemblerRiscv64Compat::convertUInt32ToFloat32(Register src,
    535                                                         FloatRegister dest) {
    536  fcvt_s_wu(dest, src);
    537 }
    538 
    539 void MacroAssemblerRiscv64Compat::convertDoubleToFloat32(FloatRegister src,
    540                                                         FloatRegister dest) {
    541  fcvt_s_d(dest, src);
    542 }
    543 
    544 void MacroAssemblerRiscv64Compat::minMax32(Register lhs, Register rhs,
    545                                           Register dest, bool isMax) {
    546  if (rhs == dest) {
    547    std::swap(lhs, rhs);
    548  }
    549 
    550  auto cond = isMax ? Assembler::GreaterThan : Assembler::LessThan;
    551  if (lhs != dest) {
    552    move32(lhs, dest);
    553  }
    554  asMasm().cmp32Move32(cond, rhs, lhs, rhs, dest);
    555 }
    556 
    557 void MacroAssemblerRiscv64Compat::minMax32(Register lhs, Imm32 rhs,
    558                                           Register dest, bool isMax) {
    559  if (rhs.value == 0) {
    560    UseScratchRegisterScope temps(this);
    561    Register scratch = temps.Acquire();
    562 
    563    if (isMax) {
    564      // dest = -(lhs > 0 ? 1 : 0) & lhs
    565      sgtz(scratch, lhs);
    566      neg(scratch, scratch);
    567      and_(dest, lhs, scratch);
    568    } else {
    569      // dest = (lhs >> 31) & lhs
    570      sraiw(scratch, lhs, 31);
    571      and_(dest, lhs, scratch);
    572    }
    573    return;
    574  }
    575 
    576  // Uses branches because "Zicond" extension isn't yet supported.
    577 
    578  auto cond =
    579      isMax ? Assembler::GreaterThanOrEqual : Assembler::LessThanOrEqual;
    580  if (lhs != dest) {
    581    move32(lhs, dest);
    582  }
    583  Label done;
    584  asMasm().branch32(cond, lhs, rhs, &done);
    585  move32(rhs, dest);
    586  bind(&done);
    587 }
    588 
    589 void MacroAssemblerRiscv64Compat::minMaxPtr(Register lhs, Register rhs,
    590                                            Register dest, bool isMax) {
    591  if (rhs == dest) {
    592    std::swap(lhs, rhs);
    593  }
    594 
    595  auto cond = isMax ? Assembler::GreaterThan : Assembler::LessThan;
    596  if (lhs != dest) {
    597    movePtr(lhs, dest);
    598  }
    599  asMasm().cmpPtrMovePtr(cond, rhs, lhs, rhs, dest);
    600 }
    601 
    602 void MacroAssemblerRiscv64Compat::minMaxPtr(Register lhs, ImmWord rhs,
    603                                            Register dest, bool isMax) {
    604  if (rhs.value == 0) {
    605    UseScratchRegisterScope temps(this);
    606    Register scratch = temps.Acquire();
    607 
    608    if (isMax) {
    609      // dest = -(lhs > 0 ? 1 : 0) & lhs
    610      sgtz(scratch, lhs);
    611      neg(scratch, scratch);
    612      and_(dest, lhs, scratch);
    613    } else {
    614      // dest = (lhs >> 63) & lhs
    615      srai(scratch, lhs, 63);
    616      and_(dest, lhs, scratch);
    617    }
    618    return;
    619  }
    620 
    621  // Uses branches because "Zicond" extension isn't yet supported.
    622 
    623  auto cond =
    624      isMax ? Assembler::GreaterThanOrEqual : Assembler::LessThanOrEqual;
    625  if (lhs != dest) {
    626    movePtr(lhs, dest);
    627  }
    628  Label done;
    629  asMasm().branchPtr(cond, lhs, rhs, &done);
    630  movePtr(rhs, dest);
    631  bind(&done);
    632 }
    633 
    634 template <typename F>
    635 void MacroAssemblerRiscv64::RoundHelper(FPURegister dst, FPURegister src,
    636                                        FPURegister fpu_scratch,
    637                                        FPURoundingMode frm) {
    638  BlockTrampolinePoolScope block_trampoline_pool(this, 20, 2);
    639  UseScratchRegisterScope temps(this);
    640  Register scratch2 = temps.Acquire();
    641 
    642  MOZ_ASSERT((std::is_same<float, F>::value) ||
    643             (std::is_same<double, F>::value));
    644  // Need at least two FPRs, so check against dst == src == fpu_scratch
    645  MOZ_ASSERT(!(dst == src && dst == fpu_scratch));
    646 
    647  const int kFloatMantissaBits =
    648      sizeof(F) == 4 ? kFloat32MantissaBits : kFloat64MantissaBits;
    649  const int kFloatExponentBits =
    650      sizeof(F) == 4 ? kFloat32ExponentBits : kFloat64ExponentBits;
    651  const int kFloatExponentBias =
    652      sizeof(F) == 4 ? kFloat32ExponentBias : kFloat64ExponentBias;
    653  Label done;
    654 
    655  {
    656    UseScratchRegisterScope temps2(this);
    657    Register scratch = temps2.Acquire();
    658    // extract exponent value of the source floating-point to scratch
    659    if (std::is_same<F, double>::value) {
    660      fmv_x_d(scratch, src);
    661    } else {
    662      fmv_x_w(scratch, src);
    663    }
    664    ExtractBits(scratch2, scratch, kFloatMantissaBits, kFloatExponentBits);
    665  }
    666 
    667  // if src is NaN/+-Infinity/+-Zero or if the exponent is larger than # of bits
    668  // in mantissa, the result is the same as src, so move src to dest  (to avoid
    669  // generating another branch)
    670  if (dst != src) {
    671    if (std::is_same<F, double>::value) {
    672      fmv_d(dst, src);
    673    } else {
    674      fmv_s(dst, src);
    675    }
    676  }
    677  {
    678    Label not_NaN;
    679    UseScratchRegisterScope temps2(this);
    680    Register scratch = temps2.Acquire();
    681    // According to the wasm spec
    682    // (https://webassembly.github.io/spec/core/exec/numerics.html#aux-nans)
    683    // if input is canonical NaN, then output is canonical NaN, and if input is
    684    // any other NaN, then output is any NaN with most significant bit of
    685    // payload is 1. In RISC-V, feq_d will set scratch to 0 if src is a NaN. If
    686    // src is not a NaN, branch to the label and do nothing, but if it is,
    687    // fmin_d will set dst to the canonical NaN.
    688    if (std::is_same<F, double>::value) {
    689      feq_d(scratch, src, src);
    690      bnez(scratch, &not_NaN);
    691      fmin_d(dst, src, src);
    692    } else {
    693      feq_s(scratch, src, src);
    694      bnez(scratch, &not_NaN);
    695      fmin_s(dst, src, src);
    696    }
    697    bind(&not_NaN);
    698  }
    699 
    700  // If real exponent (i.e., scratch2 - kFloatExponentBias) is greater than
    701  // kFloat32MantissaBits, it means the floating-point value has no fractional
    702  // part, thus the input is already rounded, jump to done. Note that, NaN and
    703  // Infinity in floating-point representation sets maximal exponent value, so
    704  // they also satisfy (scratch2 - kFloatExponentBias >= kFloatMantissaBits),
    705  // and JS round semantics specify that rounding of NaN (Infinity) returns NaN
    706  // (Infinity), so NaN and Infinity are considered rounded value too.
    707  ma_branch(&done, GreaterThanOrEqual, scratch2,
    708            Operand(kFloatExponentBias + kFloatMantissaBits));
    709 
    710  // Actual rounding is needed along this path
    711 
    712  // old_src holds the original input, needed for the case of src == dst
    713  FPURegister old_src = src;
    714  if (src == dst) {
    715    MOZ_ASSERT(fpu_scratch != dst);
    716    fmv_d(fpu_scratch, src);
    717    old_src = fpu_scratch;
    718  }
    719 
    720  // Since only input whose real exponent value is less than kMantissaBits
    721  // (i.e., 23 or 52-bits) falls into this path, the value range of the input
    722  // falls into that of 23- or 53-bit integers. So we round the input to integer
    723  // values, then convert them back to floating-point.
    724  {
    725    UseScratchRegisterScope temps(this);
    726    Register scratch = temps.Acquire();
    727    if (std::is_same<F, double>::value) {
    728      fcvt_l_d(scratch, src, frm);
    729      fcvt_d_l(dst, scratch, frm);
    730    } else {
    731      fcvt_w_s(scratch, src, frm);
    732      fcvt_s_w(dst, scratch, frm);
    733    }
    734  }
    735  // A special handling is needed if the input is a very small positive/negative
    736  // number that rounds to zero. JS semantics requires that the rounded result
    737  // retains the sign of the input, so a very small positive (negative)
    738  // floating-point number should be rounded to positive (negative) 0.
    739  // Therefore, we use sign-bit injection to produce +/-0 correctly. Instead of
    740  // testing for zero w/ a branch, we just insert sign-bit for everyone on this
    741  // path (this is where old_src is needed)
    742  if (std::is_same<F, double>::value) {
    743    fsgnj_d(dst, dst, old_src);
    744  } else {
    745    fsgnj_s(dst, dst, old_src);
    746  }
    747 
    748  bind(&done);
    749 }
    750 
    751 template <typename CvtFunc>
    752 void MacroAssemblerRiscv64::RoundFloatingPointToInteger(Register rd,
    753                                                        FPURegister fs,
    754                                                        Register result,
    755                                                        CvtFunc fcvt_generator,
    756                                                        bool Inexact) {
    757  // Save csr_fflags to scratch & clear exception flags
    758  if (result != Register::Invalid()) {
    759    BlockTrampolinePoolScope block_trampoline_pool(this, 6);
    760    UseScratchRegisterScope temps(this);
    761    Register scratch = temps.Acquire();
    762 
    763    int exception_flags = kInvalidOperation;
    764    if (Inexact) exception_flags |= kInexact;
    765    csrrci(scratch, csr_fflags, exception_flags);
    766 
    767    // actual conversion instruction
    768    fcvt_generator(this, rd, fs);
    769 
    770    // check kInvalidOperation flag (out-of-range, NaN)
    771    // set result to 1 if normal, otherwise set result to 0 for abnormal
    772    frflags(result);
    773    andi(result, result, exception_flags);
    774    seqz(result, result);  // result <-- 1 (normal), result <-- 0 (abnormal)
    775 
    776    // restore csr_fflags
    777    csrw(csr_fflags, scratch);
    778  } else {
    779    // actual conversion instruction
    780    fcvt_generator(this, rd, fs);
    781  }
    782 }
    783 
    784 void MacroAssemblerRiscv64::Trunc_uw_d(Register rd, FPURegister fs,
    785                                       Register result, bool Inexact) {
    786  RoundFloatingPointToInteger(
    787      rd, fs, result,
    788      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    789        masm->fcvt_wu_d(dst, src, RTZ);
    790      },
    791      Inexact);
    792 }
    793 
    794 void MacroAssemblerRiscv64::Trunc_w_d(Register rd, FPURegister fs,
    795                                      Register result, bool Inexact) {
    796  RoundFloatingPointToInteger(
    797      rd, fs, result,
    798      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    799        masm->fcvt_w_d(dst, src, RTZ);
    800      },
    801      Inexact);
    802 }
    803 
    804 void MacroAssemblerRiscv64::Trunc_uw_s(Register rd, FPURegister fs,
    805                                       Register result, bool Inexact) {
    806  RoundFloatingPointToInteger(
    807      rd, fs, result,
    808      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    809        masm->fcvt_wu_s(dst, src, RTZ);
    810      },
    811      Inexact);
    812 }
    813 
    814 void MacroAssemblerRiscv64::Trunc_w_s(Register rd, FPURegister fs,
    815                                      Register result, bool Inexact) {
    816  RoundFloatingPointToInteger(
    817      rd, fs, result,
    818      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    819        masm->fcvt_w_s(dst, src, RTZ);
    820      },
    821      Inexact);
    822 }
    823 void MacroAssemblerRiscv64::Trunc_ul_d(Register rd, FPURegister fs,
    824                                       Register result, bool Inexact) {
    825  RoundFloatingPointToInteger(
    826      rd, fs, result,
    827      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    828        masm->fcvt_lu_d(dst, src, RTZ);
    829      },
    830      Inexact);
    831 }
    832 
    833 void MacroAssemblerRiscv64::Trunc_l_d(Register rd, FPURegister fs,
    834                                      Register result, bool Inexact) {
    835  RoundFloatingPointToInteger(
    836      rd, fs, result,
    837      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    838        masm->fcvt_l_d(dst, src, RTZ);
    839      },
    840      Inexact);
    841 }
    842 
    843 void MacroAssemblerRiscv64::Trunc_ul_s(Register rd, FPURegister fs,
    844                                       Register result, bool Inexact) {
    845  RoundFloatingPointToInteger(
    846      rd, fs, result,
    847      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    848        masm->fcvt_lu_s(dst, src, RTZ);
    849      },
    850      Inexact);
    851 }
    852 
    853 void MacroAssemblerRiscv64::Trunc_l_s(Register rd, FPURegister fs,
    854                                      Register result, bool Inexact) {
    855  RoundFloatingPointToInteger(
    856      rd, fs, result,
    857      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    858        masm->fcvt_l_s(dst, src, RTZ);
    859      },
    860      Inexact);
    861 }
    862 
    863 void MacroAssemblerRiscv64::Floor_d_d(FPURegister dst, FPURegister src,
    864                                      FPURegister fpu_scratch) {
    865  RoundHelper<double>(dst, src, fpu_scratch, RDN);
    866 }
    867 
    868 void MacroAssemblerRiscv64::Ceil_d_d(FPURegister dst, FPURegister src,
    869                                     FPURegister fpu_scratch) {
    870  RoundHelper<double>(dst, src, fpu_scratch, RUP);
    871 }
    872 
    873 void MacroAssemblerRiscv64::Trunc_d_d(FPURegister dst, FPURegister src,
    874                                      FPURegister fpu_scratch) {
    875  RoundHelper<double>(dst, src, fpu_scratch, RTZ);
    876 }
    877 
    878 void MacroAssemblerRiscv64::Round_d_d(FPURegister dst, FPURegister src,
    879                                      FPURegister fpu_scratch) {
    880  RoundHelper<double>(dst, src, fpu_scratch, RNE);
    881 }
    882 
    883 void MacroAssemblerRiscv64::Floor_s_s(FPURegister dst, FPURegister src,
    884                                      FPURegister fpu_scratch) {
    885  RoundHelper<float>(dst, src, fpu_scratch, RDN);
    886 }
    887 
    888 void MacroAssemblerRiscv64::Ceil_s_s(FPURegister dst, FPURegister src,
    889                                     FPURegister fpu_scratch) {
    890  RoundHelper<float>(dst, src, fpu_scratch, RUP);
    891 }
    892 
    893 void MacroAssemblerRiscv64::Trunc_s_s(FPURegister dst, FPURegister src,
    894                                      FPURegister fpu_scratch) {
    895  RoundHelper<float>(dst, src, fpu_scratch, RTZ);
    896 }
    897 
    898 void MacroAssemblerRiscv64::Round_s_s(FPURegister dst, FPURegister src,
    899                                      FPURegister fpu_scratch) {
    900  RoundHelper<float>(dst, src, fpu_scratch, RNE);
    901 }
    902 
    903 void MacroAssemblerRiscv64::Round_w_s(Register rd, FPURegister fs,
    904                                      Register result, bool Inexact) {
    905  RoundFloatingPointToInteger(
    906      rd, fs, result,
    907      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    908        masm->fcvt_w_s(dst, src, RNE);
    909      },
    910      Inexact);
    911 }
    912 
    913 void MacroAssemblerRiscv64::Round_w_d(Register rd, FPURegister fs,
    914                                      Register result, bool Inexact) {
    915  RoundFloatingPointToInteger(
    916      rd, fs, result,
    917      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    918        masm->fcvt_w_d(dst, src, RNE);
    919      },
    920      Inexact);
    921 }
    922 
    923 void MacroAssemblerRiscv64::Ceil_w_s(Register rd, FPURegister fs,
    924                                     Register result, bool Inexact) {
    925  RoundFloatingPointToInteger(
    926      rd, fs, result,
    927      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    928        masm->fcvt_w_s(dst, src, RUP);
    929      },
    930      Inexact);
    931 }
    932 
    933 void MacroAssemblerRiscv64::Ceil_l_d(Register rd, FPURegister fs,
    934                                     Register result, bool Inexact) {
    935  RoundFloatingPointToInteger(
    936      rd, fs, result,
    937      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    938        masm->fcvt_l_d(dst, src, RUP);
    939      },
    940      Inexact);
    941 }
    942 
    943 void MacroAssemblerRiscv64::Ceil_l_s(Register rd, FPURegister fs,
    944                                     Register result, bool Inexact) {
    945  RoundFloatingPointToInteger(
    946      rd, fs, result,
    947      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    948        masm->fcvt_l_s(dst, src, RUP);
    949      },
    950      Inexact);
    951 }
    952 
    953 void MacroAssemblerRiscv64::Ceil_w_d(Register rd, FPURegister fs,
    954                                     Register result, bool Inexact) {
    955  RoundFloatingPointToInteger(
    956      rd, fs, result,
    957      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    958        masm->fcvt_w_d(dst, src, RUP);
    959      },
    960      Inexact);
    961 }
    962 
    963 void MacroAssemblerRiscv64::Floor_w_s(Register rd, FPURegister fs,
    964                                      Register result, bool Inexact) {
    965  RoundFloatingPointToInteger(
    966      rd, fs, result,
    967      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    968        masm->fcvt_w_s(dst, src, RDN);
    969      },
    970      Inexact);
    971 }
    972 
    973 void MacroAssemblerRiscv64::Floor_w_d(Register rd, FPURegister fs,
    974                                      Register result, bool Inexact) {
    975  RoundFloatingPointToInteger(
    976      rd, fs, result,
    977      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    978        masm->fcvt_w_d(dst, src, RDN);
    979      },
    980      Inexact);
    981 }
    982 
    983 void MacroAssemblerRiscv64::Floor_l_s(Register rd, FPURegister fs,
    984                                      Register result, bool Inexact) {
    985  RoundFloatingPointToInteger(
    986      rd, fs, result,
    987      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    988        masm->fcvt_l_s(dst, src, RDN);
    989      },
    990      Inexact);
    991 }
    992 
    993 void MacroAssemblerRiscv64::Floor_l_d(Register rd, FPURegister fs,
    994                                      Register result, bool Inexact) {
    995  RoundFloatingPointToInteger(
    996      rd, fs, result,
    997      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
    998        masm->fcvt_l_d(dst, src, RDN);
    999      },
   1000      Inexact);
   1001 }
   1002 
   1003 void MacroAssemblerRiscv64::RoundMaxMag_l_s(Register rd, FPURegister fs,
   1004                                            Register result, bool Inexact) {
   1005  RoundFloatingPointToInteger(
   1006      rd, fs, result,
   1007      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
   1008        masm->fcvt_l_s(dst, src, RMM);
   1009      },
   1010      Inexact);
   1011 }
   1012 
   1013 void MacroAssemblerRiscv64::RoundMaxMag_l_d(Register rd, FPURegister fs,
   1014                                            Register result, bool Inexact) {
   1015  RoundFloatingPointToInteger(
   1016      rd, fs, result,
   1017      [](MacroAssemblerRiscv64* masm, Register dst, FPURegister src) {
   1018        masm->fcvt_l_d(dst, src, RMM);
   1019      },
   1020      Inexact);
   1021 }
   1022 
   1023 // Checks whether a double is representable as a 32-bit integer. If so, the
   1024 // integer is written to the output register. Otherwise, a bailout is taken to
   1025 // the given snapshot. This function overwrites the scratch float register.
   1026 void MacroAssemblerRiscv64Compat::convertDoubleToInt32(FloatRegister src,
   1027                                                       Register dest,
   1028                                                       Label* fail,
   1029                                                       bool negativeZeroCheck) {
   1030  if (negativeZeroCheck) {
   1031    fclass_d(dest, src);
   1032    ma_b(dest, Imm32(kNegativeZero), fail, Equal);
   1033  }
   1034  UseScratchRegisterScope temps(this);
   1035  Register scratch = temps.Acquire();
   1036  Trunc_w_d(dest, src, scratch, true);
   1037  ma_b(scratch, Imm32(0), fail, Equal);
   1038 }
   1039 
   1040 void MacroAssemblerRiscv64Compat::convertDoubleToPtr(FloatRegister src,
   1041                                                     Register dest, Label* fail,
   1042                                                     bool negativeZeroCheck) {
   1043  if (negativeZeroCheck) {
   1044    fclass_d(dest, src);
   1045    ma_b(dest, Imm32(kNegativeZero), fail, Equal);
   1046  }
   1047  UseScratchRegisterScope temps(this);
   1048  Register scratch = temps.Acquire();
   1049  Trunc_l_d(dest, src, scratch, true);
   1050  ma_b(scratch, Imm32(0), fail, Equal);
   1051 }
   1052 
   1053 // Checks whether a float32 is representable as a 32-bit integer. If so, the
   1054 // integer is written to the output register. Otherwise, a bailout is taken to
   1055 // the given snapshot. This function overwrites the scratch float register.
   1056 void MacroAssemblerRiscv64Compat::convertFloat32ToInt32(
   1057    FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck) {
   1058  if (negativeZeroCheck) {
   1059    fclass_d(dest, src);
   1060    ma_b(dest, Imm32(kNegativeZero), fail, Equal);
   1061  }
   1062  UseScratchRegisterScope temps(this);
   1063  Register scratch = temps.Acquire();
   1064  Trunc_w_s(dest, src, scratch, true);
   1065  ma_b(scratch, Imm32(0), fail, Equal);
   1066 }
   1067 
   1068 void MacroAssemblerRiscv64Compat::convertFloat32ToDouble(FloatRegister src,
   1069                                                         FloatRegister dest) {
   1070  fcvt_d_s(dest, src);
   1071 }
   1072 
   1073 void MacroAssemblerRiscv64Compat::convertInt32ToFloat32(Register src,
   1074                                                        FloatRegister dest) {
   1075  fcvt_s_w(dest, src);
   1076 }
   1077 
   1078 void MacroAssemblerRiscv64Compat::convertInt32ToFloat32(const Address& src,
   1079                                                        FloatRegister dest) {
   1080  UseScratchRegisterScope temps(this);
   1081  Register scratch = temps.Acquire();
   1082  load32(src, scratch);
   1083  fcvt_s_w(dest, scratch);
   1084 }
   1085 
   1086 void MacroAssemblerRiscv64Compat::truncateFloat32ModUint32(FloatRegister src,
   1087                                                           Register dest) {
   1088  UseScratchRegisterScope temps(this);
   1089  Register scratch = temps.Acquire();
   1090 
   1091  // Convert scalar to signed 64-bit fixed-point, rounding toward zero.
   1092  // In the case of overflow or NaN, the output is saturated.
   1093  // In the case of -0, the output is zero.
   1094  Trunc_l_s(dest, src);
   1095 
   1096  // Unsigned subtraction of INT64_MAX returns 1 resp. 0 for INT64_{MIN,MAX}.
   1097  ma_li(scratch, Imm64(0x7fff'ffff'ffff'ffff));
   1098  ma_sub64(scratch, dest, scratch);
   1099 
   1100  // If scratch u< 2, then scratch = 0; else scratch = -1.
   1101  ma_sltu(scratch, scratch, Imm32(2));
   1102  ma_add32(scratch, scratch, Imm32(-1));
   1103 
   1104  // Clear |dest| if the truncation result was saturated.
   1105  ma_and(dest, dest, scratch);
   1106 
   1107  // Clear upper 32 bits.
   1108  SignExtendWord(dest, dest);
   1109 }
   1110 
   1111 // Memory.
   1112 FaultingCodeOffset MacroAssemblerRiscv64::ma_loadDouble(FloatRegister dest,
   1113                                                        Address address) {
   1114  UseScratchRegisterScope temps(this);
   1115  int16_t encodedOffset;
   1116  Register base;
   1117 
   1118  if (!is_int12(address.offset)) {
   1119    Register scratch = temps.Acquire();
   1120    ma_li(scratch, Imm32(address.offset));
   1121    add(scratch, address.base, scratch);
   1122    base = scratch;
   1123    encodedOffset = 0;
   1124  } else {
   1125    encodedOffset = address.offset;
   1126    base = address.base;
   1127  }
   1128  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   1129  fld(dest, base, encodedOffset);
   1130  return fco;
   1131 }
   1132 
   1133 FaultingCodeOffset MacroAssemblerRiscv64::ma_loadFloat(FloatRegister dest,
   1134                                                       Address address) {
   1135  UseScratchRegisterScope temps(this);
   1136  int16_t encodedOffset;
   1137  Register base;
   1138 
   1139  if (!is_int12(address.offset)) {
   1140    Register scratch = temps.Acquire();
   1141    ma_li(scratch, Imm32(address.offset));
   1142    add(scratch, address.base, scratch);
   1143    base = scratch;
   1144    encodedOffset = 0;
   1145  } else {
   1146    encodedOffset = address.offset;
   1147    base = address.base;
   1148  }
   1149  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   1150  flw(dest, base, encodedOffset);
   1151  return fco;
   1152 }
   1153 
   1154 FaultingCodeOffset MacroAssemblerRiscv64::ma_load(
   1155    Register dest, Address address, LoadStoreSize size,
   1156    LoadStoreExtension extension) {
   1157  UseScratchRegisterScope temps(this);
   1158  int16_t encodedOffset;
   1159  Register base;
   1160 
   1161  if (!is_int12(address.offset)) {
   1162    Register scratch = temps.Acquire();
   1163    ma_li(scratch, Imm32(address.offset));
   1164    add(scratch, address.base, scratch);
   1165    base = scratch;
   1166    encodedOffset = 0;
   1167  } else {
   1168    encodedOffset = address.offset;
   1169    base = address.base;
   1170  }
   1171  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   1172  switch (size) {
   1173    case SizeByte:
   1174      if (ZeroExtend == extension) {
   1175        lbu(dest, base, encodedOffset);
   1176      } else {
   1177        lb(dest, base, encodedOffset);
   1178      }
   1179      break;
   1180    case SizeHalfWord:
   1181      if (ZeroExtend == extension) {
   1182        lhu(dest, base, encodedOffset);
   1183      } else {
   1184        lh(dest, base, encodedOffset);
   1185      }
   1186      break;
   1187    case SizeWord:
   1188      if (ZeroExtend == extension) {
   1189        lwu(dest, base, encodedOffset);
   1190      } else {
   1191        lw(dest, base, encodedOffset);
   1192      }
   1193      break;
   1194    case SizeDouble:
   1195      ld(dest, base, encodedOffset);
   1196      break;
   1197    default:
   1198      MOZ_CRASH("Invalid argument for ma_load");
   1199  }
   1200  return fco;
   1201 }
   1202 
   1203 FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
   1204    Register data, const BaseIndex& dest, LoadStoreSize size,
   1205    LoadStoreExtension extension) {
   1206  UseScratchRegisterScope temps(this);
   1207  Register scratch2 = temps.Acquire();
   1208  computeScaledAddress(dest, scratch2);
   1209  return ma_store(data, Address(scratch2, dest.offset), size, extension);
   1210 }
   1211 
   1212 FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
   1213    Imm32 imm, const BaseIndex& dest, LoadStoreSize size,
   1214    LoadStoreExtension extension) {
   1215  UseScratchRegisterScope temps(this);
   1216 
   1217  Register address = temps.Acquire();
   1218  computeScaledAddress(dest, address);
   1219 
   1220  Register scratch = temps.Acquire();
   1221  ma_li(scratch, imm);
   1222 
   1223  return ma_store(scratch, Address(address, dest.offset), size, extension);
   1224 }
   1225 
   1226 FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
   1227    Imm32 imm, Address address, LoadStoreSize size,
   1228    LoadStoreExtension extension) {
   1229  UseScratchRegisterScope temps(this);
   1230  Register scratch = temps.Acquire();
   1231  ma_li(scratch, imm);
   1232  return ma_store(scratch, address, size, extension);
   1233 }
   1234 
   1235 FaultingCodeOffset MacroAssemblerRiscv64::ma_store(
   1236    Register data, Address address, LoadStoreSize size,
   1237    LoadStoreExtension extension) {
   1238  UseScratchRegisterScope temps(this);
   1239 
   1240  int16_t encodedOffset;
   1241  Register base;
   1242 
   1243  if (!is_int12(address.offset)) {
   1244    Register scratch = temps.Acquire();
   1245    ma_li(scratch, Imm32(address.offset));
   1246    add(scratch, address.base, scratch);
   1247    base = scratch;
   1248    encodedOffset = 0;
   1249  } else {
   1250    encodedOffset = address.offset;
   1251    base = address.base;
   1252  }
   1253  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   1254  switch (size) {
   1255    case SizeByte:
   1256      sb(data, base, encodedOffset);
   1257      break;
   1258    case SizeHalfWord:
   1259      sh(data, base, encodedOffset);
   1260      break;
   1261    case SizeWord:
   1262      sw(data, base, encodedOffset);
   1263      break;
   1264    case SizeDouble:
   1265      sd(data, base, encodedOffset);
   1266      break;
   1267    default:
   1268      MOZ_CRASH("Invalid argument for ma_store");
   1269  }
   1270  return fco;
   1271 }
   1272 
   1273 // Memory.
   1274 void MacroAssemblerRiscv64::ma_storeDouble(FloatRegister dest,
   1275                                           Address address) {
   1276  UseScratchRegisterScope temps(this);
   1277  int16_t encodedOffset;
   1278  Register base;
   1279 
   1280  if (!is_int12(address.offset)) {
   1281    Register scratch = temps.Acquire();
   1282    ma_li(scratch, Imm32(address.offset));
   1283    add(scratch, address.base, scratch);
   1284    base = scratch;
   1285    encodedOffset = 0;
   1286  } else {
   1287    encodedOffset = address.offset;
   1288    base = address.base;
   1289  }
   1290  fsd(dest, base, encodedOffset);
   1291 }
   1292 
   1293 void MacroAssemblerRiscv64::ma_storeFloat(FloatRegister dest, Address address) {
   1294  UseScratchRegisterScope temps(this);
   1295  int16_t encodedOffset;
   1296  Register base;
   1297 
   1298  if (!is_int12(address.offset)) {
   1299    Register scratch = temps.Acquire();
   1300    ma_li(scratch, Imm32(address.offset));
   1301    add(scratch, address.base, scratch);
   1302    base = scratch;
   1303    encodedOffset = 0;
   1304  } else {
   1305    encodedOffset = address.offset;
   1306    base = address.base;
   1307  }
   1308  fsw(dest, base, encodedOffset);
   1309 }
   1310 
   1311 void MacroAssemblerRiscv64::computeScaledAddress(const BaseIndex& address,
   1312                                                 Register dest) {
   1313  Register base = address.base;
   1314  Register index = address.index;
   1315  int32_t shift = Imm32::ShiftOf(address.scale).value;
   1316  UseScratchRegisterScope temps(this);
   1317  if (shift && base == zero) {
   1318    MOZ_ASSERT(shift <= 4);
   1319    slli(dest, index, shift);
   1320  } else if (shift) {
   1321    Register tmp = dest == base ? temps.Acquire() : dest;
   1322    MOZ_ASSERT(shift <= 4);
   1323    slli(tmp, index, shift);
   1324    add(dest, base, tmp);
   1325  } else {
   1326    add(dest, base, index);
   1327  }
   1328 }
   1329 
   1330 void MacroAssemblerRiscv64::computeScaledAddress32(const BaseIndex& address,
   1331                                                   Register dest) {
   1332  Register base = address.base;
   1333  Register index = address.index;
   1334  int32_t shift = Imm32::ShiftOf(address.scale).value;
   1335  UseScratchRegisterScope temps(this);
   1336  if (shift && base == zero) {
   1337    MOZ_ASSERT(shift <= 4);
   1338    slliw(dest, index, shift);
   1339  } else if (shift) {
   1340    Register tmp = dest == base ? temps.Acquire() : dest;
   1341    MOZ_ASSERT(shift <= 4);
   1342    slliw(tmp, index, shift);
   1343    addw(dest, base, tmp);
   1344  } else {
   1345    addw(dest, base, index);
   1346  }
   1347 }
   1348 
   1349 void MacroAssemblerRiscv64Compat::wasmLoadI64Impl(
   1350    const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
   1351    Register ptrScratch, Register64 output, Register tmp) {
   1352  access.assertOffsetInGuardPages();
   1353  uint32_t offset = access.offset32();
   1354  MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
   1355 
   1356  // Maybe add the offset.
   1357  if (offset) {
   1358    asMasm().addPtr(ImmWord(offset), ptrScratch);
   1359    ptr = ptrScratch;
   1360  }
   1361 
   1362  asMasm().memoryBarrierBefore(access.sync());
   1363  UseScratchRegisterScope temps(this);
   1364  Register scratch = temps.Acquire();
   1365  FaultingCodeOffset fco;
   1366  switch (access.type()) {
   1367    case Scalar::Int8:
   1368      add(scratch, memoryBase, ptr);
   1369      fco = FaultingCodeOffset(currentOffset());
   1370      lb(output.reg, scratch, 0);
   1371      break;
   1372    case Scalar::Uint8:
   1373      add(scratch, memoryBase, ptr);
   1374      fco = FaultingCodeOffset(currentOffset());
   1375      lbu(output.reg, scratch, 0);
   1376      break;
   1377    case Scalar::Int16:
   1378      add(scratch, memoryBase, ptr);
   1379      fco = FaultingCodeOffset(currentOffset());
   1380      lh(output.reg, scratch, 0);
   1381      break;
   1382    case Scalar::Uint16:
   1383      add(scratch, memoryBase, ptr);
   1384      fco = FaultingCodeOffset(currentOffset());
   1385      lhu(output.reg, scratch, 0);
   1386      break;
   1387    case Scalar::Int32:
   1388      add(scratch, memoryBase, ptr);
   1389      fco = FaultingCodeOffset(currentOffset());
   1390      lw(output.reg, scratch, 0);
   1391      break;
   1392    case Scalar::Uint32:
   1393      // TODO(riscv): Why need zero-extension here?
   1394      add(scratch, memoryBase, ptr);
   1395      fco = FaultingCodeOffset(currentOffset());
   1396      lwu(output.reg, scratch, 0);
   1397      break;
   1398    case Scalar::Int64:
   1399      add(scratch, memoryBase, ptr);
   1400      fco = FaultingCodeOffset(currentOffset());
   1401      ld(output.reg, scratch, 0);
   1402      break;
   1403    default:
   1404      MOZ_CRASH("unexpected array type");
   1405  }
   1406 
   1407  append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()), fco);
   1408  asMasm().memoryBarrierAfter(access.sync());
   1409 }
   1410 
   1411 void MacroAssemblerRiscv64Compat::wasmStoreI64Impl(
   1412    const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase,
   1413    Register ptr, Register ptrScratch, Register tmp) {
   1414  access.assertOffsetInGuardPages();
   1415  uint32_t offset = access.offset32();
   1416  MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
   1417 
   1418  // Maybe add the offset.
   1419  if (offset) {
   1420    asMasm().addPtr(ImmWord(offset), ptrScratch);
   1421    ptr = ptrScratch;
   1422  }
   1423 
   1424  asMasm().memoryBarrierBefore(access.sync());
   1425  UseScratchRegisterScope temps(this);
   1426  Register scratch = temps.Acquire();
   1427  FaultingCodeOffset fco;
   1428  switch (access.type()) {
   1429    case Scalar::Int8:
   1430    case Scalar::Uint8:
   1431      add(scratch, memoryBase, ptr);
   1432      fco = FaultingCodeOffset(currentOffset());
   1433      sb(value.reg, scratch, 0);
   1434      break;
   1435    case Scalar::Int16:
   1436    case Scalar::Uint16:
   1437      add(scratch, memoryBase, ptr);
   1438      fco = FaultingCodeOffset(currentOffset());
   1439      sh(value.reg, scratch, 0);
   1440      break;
   1441    case Scalar::Int32:
   1442    case Scalar::Uint32:
   1443      add(scratch, memoryBase, ptr);
   1444      fco = FaultingCodeOffset(currentOffset());
   1445      sw(value.reg, scratch, 0);
   1446      break;
   1447    case Scalar::Int64:
   1448      add(scratch, memoryBase, ptr);
   1449      fco = FaultingCodeOffset(currentOffset());
   1450      sd(value.reg, scratch, 0);
   1451      break;
   1452    default:
   1453      MOZ_CRASH("unexpected array type");
   1454  }
   1455 
   1456  append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()), fco);
   1457  asMasm().memoryBarrierAfter(access.sync());
   1458 }
   1459 
   1460 void MacroAssemblerRiscv64Compat::profilerEnterFrame(Register framePtr,
   1461                                                     Register scratch) {
   1462  asMasm().loadJSContext(scratch);
   1463  loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
   1464  storePtr(framePtr,
   1465           Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
   1466  storePtr(ImmPtr(nullptr),
   1467           Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
   1468 }
   1469 
   1470 void MacroAssemblerRiscv64Compat::profilerExitFrame() {
   1471  jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
   1472 }
   1473 
   1474 void MacroAssemblerRiscv64Compat::move32(Imm32 imm, Register dest) {
   1475  ma_li(dest, imm);
   1476 }
   1477 
   1478 void MacroAssemblerRiscv64Compat::move32(Register src, Register dest) {
   1479  SignExtendWord(dest, src);
   1480 }
   1481 
   1482 FaultingCodeOffset MacroAssemblerRiscv64Compat::load8ZeroExtend(
   1483    const Address& address, Register dest) {
   1484  return ma_load(dest, address, SizeByte, ZeroExtend);
   1485 }
   1486 
   1487 FaultingCodeOffset MacroAssemblerRiscv64Compat::load8ZeroExtend(
   1488    const BaseIndex& src, Register dest) {
   1489  return ma_load(dest, src, SizeByte, ZeroExtend);
   1490 }
   1491 
   1492 FaultingCodeOffset MacroAssemblerRiscv64Compat::load8SignExtend(
   1493    const Address& address, Register dest) {
   1494  return ma_load(dest, address, SizeByte, SignExtend);
   1495 }
   1496 
   1497 FaultingCodeOffset MacroAssemblerRiscv64Compat::load8SignExtend(
   1498    const BaseIndex& src, Register dest) {
   1499  return ma_load(dest, src, SizeByte, SignExtend);
   1500 }
   1501 
   1502 FaultingCodeOffset MacroAssemblerRiscv64Compat::load16ZeroExtend(
   1503    const Address& address, Register dest) {
   1504  return ma_load(dest, address, SizeHalfWord, ZeroExtend);
   1505 }
   1506 
   1507 FaultingCodeOffset MacroAssemblerRiscv64Compat::load16ZeroExtend(
   1508    const BaseIndex& src, Register dest) {
   1509  return ma_load(dest, src, SizeHalfWord, ZeroExtend);
   1510 }
   1511 
   1512 FaultingCodeOffset MacroAssemblerRiscv64Compat::load16SignExtend(
   1513    const Address& address, Register dest) {
   1514  return ma_load(dest, address, SizeHalfWord, SignExtend);
   1515 }
   1516 
   1517 FaultingCodeOffset MacroAssemblerRiscv64Compat::load16SignExtend(
   1518    const BaseIndex& src, Register dest) {
   1519  return ma_load(dest, src, SizeHalfWord, SignExtend);
   1520 }
   1521 
   1522 FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(const Address& address,
   1523                                                       Register dest) {
   1524  return ma_load(dest, address, SizeWord);
   1525 }
   1526 
   1527 FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(const BaseIndex& address,
   1528                                                       Register dest) {
   1529  return ma_load(dest, address, SizeWord);
   1530 }
   1531 
   1532 FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(AbsoluteAddress address,
   1533                                                       Register dest) {
   1534  UseScratchRegisterScope temps(this);
   1535  Register scratch = temps.Acquire();
   1536  movePtr(ImmPtr(address.addr), scratch);
   1537  return load32(Address(scratch, 0), dest);
   1538 }
   1539 
   1540 FaultingCodeOffset MacroAssemblerRiscv64Compat::load32(
   1541    wasm::SymbolicAddress address, Register dest) {
   1542  UseScratchRegisterScope temps(this);
   1543  Register scratch = temps.Acquire();
   1544  movePtr(address, scratch);
   1545  return load32(Address(scratch, 0), dest);
   1546 }
   1547 
   1548 FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(const Address& address,
   1549                                                        Register dest) {
   1550  return ma_load(dest, address, SizeDouble);
   1551 }
   1552 
   1553 FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(const BaseIndex& src,
   1554                                                        Register dest) {
   1555  return ma_load(dest, src, SizeDouble);
   1556 }
   1557 
   1558 FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(AbsoluteAddress address,
   1559                                                        Register dest) {
   1560  UseScratchRegisterScope temps(this);
   1561  Register scratch = temps.Acquire();
   1562  movePtr(ImmPtr(address.addr), scratch);
   1563  return loadPtr(Address(scratch, 0), dest);
   1564 }
   1565 
   1566 FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPtr(
   1567    wasm::SymbolicAddress address, Register dest) {
   1568  UseScratchRegisterScope temps(this);
   1569  Register scratch = temps.Acquire();
   1570  movePtr(address, scratch);
   1571  return loadPtr(Address(scratch, 0), dest);
   1572 }
   1573 
   1574 FaultingCodeOffset MacroAssemblerRiscv64Compat::loadPrivate(
   1575    const Address& address, Register dest) {
   1576  return loadPtr(address, dest);
   1577 }
   1578 
   1579 FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(Imm32 imm,
   1580                                                       const Address& address) {
   1581  UseScratchRegisterScope temps(this);
   1582  Register scratch = temps.Acquire();
   1583  ma_li(scratch, imm);
   1584  return ma_store(scratch, address, SizeByte);
   1585 }
   1586 
   1587 FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(Register src,
   1588                                                       const Address& address) {
   1589  return ma_store(src, address, SizeByte);
   1590 }
   1591 
   1592 FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(Imm32 imm,
   1593                                                       const BaseIndex& dest) {
   1594  return ma_store(imm, dest, SizeByte);
   1595 }
   1596 
   1597 FaultingCodeOffset MacroAssemblerRiscv64Compat::store8(Register src,
   1598                                                       const BaseIndex& dest) {
   1599  return ma_store(src, dest, SizeByte);
   1600 }
   1601 
   1602 FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
   1603    Imm32 imm, const Address& address) {
   1604  UseScratchRegisterScope temps(this);
   1605  Register scratch = temps.Acquire();
   1606  ma_li(scratch, imm);
   1607  return ma_store(scratch, address, SizeHalfWord);
   1608 }
   1609 
   1610 FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
   1611    Register src, const Address& address) {
   1612  return ma_store(src, address, SizeHalfWord);
   1613 }
   1614 
   1615 FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(Imm32 imm,
   1616                                                        const BaseIndex& dest) {
   1617  return ma_store(imm, dest, SizeHalfWord);
   1618 }
   1619 
   1620 FaultingCodeOffset MacroAssemblerRiscv64Compat::store16(
   1621    Register src, const BaseIndex& address) {
   1622  return ma_store(src, address, SizeHalfWord);
   1623 }
   1624 
   1625 FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
   1626    Register src, AbsoluteAddress address) {
   1627  UseScratchRegisterScope temps(this);
   1628  Register scratch = temps.Acquire();
   1629  movePtr(ImmPtr(address.addr), scratch);
   1630  return store32(src, Address(scratch, 0));
   1631 }
   1632 
   1633 FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
   1634    Register src, const Address& address) {
   1635  return ma_store(src, address, SizeWord);
   1636 }
   1637 
   1638 FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(
   1639    Imm32 src, const Address& address) {
   1640  UseScratchRegisterScope temps(this);
   1641  Register scratch = temps.Acquire();
   1642  move32(src, scratch);
   1643  return ma_store(scratch, address, SizeWord);
   1644 }
   1645 
   1646 FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(Imm32 imm,
   1647                                                        const BaseIndex& dest) {
   1648  return ma_store(imm, dest, SizeWord);
   1649 }
   1650 
   1651 FaultingCodeOffset MacroAssemblerRiscv64Compat::store32(Register src,
   1652                                                        const BaseIndex& dest) {
   1653  return ma_store(src, dest, SizeWord);
   1654 }
   1655 
   1656 template <typename T>
   1657 FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(ImmWord imm,
   1658                                                         T address) {
   1659  UseScratchRegisterScope temps(this);
   1660  Register scratch = temps.Acquire();
   1661  ma_li(scratch, imm);
   1662  return ma_store(scratch, address, SizeDouble);
   1663 }
   1664 
   1665 template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<Address>(
   1666    ImmWord imm, Address address);
   1667 template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<BaseIndex>(
   1668    ImmWord imm, BaseIndex address);
   1669 
   1670 template <typename T>
   1671 FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(ImmPtr imm,
   1672                                                         T address) {
   1673  return storePtr(ImmWord(uintptr_t(imm.value)), address);
   1674 }
   1675 
   1676 template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<Address>(
   1677    ImmPtr imm, Address address);
   1678 template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<BaseIndex>(
   1679    ImmPtr imm, BaseIndex address);
   1680 
   1681 template <typename T>
   1682 FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(ImmGCPtr imm,
   1683                                                         T address) {
   1684  UseScratchRegisterScope temps(this);
   1685  Register scratch = temps.Acquire();
   1686  movePtr(imm, scratch);
   1687  return storePtr(scratch, address);
   1688 }
   1689 
   1690 template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<Address>(
   1691    ImmGCPtr imm, Address address);
   1692 template FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr<BaseIndex>(
   1693    ImmGCPtr imm, BaseIndex address);
   1694 
   1695 FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(
   1696    Register src, const Address& address) {
   1697  return ma_store(src, address, SizeDouble);
   1698 }
   1699 
   1700 FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(
   1701    Register src, const BaseIndex& address) {
   1702  return ma_store(src, address, SizeDouble);
   1703 }
   1704 
   1705 FaultingCodeOffset MacroAssemblerRiscv64Compat::storePtr(Register src,
   1706                                                         AbsoluteAddress dest) {
   1707  UseScratchRegisterScope temps(this);
   1708  Register scratch = temps.Acquire();
   1709  movePtr(ImmPtr(dest.addr), scratch);
   1710  return storePtr(src, Address(scratch, 0));
   1711 }
   1712 
   1713 void MacroAssemblerRiscv64Compat::testNullSet(Condition cond,
   1714                                              const ValueOperand& value,
   1715                                              Register dest) {
   1716  MOZ_ASSERT(cond == Equal || cond == NotEqual);
   1717  UseScratchRegisterScope temps(this);
   1718  Register scratch = temps.Acquire();
   1719  splitTag(value, scratch);
   1720  ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_NULL), cond);
   1721 }
   1722 
   1723 void MacroAssemblerRiscv64Compat::testObjectSet(Condition cond,
   1724                                                const ValueOperand& value,
   1725                                                Register dest) {
   1726  MOZ_ASSERT(cond == Equal || cond == NotEqual);
   1727  UseScratchRegisterScope temps(this);
   1728  Register scratch = temps.Acquire();
   1729  splitTag(value, scratch);
   1730  ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_OBJECT), cond);
   1731 }
   1732 
   1733 void MacroAssemblerRiscv64Compat::testUndefinedSet(Condition cond,
   1734                                                   const ValueOperand& value,
   1735                                                   Register dest) {
   1736  MOZ_ASSERT(cond == Equal || cond == NotEqual);
   1737  UseScratchRegisterScope temps(this);
   1738  Register scratch = temps.Acquire();
   1739  splitTag(value, scratch);
   1740  ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_UNDEFINED), cond);
   1741 }
   1742 
   1743 void MacroAssemblerRiscv64Compat::unboxInt32(const ValueOperand& operand,
   1744                                             Register dest) {
   1745  SignExtendWord(dest, operand.valueReg());
   1746 }
   1747 
   1748 void MacroAssemblerRiscv64Compat::unboxInt32(Register src, Register dest) {
   1749  SignExtendWord(dest, src);
   1750 }
   1751 
   1752 void MacroAssemblerRiscv64Compat::unboxInt32(const Address& src,
   1753                                             Register dest) {
   1754  load32(Address(src.base, src.offset), dest);
   1755 }
   1756 
   1757 void MacroAssemblerRiscv64Compat::unboxInt32(const BaseIndex& src,
   1758                                             Register dest) {
   1759  UseScratchRegisterScope temps(this);
   1760  Register scratch = temps.Acquire();
   1761  computeScaledAddress(src, scratch);
   1762  load32(Address(scratch, src.offset), dest);
   1763 }
   1764 
   1765 void MacroAssemblerRiscv64Compat::unboxBoolean(const ValueOperand& operand,
   1766                                               Register dest) {
   1767  ExtractBits(dest, operand.valueReg(), 0, 32);
   1768 }
   1769 
   1770 void MacroAssemblerRiscv64Compat::unboxBoolean(Register src, Register dest) {
   1771  ExtractBits(dest, src, 0, 32);
   1772 }
   1773 
   1774 void MacroAssemblerRiscv64Compat::unboxBoolean(const Address& src,
   1775                                               Register dest) {
   1776  ma_load(dest, Address(src.base, src.offset), SizeWord, ZeroExtend);
   1777 }
   1778 
   1779 void MacroAssemblerRiscv64Compat::unboxBoolean(const BaseIndex& src,
   1780                                               Register dest) {
   1781  UseScratchRegisterScope temps(this);
   1782  Register scratch = temps.Acquire();
   1783  computeScaledAddress(src, scratch);
   1784  ma_load(dest, Address(scratch, src.offset), SizeWord, ZeroExtend);
   1785 }
   1786 
   1787 void MacroAssemblerRiscv64Compat::unboxDouble(const ValueOperand& operand,
   1788                                              FloatRegister dest) {
   1789  fmv_d_x(dest, operand.valueReg());
   1790 }
   1791 
   1792 void MacroAssemblerRiscv64Compat::unboxDouble(const Address& src,
   1793                                              FloatRegister dest) {
   1794  ma_loadDouble(dest, Address(src.base, src.offset));
   1795 }
   1796 
   1797 void MacroAssemblerRiscv64Compat::unboxDouble(const BaseIndex& src,
   1798                                              FloatRegister dest) {
   1799  UseScratchRegisterScope temps(this);
   1800  Register scratch = temps.Acquire();
   1801  loadPtr(src, scratch);
   1802  unboxDouble(ValueOperand(scratch), dest);
   1803 }
   1804 
   1805 void MacroAssemblerRiscv64Compat::unboxString(const ValueOperand& operand,
   1806                                              Register dest) {
   1807  unboxNonDouble(operand, dest, JSVAL_TYPE_STRING);
   1808 }
   1809 
   1810 void MacroAssemblerRiscv64Compat::unboxString(Register src, Register dest) {
   1811  unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
   1812 }
   1813 
   1814 void MacroAssemblerRiscv64Compat::unboxString(const Address& src,
   1815                                              Register dest) {
   1816  unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
   1817 }
   1818 
   1819 void MacroAssemblerRiscv64Compat::unboxSymbol(const ValueOperand& operand,
   1820                                              Register dest) {
   1821  unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL);
   1822 }
   1823 
   1824 void MacroAssemblerRiscv64Compat::unboxSymbol(Register src, Register dest) {
   1825  unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
   1826 }
   1827 
   1828 void MacroAssemblerRiscv64Compat::unboxSymbol(const Address& src,
   1829                                              Register dest) {
   1830  unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
   1831 }
   1832 
   1833 void MacroAssemblerRiscv64Compat::unboxBigInt(const ValueOperand& operand,
   1834                                              Register dest) {
   1835  unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT);
   1836 }
   1837 
   1838 void MacroAssemblerRiscv64Compat::unboxBigInt(Register src, Register dest) {
   1839  unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
   1840 }
   1841 
   1842 void MacroAssemblerRiscv64Compat::unboxBigInt(const Address& src,
   1843                                              Register dest) {
   1844  unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
   1845 }
   1846 
   1847 void MacroAssemblerRiscv64Compat::unboxObject(const ValueOperand& src,
   1848                                              Register dest) {
   1849  unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
   1850 }
   1851 
   1852 void MacroAssemblerRiscv64Compat::unboxObject(Register src, Register dest) {
   1853  unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
   1854 }
   1855 
   1856 void MacroAssemblerRiscv64Compat::unboxObject(const Address& src,
   1857                                              Register dest) {
   1858  unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
   1859 }
   1860 
   1861 void MacroAssemblerRiscv64Compat::unboxValue(const ValueOperand& src,
   1862                                             AnyRegister dest,
   1863                                             JSValueType type) {
   1864  if (dest.isFloat()) {
   1865    Label notInt32, end;
   1866    asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
   1867    convertInt32ToDouble(src.valueReg(), dest.fpu());
   1868    ma_branch(&end);
   1869    bind(&notInt32);
   1870    unboxDouble(src, dest.fpu());
   1871    bind(&end);
   1872  } else {
   1873    unboxNonDouble(src, dest.gpr(), type);
   1874  }
   1875 }
   1876 
   1877 void MacroAssemblerRiscv64Compat::boxDouble(FloatRegister src,
   1878                                            const ValueOperand& dest,
   1879                                            FloatRegister) {
   1880  fmv_x_d(dest.valueReg(), src);
   1881 }
   1882 
   1883 #ifdef DEBUG
   1884 static constexpr int32_t PayloadSize(JSValueType type) {
   1885  switch (type) {
   1886    case JSVAL_TYPE_UNDEFINED:
   1887    case JSVAL_TYPE_NULL:
   1888      return 0;
   1889    case JSVAL_TYPE_BOOLEAN:
   1890      return 1;
   1891    case JSVAL_TYPE_INT32:
   1892    case JSVAL_TYPE_MAGIC:
   1893      return 32;
   1894    case JSVAL_TYPE_STRING:
   1895    case JSVAL_TYPE_SYMBOL:
   1896    case JSVAL_TYPE_PRIVATE_GCTHING:
   1897    case JSVAL_TYPE_BIGINT:
   1898    case JSVAL_TYPE_OBJECT:
   1899      return JSVAL_TAG_SHIFT;
   1900    case JSVAL_TYPE_DOUBLE:
   1901    case JSVAL_TYPE_UNKNOWN:
   1902      break;
   1903  }
   1904  MOZ_CRASH("bad value type");
   1905 }
   1906 #endif
   1907 
   1908 static void AssertValidPayload(MacroAssemblerRiscv64Compat& masm,
   1909                               JSValueType type, Register payload,
   1910                               Register scratch) {
   1911 #ifdef DEBUG
   1912  if (type == JSVAL_TYPE_INT32) {
   1913    // Ensure the payload is a properly sign-extended int32.
   1914    Label signExtended;
   1915    masm.SignExtendWord(scratch, payload);
   1916    masm.ma_b(payload, scratch, &signExtended, Assembler::Equal, ShortJump);
   1917    masm.breakpoint();
   1918    masm.bind(&signExtended);
   1919  } else {
   1920    // All bits above the payload must be zeroed.
   1921    Label zeroed;
   1922    masm.srli(scratch, payload, PayloadSize(type));
   1923    masm.ma_b(scratch, Imm32(0), &zeroed, Assembler::Equal, ShortJump);
   1924    masm.breakpoint();
   1925    masm.bind(&zeroed);
   1926  }
   1927 #endif
   1928 }
   1929 
   1930 void MacroAssemblerRiscv64Compat::boxValue(JSValueType type, Register src,
   1931                                           Register dest) {
   1932  MOZ_ASSERT(type != JSVAL_TYPE_UNDEFINED && type != JSVAL_TYPE_NULL);
   1933  MOZ_ASSERT(src != dest);
   1934 
   1935  AssertValidPayload(*this, type, src, dest);
   1936 
   1937  switch (type) {
   1938    case JSVAL_TYPE_INT32: {
   1939      // Loading the shifted tag requires only two instructions.
   1940      ma_li(dest, ImmShiftedTag(type));
   1941 
   1942      UseScratchRegisterScope temps(this);
   1943      Register scratch = temps.Acquire();
   1944 
   1945      // Insert low 32 bits as payload, removing all high bits from |src|.
   1946      ZeroExtendWord(scratch, src);
   1947      or_(dest, dest, scratch);
   1948      return;
   1949    }
   1950    case JSVAL_TYPE_BOOLEAN:
   1951    case JSVAL_TYPE_MAGIC:
   1952    case JSVAL_TYPE_STRING:
   1953    case JSVAL_TYPE_SYMBOL:
   1954    case JSVAL_TYPE_PRIVATE_GCTHING:
   1955    case JSVAL_TYPE_BIGINT:
   1956    case JSVAL_TYPE_OBJECT: {
   1957      // Loading the shifted tag requires only two instructions.
   1958      ma_li(dest, ImmShiftedTag(type));
   1959 
   1960      // Insert payload.
   1961      or_(dest, dest, src);
   1962      return;
   1963    }
   1964    case JSVAL_TYPE_DOUBLE:
   1965    case JSVAL_TYPE_UNDEFINED:
   1966    case JSVAL_TYPE_NULL:
   1967    case JSVAL_TYPE_UNKNOWN:
   1968      break;
   1969  }
   1970  MOZ_CRASH("bad value type");
   1971 }
   1972 
   1973 void MacroAssemblerRiscv64Compat::boxValue(Register type, Register src,
   1974                                           Register dest) {
   1975  MOZ_ASSERT(src != dest);
   1976 
   1977 #ifdef DEBUG
   1978  Label done, isNullOrUndefined, isBoolean, isInt32OrMagic, isPointerSized;
   1979 
   1980  asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_NULL),
   1981                    &isNullOrUndefined);
   1982  asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_UNDEFINED),
   1983                    &isNullOrUndefined);
   1984  asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_BOOLEAN),
   1985                    &isBoolean);
   1986  asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_INT32),
   1987                    &isInt32OrMagic);
   1988  asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_MAGIC),
   1989                    &isInt32OrMagic);
   1990  // GCThing types aren't currently supported, because SignExtendWord truncates
   1991  // payloads above UINT32_MAX.
   1992  breakpoint();
   1993  {
   1994    bind(&isNullOrUndefined);
   1995 
   1996    // Ensure no payload for null and undefined.
   1997    ma_b(src, src, &done, Assembler::Zero, ShortJump);
   1998    breakpoint();
   1999  }
   2000  {
   2001    bind(&isBoolean);
   2002 
   2003    // Ensure boolean values are either 0 or 1.
   2004    ma_b(src, Imm32(1), &done, Assembler::BelowOrEqual, ShortJump);
   2005    breakpoint();
   2006  }
   2007  {
   2008    bind(&isInt32OrMagic);
   2009 
   2010    // Ensure |src| is sign-extended.
   2011    UseScratchRegisterScope temps(this);
   2012    Register scratch = temps.Acquire();
   2013    SignExtendWord(scratch, src);
   2014    ma_b(src, scratch, &done, Assembler::Equal, ShortJump);
   2015    breakpoint();
   2016  }
   2017  bind(&done);
   2018 #endif
   2019 
   2020  // JSVAL_TAG_MAX_DOUBLE can't be directly encoded in a single `ori`
   2021  // instruction. Sign-extend the tag, taking the bits into account which will
   2022  // later be shifted out, into a shorter immediate which fits into `ori`.
   2023  constexpr int64_t tag =
   2024      int64_t(uint64_t(JSVAL_TAG_MAX_DOUBLE) << JSVAL_TAG_SHIFT) >>
   2025      JSVAL_TAG_SHIFT;
   2026  static_assert(is_int12(tag), "ori requires int12 immediate");
   2027 
   2028  ori(dest, type, tag);
   2029  slli(dest, dest, JSVAL_TAG_SHIFT);
   2030 
   2031  // Insert low 32 bits as payload, removing all high bits from |src|.
   2032  UseScratchRegisterScope temps(this);
   2033  Register scratch = temps.Acquire();
   2034  ZeroExtendWord(scratch, src);
   2035 
   2036  ma_or(dest, dest, scratch);
   2037 }
   2038 
   2039 void MacroAssemblerRiscv64Compat::loadConstantFloat32(float f,
   2040                                                      FloatRegister dest) {
   2041  ma_lis(dest, f);
   2042 }
   2043 
   2044 void MacroAssemblerRiscv64Compat::loadInt32OrDouble(const Address& src,
   2045                                                    FloatRegister dest) {
   2046  UseScratchRegisterScope temps(this);
   2047  Register scratch = temps.Acquire();
   2048 
   2049  Label notInt32, end;
   2050  {
   2051    // Inlined |branchTestInt32| to use a short-jump.
   2052    Register tag = extractTag(src, scratch);
   2053    ma_b(tag, ImmTag(JSVAL_TAG_INT32), &notInt32, Assembler::NotEqual,
   2054         ShortJump);
   2055  }
   2056  {
   2057    // If it's an int, convert it to double.
   2058    unboxInt32(src, scratch);
   2059    convertInt32ToDouble(scratch, dest);
   2060    ma_branch(&end);
   2061  }
   2062  bind(&notInt32);
   2063  {
   2064    // Not an int, just load as double.
   2065    unboxDouble(src, dest);
   2066  }
   2067  bind(&end);
   2068 }
   2069 
   2070 void MacroAssemblerRiscv64Compat::loadInt32OrDouble(const BaseIndex& addr,
   2071                                                    FloatRegister dest) {
   2072  UseScratchRegisterScope temps(this);
   2073  Register scratch = temps.Acquire();
   2074 
   2075  computeScaledAddress(addr, scratch);
   2076  loadInt32OrDouble(Address(scratch, addr.offset), dest);
   2077 }
   2078 
   2079 void MacroAssemblerRiscv64Compat::loadConstantDouble(double dp,
   2080                                                     FloatRegister dest) {
   2081  ma_lid(dest, dp);
   2082 }
   2083 
   2084 Register MacroAssemblerRiscv64Compat::extractObject(const Address& address,
   2085                                                    Register scratch) {
   2086  loadPtr(address, scratch);
   2087  ExtractBits(scratch, scratch, 0, JSVAL_TAG_SHIFT);
   2088  return scratch;
   2089 }
   2090 
   2091 Register MacroAssemblerRiscv64Compat::extractTag(const Address& address,
   2092                                                 Register scratch) {
   2093  loadPtr(address, scratch);
   2094  ExtractBits(scratch, scratch, JSVAL_TAG_SHIFT, 64 - JSVAL_TAG_SHIFT);
   2095  return scratch;
   2096 }
   2097 
   2098 Register MacroAssemblerRiscv64Compat::extractTag(const BaseIndex& address,
   2099                                                 Register scratch) {
   2100  computeScaledAddress(address, scratch);
   2101  return extractTag(Address(scratch, address.offset), scratch);
   2102 }
   2103 
   2104 /////////////////////////////////////////////////////////////////
   2105 // X86/X64-common/ARM/LoongArch interface.
   2106 /////////////////////////////////////////////////////////////////
   2107 /////////////////////////////////////////////////////////////////
   2108 // X86/X64-common/ARM/MIPS interface.
   2109 /////////////////////////////////////////////////////////////////
   2110 void MacroAssemblerRiscv64Compat::storeValue(ValueOperand val,
   2111                                             const BaseIndex& dest) {
   2112  UseScratchRegisterScope temps(this);
   2113  Register scratch = temps.Acquire();
   2114  computeScaledAddress(dest, scratch);
   2115  storeValue(val, Address(scratch, dest.offset));
   2116 }
   2117 
   2118 void MacroAssemblerRiscv64Compat::storeValue(JSValueType type, Register reg,
   2119                                             BaseIndex dest) {
   2120  UseScratchRegisterScope temps(this);
   2121  Register scratch = temps.Acquire();
   2122 
   2123  computeScaledAddress(dest, scratch);
   2124 
   2125  int32_t offset = dest.offset;
   2126  if (!is_int12(offset)) {
   2127    UseScratchRegisterScope temps(this);
   2128    Register scratch2 = temps.Acquire();
   2129    ma_li(scratch2, Imm32(offset));
   2130    add(scratch, scratch, scratch2);
   2131    offset = 0;
   2132  }
   2133 
   2134  storeValue(type, reg, Address(scratch, offset));
   2135 }
   2136 
   2137 void MacroAssemblerRiscv64Compat::storeValue(ValueOperand val,
   2138                                             const Address& dest) {
   2139  storePtr(val.valueReg(), Address(dest.base, dest.offset));
   2140 }
   2141 
   2142 void MacroAssemblerRiscv64Compat::storeValue(JSValueType type, Register reg,
   2143                                             Address dest) {
   2144  if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
   2145 #ifdef DEBUG
   2146    {
   2147      UseScratchRegisterScope temps(this);
   2148      Register scratch = temps.Acquire();
   2149 
   2150      AssertValidPayload(*this, type, reg, scratch);
   2151    }
   2152 #endif
   2153 
   2154    store32(reg, dest);
   2155    JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
   2156    store32(Imm64(tag).hi(), Address(dest.base, dest.offset + 4));
   2157  } else {
   2158    UseScratchRegisterScope temps(this);
   2159    Register scratch = temps.Acquire();
   2160    MOZ_ASSERT(dest.base != scratch);
   2161    boxValue(type, reg, scratch);
   2162    storePtr(scratch, Address(dest.base, dest.offset));
   2163  }
   2164 }
   2165 
   2166 void MacroAssemblerRiscv64Compat::storeValue(const Value& val, Address dest) {
   2167  UseScratchRegisterScope temps(this);
   2168  Register scratch2 = temps.Acquire();
   2169  if (val.isGCThing()) {
   2170    writeDataRelocation(val);
   2171    movWithPatch(ImmWord(val.asRawBits()), scratch2);
   2172  } else {
   2173    ma_li(scratch2, ImmWord(val.asRawBits()));
   2174  }
   2175  storePtr(scratch2, Address(dest.base, dest.offset));
   2176 }
   2177 
   2178 void MacroAssemblerRiscv64Compat::storeValue(const Value& val, BaseIndex dest) {
   2179  UseScratchRegisterScope temps(this);
   2180  Register scratch = temps.Acquire();
   2181  computeScaledAddress(dest, scratch);
   2182 
   2183  int32_t offset = dest.offset;
   2184  if (!is_int12(offset)) {
   2185    Register scratch2 = temps.Acquire();
   2186    ma_li(scratch2, Imm32(offset));
   2187    add(scratch, scratch, scratch2);
   2188    offset = 0;
   2189  }
   2190  storeValue(val, Address(scratch, offset));
   2191 }
   2192 
   2193 void MacroAssemblerRiscv64Compat::loadValue(const BaseIndex& addr,
   2194                                            ValueOperand val) {
   2195  UseScratchRegisterScope temps(this);
   2196  Register scratch = temps.Acquire();
   2197  computeScaledAddress(addr, scratch);
   2198  loadValue(Address(scratch, addr.offset), val);
   2199 }
   2200 
   2201 void MacroAssemblerRiscv64Compat::loadValue(Address src, ValueOperand val) {
   2202  loadPtr(Address(src.base, src.offset), val.valueReg());
   2203 }
   2204 
   2205 void MacroAssemblerRiscv64Compat::tagValue(JSValueType type, Register payload,
   2206                                           ValueOperand dest) {
   2207  MOZ_ASSERT(type != JSVAL_TYPE_UNDEFINED && type != JSVAL_TYPE_NULL);
   2208 
   2209  JitSpew(JitSpew_Codegen, "[ tagValue");
   2210 
   2211  if (payload == dest.valueReg()) {
   2212    UseScratchRegisterScope temps(this);
   2213    Register scratch = temps.Acquire();
   2214    MOZ_ASSERT(dest.valueReg() != scratch);
   2215 
   2216    AssertValidPayload(*this, type, payload, scratch);
   2217 
   2218    switch (type) {
   2219      case JSVAL_TYPE_INT32: {
   2220        // Loading the shifted tag requires only two instructions.
   2221        ma_li(scratch, ImmShiftedTag(type));
   2222 
   2223        // Insert low 32 bits as payload, removing all high bits from |payload|.
   2224        ZeroExtendWord(payload, payload);
   2225        or_(dest.valueReg(), payload, scratch);
   2226        break;
   2227      }
   2228      case JSVAL_TYPE_BOOLEAN:
   2229      case JSVAL_TYPE_MAGIC:
   2230      case JSVAL_TYPE_STRING:
   2231      case JSVAL_TYPE_SYMBOL:
   2232      case JSVAL_TYPE_PRIVATE_GCTHING:
   2233      case JSVAL_TYPE_BIGINT:
   2234      case JSVAL_TYPE_OBJECT: {
   2235        // Loading the shifted tag requires only two instructions.
   2236        ma_li(scratch, ImmShiftedTag(type));
   2237 
   2238        // Insert payload.
   2239        or_(dest.valueReg(), payload, scratch);
   2240        break;
   2241      }
   2242      case JSVAL_TYPE_DOUBLE:
   2243      case JSVAL_TYPE_UNDEFINED:
   2244      case JSVAL_TYPE_NULL:
   2245      case JSVAL_TYPE_UNKNOWN:
   2246        MOZ_CRASH("bad value type");
   2247    }
   2248  } else {
   2249    boxNonDouble(type, payload, dest);
   2250  }
   2251 
   2252  JitSpew(JitSpew_Codegen, "]");
   2253 }
   2254 
   2255 void MacroAssemblerRiscv64Compat::pushValue(ValueOperand val) {
   2256  // Allocate stack slots for Value. One for each.
   2257  asMasm().subPtr(Imm32(sizeof(Value)), StackPointer);
   2258  // Store Value
   2259  storeValue(val, Address(StackPointer, 0));
   2260 }
   2261 
   2262 void MacroAssemblerRiscv64Compat::pushValue(const Address& addr) {
   2263  // Load value before allocate stack, addr.base may be is sp.
   2264  UseScratchRegisterScope temps(this);
   2265  Register scratch = temps.Acquire();
   2266  loadPtr(Address(addr.base, addr.offset), scratch);
   2267  ma_sub64(StackPointer, StackPointer, Imm32(sizeof(Value)));
   2268  storePtr(scratch, Address(StackPointer, 0));
   2269 }
   2270 
   2271 void MacroAssemblerRiscv64Compat::popValue(ValueOperand val) {
   2272  ld(val.valueReg(), StackPointer, 0);
   2273  ma_add64(StackPointer, StackPointer, Imm32(sizeof(Value)));
   2274 }
   2275 
   2276 void MacroAssemblerRiscv64Compat::breakpoint(uint32_t value) { break_(value); }
   2277 
   2278 void MacroAssemblerRiscv64Compat::handleFailureWithHandlerTail(
   2279    Label* profilerExitTail, Label* bailoutTail,
   2280    uint32_t* returnValueCheckOffset) {
   2281  // Reserve space for exception information.
   2282  int size = (sizeof(ResumeFromException) + ABIStackAlignment) &
   2283             ~(ABIStackAlignment - 1);
   2284  asMasm().subPtr(Imm32(size), StackPointer);
   2285  mv(a0, StackPointer);  // Use a0 since it is a first function argument
   2286 
   2287  // Call the handler.
   2288  using Fn = void (*)(ResumeFromException* rfe);
   2289  asMasm().setupUnalignedABICall(a1);
   2290  asMasm().passABIArg(a0);
   2291  asMasm().callWithABI<Fn, HandleException>(
   2292      ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
   2293 
   2294  *returnValueCheckOffset = currentOffset();
   2295 
   2296  Label entryFrame;
   2297  Label catch_;
   2298  Label finally;
   2299  Label returnBaseline;
   2300  Label returnIon;
   2301  Label bailout;
   2302  Label wasmInterpEntry;
   2303  Label wasmCatch;
   2304 
   2305  // Already clobbered a0, so use it...
   2306  load32(Address(StackPointer, ResumeFromException::offsetOfKind()), a0);
   2307  asMasm().branch32(Assembler::Equal, a0,
   2308                    Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
   2309  asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Catch),
   2310                    &catch_);
   2311  asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Finally),
   2312                    &finally);
   2313  asMasm().branch32(Assembler::Equal, a0,
   2314                    Imm32(ExceptionResumeKind::ForcedReturnBaseline),
   2315                    &returnBaseline);
   2316  asMasm().branch32(Assembler::Equal, a0,
   2317                    Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
   2318  asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Bailout),
   2319                    &bailout);
   2320  asMasm().branch32(Assembler::Equal, a0,
   2321                    Imm32(ExceptionResumeKind::WasmInterpEntry),
   2322                    &wasmInterpEntry);
   2323  asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::WasmCatch),
   2324                    &wasmCatch);
   2325 
   2326  breakpoint();  // Invalid kind.
   2327 
   2328  // No exception handler. Load the error value, restore state and return from
   2329  // the entry frame.
   2330  bind(&entryFrame);
   2331  asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
   2332  loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
   2333          FramePointer);
   2334  loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
   2335          StackPointer);
   2336 
   2337  // We're going to be returning by the ion calling convention
   2338  ma_pop(ra);
   2339  jump(ra);
   2340  nop();
   2341 
   2342  // If we found a catch handler, this must be a baseline frame. Restore
   2343  // state and jump to the catch block.
   2344  bind(&catch_);
   2345  loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), a0);
   2346  loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
   2347          FramePointer);
   2348  loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
   2349          StackPointer);
   2350  jump(a0);
   2351 
   2352  // If we found a finally block, this must be a baseline frame. Push three
   2353  // values expected by the finally block: the exception, the exception stack,
   2354  // and BooleanValue(true).
   2355  bind(&finally);
   2356  ValueOperand exception = ValueOperand(a1);
   2357  loadValue(Address(sp, ResumeFromException::offsetOfException()), exception);
   2358 
   2359  ValueOperand exceptionStack = ValueOperand(a2);
   2360  loadValue(Address(sp, ResumeFromException::offsetOfExceptionStack()),
   2361            exceptionStack);
   2362 
   2363  loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a0);
   2364  loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()),
   2365          FramePointer);
   2366  loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp);
   2367 
   2368  pushValue(exception);
   2369  pushValue(exceptionStack);
   2370  pushValue(BooleanValue(true));
   2371  jump(a0);
   2372 
   2373  // Return BaselineFrame->returnValue() to the caller.
   2374  // Used in debug mode and for GeneratorReturn.
   2375  Label profilingInstrumentation;
   2376  bind(&returnBaseline);
   2377  loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
   2378          FramePointer);
   2379  loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
   2380          StackPointer);
   2381  loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
   2382            JSReturnOperand);
   2383  jump(&profilingInstrumentation);
   2384 
   2385  // Return the given value to the caller.
   2386  bind(&returnIon);
   2387  loadValue(Address(StackPointer, ResumeFromException::offsetOfException()),
   2388            JSReturnOperand);
   2389  loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
   2390          FramePointer);
   2391  loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
   2392          StackPointer);
   2393 
   2394  // If profiling is enabled, then update the lastProfilingFrame to refer to
   2395  // caller frame before returning. This code is shared by ForcedReturnIon
   2396  // and ForcedReturnBaseline.
   2397  bind(&profilingInstrumentation);
   2398  {
   2399    Label skipProfilingInstrumentation;
   2400    // Test if profiler enabled.
   2401    AbsoluteAddress addressOfEnabled(
   2402        asMasm().runtime()->geckoProfiler().addressOfEnabled());
   2403    asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
   2404                      &skipProfilingInstrumentation);
   2405    jump(profilerExitTail);
   2406    bind(&skipProfilingInstrumentation);
   2407  }
   2408 
   2409  mv(StackPointer, FramePointer);
   2410  pop(FramePointer);
   2411  ret();
   2412 
   2413  // If we are bailing out to baseline to handle an exception, jump to
   2414  // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
   2415  bind(&bailout);
   2416  loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), a2);
   2417  loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
   2418          StackPointer);
   2419  ma_li(ReturnReg, Imm32(1));
   2420  jump(bailoutTail);
   2421 
   2422  // Reset SP and FP; SP is pointing to the unwound return address to the wasm
   2423  // interpreter entry, so we can just ret().
   2424  bind(&wasmInterpEntry);
   2425  loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
   2426          FramePointer);
   2427  loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
   2428          StackPointer);
   2429  ma_li(InstanceReg, ImmWord(wasm::InterpFailInstanceReg));
   2430  ret();
   2431 
   2432  // Found a wasm catch handler, restore state and jump to it.
   2433  bind(&wasmCatch);
   2434  wasm::GenerateJumpToCatchHandler(asMasm(), sp, a1, a2);
   2435 }
   2436 
   2437 CodeOffset MacroAssemblerRiscv64Compat::toggledJump(Label* label) {
   2438  CodeOffset ret(nextOffset().getOffset());
   2439  BranchShort(label);
   2440  return ret;
   2441 }
   2442 
   2443 CodeOffset MacroAssemblerRiscv64Compat::toggledCall(JitCode* target,
   2444                                                    bool enabled) {
   2445  DEBUG_PRINTF("\ttoggledCall\n");
   2446  UseScratchRegisterScope temps(this);
   2447  Register scratch = temps.Acquire();
   2448  BlockTrampolinePoolScope block_trampoline_pool(this, 8);
   2449  BufferOffset bo = nextOffset();
   2450  CodeOffset offset(bo.getOffset());
   2451  addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
   2452  ma_liPatchable(scratch, ImmPtr(target->raw()));
   2453  if (enabled) {
   2454    jalr(scratch);
   2455  } else {
   2456    nop();
   2457  }
   2458  MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() ==
   2459                            ToggledCallSize(nullptr));
   2460  return offset;
   2461 }
   2462 
   2463 void MacroAssembler::subFromStackPtr(Imm32 imm32) {
   2464  if (imm32.value) {
   2465    asMasm().subPtr(imm32, StackPointer);
   2466  }
   2467 }
   2468 
   2469 void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
   2470  Round_w_d(output, input);
   2471  Clear_if_nan_d(output, input);
   2472  clampIntToUint8(output);
   2473 }
   2474 
   2475 //{{{ check_macroassembler_style
   2476 // ===============================================================
   2477 // MacroAssembler high-level usage.
   2478 bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
   2479 CodeOffset MacroAssembler::call(Label* label) { return BranchAndLink(label); }
   2480 CodeOffset MacroAssembler::call(Register reg) {
   2481  jalr(reg, 0);
   2482  return CodeOffset(currentOffset());
   2483 }
   2484 CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
   2485  UseScratchRegisterScope temps(this);
   2486  temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
   2487  movePtr(target, CallReg);
   2488  return call(CallReg);
   2489 }
   2490 CodeOffset MacroAssembler::farJumpWithPatch() {
   2491  UseScratchRegisterScope temps(this);
   2492  Register scratch = temps.Acquire();
   2493  Register scratch2 = temps.Acquire();
   2494  // Allocate space which will be patched by patchFarJump().
   2495  CodeOffset farJump(nextInstrOffset(5).getOffset());
   2496  auipc(scratch, 0);
   2497  lw(scratch2, scratch, 4 * sizeof(Instr));
   2498  add(scratch, scratch, scratch2);
   2499  jr(scratch, 0);
   2500  spew(".space 32bit initValue 0xffff ffff");
   2501  emit(UINT32_MAX);
   2502  return farJump;
   2503 }
   2504 CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
   2505  return movWithPatch(ImmPtr(nullptr), dest);
   2506 }
   2507 CodeOffset MacroAssembler::nopPatchableToCall() {
   2508  BlockTrampolinePoolScope block_trampoline_pool(this, 7);
   2509  // riscv64
   2510  nop();  // lui(rd, (int32_t)high_20);
   2511  nop();  // addi(rd, rd, low_12);  // 31 bits in rd.
   2512  nop();  // slli(rd, rd, 11);      // Space for next 11 bis
   2513  nop();  // ori(rd, rd, b11);      // 11 bits are put in. 42 bit in rd
   2514  nop();  // slli(rd, rd, 6);       // Space for next 6 bits
   2515  nop();  // ori(rd, rd, a6);       // 6 bits are put in. 48 bis in rd
   2516  nop();  // jirl
   2517  return CodeOffset(currentOffset());
   2518 }
   2519 FaultingCodeOffset MacroAssembler::wasmTrapInstruction() {
   2520  BlockTrampolinePoolScope block_trampoline_pool(this, 2);
   2521  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   2522  illegal_trap(kWasmTrapCode);
   2523  ebreak();
   2524  return fco;
   2525 }
   2526 size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
   2527  return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
   2528 }
   2529 
   2530 template <typename T>
   2531 void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
   2532                                                  const T& value, Register temp,
   2533                                                  Label* label) {
   2534  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
   2535  Label done;
   2536  branchTestGCThing(Assembler::NotEqual, value,
   2537                    cond == Assembler::Equal ? &done : label);
   2538 
   2539  // temp may be InvalidReg, use scratch2 instead.
   2540  UseScratchRegisterScope temps(this);
   2541  Register scratch2 = temps.Acquire();
   2542 
   2543  getGCThingValueChunk(value, scratch2);
   2544  loadPtr(Address(scratch2, gc::ChunkStoreBufferOffset), scratch2);
   2545  branchPtr(InvertCondition(cond), scratch2, ImmWord(0), label);
   2546 
   2547  bind(&done);
   2548 }
   2549 
   2550 template <typename T>
   2551 void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
   2552                                       MIRType valueType, const T& dest) {
   2553  MOZ_ASSERT(valueType < MIRType::Value);
   2554 
   2555  if (valueType == MIRType::Double) {
   2556    boxDouble(value.reg().typedReg().fpu(), dest);
   2557    return;
   2558  }
   2559 
   2560  if (value.constant()) {
   2561    storeValue(value.value(), dest);
   2562  } else {
   2563    storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
   2564               dest);
   2565  }
   2566 }
   2567 
   2568 template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
   2569                                                MIRType valueType,
   2570                                                const Address& dest);
   2571 template void MacroAssembler::storeUnboxedValue(
   2572    const ConstantOrRegister& value, MIRType valueType,
   2573    const BaseObjectElementIndex& dest);
   2574 
   2575 // ===============================================================
   2576 // Jit Frames.
   2577 
   2578 uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
   2579  CodeLabel cl;
   2580 
   2581  ma_li(scratch, &cl);
   2582  Push(scratch);
   2583  bind(&cl);
   2584  uint32_t retAddr = currentOffset();
   2585 
   2586  addCodeLabel(cl);
   2587  return retAddr;
   2588 }
   2589 
   2590 //===============================
   2591 // AtomicOp
   2592 
   2593 template <typename T>
   2594 static void AtomicExchange(MacroAssembler& masm,
   2595                           const wasm::MemoryAccessDesc* access,
   2596                           Scalar::Type type, Synchronization sync,
   2597                           const T& mem, Register value, Register valueTemp,
   2598                           Register offsetTemp, Register maskTemp,
   2599                           Register output) {
   2600  bool signExtend = Scalar::isSignedIntType(type);
   2601  unsigned nbytes = Scalar::byteSize(type);
   2602 
   2603  UseScratchRegisterScope temps(&masm);
   2604 
   2605  switch (nbytes) {
   2606    case 1:
   2607    case 2:
   2608      break;
   2609    case 4:
   2610      MOZ_ASSERT(valueTemp == InvalidReg);
   2611      MOZ_ASSERT(offsetTemp == InvalidReg);
   2612      MOZ_ASSERT(maskTemp == InvalidReg);
   2613      break;
   2614    default:
   2615      MOZ_CRASH();
   2616  }
   2617 
   2618  Label again;
   2619 
   2620  Register scratch = temps.Acquire();
   2621  masm.computeEffectiveAddress(mem, scratch);
   2622 
   2623  Register scratch2 = temps.Acquire();
   2624 
   2625  if (nbytes == 4) {
   2626    masm.memoryBarrierBefore(sync);
   2627    masm.bind(&again);
   2628    BlockTrampolinePoolScope block_trampoline_pool(&masm,
   2629                                                   /* 1 + 1 + 1 + 4 + 1 = */ 8,
   2630                                                   1);
   2631    if (access) {
   2632      masm.append(*access, wasm::TrapMachineInsn::Atomic,
   2633                  FaultingCodeOffset(masm.currentOffset()));
   2634    }
   2635 
   2636    masm.lr_w(true, true, output, scratch);
   2637    masm.or_(scratch2, value, zero);
   2638    masm.sc_w(true, true, scratch2, scratch, scratch2);
   2639    masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
   2640              ShortJump);
   2641 
   2642    masm.memoryBarrierAfter(sync);
   2643 
   2644    return;
   2645  }
   2646 
   2647  masm.andi(offsetTemp, scratch, 3);
   2648  masm.subPtr(offsetTemp, scratch);
   2649  masm.slliw(offsetTemp, offsetTemp, 3);
   2650  masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
   2651  masm.sllw(maskTemp, maskTemp, offsetTemp);
   2652  masm.not_(maskTemp, maskTemp);
   2653  switch (nbytes) {
   2654    case 1:
   2655      masm.andi(valueTemp, value, 0xff);
   2656      break;
   2657    case 2:
   2658      masm.ma_and(valueTemp, value, Imm32(0xffff));
   2659      break;
   2660  }
   2661  masm.sllw(valueTemp, valueTemp, offsetTemp);
   2662 
   2663  masm.memoryBarrierBefore(sync);
   2664 
   2665  masm.bind(&again);
   2666 
   2667  BlockTrampolinePoolScope block_trampoline_pool(
   2668      &masm, /* 1 + 1 + 1 + 1 + 4 + 1 + 2 + 1 = */ 12, 1);
   2669  if (access) {
   2670    masm.append(*access, wasm::TrapMachineInsn::Atomic,
   2671                FaultingCodeOffset(masm.currentOffset()));
   2672  }
   2673 
   2674  masm.lr_w(true, true, output, scratch);
   2675  masm.and_(scratch2, output, maskTemp);
   2676  masm.or_(scratch2, scratch2, valueTemp);
   2677 
   2678  masm.sc_w(true, true, scratch2, scratch, scratch2);
   2679 
   2680  masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
   2681            ShortJump);
   2682 
   2683  masm.srlw(output, output, offsetTemp);
   2684 
   2685  switch (nbytes) {
   2686    case 1:
   2687      if (signExtend) {
   2688        masm.SignExtendByte(output, output);
   2689      } else {
   2690        masm.andi(output, output, 0xff);
   2691      }
   2692      break;
   2693    case 2:
   2694      if (signExtend) {
   2695        masm.SignExtendShort(output, output);
   2696      } else {
   2697        masm.ma_and(output, Imm32(0xffff));
   2698      }
   2699      break;
   2700  }
   2701 
   2702  masm.memoryBarrierAfter(sync);
   2703 }
   2704 
   2705 template <typename T>
   2706 static void AtomicExchange64(MacroAssembler& masm,
   2707                             const wasm::MemoryAccessDesc* access,
   2708                             Synchronization sync, const T& mem,
   2709                             Register64 value, Register64 output) {
   2710  MOZ_ASSERT(value != output);
   2711  UseScratchRegisterScope temps(&masm);
   2712  Register scratch = temps.Acquire();
   2713  Register scratch2 = temps.Acquire();
   2714  masm.computeEffectiveAddress(mem, scratch2);
   2715 
   2716  Label tryAgain;
   2717 
   2718  masm.memoryBarrierBefore(sync);
   2719 
   2720  masm.bind(&tryAgain);
   2721  BlockTrampolinePoolScope block_trampoline_pool(&masm,
   2722                                                 /* 1 + 1 + 1 + 4 + 1 = */ 8,
   2723                                                 1);
   2724  if (access) {
   2725    masm.append(*access, js::wasm::TrapMachineInsn::Load64,
   2726                FaultingCodeOffset(masm.currentOffset()));
   2727  }
   2728 
   2729  masm.lr_d(true, true, output.reg, scratch2);
   2730  masm.movePtr(value.reg, scratch);
   2731  masm.sc_d(true, true, scratch, scratch2, scratch);
   2732  masm.ma_b(scratch, scratch, &tryAgain, Assembler::NonZero, ShortJump);
   2733 
   2734  masm.memoryBarrierAfter(sync);
   2735 }
   2736 
   2737 template <typename T>
   2738 static void AtomicFetchOp64(MacroAssembler& masm,
   2739                            const wasm::MemoryAccessDesc* access,
   2740                            Synchronization sync, AtomicOp op, Register64 value,
   2741                            const T& mem, Register64 temp, Register64 output) {
   2742  MOZ_ASSERT(value != output);
   2743  MOZ_ASSERT(value != temp);
   2744  UseScratchRegisterScope temps(&masm);
   2745  Register scratch2 = temps.Acquire();
   2746  masm.computeEffectiveAddress(mem, scratch2);
   2747 
   2748  Label tryAgain;
   2749 
   2750  masm.memoryBarrierBefore(sync);
   2751 
   2752  masm.bind(&tryAgain);
   2753  BlockTrampolinePoolScope block_trampoline_pool(&masm,
   2754                                                 /* 1 + 1 + 1 + 4 + 1 = */ 8,
   2755                                                 1);
   2756  if (access) {
   2757    masm.append(*access, js::wasm::TrapMachineInsn::Load64,
   2758                FaultingCodeOffset(masm.currentOffset()));
   2759  }
   2760 
   2761  masm.lr_d(true, true, output.reg, scratch2);
   2762 
   2763  switch (op) {
   2764    case AtomicOp::Add:
   2765      masm.add(temp.reg, output.reg, value.reg);
   2766      break;
   2767    case AtomicOp::Sub:
   2768      masm.sub(temp.reg, output.reg, value.reg);
   2769      break;
   2770    case AtomicOp::And:
   2771      masm.and_(temp.reg, output.reg, value.reg);
   2772      break;
   2773    case AtomicOp::Or:
   2774      masm.or_(temp.reg, output.reg, value.reg);
   2775      break;
   2776    case AtomicOp::Xor:
   2777      masm.xor_(temp.reg, output.reg, value.reg);
   2778      break;
   2779    default:
   2780      MOZ_CRASH();
   2781  }
   2782 
   2783  masm.sc_d(true, true, temp.reg, scratch2, temp.reg);
   2784  masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::NonZero, ShortJump);
   2785 
   2786  masm.memoryBarrierAfter(sync);
   2787 }
   2788 
   2789 template <typename T>
   2790 static void AtomicEffectOp(MacroAssembler& masm,
   2791                           const wasm::MemoryAccessDesc* access,
   2792                           Scalar::Type type, Synchronization sync, AtomicOp op,
   2793                           const T& mem, Register value, Register valueTemp,
   2794                           Register offsetTemp, Register maskTemp) {
   2795  UseScratchRegisterScope temps(&masm);
   2796  unsigned nbytes = Scalar::byteSize(type);
   2797 
   2798  switch (nbytes) {
   2799    case 1:
   2800    case 2:
   2801      break;
   2802    case 4:
   2803      MOZ_ASSERT(valueTemp == InvalidReg);
   2804      MOZ_ASSERT(offsetTemp == InvalidReg);
   2805      MOZ_ASSERT(maskTemp == InvalidReg);
   2806      break;
   2807    default:
   2808      MOZ_CRASH();
   2809  }
   2810 
   2811  Label again;
   2812 
   2813  Register scratch = temps.Acquire();
   2814  masm.computeEffectiveAddress(mem, scratch);
   2815 
   2816  Register scratch2 = temps.Acquire();
   2817 
   2818  if (nbytes == 4) {
   2819    masm.memoryBarrierBefore(sync);
   2820    masm.bind(&again);
   2821 
   2822    if (access) {
   2823      masm.append(*access, wasm::TrapMachineInsn::Atomic,
   2824                  FaultingCodeOffset(masm.currentOffset()));
   2825    }
   2826 
   2827    masm.lr_w(true, true, scratch2, scratch);
   2828 
   2829    switch (op) {
   2830      case AtomicOp::Add:
   2831        masm.addw(scratch2, scratch2, value);
   2832        break;
   2833      case AtomicOp::Sub:
   2834        masm.subw(scratch2, scratch2, value);
   2835        break;
   2836      case AtomicOp::And:
   2837        masm.and_(scratch2, scratch2, value);
   2838        break;
   2839      case AtomicOp::Or:
   2840        masm.or_(scratch2, scratch2, value);
   2841        break;
   2842      case AtomicOp::Xor:
   2843        masm.xor_(scratch2, scratch2, value);
   2844        break;
   2845      default:
   2846        MOZ_CRASH();
   2847    }
   2848 
   2849    masm.sc_w(true, true, scratch2, scratch, scratch2);
   2850    masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
   2851              ShortJump);
   2852 
   2853    masm.memoryBarrierAfter(sync);
   2854 
   2855    return;
   2856  }
   2857 
   2858  masm.andi(offsetTemp, scratch, 3);
   2859  masm.subPtr(offsetTemp, scratch);
   2860  masm.slliw(offsetTemp, offsetTemp, 3);
   2861  masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
   2862  masm.sllw(maskTemp, maskTemp, offsetTemp);
   2863  masm.not_(maskTemp, maskTemp);
   2864 
   2865  masm.memoryBarrierBefore(sync);
   2866 
   2867  masm.bind(&again);
   2868 
   2869  if (access) {
   2870    masm.append(*access, wasm::TrapMachineInsn::Atomic,
   2871                FaultingCodeOffset(masm.currentOffset()));
   2872  }
   2873 
   2874  masm.lr_w(true, true, scratch2, scratch);
   2875  masm.srlw(valueTemp, scratch2, offsetTemp);
   2876 
   2877  switch (op) {
   2878    case AtomicOp::Add:
   2879      masm.addw(valueTemp, valueTemp, value);
   2880      break;
   2881    case AtomicOp::Sub:
   2882      masm.subw(valueTemp, valueTemp, value);
   2883      break;
   2884    case AtomicOp::And:
   2885      masm.and_(valueTemp, valueTemp, value);
   2886      break;
   2887    case AtomicOp::Or:
   2888      masm.or_(valueTemp, valueTemp, value);
   2889      break;
   2890    case AtomicOp::Xor:
   2891      masm.xor_(valueTemp, valueTemp, value);
   2892      break;
   2893    default:
   2894      MOZ_CRASH();
   2895  }
   2896 
   2897  switch (nbytes) {
   2898    case 1:
   2899      masm.andi(valueTemp, valueTemp, 0xff);
   2900      break;
   2901    case 2:
   2902      masm.ma_and(valueTemp, valueTemp, Imm32(0xffff));
   2903      break;
   2904  }
   2905 
   2906  masm.sllw(valueTemp, valueTemp, offsetTemp);
   2907 
   2908  masm.and_(scratch2, scratch2, maskTemp);
   2909  masm.or_(scratch2, scratch2, valueTemp);
   2910 
   2911  masm.sc_w(true, true, scratch2, scratch, scratch2);
   2912 
   2913  masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
   2914            ShortJump);
   2915 
   2916  masm.memoryBarrierAfter(sync);
   2917 }
   2918 
   2919 template <typename T>
   2920 static void AtomicFetchOp(MacroAssembler& masm,
   2921                          const wasm::MemoryAccessDesc* access,
   2922                          Scalar::Type type, Synchronization sync, AtomicOp op,
   2923                          const T& mem, Register value, Register valueTemp,
   2924                          Register offsetTemp, Register maskTemp,
   2925                          Register output) {
   2926  UseScratchRegisterScope temps(&masm);
   2927  bool signExtend = Scalar::isSignedIntType(type);
   2928  unsigned nbytes = Scalar::byteSize(type);
   2929 
   2930  switch (nbytes) {
   2931    case 1:
   2932    case 2:
   2933      break;
   2934    case 4:
   2935      MOZ_ASSERT(valueTemp == InvalidReg);
   2936      MOZ_ASSERT(offsetTemp == InvalidReg);
   2937      MOZ_ASSERT(maskTemp == InvalidReg);
   2938      break;
   2939    default:
   2940      MOZ_CRASH();
   2941  }
   2942 
   2943  Label again;
   2944 
   2945  Register scratch = temps.Acquire();
   2946  masm.computeEffectiveAddress(mem, scratch);
   2947 
   2948  Register scratch2 = temps.Acquire();
   2949 
   2950  if (nbytes == 4) {
   2951    masm.memoryBarrierBefore(sync);
   2952    masm.bind(&again);
   2953 
   2954    if (access) {
   2955      masm.append(*access, wasm::TrapMachineInsn::Atomic,
   2956                  FaultingCodeOffset(masm.currentOffset()));
   2957    }
   2958 
   2959    masm.lr_w(true, true, output, scratch);
   2960 
   2961    switch (op) {
   2962      case AtomicOp::Add:
   2963        masm.addw(scratch2, output, value);
   2964        break;
   2965      case AtomicOp::Sub:
   2966        masm.subw(scratch2, output, value);
   2967        break;
   2968      case AtomicOp::And:
   2969        masm.and_(scratch2, output, value);
   2970        break;
   2971      case AtomicOp::Or:
   2972        masm.or_(scratch2, output, value);
   2973        break;
   2974      case AtomicOp::Xor:
   2975        masm.xor_(scratch2, output, value);
   2976        break;
   2977      default:
   2978        MOZ_CRASH();
   2979    }
   2980 
   2981    masm.sc_w(true, true, scratch2, scratch, scratch2);
   2982    masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
   2983              ShortJump);
   2984 
   2985    masm.memoryBarrierAfter(sync);
   2986 
   2987    return;
   2988  }
   2989 
   2990  masm.andi(offsetTemp, scratch, 3);
   2991  masm.subPtr(offsetTemp, scratch);
   2992  masm.slliw(offsetTemp, offsetTemp, 3);
   2993  masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
   2994  masm.sllw(maskTemp, maskTemp, offsetTemp);
   2995  masm.not_(maskTemp, maskTemp);
   2996 
   2997  masm.memoryBarrierBefore(sync);
   2998 
   2999  masm.bind(&again);
   3000 
   3001  if (access) {
   3002    masm.append(*access, wasm::TrapMachineInsn::Atomic,
   3003                FaultingCodeOffset(masm.currentOffset()));
   3004  }
   3005 
   3006  masm.lr_w(true, true, scratch2, scratch);
   3007  masm.srlw(output, scratch2, offsetTemp);
   3008 
   3009  switch (op) {
   3010    case AtomicOp::Add:
   3011      masm.addw(valueTemp, output, value);
   3012      break;
   3013    case AtomicOp::Sub:
   3014      masm.subw(valueTemp, output, value);
   3015      break;
   3016    case AtomicOp::And:
   3017      masm.and_(valueTemp, output, value);
   3018      break;
   3019    case AtomicOp::Or:
   3020      masm.or_(valueTemp, output, value);
   3021      break;
   3022    case AtomicOp::Xor:
   3023      masm.xor_(valueTemp, output, value);
   3024      break;
   3025    default:
   3026      MOZ_CRASH();
   3027  }
   3028 
   3029  switch (nbytes) {
   3030    case 1:
   3031      masm.andi(valueTemp, valueTemp, 0xff);
   3032      break;
   3033    case 2:
   3034      masm.ma_and(valueTemp, Imm32(0xffff));
   3035      break;
   3036  }
   3037 
   3038  masm.sllw(valueTemp, valueTemp, offsetTemp);
   3039 
   3040  masm.and_(scratch2, scratch2, maskTemp);
   3041  masm.or_(scratch2, scratch2, valueTemp);
   3042 
   3043  masm.sc_w(true, true, scratch2, scratch, scratch2);
   3044 
   3045  masm.ma_b(scratch2, Register(scratch2), &again, Assembler::NonZero,
   3046            ShortJump);
   3047 
   3048  switch (nbytes) {
   3049    case 1:
   3050      if (signExtend) {
   3051        masm.SignExtendByte(output, output);
   3052      } else {
   3053        masm.andi(output, output, 0xff);
   3054      }
   3055      break;
   3056    case 2:
   3057      if (signExtend) {
   3058        masm.SignExtendShort(output, output);
   3059      } else {
   3060        masm.ma_and(output, Imm32(0xffff));
   3061      }
   3062      break;
   3063  }
   3064 
   3065  masm.memoryBarrierAfter(sync);
   3066 }
   3067 
   3068 // ========================================================================
   3069 // JS atomic operations.
   3070 
   3071 template <typename T>
   3072 static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
   3073                              Synchronization sync, const T& mem,
   3074                              Register oldval, Register newval,
   3075                              Register valueTemp, Register offsetTemp,
   3076                              Register maskTemp, Register temp,
   3077                              AnyRegister output) {
   3078  if (arrayType == Scalar::Uint32) {
   3079    masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
   3080                         offsetTemp, maskTemp, temp);
   3081    masm.convertUInt32ToDouble(temp, output.fpu());
   3082  } else {
   3083    masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
   3084                         offsetTemp, maskTemp, output.gpr());
   3085  }
   3086 }
   3087 
   3088 template <typename T>
   3089 static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
   3090                             Synchronization sync, const T& mem, Register value,
   3091                             Register valueTemp, Register offsetTemp,
   3092                             Register maskTemp, Register temp,
   3093                             AnyRegister output) {
   3094  if (arrayType == Scalar::Uint32) {
   3095    masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
   3096                        maskTemp, temp);
   3097    masm.convertUInt32ToDouble(temp, output.fpu());
   3098  } else {
   3099    masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
   3100                        maskTemp, output.gpr());
   3101  }
   3102 }
   3103 
   3104 template <typename T>
   3105 static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
   3106                            Synchronization sync, AtomicOp op, Register value,
   3107                            const T& mem, Register valueTemp,
   3108                            Register offsetTemp, Register maskTemp,
   3109                            Register temp, AnyRegister output) {
   3110  if (arrayType == Scalar::Uint32) {
   3111    masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
   3112                       maskTemp, temp);
   3113    masm.convertUInt32ToDouble(temp, output.fpu());
   3114  } else {
   3115    masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
   3116                       maskTemp, output.gpr());
   3117  }
   3118 }
   3119 
   3120 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
   3121                                      Synchronization sync, AtomicOp op,
   3122                                      Register value, const BaseIndex& mem,
   3123                                      Register valueTemp, Register offsetTemp,
   3124                                      Register maskTemp) {
   3125  AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
   3126                 offsetTemp, maskTemp);
   3127 }
   3128 
   3129 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
   3130                                      Synchronization sync, AtomicOp op,
   3131                                      Register value, const Address& mem,
   3132                                      Register valueTemp, Register offsetTemp,
   3133                                      Register maskTemp) {
   3134  AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
   3135                 offsetTemp, maskTemp);
   3136 }
   3137 void MacroAssembler::atomicExchange64(Synchronization sync, const Address& mem,
   3138                                      Register64 value, Register64 output) {
   3139  AtomicExchange64(*this, nullptr, sync, mem, value, output);
   3140 }
   3141 
   3142 void MacroAssembler::atomicExchange64(Synchronization sync,
   3143                                      const BaseIndex& mem, Register64 value,
   3144                                      Register64 output) {
   3145  AtomicExchange64(*this, nullptr, sync, mem, value, output);
   3146 }
   3147 
   3148 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
   3149                                      Synchronization sync, const Address& mem,
   3150                                      Register value, Register valueTemp,
   3151                                      Register offsetTemp, Register maskTemp,
   3152                                      Register temp, AnyRegister output) {
   3153  AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
   3154                   maskTemp, temp, output);
   3155 }
   3156 
   3157 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
   3158                                      Synchronization sync,
   3159                                      const BaseIndex& mem, Register value,
   3160                                      Register valueTemp, Register offsetTemp,
   3161                                      Register maskTemp, Register temp,
   3162                                      AnyRegister output) {
   3163  AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
   3164                   maskTemp, temp, output);
   3165 }
   3166 
   3167 void MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync,
   3168                                    const Address& mem, Register value,
   3169                                    Register valueTemp, Register offsetTemp,
   3170                                    Register maskTemp, Register output) {
   3171  AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
   3172                 maskTemp, output);
   3173 }
   3174 
   3175 void MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync,
   3176                                    const BaseIndex& mem, Register value,
   3177                                    Register valueTemp, Register offsetTemp,
   3178                                    Register maskTemp, Register output) {
   3179  AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
   3180                 maskTemp, output);
   3181 }
   3182 
   3183 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
   3184                                     Synchronization sync, AtomicOp op,
   3185                                     Register value, const Address& mem,
   3186                                     Register valueTemp, Register offsetTemp,
   3187                                     Register maskTemp, Register temp,
   3188                                     AnyRegister output) {
   3189  AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
   3190                  maskTemp, temp, output);
   3191 }
   3192 
   3193 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
   3194                                     Synchronization sync, AtomicOp op,
   3195                                     Register value, const BaseIndex& mem,
   3196                                     Register valueTemp, Register offsetTemp,
   3197                                     Register maskTemp, Register temp,
   3198                                     AnyRegister output) {
   3199  AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
   3200                  maskTemp, temp, output);
   3201 }
   3202 
   3203 void MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync,
   3204                                   AtomicOp op, Register value,
   3205                                   const Address& mem, Register valueTemp,
   3206                                   Register offsetTemp, Register maskTemp,
   3207                                   Register output) {
   3208  AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
   3209                offsetTemp, maskTemp, output);
   3210 }
   3211 
   3212 void MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync,
   3213                                   AtomicOp op, Register value,
   3214                                   const BaseIndex& mem, Register valueTemp,
   3215                                   Register offsetTemp, Register maskTemp,
   3216                                   Register output) {
   3217  AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
   3218                offsetTemp, maskTemp, output);
   3219 }
   3220 
   3221 void MacroAssembler::atomicPause() {
   3222  // `pause` hint defined in Zihintpause extension.
   3223  // It is encoded as `fence w, 0`.
   3224  fence(0b0001, 0b0000);
   3225 }
   3226 
   3227 void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
   3228                                             Register temp, Label* label) {
   3229  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
   3230  MOZ_ASSERT(ptr != temp);
   3231  MOZ_ASSERT(temp != InvalidReg);
   3232 
   3233  ma_and(temp, ptr, Imm32(int32_t(~gc::ChunkMask)));
   3234  branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
   3235            zero, label);
   3236 }
   3237 void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
   3238                                     const Value& rhs, Label* label) {
   3239  MOZ_ASSERT(cond == Equal || cond == NotEqual);
   3240  MOZ_ASSERT(!rhs.isNaN());
   3241 
   3242  if (!rhs.isGCThing()) {
   3243    ma_b(lhs.valueReg(), ImmWord(rhs.asRawBits()), label, cond);
   3244  } else {
   3245    UseScratchRegisterScope temps(this);
   3246    Register scratch = temps.Acquire();
   3247    MOZ_ASSERT(lhs.valueReg() != scratch);
   3248    moveValue(rhs, ValueOperand(scratch));
   3249    ma_b(lhs.valueReg(), scratch, label, cond);
   3250  }
   3251 }
   3252 
   3253 void MacroAssembler::branchTestNaNValue(Condition cond, const ValueOperand& val,
   3254                                        Register temp, Label* label) {
   3255  MOZ_ASSERT(cond == Equal || cond == NotEqual);
   3256  UseScratchRegisterScope temps(this);
   3257  Register scratch = temps.Acquire();
   3258  MOZ_ASSERT(val.valueReg() != scratch);
   3259 
   3260  // When testing for NaN, we want to ignore the sign bit.
   3261  // Clear the top bit by shifting left and then right.
   3262  slli(temp, val.valueReg(), 1);
   3263  srli(temp, temp, 1);
   3264 
   3265  // Compare against a NaN with sign bit 0.
   3266  static_assert(JS::detail::CanonicalizedNaNSignBit == 0);
   3267  moveValue(DoubleValue(JS::GenericNaN()), ValueOperand(scratch));
   3268  ma_b(temp, scratch, label, cond);
   3269 }
   3270 
   3271 void MacroAssembler::branchValueIsNurseryCell(Condition cond,
   3272                                              const Address& address,
   3273                                              Register temp, Label* label) {
   3274  branchValueIsNurseryCellImpl(cond, address, temp, label);
   3275 }
   3276 
   3277 void MacroAssembler::branchValueIsNurseryCell(Condition cond,
   3278                                              ValueOperand value, Register temp,
   3279                                              Label* label) {
   3280  branchValueIsNurseryCellImpl(cond, value, temp, label);
   3281 }
   3282 CodeOffset MacroAssembler::call(const Address& addr) {
   3283  UseScratchRegisterScope temps(this);
   3284  temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
   3285  loadPtr(addr, CallReg);
   3286  return call(CallReg);
   3287 }
   3288 void MacroAssembler::call(ImmPtr target) {
   3289  BufferOffset bo = m_buffer.nextOffset();
   3290  addPendingJump(bo, target, RelocationKind::HARDCODED);
   3291  ma_call(target);
   3292 }
   3293 void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); }
   3294 
   3295 void MacroAssembler::call(JitCode* c) {
   3296  DEBUG_PRINTF("[ %s\n", __FUNCTION__);
   3297  BlockTrampolinePoolScope block_trampoline_pool(this, 8);
   3298  UseScratchRegisterScope temps(this);
   3299  Register scratch = temps.Acquire();
   3300  BufferOffset bo = m_buffer.nextOffset();
   3301  addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
   3302  ma_liPatchable(scratch, ImmPtr(c->raw()));
   3303  callJitNoProfiler(scratch);
   3304  DEBUG_PRINTF("]\n");
   3305 }
   3306 
   3307 void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
   3308  MOZ_ASSERT(inCall_);
   3309  uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
   3310 
   3311  // Reserve place for $ra.
   3312  stackForCall += sizeof(intptr_t);
   3313 
   3314  if (dynamicAlignment_) {
   3315    stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment);
   3316  } else {
   3317    uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
   3318    stackForCall += ComputeByteAlignment(
   3319        stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
   3320  }
   3321 
   3322  *stackAdjust = stackForCall;
   3323  reserveStack(stackForCall);
   3324 
   3325  // Save $ra because call is going to clobber it. Restore it in
   3326  // callWithABIPost. NOTE: This is needed for calls from SharedIC.
   3327  // Maybe we can do this differently.
   3328  storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t)));
   3329 
   3330  // Position all arguments.
   3331  {
   3332    enoughMemory_ &= moveResolver_.resolve();
   3333    if (!enoughMemory_) {
   3334      return;
   3335    }
   3336 
   3337    MoveEmitter emitter(asMasm());
   3338    emitter.emit(moveResolver_);
   3339    emitter.finish();
   3340  }
   3341 
   3342  assertStackAlignment(ABIStackAlignment);
   3343 }
   3344 
   3345 void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) {
   3346  // Restore ra value (as stored in callWithABIPre()).
   3347  loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra);
   3348 
   3349  if (dynamicAlignment_) {
   3350    // Restore sp value from stack (as stored in setupUnalignedABICall()).
   3351    loadPtr(Address(StackPointer, stackAdjust), StackPointer);
   3352    // Use adjustFrame instead of freeStack because we already restored sp.
   3353    adjustFrame(-stackAdjust);
   3354  } else {
   3355    freeStack(stackAdjust);
   3356  }
   3357 
   3358 #ifdef DEBUG
   3359  MOZ_ASSERT(inCall_);
   3360  inCall_ = false;
   3361 #endif
   3362 }
   3363 
   3364 void MacroAssembler::callWithABINoProfiler(Register fun, ABIType result) {
   3365  // Load the callee in scratch2, no instruction between the movePtr and
   3366  // call should clobber it. Note that we can't use fun because it may be
   3367  // one of the IntArg registers clobbered before the call.
   3368  UseScratchRegisterScope temps(this);
   3369  temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
   3370  movePtr(fun, CallReg);
   3371 
   3372  uint32_t stackAdjust;
   3373  callWithABIPre(&stackAdjust);
   3374  call(CallReg);
   3375  callWithABIPost(stackAdjust, result);
   3376 }
   3377 
   3378 void MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result) {
   3379  // Load the callee in scratch2, as above.
   3380  UseScratchRegisterScope temps(this);
   3381  temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
   3382  loadPtr(fun, CallReg);
   3383 
   3384  uint32_t stackAdjust;
   3385  callWithABIPre(&stackAdjust);
   3386  call(CallReg);
   3387  callWithABIPost(stackAdjust, result);
   3388 }
   3389 
   3390 void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
   3391                                       Label* fail) {
   3392  UseScratchRegisterScope temps(this);
   3393  Register scratch = temps.Acquire();
   3394 
   3395  // Round toward positive infinity.
   3396  Ceil_l_d(dest, src);
   3397 
   3398  // Sign extend lower 32 bits to test if the result isn't an Int32.
   3399  {
   3400    move32SignExtendToPtr(dest, scratch);
   3401    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   3402  }
   3403 
   3404  // We have to check for (-1, -0] when the result is zero.
   3405  Label notZero;
   3406  ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
   3407  {
   3408    fmv_x_d(scratch, src);
   3409    ma_b(scratch, scratch, fail, Assembler::Signed);
   3410  }
   3411  bind(&notZero);
   3412 }
   3413 
   3414 void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
   3415                                        Label* fail) {
   3416  UseScratchRegisterScope temps(this);
   3417  Register scratch = temps.Acquire();
   3418 
   3419  // Round toward positive infinity.
   3420  Ceil_l_s(dest, src);
   3421 
   3422  // Sign extend lower 32 bits to test if the result isn't an Int32.
   3423  {
   3424    move32SignExtendToPtr(dest, scratch);
   3425    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   3426  }
   3427 
   3428  // We have to check for (-1, -0] when the result is zero.
   3429  Label notZero;
   3430  ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
   3431  {
   3432    fmv_x_w(scratch, src);
   3433    ma_b(scratch, scratch, fail, Assembler::Signed);
   3434  }
   3435  bind(&notZero);
   3436 }
   3437 
   3438 void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
   3439 
   3440 template <typename T>
   3441 static void CompareExchange64(MacroAssembler& masm,
   3442                              const wasm::MemoryAccessDesc* access,
   3443                              Synchronization sync, const T& mem,
   3444                              Register64 expect, Register64 replace,
   3445                              Register64 output) {
   3446  MOZ_ASSERT(expect != output && replace != output);
   3447  UseScratchRegisterScope temps(&masm);
   3448  Register scratch = temps.Acquire();
   3449  masm.computeEffectiveAddress(mem, scratch);
   3450 
   3451  Register scratch2 = temps.Acquire();
   3452 
   3453  Label tryAgain;
   3454  Label exit;
   3455 
   3456  masm.memoryBarrierBefore(sync);
   3457 
   3458  masm.bind(&tryAgain);
   3459 
   3460  if (access) {
   3461    masm.append(*access, wasm::TrapMachineInsn::Atomic,
   3462                FaultingCodeOffset(masm.currentOffset()));
   3463  }
   3464 
   3465  masm.lr_d(true, true, output.reg, scratch);
   3466 
   3467  masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump);
   3468  masm.movePtr(replace.reg, scratch2);
   3469  masm.sc_d(true, true, scratch2, scratch, scratch2);
   3470  masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::NonZero,
   3471            ShortJump);
   3472 
   3473  masm.memoryBarrierAfter(sync);
   3474 
   3475  masm.bind(&exit);
   3476 }
   3477 
   3478 void MacroAssembler::compareExchange64(Synchronization sync, const Address& mem,
   3479                                       Register64 expect, Register64 replace,
   3480                                       Register64 output) {
   3481  CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
   3482 }
   3483 
   3484 void MacroAssembler::compareExchange64(Synchronization sync,
   3485                                       const BaseIndex& mem, Register64 expect,
   3486                                       Register64 replace, Register64 output) {
   3487  CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
   3488 }
   3489 
   3490 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
   3491                                       Synchronization sync, const Address& mem,
   3492                                       Register oldval, Register newval,
   3493                                       Register valueTemp, Register offsetTemp,
   3494                                       Register maskTemp, Register temp,
   3495                                       AnyRegister output) {
   3496  CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp,
   3497                    offsetTemp, maskTemp, temp, output);
   3498 }
   3499 
   3500 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
   3501                                       Synchronization sync,
   3502                                       const BaseIndex& mem, Register oldval,
   3503                                       Register newval, Register valueTemp,
   3504                                       Register offsetTemp, Register maskTemp,
   3505                                       Register temp, AnyRegister output) {
   3506  CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp,
   3507                    offsetTemp, maskTemp, temp, output);
   3508 }
   3509 
   3510 void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
   3511  fcvt_d_l(dest, src.scratchReg());
   3512 }
   3513 void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) {
   3514  fcvt_s_l(dest, src.scratchReg());
   3515 }
   3516 void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
   3517  fcvt_d_l(dest, src);
   3518 }
   3519 void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
   3520                                           Register tmp) {
   3521  fcvt_d_lu(dest, src.scratchReg());
   3522 }
   3523 void MacroAssembler::convertUInt64ToFloat32(Register64 src, FloatRegister dest,
   3524                                            Register tmp) {
   3525  fcvt_s_lu(dest, src.scratchReg());
   3526 }
   3527 void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
   3528                                    FloatRegister output) {
   3529  fsgnj_d(output, lhs, rhs);
   3530 }
   3531 void MacroAssembler::copySignFloat32(FloatRegister lhs, FloatRegister rhs,
   3532                                     FloatRegister output) {
   3533  fsgnj_s(output, lhs, rhs);
   3534 }
   3535 void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
   3536                                               ExitFrameType type) {
   3537  enterFakeExitFrame(cxreg, scratch, type);
   3538 }
   3539 CodeOffset MacroAssembler::sub32FromMemAndBranchIfNegativeWithPatch(
   3540    Address address, Label* label) {
   3541  UseScratchRegisterScope temps(this);
   3542  Register scratch = temps.Acquire();
   3543  MOZ_ASSERT(scratch != address.base);
   3544  ma_load(scratch, address);
   3545  addiw(scratch, scratch, 128);
   3546  CodeOffset patchPoint = CodeOffset(currentOffset());
   3547  ma_store(scratch, address);
   3548  ma_b(scratch, scratch, label, Assembler::Signed);
   3549  return patchPoint;
   3550 }
   3551 void MacroAssembler::patchSub32FromMemAndBranchIfNegative(CodeOffset offset,
   3552                                                          Imm32 imm) {
   3553  int32_t val = imm.value;
   3554  MOZ_RELEASE_ASSERT(val >= 1 && val <= 127);
   3555  auto* inst = m_buffer.getInst(BufferOffset(offset.offset() - 4));
   3556  inst->InstructionOpcodeType();
   3557  MOZ_ASSERT(IsAddiw(inst->InstructionBits()));
   3558  /*
   3559   * | imm[11:0] | rs1 | 000 | rd | 0011011 |
   3560   */
   3561  inst->SetInstructionBits(((uint32_t)inst->InstructionBits() & ~kImm12Mask) |
   3562                           (((uint32_t)(-val) & 0xfff) << kImm12Shift));
   3563 }
   3564 void MacroAssembler::flexibleDivMod32(Register lhs, Register rhs,
   3565                                      Register divOutput, Register remOutput,
   3566                                      bool isUnsigned, const LiveRegisterSet&) {
   3567  MOZ_ASSERT(lhs != divOutput && lhs != remOutput, "lhs is preserved");
   3568  MOZ_ASSERT(rhs != divOutput && rhs != remOutput, "rhs is preserved");
   3569 
   3570  // The recommended code sequence to obtain both the quotient and remainder
   3571  // is div[u] followed by mod[u].
   3572  if (isUnsigned) {
   3573    ma_divu32(divOutput, lhs, rhs);
   3574    ma_modu32(remOutput, lhs, rhs);
   3575  } else {
   3576    ma_div32(divOutput, lhs, rhs);
   3577    ma_mod32(remOutput, lhs, rhs);
   3578  }
   3579 }
   3580 void MacroAssembler::flexibleQuotient32(Register lhs, Register rhs,
   3581                                        Register dest, bool isUnsigned,
   3582                                        const LiveRegisterSet&) {
   3583  quotient32(lhs, rhs, dest, isUnsigned);
   3584 }
   3585 
   3586 void MacroAssembler::flexibleQuotientPtr(Register lhs, Register rhs,
   3587                                         Register dest, bool isUnsigned,
   3588                                         const LiveRegisterSet&) {
   3589  quotient64(lhs, rhs, dest, isUnsigned);
   3590 }
   3591 
   3592 void MacroAssembler::flexibleRemainder32(Register lhs, Register rhs,
   3593                                         Register dest, bool isUnsigned,
   3594                                         const LiveRegisterSet&) {
   3595  remainder32(lhs, rhs, dest, isUnsigned);
   3596 }
   3597 
   3598 void MacroAssembler::flexibleRemainderPtr(Register lhs, Register rhs,
   3599                                          Register dest, bool isUnsigned,
   3600                                          const LiveRegisterSet&) {
   3601  remainder64(lhs, rhs, dest, isUnsigned);
   3602 }
   3603 
   3604 void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
   3605                                        Label* fail) {
   3606  UseScratchRegisterScope temps(this);
   3607  Register scratch = temps.Acquire();
   3608 
   3609  // Round toward negative infinity.
   3610  Floor_l_d(dest, src);
   3611 
   3612  // Sign extend lower 32 bits to test if the result isn't an Int32.
   3613  {
   3614    move32SignExtendToPtr(dest, scratch);
   3615    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   3616  }
   3617 
   3618  // Fail if the input is negative zero.
   3619  {
   3620    fclass_d(scratch, src);
   3621    ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Equal);
   3622  }
   3623 }
   3624 
   3625 void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
   3626                                         Label* fail) {
   3627  UseScratchRegisterScope temps(this);
   3628  Register scratch = temps.Acquire();
   3629 
   3630  // Round toward negative infinity.
   3631  Floor_l_s(dest, src);
   3632 
   3633  // Sign extend lower 32 bits to test if the result isn't an Int32.
   3634  {
   3635    move32SignExtendToPtr(dest, scratch);
   3636    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   3637  }
   3638 
   3639  // Fail if the input is negative zero.
   3640  {
   3641    fclass_s(scratch, src);
   3642    ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Equal);
   3643  }
   3644 }
   3645 
   3646 void MacroAssembler::flush() {}
   3647 void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
   3648  ma_and(buffer, ptr, Imm32(int32_t(~gc::ChunkMask)));
   3649  loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
   3650 }
   3651 
   3652 void MacroAssembler::moveValue(const ValueOperand& src,
   3653                               const ValueOperand& dest) {
   3654  if (src == dest) {
   3655    return;
   3656  }
   3657  movePtr(src.valueReg(), dest.valueReg());
   3658 }
   3659 
   3660 void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
   3661  if (!src.isGCThing()) {
   3662    ma_li(dest.valueReg(), ImmWord(src.asRawBits()));
   3663    return;
   3664  }
   3665 
   3666  writeDataRelocation(src);
   3667  movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
   3668 }
   3669 
   3670 void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
   3671                                     FloatRegister dest) {
   3672  MOZ_ASSERT(HasRoundInstruction(mode));
   3673 
   3674  ScratchDoubleScope2 fscratch(*this);
   3675 
   3676  switch (mode) {
   3677    case RoundingMode::Down:
   3678      Floor_d_d(dest, src, fscratch);
   3679      break;
   3680    case RoundingMode::Up:
   3681      Ceil_d_d(dest, src, fscratch);
   3682      break;
   3683    case RoundingMode::NearestTiesToEven:
   3684      Round_d_d(dest, src, fscratch);
   3685      break;
   3686    case RoundingMode::TowardsZero:
   3687      Trunc_d_d(dest, src, fscratch);
   3688      break;
   3689  }
   3690 }
   3691 
   3692 void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
   3693                                      FloatRegister dest) {
   3694  MOZ_ASSERT(HasRoundInstruction(mode));
   3695 
   3696  ScratchFloat32Scope2 fscratch(*this);
   3697 
   3698  switch (mode) {
   3699    case RoundingMode::Down:
   3700      Floor_s_s(dest, src, fscratch);
   3701      break;
   3702    case RoundingMode::Up:
   3703      Ceil_s_s(dest, src, fscratch);
   3704      break;
   3705    case RoundingMode::NearestTiesToEven:
   3706      Round_s_s(dest, src, fscratch);
   3707      break;
   3708    case RoundingMode::TowardsZero:
   3709      Trunc_s_s(dest, src, fscratch);
   3710      break;
   3711  }
   3712 }
   3713 
   3714 void MacroAssembler::oolWasmTruncateCheckF32ToI32(
   3715    FloatRegister input, Register output, TruncFlags flags,
   3716    const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
   3717  MOZ_ASSERT(!(flags & TRUNC_SATURATING));
   3718 
   3719  Label notNaN;
   3720  BranchFloat32(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
   3721  wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
   3722  bind(&notNaN);
   3723 
   3724  wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
   3725 }
   3726 
   3727 void MacroAssembler::oolWasmTruncateCheckF64ToI32(
   3728    FloatRegister input, Register output, TruncFlags flags,
   3729    const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
   3730  MOZ_ASSERT(!(flags & TRUNC_SATURATING));
   3731 
   3732  Label notNaN;
   3733  BranchFloat64(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
   3734  wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
   3735  bind(&notNaN);
   3736 
   3737  wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
   3738 }
   3739 
   3740 void MacroAssembler::oolWasmTruncateCheckF32ToI64(
   3741    FloatRegister input, Register64 output, TruncFlags flags,
   3742    const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
   3743  MOZ_ASSERT(!(flags & TRUNC_SATURATING));
   3744 
   3745  Label notNaN;
   3746  BranchFloat32(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
   3747  wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
   3748  bind(&notNaN);
   3749 
   3750  wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
   3751 }
   3752 
   3753 void MacroAssembler::oolWasmTruncateCheckF64ToI64(
   3754    FloatRegister input, Register64 output, TruncFlags flags,
   3755    const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) {
   3756  MOZ_ASSERT(!(flags & TRUNC_SATURATING));
   3757 
   3758  Label notNaN;
   3759  BranchFloat64(Assembler::DoubleOrdered, input, input, &notNaN, ShortJump);
   3760  wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc);
   3761  bind(&notNaN);
   3762 
   3763  wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc);
   3764 }
   3765 
   3766 void MacroAssembler::patchCallToNop(uint8_t* call) {
   3767  uint32_t* p = reinterpret_cast<uint32_t*>(call) - 7;
   3768  *reinterpret_cast<Instr*>(p) = kNopByte;
   3769  *reinterpret_cast<Instr*>(p + 1) = kNopByte;
   3770  *reinterpret_cast<Instr*>(p + 2) = kNopByte;
   3771  *reinterpret_cast<Instr*>(p + 3) = kNopByte;
   3772  *reinterpret_cast<Instr*>(p + 4) = kNopByte;
   3773  *reinterpret_cast<Instr*>(p + 5) = kNopByte;
   3774  *reinterpret_cast<Instr*>(p + 6) = kNopByte;
   3775 }
   3776 
   3777 CodeOffset MacroAssembler::callWithPatch() {
   3778  BlockTrampolinePoolScope block_trampoline_pool(this, 2);
   3779  DEBUG_PRINTF("\tcallWithPatch\n");
   3780  UseScratchRegisterScope temps(this);
   3781  Register scratch = temps.Acquire();
   3782  int32_t imm32 = 1 * sizeof(uint32_t);
   3783  int32_t Hi20 = ((imm32 + 0x800) >> 12);
   3784  int32_t Lo12 = imm32 << 20 >> 20;
   3785  auipc(scratch, Hi20);  // Read PC + Hi20 into scratch.
   3786  jalr(scratch, Lo12);   // jump PC + Hi20 + Lo12
   3787  DEBUG_PRINTF("\tret %d\n", currentOffset());
   3788  return CodeOffset(currentOffset());
   3789 }
   3790 
   3791 void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
   3792  DEBUG_PRINTF("\tpatchCall\n");
   3793  BufferOffset call(callerOffset - 2 * sizeof(uint32_t));
   3794  DEBUG_PRINTF("\tcallerOffset %d\n", callerOffset);
   3795  int32_t offset = BufferOffset(calleeOffset).getOffset() - call.getOffset();
   3796  if (is_int32(offset)) {
   3797    Instruction* auipc_ = (Instruction*)editSrc(call);
   3798    Instruction* jalr_ = (Instruction*)editSrc(
   3799        BufferOffset(callerOffset - 1 * sizeof(uint32_t)));
   3800    DEBUG_PRINTF("\t%p %zu\n\t", auipc_, callerOffset - 2 * sizeof(uint32_t));
   3801    disassembleInstr(auipc_->InstructionBits());
   3802    DEBUG_PRINTF("\t%p %zu\n\t", jalr_, callerOffset - 1 * sizeof(uint32_t));
   3803    disassembleInstr(jalr_->InstructionBits());
   3804    DEBUG_PRINTF("\t\n");
   3805    MOZ_ASSERT(IsJalr(jalr_->InstructionBits()) &&
   3806               IsAuipc(auipc_->InstructionBits()));
   3807    MOZ_ASSERT(auipc_->RdValue() == jalr_->Rs1Value());
   3808    int32_t Hi20 = (((int32_t)offset + 0x800) >> 12);
   3809    int32_t Lo12 = (int32_t)offset << 20 >> 20;
   3810    instr_at_put(call, SetAuipcOffset(Hi20, auipc_->InstructionBits()));
   3811    instr_at_put(BufferOffset(callerOffset - 1 * sizeof(uint32_t)),
   3812                 SetJalrOffset(Lo12, jalr_->InstructionBits()));
   3813  } else {
   3814    MOZ_CRASH();
   3815  }
   3816 }
   3817 
   3818 void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
   3819  uint32_t* u32 = reinterpret_cast<uint32_t*>(
   3820      editSrc(BufferOffset(farJump.offset() + 4 * kInstrSize)));
   3821  MOZ_ASSERT(*u32 == UINT32_MAX);
   3822  *u32 = targetOffset - farJump.offset();
   3823 }
   3824 
   3825 void MacroAssembler::patchFarJump(uint8_t* farJump, uint8_t* target) {
   3826  uint32_t* u32 = reinterpret_cast<uint32_t*>(farJump + 4 * kInstrSize);
   3827  MOZ_ASSERT(*u32 == UINT32_MAX);
   3828  *u32 = (int64_t)target - (int64_t)farJump;
   3829 }
   3830 
   3831 void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
   3832                                          CodeLocationLabel target) {
   3833  PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
   3834 }
   3835 
   3836 void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
   3837  uint32_t* p = reinterpret_cast<uint32_t*>(call) - 7;
   3838  Assembler::WriteLoad64Instructions((Instruction*)p, SavedScratchRegister,
   3839                                     (uint64_t)target);
   3840  DEBUG_PRINTF("\tpatchNopToCall %" PRIu64 " %" PRIu64 "\n", (uint64_t)target,
   3841               ExtractLoad64Value((Instruction*)p));
   3842  MOZ_ASSERT(ExtractLoad64Value((Instruction*)p) == (uint64_t)target);
   3843  Instr jalr_ = JALR | (ra.code() << kRdShift) | (0x0 << kFunct3Shift) |
   3844                (SavedScratchRegister.code() << kRs1Shift) |
   3845                (0x0 << kImm12Shift);
   3846  *reinterpret_cast<Instr*>(p + 6) = jalr_;
   3847 }
   3848 void MacroAssembler::Pop(Register reg) {
   3849  pop(reg);
   3850  adjustFrame(-int32_t(sizeof(intptr_t)));
   3851 }
   3852 
   3853 void MacroAssembler::Pop(FloatRegister f) {
   3854  pop(f);
   3855  // See MacroAssemblerRiscv64::ma_pop(FloatRegister) for why we use
   3856  // sizeof(double).
   3857  adjustFrame(-int32_t(sizeof(double)));
   3858 }
   3859 
   3860 void MacroAssembler::Pop(const ValueOperand& val) {
   3861  popValue(val);
   3862  adjustFrame(-int32_t(sizeof(Value)));
   3863 }
   3864 
   3865 void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
   3866                                         LiveRegisterSet ignore) {
   3867  int32_t diff =
   3868      set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
   3869  const int32_t reserved = diff;
   3870 
   3871  for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
   3872    diff -= sizeof(intptr_t);
   3873    if (!ignore.has(*iter)) {
   3874      loadPtr(Address(StackPointer, diff), *iter);
   3875    }
   3876  }
   3877 
   3878 #ifdef ENABLE_WASM_SIMD
   3879 #  error "Needs more careful logic if SIMD is enabled"
   3880 #endif
   3881 
   3882  for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
   3883       iter.more(); ++iter) {
   3884    diff -= sizeof(double);
   3885    if (!ignore.has(*iter)) {
   3886      loadDouble(Address(StackPointer, diff), *iter);
   3887    }
   3888  }
   3889  MOZ_ASSERT(diff == 0);
   3890  freeStack(reserved);
   3891 }
   3892 
   3893 CodeOffset MacroAssembler::move32WithPatch(Register dest) {
   3894  CodeOffset offs = CodeOffset(currentOffset());
   3895  ma_liPatchable(dest, Imm32(0));
   3896  return offs;
   3897 }
   3898 
   3899 void MacroAssembler::patchMove32(CodeOffset offset, Imm32 n) {
   3900  patchSub32FromStackPtr(offset, n);
   3901 }
   3902 
   3903 void MacroAssembler::pushReturnAddress() { push(ra); }
   3904 
   3905 void MacroAssembler::popReturnAddress() { pop(ra); }
   3906 void MacroAssembler::PopStackPtr() {
   3907  loadPtr(Address(StackPointer, 0), StackPointer);
   3908  adjustFrame(-int32_t(sizeof(intptr_t)));
   3909 }
   3910 void MacroAssembler::freeStackTo(uint32_t framePushed) {
   3911  MOZ_ASSERT(framePushed <= framePushed_);
   3912  ma_sub64(StackPointer, FramePointer, Imm32(framePushed));
   3913  framePushed_ = framePushed;
   3914 }
   3915 void MacroAssembler::PushBoxed(FloatRegister reg) {
   3916  subFromStackPtr(Imm32(sizeof(double)));
   3917  boxDouble(reg, Address(getStackPointer(), 0));
   3918  adjustFrame(sizeof(double));
   3919 }
   3920 
   3921 void MacroAssembler::Push(Register reg) {
   3922  push(reg);
   3923  adjustFrame(int32_t(sizeof(intptr_t)));
   3924 }
   3925 
   3926 void MacroAssembler::Push(const Imm32 imm) {
   3927  push(imm);
   3928  adjustFrame(int32_t(sizeof(intptr_t)));
   3929 }
   3930 
   3931 void MacroAssembler::Push(const ImmWord imm) {
   3932  push(imm);
   3933  adjustFrame(int32_t(sizeof(intptr_t)));
   3934 }
   3935 
   3936 void MacroAssembler::Push(const ImmPtr imm) {
   3937  Push(ImmWord(uintptr_t(imm.value)));
   3938 }
   3939 
   3940 void MacroAssembler::Push(const ImmGCPtr ptr) {
   3941  push(ptr);
   3942  adjustFrame(int32_t(sizeof(intptr_t)));
   3943 }
   3944 
   3945 void MacroAssembler::Push(FloatRegister f) {
   3946  push(f);
   3947  // See MacroAssemblerRiscv64::ma_push(FloatRegister) for why we use
   3948  // sizeof(double).
   3949  adjustFrame(int32_t(sizeof(double)));
   3950 }
   3951 
   3952 void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
   3953  int32_t diff =
   3954      set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
   3955  const int32_t reserved = diff;
   3956 
   3957  reserveStack(reserved);
   3958  for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
   3959    diff -= sizeof(intptr_t);
   3960    storePtr(*iter, Address(StackPointer, diff));
   3961  }
   3962 
   3963 #ifdef ENABLE_WASM_SIMD
   3964 #  error "Needs more careful logic if SIMD is enabled"
   3965 #endif
   3966 
   3967  for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
   3968       iter.more(); ++iter) {
   3969    diff -= sizeof(double);
   3970    storeDouble(*iter, Address(StackPointer, diff));
   3971  }
   3972  MOZ_ASSERT(diff == 0);
   3973 }
   3974 
   3975 void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
   3976                                         FloatRegister temp, Label* fail) {
   3977  JitSpew(JitSpew_Codegen, "[ %s", __FUNCTION__);
   3978  Label negative, done;
   3979 
   3980  // Branch to a slow path if input < 0.0 due to complicated rounding rules.
   3981  {
   3982    loadConstantFloat32(0.0f, temp);
   3983    BranchFloat32(Assembler::DoubleLessThan, src, temp, &negative, ShortJump);
   3984  }
   3985 
   3986  // Fail if the input is negative zero.
   3987  {
   3988    UseScratchRegisterScope temps(this);
   3989    Register scratch = temps.Acquire();
   3990 
   3991    fclass_s(scratch, src);
   3992    ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Assembler::Equal);
   3993  }
   3994 
   3995  // Handle the simple case of a positive input and NaN.
   3996  // Rounding proceeds with consideration of the fractional part of the input:
   3997  // 1. If > 0.5, round to integer with higher absolute value (so, up).
   3998  // 2. If < 0.5, round to integer with lower absolute value (so, down).
   3999  // 3. If = 0.5, round to +Infinity (so, up).
   4000  {
   4001    // Round, ties away from zero.
   4002    RoundMaxMag_l_s(dest, src);
   4003    ma_branch(&done);
   4004  }
   4005 
   4006  // Handle the complicated case of a negative input.
   4007  // Rounding proceeds with consideration of the fractional part of the input:
   4008  // 1. If > 0.5, round to integer with higher absolute value (so, down).
   4009  // 2. If < 0.5, round to integer with lower absolute value (so, up).
   4010  // 3. If = 0.5, round to +Infinity (so, up).
   4011  bind(&negative);
   4012  {
   4013    // Inputs in [-0.5, 0) are rounded to -0. Fail.
   4014    loadConstantFloat32(-0.5f, temp);
   4015    branchFloat(Assembler::DoubleGreaterThanOrEqual, src, temp, fail);
   4016 
   4017    // Other negative inputs need the biggest float less than 0.5 added.
   4018    loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
   4019    fadd_s(temp, src, temp);
   4020 
   4021    // Round toward negative infinity.
   4022    Floor_l_s(dest, temp);
   4023  }
   4024 
   4025  // Sign extend lower 32 bits to test if the result isn't an Int32.
   4026  bind(&done);
   4027  {
   4028    UseScratchRegisterScope temps(this);
   4029    Register scratch = temps.Acquire();
   4030 
   4031    move32SignExtendToPtr(dest, scratch);
   4032    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   4033  }
   4034  JitSpew(JitSpew_Codegen, "]");
   4035 }
   4036 
   4037 void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
   4038                                        FloatRegister temp, Label* fail) {
   4039  JitSpew(JitSpew_Codegen, "[ %s", __FUNCTION__);
   4040  Label negative, done;
   4041 
   4042  // Branch to a slow path if input < 0.0 due to complicated rounding rules.
   4043  {
   4044    loadConstantDouble(0.0, temp);
   4045    BranchFloat64(Assembler::DoubleLessThan, src, temp, &negative, ShortJump);
   4046  }
   4047 
   4048  // Fail if the input is negative zero.
   4049  {
   4050    UseScratchRegisterScope temps(this);
   4051    Register scratch = temps.Acquire();
   4052 
   4053    fclass_d(scratch, src);
   4054    ma_b(scratch, Imm32(FClassFlag::kNegativeZero), fail, Equal);
   4055  }
   4056 
   4057  // Handle the simple case of a positive input and NaN.
   4058  // Rounding proceeds with consideration of the fractional part of the input:
   4059  // 1. If > 0.5, round to integer with higher absolute value (so, up).
   4060  // 2. If < 0.5, round to integer with lower absolute value (so, down).
   4061  // 3. If = 0.5, round to +Infinity (so, up).
   4062  {
   4063    // Round, ties away from zero.
   4064    RoundMaxMag_l_d(dest, src);
   4065    ma_branch(&done);
   4066  }
   4067 
   4068  // Handle the complicated case of a negative input.
   4069  // Rounding proceeds with consideration of the fractional part of the input:
   4070  // 1. If > 0.5, round to integer with higher absolute value (so, down).
   4071  // 2. If < 0.5, round to integer with lower absolute value (so, up).
   4072  // 3. If = 0.5, round to +Infinity (so, up).
   4073  bind(&negative);
   4074  {
   4075    // Inputs in [-0.5, 0) are rounded to -0. Fail.
   4076    loadConstantDouble(-0.5, temp);
   4077    branchDouble(Assembler::DoubleGreaterThanOrEqual, src, temp, fail);
   4078 
   4079    // Other negative inputs need the biggest double less than 0.5 added.
   4080    loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
   4081    fadd_d(temp, src, temp);
   4082 
   4083    // Round toward negative infinity.
   4084    Floor_l_d(dest, temp);
   4085  }
   4086 
   4087  // Sign extend lower 32 bits to test if the result isn't an Int32.
   4088  bind(&done);
   4089  {
   4090    UseScratchRegisterScope temps(this);
   4091    Register scratch = temps.Acquire();
   4092 
   4093    move32SignExtendToPtr(dest, scratch);
   4094    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   4095  }
   4096  JitSpew(JitSpew_Codegen, "]");
   4097 }
   4098 
   4099 void MacroAssembler::setupUnalignedABICall(Register scratch) {
   4100  MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls");
   4101  setupNativeABICall();
   4102  dynamicAlignment_ = true;
   4103 
   4104  or_(scratch, StackPointer, zero);
   4105 
   4106  // Force sp to be aligned
   4107  asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
   4108  ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1)));
   4109  storePtr(scratch, Address(StackPointer, 0));
   4110 }
   4111 void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
   4112                                        Register pointer) {
   4113  if (IsShiftInScaleRange(shift)) {
   4114    computeEffectiveAddress(
   4115        BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
   4116    return;
   4117  }
   4118  lshift32(Imm32(shift), indexTemp32);
   4119  addPtr(indexTemp32, pointer);
   4120 }
   4121 void MacroAssembler::speculationBarrier() { MOZ_CRASH(); }
   4122 void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
   4123                                     Register) {
   4124  FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
   4125  mozilla::DebugOnly<unsigned> numFpu = fpuSet.size();
   4126  int32_t diffF = fpuSet.getPushSizeInBytes();
   4127  mozilla::DebugOnly<int32_t> diffG = set.gprs().size() * sizeof(intptr_t);
   4128 
   4129  MOZ_ASSERT(dest.offset >= diffG + diffF);
   4130 
   4131  for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
   4132    diffG -= sizeof(intptr_t);
   4133    dest.offset -= sizeof(intptr_t);
   4134    storePtr(*iter, dest);
   4135  }
   4136  MOZ_ASSERT(diffG == 0);
   4137 
   4138 #ifdef ENABLE_WASM_SIMD
   4139 #  error "Needs more careful logic if SIMD is enabled"
   4140 #endif
   4141 
   4142  for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
   4143    FloatRegister reg = *iter;
   4144    diffF -= reg.size();
   4145    numFpu -= 1;
   4146    dest.offset -= reg.size();
   4147    if (reg.isDouble()) {
   4148      storeDouble(reg, dest);
   4149    } else if (reg.isSingle()) {
   4150      storeFloat32(reg, dest);
   4151    } else {
   4152      MOZ_CRASH("Unknown register type.");
   4153    }
   4154  }
   4155  MOZ_ASSERT(numFpu == 0);
   4156 
   4157  diffF -= diffF % sizeof(uintptr_t);
   4158  MOZ_ASSERT(diffF == 0);
   4159 }
   4160 void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
   4161                                        Label* fail) {
   4162  UseScratchRegisterScope temps(*this);
   4163  Register scratch = temps.Acquire();
   4164 
   4165  // Round toward zero.
   4166  Trunc_l_d(dest, src);
   4167 
   4168  // Sign extend lower 32 bits to test if the result isn't an Int32.
   4169  {
   4170    move32SignExtendToPtr(dest, scratch);
   4171    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   4172  }
   4173 
   4174  // We have to check for (-1, -0] when the result is zero.
   4175  Label notZero;
   4176  ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
   4177  {
   4178    fmv_x_d(scratch, src);
   4179    ma_b(scratch, scratch, fail, Assembler::Signed);
   4180  }
   4181  bind(&notZero);
   4182 }
   4183 void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
   4184                                         Label* fail) {
   4185  UseScratchRegisterScope temps(this);
   4186  Register scratch = temps.Acquire();
   4187 
   4188  // Round toward zero.
   4189  Trunc_l_s(dest, src);
   4190 
   4191  // Sign extend lower 32 bits to test if the result isn't an Int32.
   4192  {
   4193    move32SignExtendToPtr(dest, scratch);
   4194    branchPtr(Assembler::NotEqual, dest, scratch, fail);
   4195  }
   4196 
   4197  // We have to check for (-1, -0] when the result is zero.
   4198  Label notZero;
   4199  ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
   4200  {
   4201    fmv_x_w(scratch, src);
   4202    ma_b(scratch, scratch, fail, Assembler::Signed);
   4203  }
   4204  bind(&notZero);
   4205 }
   4206 
   4207 void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
   4208                                        AtomicOp op, Register value,
   4209                                        const Address& mem, Register valueTemp,
   4210                                        Register offsetTemp,
   4211                                        Register maskTemp) {
   4212  AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
   4213                 valueTemp, offsetTemp, maskTemp);
   4214 }
   4215 
   4216 void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
   4217                                        AtomicOp op, Register value,
   4218                                        const BaseIndex& mem,
   4219                                        Register valueTemp, Register offsetTemp,
   4220                                        Register maskTemp) {
   4221  AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
   4222                 valueTemp, offsetTemp, maskTemp);
   4223 }
   4224 template <typename T>
   4225 static void WasmAtomicExchange64(MacroAssembler& masm,
   4226                                 const wasm::MemoryAccessDesc& access,
   4227                                 const T& mem, Register64 value,
   4228                                 Register64 output) {
   4229  AtomicExchange64(masm, &access, access.sync(), mem, value, output);
   4230 }
   4231 
   4232 void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
   4233                                          const Address& mem, Register64 src,
   4234                                          Register64 output) {
   4235  WasmAtomicExchange64(*this, access, mem, src, output);
   4236 }
   4237 
   4238 void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
   4239                                          const BaseIndex& mem, Register64 src,
   4240                                          Register64 output) {
   4241  WasmAtomicExchange64(*this, access, mem, src, output);
   4242 }
   4243 void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
   4244                                        const Address& mem, Register value,
   4245                                        Register valueTemp, Register offsetTemp,
   4246                                        Register maskTemp, Register output) {
   4247  AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
   4248                 valueTemp, offsetTemp, maskTemp, output);
   4249 }
   4250 
   4251 void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
   4252                                        const BaseIndex& mem, Register value,
   4253                                        Register valueTemp, Register offsetTemp,
   4254                                        Register maskTemp, Register output) {
   4255  AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
   4256                 valueTemp, offsetTemp, maskTemp, output);
   4257 }
   4258 void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
   4259                                         AtomicOp op, Register64 value,
   4260                                         const Address& mem, Register64 temp,
   4261                                         Register64 output) {
   4262  AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
   4263 }
   4264 void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
   4265                                         AtomicOp op, Register64 value,
   4266                                         const BaseIndex& mem, Register64 temp,
   4267                                         Register64 output) {
   4268  AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
   4269 }
   4270 
   4271 void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op,
   4272                                     Register64 value, const Address& mem,
   4273                                     Register64 temp, Register64 output) {
   4274  AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
   4275 }
   4276 
   4277 void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op,
   4278                                     Register64 value, const BaseIndex& mem,
   4279                                     Register64 temp, Register64 output) {
   4280  AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
   4281 }
   4282 
   4283 void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op,
   4284                                      Register64 value, const Address& mem,
   4285                                      Register64 temp) {
   4286  AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
   4287 }
   4288 
   4289 void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op,
   4290                                      Register64 value, const BaseIndex& mem,
   4291                                      Register64 temp) {
   4292  AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
   4293 }
   4294 void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
   4295                                       AtomicOp op, Register value,
   4296                                       const Address& mem, Register valueTemp,
   4297                                       Register offsetTemp, Register maskTemp,
   4298                                       Register output) {
   4299  AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
   4300                valueTemp, offsetTemp, maskTemp, output);
   4301 }
   4302 
   4303 void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
   4304                                       AtomicOp op, Register value,
   4305                                       const BaseIndex& mem, Register valueTemp,
   4306                                       Register offsetTemp, Register maskTemp,
   4307                                       Register output) {
   4308  AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
   4309                valueTemp, offsetTemp, maskTemp, output);
   4310 }
   4311 void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
   4312                                       Register boundsCheckLimit,
   4313                                       Label* label) {
   4314  ma_b(index, boundsCheckLimit, label, cond);
   4315 }
   4316 
   4317 void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
   4318                                       Address boundsCheckLimit, Label* label) {
   4319  UseScratchRegisterScope temps(this);
   4320  Register scratch2 = temps.Acquire();
   4321  load32(boundsCheckLimit, scratch2);
   4322  ma_b(index, Register(scratch2), label, cond);
   4323 }
   4324 
   4325 void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
   4326                                       Register64 boundsCheckLimit,
   4327                                       Label* label) {
   4328  ma_b(index.reg, boundsCheckLimit.reg, label, cond);
   4329 }
   4330 
   4331 void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
   4332                                       Address boundsCheckLimit, Label* label) {
   4333  UseScratchRegisterScope temps(this);
   4334  Register scratch2 = temps.Acquire();
   4335  loadPtr(boundsCheckLimit, scratch2);
   4336  ma_b(index.reg, scratch2, label, cond);
   4337 }
   4338 
   4339 void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
   4340                                           const Address& mem,
   4341                                           Register64 expect,
   4342                                           Register64 replace,
   4343                                           Register64 output) {
   4344  CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
   4345                    output);
   4346 }
   4347 
   4348 void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
   4349                                           const BaseIndex& mem,
   4350                                           Register64 expect,
   4351                                           Register64 replace,
   4352                                           Register64 output) {
   4353  CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
   4354                    output);
   4355 }
   4356 
   4357 template <typename T>
   4358 static void CompareExchange(MacroAssembler& masm,
   4359                            const wasm::MemoryAccessDesc* access,
   4360                            Scalar::Type type, Synchronization sync,
   4361                            const T& mem, Register oldval, Register newval,
   4362                            Register valueTemp, Register offsetTemp,
   4363                            Register maskTemp, Register output) {
   4364  bool signExtend = Scalar::isSignedIntType(type);
   4365  unsigned nbytes = Scalar::byteSize(type);
   4366 
   4367  switch (nbytes) {
   4368    case 1:
   4369    case 2:
   4370      break;
   4371    case 4:
   4372      MOZ_ASSERT(valueTemp == InvalidReg);
   4373      MOZ_ASSERT(offsetTemp == InvalidReg);
   4374      MOZ_ASSERT(maskTemp == InvalidReg);
   4375      break;
   4376    default:
   4377      MOZ_CRASH();
   4378  }
   4379 
   4380  Label again, end;
   4381  UseScratchRegisterScope temps(&masm);
   4382  Register scratch1 = temps.Acquire();
   4383  Register scratch2 = temps.Acquire();
   4384  masm.computeEffectiveAddress(mem, scratch2);
   4385 
   4386  if (nbytes == 4) {
   4387    masm.memoryBarrierBefore(sync);
   4388    masm.bind(&again);
   4389 
   4390    if (access) {
   4391      masm.append(*access, wasm::TrapMachineInsn::Atomic,
   4392                  FaultingCodeOffset(masm.currentOffset()));
   4393    }
   4394 
   4395    masm.lr_w(true, true, output, scratch2);
   4396    masm.SignExtendWord(scratch1, oldval);
   4397    masm.ma_b(output, scratch1, &end, Assembler::NotEqual, ShortJump);
   4398    masm.mv(scratch1, newval);
   4399    masm.sc_w(true, true, scratch1, scratch2, scratch1);
   4400    masm.ma_b(scratch1, scratch1, &again, Assembler::NonZero, ShortJump);
   4401 
   4402    masm.memoryBarrierAfter(sync);
   4403    masm.bind(&end);
   4404 
   4405    return;
   4406  }
   4407 
   4408  masm.andi(offsetTemp, scratch2, 3);
   4409  masm.subPtr(offsetTemp, scratch2);
   4410 #if !MOZ_LITTLE_ENDIAN()
   4411  masm.as_xori(offsetTemp, offsetTemp, 3);
   4412 #endif
   4413  masm.slli(offsetTemp, offsetTemp, 3);
   4414  masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
   4415  masm.sll(maskTemp, maskTemp, offsetTemp);
   4416  masm.not_(maskTemp, maskTemp);
   4417 
   4418  masm.memoryBarrierBefore(sync);
   4419 
   4420  masm.bind(&again);
   4421 
   4422  if (access) {
   4423    masm.append(*access, wasm::TrapMachineInsn::Atomic,
   4424                FaultingCodeOffset(masm.currentOffset()));
   4425  }
   4426 
   4427  masm.lr_w(true, true, scratch1, scratch2);
   4428 
   4429  masm.srl(output, scratch1, offsetTemp);
   4430 
   4431  switch (nbytes) {
   4432    case 1:
   4433      if (signExtend) {
   4434        masm.SignExtendByte(valueTemp, oldval);
   4435        masm.SignExtendByte(output, output);
   4436        masm.SignExtendByte(newval, newval);
   4437      } else {
   4438        masm.andi(valueTemp, oldval, 0xff);
   4439        masm.andi(output, output, 0xff);
   4440        masm.andi(newval, newval, 0xff);
   4441      }
   4442      break;
   4443    case 2:
   4444      if (signExtend) {
   4445        masm.SignExtendShort(valueTemp, oldval);
   4446        masm.SignExtendShort(output, output);
   4447        masm.SignExtendShort(newval, newval);
   4448      } else {
   4449        UseScratchRegisterScope temps(&masm);
   4450        Register mask = temps.Acquire();
   4451        masm.ma_li(mask, Imm32(0xffff));
   4452        masm.and_(valueTemp, oldval, mask);
   4453        masm.and_(output, output, mask);
   4454        masm.and_(newval, newval, mask);
   4455      }
   4456      break;
   4457  }
   4458 
   4459  masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump);
   4460 
   4461  masm.sllw(valueTemp, newval, offsetTemp);
   4462  masm.and_(scratch1, scratch1, maskTemp);
   4463  masm.or_(scratch1, scratch1, valueTemp);
   4464  masm.sc_w(true, true, scratch1, scratch2, scratch1);
   4465 
   4466  masm.ma_b(scratch1, scratch1, &again, Assembler::NonZero, ShortJump);
   4467 
   4468  masm.memoryBarrierAfter(sync);
   4469 
   4470  masm.bind(&end);
   4471 }
   4472 
   4473 void MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync,
   4474                                     const Address& mem, Register oldval,
   4475                                     Register newval, Register valueTemp,
   4476                                     Register offsetTemp, Register maskTemp,
   4477                                     Register output) {
   4478  CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp,
   4479                  offsetTemp, maskTemp, output);
   4480 }
   4481 
   4482 void MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync,
   4483                                     const BaseIndex& mem, Register oldval,
   4484                                     Register newval, Register valueTemp,
   4485                                     Register offsetTemp, Register maskTemp,
   4486                                     Register output) {
   4487  CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp,
   4488                  offsetTemp, maskTemp, output);
   4489 }
   4490 
   4491 void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
   4492                                         const Address& mem, Register oldval,
   4493                                         Register newval, Register valueTemp,
   4494                                         Register offsetTemp, Register maskTemp,
   4495                                         Register output) {
   4496  CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
   4497                  newval, valueTemp, offsetTemp, maskTemp, output);
   4498 }
   4499 
   4500 void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
   4501                                         const BaseIndex& mem, Register oldval,
   4502                                         Register newval, Register valueTemp,
   4503                                         Register offsetTemp, Register maskTemp,
   4504                                         Register output) {
   4505  CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
   4506                  newval, valueTemp, offsetTemp, maskTemp, output);
   4507 }
   4508 
   4509 void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
   4510                              Register memoryBase, Register ptr,
   4511                              Register ptrScratch, AnyRegister output) {
   4512  wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
   4513 }
   4514 
   4515 void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
   4516                                 Register memoryBase, Register ptr,
   4517                                 Register ptrScratch, Register64 output) {
   4518  wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
   4519 }
   4520 
   4521 void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
   4522                               AnyRegister value, Register memoryBase,
   4523                               Register ptr, Register ptrScratch) {
   4524  wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
   4525 }
   4526 
   4527 void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
   4528                                  Register64 value, Register memoryBase,
   4529                                  Register ptr, Register ptrScratch) {
   4530  wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
   4531 }
   4532 
   4533 void MacroAssemblerRiscv64::Clear_if_nan_d(Register rd, FPURegister fs) {
   4534  UseScratchRegisterScope temps(this);
   4535  Register scratch = temps.Acquire();
   4536 
   4537  feq_d(scratch, fs, fs);
   4538  neg(scratch, scratch);
   4539  and_(rd, rd, scratch);
   4540 }
   4541 
   4542 void MacroAssemblerRiscv64::Clear_if_nan_s(Register rd, FPURegister fs) {
   4543  UseScratchRegisterScope temps(this);
   4544  Register scratch = temps.Acquire();
   4545 
   4546  feq_s(scratch, fs, fs);
   4547  neg(scratch, scratch);
   4548  and_(rd, rd, scratch);
   4549 }
   4550 
   4551 void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
   4552                                               Register output,
   4553                                               bool isSaturating,
   4554                                               Label* oolEntry) {
   4555  if (isSaturating) {
   4556    Trunc_w_d(output, input);
   4557    Clear_if_nan_d(output, input);
   4558  } else {
   4559    UseScratchRegisterScope temps(this);
   4560    Register scratch = temps.Acquire();
   4561 
   4562    Trunc_l_d(output, input);
   4563 
   4564    // Sign extend lower 32 bits to test if the result isn't an Int32.
   4565    move32SignExtendToPtr(output, scratch);
   4566    branchPtr(Assembler::NotEqual, output, scratch, oolEntry);
   4567  }
   4568 }
   4569 
   4570 void MacroAssembler::wasmTruncateDoubleToInt64(
   4571    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
   4572    Label* oolRejoin, FloatRegister tempDouble) {
   4573  if (isSaturating) {
   4574    Trunc_l_d(output.reg, input);
   4575    Clear_if_nan_d(output.reg, input);
   4576  } else {
   4577    UseScratchRegisterScope temps(this);
   4578    Register scratch = temps.Acquire();
   4579    Trunc_l_d(output.reg, input, scratch);
   4580    ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
   4581  }
   4582 }
   4583 
   4584 void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
   4585                                                Register output,
   4586                                                bool isSaturating,
   4587                                                Label* oolEntry) {
   4588  if (isSaturating) {
   4589    Trunc_uw_d(output, input);
   4590    Clear_if_nan_d(output, input);
   4591  } else {
   4592    UseScratchRegisterScope temps(this);
   4593    Register scratch = temps.Acquire();
   4594    Trunc_uw_d(output, input, scratch);
   4595    ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
   4596  }
   4597 }
   4598 
   4599 void MacroAssembler::wasmTruncateDoubleToUInt64(
   4600    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
   4601    Label* oolRejoin, FloatRegister tempDouble) {
   4602  if (isSaturating) {
   4603    Trunc_ul_d(output.reg, input);
   4604    Clear_if_nan_d(output.reg, input);
   4605  } else {
   4606    UseScratchRegisterScope temps(this);
   4607    Register scratch = temps.Acquire();
   4608    Trunc_ul_d(output.reg, input, scratch);
   4609    ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
   4610  }
   4611 }
   4612 
   4613 void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
   4614                                                Register output,
   4615                                                bool isSaturating,
   4616                                                Label* oolEntry) {
   4617  if (isSaturating) {
   4618    Trunc_w_s(output, input);
   4619    Clear_if_nan_s(output, input);
   4620  } else {
   4621    UseScratchRegisterScope temps(this);
   4622    Register scratch = temps.Acquire();
   4623 
   4624    Trunc_l_s(output, input, scratch);
   4625 
   4626    // Sign extend lower 32 bits to test if the result isn't an Int32.
   4627    move32SignExtendToPtr(output, scratch);
   4628    branchPtr(Assembler::NotEqual, output, scratch, oolEntry);
   4629  }
   4630 }
   4631 
   4632 void MacroAssembler::wasmTruncateFloat32ToInt64(
   4633    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
   4634    Label* oolRejoin, FloatRegister tempFloat) {
   4635  if (isSaturating) {
   4636    Trunc_l_s(output.reg, input);
   4637    Clear_if_nan_s(output.reg, input);
   4638  } else {
   4639    UseScratchRegisterScope temps(this);
   4640    Register scratch = temps.Acquire();
   4641    Trunc_l_s(output.reg, input, scratch);
   4642    ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
   4643  }
   4644 }
   4645 
   4646 void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
   4647                                                 Register output,
   4648                                                 bool isSaturating,
   4649                                                 Label* oolEntry) {
   4650  if (isSaturating) {
   4651    Trunc_uw_s(output, input);
   4652    Clear_if_nan_s(output, input);
   4653  } else {
   4654    UseScratchRegisterScope temps(this);
   4655    Register scratch = temps.Acquire();
   4656    Trunc_uw_s(output, input, scratch);
   4657    ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
   4658  }
   4659 }
   4660 
   4661 void MacroAssembler::wasmTruncateFloat32ToUInt64(
   4662    FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
   4663    Label* oolRejoin, FloatRegister tempFloat) {
   4664  if (isSaturating) {
   4665    Trunc_ul_s(output.reg, input);
   4666    Clear_if_nan_s(output.reg, input);
   4667  } else {
   4668    UseScratchRegisterScope temps(this);
   4669    Register scratch = temps.Acquire();
   4670    Trunc_ul_s(output.reg, input, scratch);
   4671    ma_b(scratch, Imm32(0), oolEntry, Assembler::Equal);
   4672  }
   4673 }
   4674 
   4675 // TODO(riscv64): widenInt32 should be nop?
   4676 void MacroAssembler::widenInt32(Register r) {
   4677  move32To64SignExtend(r, Register64(r));
   4678 }
   4679 
   4680 void MacroAssembler::wasmMarkCallAsSlow() { mv(ra, ra); }
   4681 
   4682 const int32_t SlowCallMarker = 0x8093;  // addi ra, ra, 0
   4683 
   4684 void MacroAssembler::wasmCheckSlowCallsite(Register ra_, Label* notSlow,
   4685                                           Register temp1, Register temp2) {
   4686  MOZ_ASSERT(ra_ != temp2);
   4687 
   4688  UseScratchRegisterScope temps(*this);
   4689  // temp1 aliases ra_, so allocating a new register.
   4690  const Register scratchMarker = temps.Acquire();
   4691  move32(Imm32(SlowCallMarker), scratchMarker);
   4692 
   4693  Label slow;
   4694  // Handle `jalr; (ra_ here) marker`.
   4695  load32(Address(ra_, 0), temp2);
   4696  branch32(Assembler::Equal, temp2, scratchMarker, &slow);
   4697  // Handle `jal; (ra_ here) nop; marker`.
   4698  // See also: AssemblerRISCVI::jal(Register rd, int32_t imm21); Bug 1996840
   4699  branch32(Assembler::NotEqual, temp2, Imm32(kNopByte), notSlow);
   4700  load32(Address(ra_, 4), temp2);
   4701  branch32(Assembler::NotEqual, temp2, scratchMarker, notSlow);
   4702  bind(&slow);
   4703 }
   4704 
   4705 CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc,
   4706                                              const Register reg) {
   4707  BlockTrampolinePoolScope block_trampoline_pool(this, 2);
   4708  CodeOffset offset = call(desc, reg);
   4709  wasmMarkCallAsSlow();
   4710  return offset;
   4711 }
   4712 //}}} check_macroassembler_style
   4713 
   4714 // This method generates lui, dsll and ori instruction block that can be
   4715 // modified by UpdateLoad64Value, either during compilation (eg.
   4716 // Assembler::bind), or during execution (eg. jit::PatchJump).
   4717 void MacroAssemblerRiscv64::ma_liPatchable(Register dest, Imm32 imm) {
   4718  m_buffer.ensureSpace(2 * sizeof(uint32_t));
   4719  int64_t value = imm.value;
   4720  int64_t high_20 = ((value + 0x800) >> 12);
   4721  int64_t low_12 = value << 52 >> 52;
   4722  lui(dest, high_20);
   4723  addi(dest, dest, low_12);
   4724 }
   4725 
   4726 void MacroAssemblerRiscv64::ma_liPatchable(Register dest, ImmPtr imm) {
   4727  return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value)));
   4728 }
   4729 
   4730 void MacroAssemblerRiscv64::ma_liPatchable(Register dest, ImmWord imm,
   4731                                           LiFlags flags) {
   4732  DEBUG_PRINTF("\tma_liPatchable\n");
   4733  if (Li64 == flags) {
   4734    li_constant(dest, imm.value);
   4735  } else {
   4736    li_ptr(dest, imm.value);
   4737  }
   4738 }
   4739 
   4740 void MacroAssemblerRiscv64::ma_li(Register dest, ImmGCPtr ptr) {
   4741  BlockTrampolinePoolScope block_trampoline_pool(this, 6);
   4742  writeDataRelocation(ptr);
   4743  ma_liPatchable(dest, ImmPtr(ptr.value));
   4744 }
   4745 void MacroAssemblerRiscv64::ma_li(Register dest, Imm32 imm) {
   4746  RV_li(dest, imm.value);
   4747 }
   4748 void MacroAssemblerRiscv64::ma_li(Register dest, Imm64 imm) {
   4749  RV_li(dest, imm.value);
   4750 }
   4751 void MacroAssemblerRiscv64::ma_li(Register dest, CodeLabel* label) {
   4752  DEBUG_PRINTF("[ %s\n", __FUNCTION__);
   4753  BlockTrampolinePoolScope block_trampoline_pool(this, 7);
   4754  BufferOffset bo = m_buffer.nextOffset();
   4755  JitSpew(JitSpew_Codegen, ".load CodeLabel %p", label);
   4756  ma_liPatchable(dest, ImmWord(/* placeholder */ 0));
   4757  label->patchAt()->bind(bo.getOffset());
   4758  label->setLinkMode(CodeLabel::MoveImmediate);
   4759  DEBUG_PRINTF("]\n");
   4760 }
   4761 void MacroAssemblerRiscv64::ma_li(Register dest, ImmWord imm) {
   4762  RV_li(dest, imm.value);
   4763 }
   4764 
   4765 // Shortcut for when we know we're transferring 32 bits of data.
   4766 void MacroAssemblerRiscv64::ma_pop(Register r) {
   4767  ld(r, StackPointer, 0);
   4768  addi(StackPointer, StackPointer, sizeof(intptr_t));
   4769 }
   4770 
   4771 void MacroAssemblerRiscv64::ma_push(Register r) {
   4772  UseScratchRegisterScope temps(this);
   4773  if (r == sp) {
   4774    Register scratch = temps.Acquire();
   4775    // Pushing sp requires one more instruction.
   4776    mv(scratch, sp);
   4777    r = scratch;
   4778  }
   4779 
   4780  addi(StackPointer, StackPointer, (int32_t)-sizeof(intptr_t));
   4781  sd(r, StackPointer, 0);
   4782 }
   4783 
   4784 void MacroAssemblerRiscv64::ma_mul32TestOverflow(Register rd, Register rj,
   4785                                                 Register rk, Label* overflow) {
   4786  UseScratchRegisterScope temps(this);
   4787  Register scratch = temps.Acquire();
   4788  MOZ_ASSERT(rd != scratch);
   4789 
   4790  mul(rd, rj, rk);
   4791  sext_w(scratch, rd);
   4792  ma_b(scratch, rd, overflow, Assembler::NotEqual);
   4793 }
   4794 void MacroAssemblerRiscv64::ma_mul32TestOverflow(Register rd, Register rj,
   4795                                                 Imm32 imm, Label* overflow) {
   4796  UseScratchRegisterScope temps(this);
   4797  Register scratch = temps.Acquire();
   4798  MOZ_ASSERT(rd != scratch && rj != scratch);
   4799 
   4800  ma_li(scratch, imm);
   4801 
   4802  mul(rd, rj, scratch);
   4803  sext_w(scratch, rd);
   4804  ma_b(scratch, rd, overflow, Assembler::NotEqual);
   4805 }
   4806 
   4807 void MacroAssemblerRiscv64::ma_mulPtrTestOverflow(Register rd, Register rj,
   4808                                                  Register rk,
   4809                                                  Label* overflow) {
   4810  UseScratchRegisterScope temps(this);
   4811  Register scratch = temps.Acquire();
   4812  Register scratch2 = temps.Acquire();
   4813  MOZ_ASSERT(rd != scratch);
   4814 
   4815  if (rd == rj) {
   4816    or_(scratch, rj, zero);
   4817    rj = scratch;
   4818    rk = (rd == rk) ? rj : rk;
   4819  } else if (rd == rk) {
   4820    or_(scratch, rk, zero);
   4821    rk = scratch;
   4822  }
   4823 
   4824  mul(rd, rj, rk);
   4825  mulh(scratch, rj, rk);
   4826  srai(scratch2, rd, 63);
   4827  ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
   4828 }
   4829 
   4830 int32_t MacroAssemblerRiscv64::GetOffset(int32_t offset, Label* L,
   4831                                         OffsetSize bits) {
   4832  if (L) {
   4833    offset = branchOffsetHelper(L, bits);
   4834  } else {
   4835    MOZ_ASSERT(is_intn(offset, bits));
   4836  }
   4837  return offset;
   4838 }
   4839 
   4840 bool MacroAssemblerRiscv64::CalculateOffset(Label* L, OffsetSize bits,
   4841                                            int32_t* offset) {
   4842  if (!is_near(L, bits)) return false;
   4843  *offset = GetOffset(*offset, L, bits);
   4844  return true;
   4845 }
   4846 
   4847 void MacroAssemblerRiscv64::BranchShortHelper(int32_t offset, Label* L) {
   4848  MOZ_ASSERT(L == nullptr || offset == 0);
   4849  BlockTrampolinePoolScope block_trampoline_pool(this, 2, 1);
   4850  offset = GetOffset(offset, L, OffsetSize::kOffset21);
   4851  Assembler::j(offset);
   4852 }
   4853 
   4854 bool MacroAssemblerRiscv64::BranchShortHelper(int32_t offset, Label* L,
   4855                                              Condition cond, Register rs,
   4856                                              const Operand& rt) {
   4857  MOZ_ASSERT(L == nullptr || offset == 0);
   4858  MOZ_ASSERT(rt.is_reg() || rt.is_imm());
   4859  UseScratchRegisterScope temps(this);
   4860  Register scratch = Register();
   4861  if (rt.is_imm()) {
   4862    if (rt.immediate() == 0) {
   4863      scratch = zero;
   4864    } else {
   4865      scratch = temps.Acquire();
   4866      ma_li(scratch, Imm64(rt.immediate()));
   4867    }
   4868  } else {
   4869    MOZ_ASSERT(rt.is_reg());
   4870    scratch = rt.rm();
   4871  }
   4872  {
   4873    BlockTrampolinePoolScope block_trampoline_pool(this, 2, 1);
   4874    switch (cond) {
   4875      case Always:
   4876        if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
   4877        Assembler::j(offset);
   4878        break;
   4879      case Equal:
   4880        // rs == rt
   4881        if (rt.is_reg() && rs == rt.rm()) {
   4882          if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
   4883          Assembler::j(offset);
   4884        } else {
   4885          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4886          Assembler::beq(rs, scratch, offset);
   4887        }
   4888        break;
   4889      case NotEqual:
   4890        // rs != rt
   4891        if (rt.is_reg() && rs == rt.rm()) {
   4892          break;  // No code needs to be emitted
   4893        } else {
   4894          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4895          Assembler::bne(rs, scratch, offset);
   4896        }
   4897        break;
   4898 
   4899      // Signed comparison.
   4900      case GreaterThan:
   4901        // rs > rt
   4902        if (rt.is_reg() && rs == rt.rm()) {
   4903          break;  // No code needs to be emitted.
   4904        } else {
   4905          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4906          Assembler::bgt(rs, scratch, offset);
   4907        }
   4908        break;
   4909      case GreaterThanOrEqual:
   4910        // rs >= rt
   4911        if (rt.is_reg() && rs == rt.rm()) {
   4912          if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
   4913          Assembler::j(offset);
   4914        } else {
   4915          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4916          Assembler::bge(rs, scratch, offset);
   4917        }
   4918        break;
   4919      case LessThan:
   4920        // rs < rt
   4921        if (rt.is_reg() && rs == rt.rm()) {
   4922          break;  // No code needs to be emitted.
   4923        } else {
   4924          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4925          Assembler::blt(rs, scratch, offset);
   4926        }
   4927        break;
   4928      case LessThanOrEqual:
   4929        // rs <= rt
   4930        if (rt.is_reg() && rs == rt.rm()) {
   4931          if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
   4932          Assembler::j(offset);
   4933        } else {
   4934          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4935          Assembler::ble(rs, scratch, offset);
   4936        }
   4937        break;
   4938 
   4939      // Unsigned comparison.
   4940      case Above:
   4941        // rs > rt
   4942        if (rt.is_reg() && rs == rt.rm()) {
   4943          break;  // No code needs to be emitted.
   4944        } else {
   4945          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4946          Assembler::bgtu(rs, scratch, offset);
   4947        }
   4948        break;
   4949      case AboveOrEqual:
   4950        // rs >= rt
   4951        if (rt.is_reg() && rs == rt.rm()) {
   4952          if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
   4953          Assembler::j(offset);
   4954        } else {
   4955          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4956          Assembler::bgeu(rs, scratch, offset);
   4957        }
   4958        break;
   4959      case Below:
   4960        // rs < rt
   4961        if (rt.is_reg() && rs == rt.rm()) {
   4962          break;  // No code needs to be emitted.
   4963        } else {
   4964          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4965          bltu(rs, scratch, offset);
   4966        }
   4967        break;
   4968      case BelowOrEqual:
   4969        // rs <= rt
   4970        if (rt.is_reg() && rs == rt.rm()) {
   4971          if (!CalculateOffset(L, OffsetSize::kOffset21, &offset)) return false;
   4972          Assembler::j(offset);
   4973        } else {
   4974          if (!CalculateOffset(L, OffsetSize::kOffset13, &offset)) return false;
   4975          Assembler::bleu(rs, scratch, offset);
   4976        }
   4977        break;
   4978      default:
   4979        MOZ_CRASH("UNREACHABLE");
   4980    }
   4981  }
   4982  return true;
   4983 }
   4984 
   4985 // BRANCH_ARGS_CHECK checks that conditional jump arguments are correct.
   4986 #define BRANCH_ARGS_CHECK(cond, rs, rt)                           \
   4987  MOZ_ASSERT((cond == Always && rs == zero && rt.rm() == zero) || \
   4988             (cond != Always && (rs != zero || rt.rm() != zero)))
   4989 
   4990 bool MacroAssemblerRiscv64::BranchShortCheck(int32_t offset, Label* L,
   4991                                             Condition cond, Register rs,
   4992                                             const Operand& rt) {
   4993  BRANCH_ARGS_CHECK(cond, rs, rt);
   4994 
   4995  if (!L) {
   4996    MOZ_ASSERT(is_int13(offset));
   4997    return BranchShortHelper(offset, nullptr, cond, rs, rt);
   4998  } else {
   4999    MOZ_ASSERT(offset == 0);
   5000    return BranchShortHelper(0, L, cond, rs, rt);
   5001  }
   5002 }
   5003 
   5004 void MacroAssemblerRiscv64::BranchShort(Label* L) { BranchShortHelper(0, L); }
   5005 
   5006 bool MacroAssemblerRiscv64::BranchShort(int32_t offset, Condition cond,
   5007                                        Register rs, const Operand& rt) {
   5008  return BranchShortCheck(offset, nullptr, cond, rs, rt);
   5009 }
   5010 
   5011 bool MacroAssemblerRiscv64::BranchShort(Label* L, Condition cond, Register rs,
   5012                                        const Operand& rt) {
   5013  return BranchShortCheck(0, L, cond, rs, rt);
   5014 }
   5015 
   5016 void MacroAssemblerRiscv64::BranchLong(Label* L) {
   5017  // Generate position independent long branch.
   5018  UseScratchRegisterScope temps(this);
   5019  Register scratch = temps.Acquire();
   5020  int32_t imm = branchLongOffsetHelper(L);
   5021  GenPCRelativeJump(scratch, imm);
   5022 }
   5023 
   5024 CodeOffset MacroAssemblerRiscv64::BranchAndLinkLong(Label* L) {
   5025  // Generate position independent long branch and link.
   5026  int32_t imm = branchLongOffsetHelper(L);
   5027  UseScratchRegisterScope temps(this);
   5028  Register scratch = temps.Acquire();
   5029  GenPCRelativeJumpAndLink(scratch, imm);
   5030  return CodeOffset(currentOffset());
   5031 }
   5032 
   5033 void MacroAssemblerRiscv64::ma_branch(Label* L, Condition cond, Register rs,
   5034                                      const Operand& rt, JumpKind jumpKind) {
   5035  // Always prefer short jumps when the label is already bound. (If the label is
   5036  // bound, BranchShort can cheaply determine if short jumps are possible.)
   5037  if (L->bound()) {
   5038    jumpKind = ShortJump;
   5039  }
   5040 
   5041  if (jumpKind == ShortJump && BranchShort(L, cond, rs, rt)) {
   5042    return;
   5043  }
   5044 
   5045  if (cond != Always) {
   5046    Label skip;
   5047    Condition neg_cond = InvertCondition(cond);
   5048    MOZ_ALWAYS_TRUE(
   5049        BranchShort(&skip, neg_cond, rs, rt));  // Guaranteed to be short.
   5050    BranchLong(L);
   5051    bind(&skip);
   5052  } else {
   5053    BranchLong(L);
   5054  }
   5055 }
   5056 
   5057 // Branches when done from within riscv code.
   5058 void MacroAssemblerRiscv64::ma_b(Register lhs, Address addr, Label* label,
   5059                                 Condition c, JumpKind jumpKind) {
   5060  UseScratchRegisterScope temps(this);
   5061  Register scratch = temps.Acquire();
   5062  MOZ_ASSERT(lhs != scratch);
   5063  ma_load(scratch, addr, SizeDouble);
   5064  ma_b(lhs, Register(scratch), label, c, jumpKind);
   5065 }
   5066 
   5067 void MacroAssemblerRiscv64::ma_b(Register lhs, ImmPtr imm, Label* l,
   5068                                 Condition c, JumpKind jumpKind) {
   5069  ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind);
   5070 }
   5071 
   5072 // Branches when done from within riscv code.
   5073 void MacroAssemblerRiscv64::ma_b(Register lhs, ImmWord imm, Label* label,
   5074                                 Condition c, JumpKind jumpKind) {
   5075  switch (c) {
   5076    case Always:
   5077      ma_branch(label, c, zero, Operand(zero), jumpKind);
   5078      break;
   5079    case Zero:
   5080    case NonZero:
   5081    case Signed:
   5082    case NotSigned:
   5083      MOZ_ASSERT(imm.value == 0);
   5084      ma_b(lhs, lhs, label, c, jumpKind);
   5085      break;
   5086    default:
   5087      ma_branch(label, c, lhs, Operand(imm.value), jumpKind);
   5088      break;
   5089  }
   5090 }
   5091 
   5092 void MacroAssemblerRiscv64::ma_b(Register lhs, Imm32 imm, Label* label,
   5093                                 Condition c, JumpKind jumpKind) {
   5094  switch (c) {
   5095    case Always:
   5096      ma_branch(label, c, zero, Operand(zero), jumpKind);
   5097      break;
   5098    case Zero:
   5099    case NonZero:
   5100    case Signed:
   5101    case NotSigned:
   5102      MOZ_ASSERT(imm.value == 0);
   5103      ma_b(lhs, lhs, label, c, jumpKind);
   5104      break;
   5105    default:
   5106      ma_branch(label, c, lhs, Operand(imm.value), jumpKind);
   5107      break;
   5108  }
   5109 }
   5110 
   5111 void MacroAssemblerRiscv64::ma_b(Address addr, Imm32 imm, Label* label,
   5112                                 Condition c, JumpKind jumpKind) {
   5113  UseScratchRegisterScope temps(this);
   5114  Register scratch2 = temps.Acquire();
   5115  ma_load(scratch2, addr);
   5116  ma_b(Register(scratch2), imm, label, c, jumpKind);
   5117 }
   5118 
   5119 void MacroAssemblerRiscv64::ma_b(Register lhs, Register rhs, Label* label,
   5120                                 Condition c, JumpKind jumpKind) {
   5121  switch (c) {
   5122    case Always:
   5123      ma_branch(label, c, zero, Operand(zero), jumpKind);
   5124      break;
   5125    case Zero:
   5126      MOZ_ASSERT(lhs == rhs);
   5127      ma_branch(label, Equal, lhs, Operand(zero), jumpKind);
   5128      break;
   5129    case NonZero:
   5130      MOZ_ASSERT(lhs == rhs);
   5131      ma_branch(label, NotEqual, lhs, Operand(zero), jumpKind);
   5132      break;
   5133    case Signed:
   5134      MOZ_ASSERT(lhs == rhs);
   5135      ma_branch(label, LessThan, lhs, Operand(zero), jumpKind);
   5136      break;
   5137    case NotSigned:
   5138      MOZ_ASSERT(lhs == rhs);
   5139      ma_branch(label, GreaterThanOrEqual, lhs, Operand(zero), jumpKind);
   5140      break;
   5141    default:
   5142      ma_branch(label, c, lhs, rhs, jumpKind);
   5143      break;
   5144  }
   5145 }
   5146 
   5147 void MacroAssemblerRiscv64::ExtractBits(Register rt, Register rs, uint16_t pos,
   5148                                        uint16_t size, bool sign_extend) {
   5149 #if JS_CODEGEN_RISCV64
   5150  constexpr uint16_t MaxBits = 64;
   5151 #elif JS_CODEGEN_RISCV32
   5152  constexpr uint16_t MaxBits = 32;
   5153 #endif
   5154 
   5155  MOZ_ASSERT(pos < MaxBits);
   5156  MOZ_ASSERT(size > 0);
   5157  MOZ_ASSERT(size <= MaxBits);
   5158  MOZ_ASSERT((pos + size) > 0);
   5159  MOZ_ASSERT((pos + size) <= MaxBits);
   5160 
   5161  Register src;
   5162  if (uint16_t shift = MaxBits - (pos + size)) {
   5163    slli(rt, rs, shift);
   5164    src = rt;
   5165  } else {
   5166    src = rs;
   5167  }
   5168 
   5169  if (sign_extend) {
   5170    srai(rt, src, MaxBits - size);
   5171  } else {
   5172    srli(rt, src, MaxBits - size);
   5173  }
   5174 }
   5175 
   5176 void MacroAssemblerRiscv64::InsertBits(Register dest, Register source, int pos,
   5177                                       int size) {
   5178 #if JS_CODEGEN_RISCV64
   5179  MOZ_ASSERT(size < 64);
   5180 #elif JS_CODEGEN_RISCV32
   5181  MOZ_ASSERT(size < 32);
   5182 #endif
   5183  UseScratchRegisterScope temps(this);
   5184  BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5185  Register source_ = temps.Acquire();
   5186  if (pos != 0) {
   5187    Register mask = temps.Acquire();
   5188    // Create a mask of the length=size.
   5189    ma_li(mask, Imm32(1));
   5190    slli(mask, mask, size);
   5191    addi(mask, mask, -1);
   5192    and_(source_, mask, source);
   5193    slli(source_, source_, pos);
   5194    // Make a mask containing 0's. 0's start at "pos" with length=size.
   5195    slli(mask, mask, pos);
   5196    not_(mask, mask);
   5197    // cut area for insertion of source.
   5198    and_(dest, mask, dest);
   5199  } else {
   5200    // clear top bits from source and bottom bits from dest.
   5201    slli(source_, source, 64 - size);
   5202    srli(source_, source_, 64 - size);
   5203    srli(dest, dest, size);
   5204    slli(dest, dest, size);
   5205  }
   5206  // insert source
   5207  or_(dest, dest, source_);
   5208 }
   5209 
   5210 void MacroAssemblerRiscv64::InsertBits(Register dest, Register source,
   5211                                       Register pos, int size) {
   5212 #if JS_CODEGEN_RISCV64
   5213  MOZ_ASSERT(size < 64);
   5214 #elif JS_CODEGEN_RISCV32
   5215  MOZ_ASSERT(size < 32);
   5216 #endif
   5217  UseScratchRegisterScope temps(this);
   5218  Register mask = temps.Acquire();
   5219  BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5220  Register source_ = temps.Acquire();
   5221  // Create a mask of the length=size.
   5222  ma_li(mask, Imm32(1));
   5223  slli(mask, mask, size);
   5224  addi(mask, mask, -1);
   5225  and_(source_, mask, source);
   5226  sll(source_, source_, pos);
   5227  // Make a mask containing 0's. 0's start at "pos" with length=size.
   5228  sll(mask, mask, pos);
   5229  not_(mask, mask);
   5230  // cut area for insertion of source.
   5231  and_(dest, mask, dest);
   5232  // insert source
   5233  or_(dest, dest, source_);
   5234 }
   5235 
   5236 void MacroAssemblerRiscv64::ma_add32(Register rd, Register rs, Operand rt) {
   5237  if (rt.is_imm()) {
   5238    if (is_int12(rt.immediate())) {
   5239      addiw(rd, rs, static_cast<int32_t>(rt.immediate()));
   5240    } else if ((-4096 <= rt.immediate() && rt.immediate() <= -2049) ||
   5241               (2048 <= rt.immediate() && rt.immediate() <= 4094)) {
   5242      addiw(rd, rs, rt.immediate() / 2);
   5243      addiw(rd, rd, rt.immediate() - (rt.immediate() / 2));
   5244    } else {
   5245      // li handles the relocation.
   5246      UseScratchRegisterScope temps(this);
   5247      Register scratch = temps.Acquire();
   5248      BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5249      ma_li(scratch, rt.immediate());
   5250      addw(rd, rs, scratch);
   5251    }
   5252  } else {
   5253    MOZ_ASSERT(rt.is_reg());
   5254    addw(rd, rs, rt.rm());
   5255  }
   5256 }
   5257 
   5258 void MacroAssemblerRiscv64::ma_add64(Register rd, Register rs, Operand rt) {
   5259  if (rt.is_imm()) {
   5260    if (is_int12(rt.immediate())) {
   5261      addi(rd, rs, static_cast<int32_t>(rt.immediate()));
   5262    } else if ((-4096 <= rt.immediate() && rt.immediate() <= -2049) ||
   5263               (2048 <= rt.immediate() && rt.immediate() <= 4094)) {
   5264      addi(rd, rs, rt.immediate() / 2);
   5265      addi(rd, rd, rt.immediate() - (rt.immediate() / 2));
   5266    } else {
   5267      // li handles the relocation.
   5268      UseScratchRegisterScope temps(this);
   5269      Register scratch = temps.Acquire();
   5270      BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5271      ma_li(scratch, rt.immediate());
   5272      add(rd, rs, scratch);
   5273    }
   5274  } else {
   5275    MOZ_ASSERT(rt.is_reg());
   5276    add(rd, rs, rt.rm());
   5277  }
   5278 }
   5279 
   5280 void MacroAssemblerRiscv64::ma_sub32(Register rd, Register rs, Operand rt) {
   5281  if (rt.is_imm()) {
   5282    if (is_int12(-rt.immediate())) {
   5283      addiw(rd, rs,
   5284            static_cast<int32_t>(
   5285                -rt.immediate()));  // No subi instr, use addi(x, y, -imm).
   5286    } else if ((-4096 <= -rt.immediate() && -rt.immediate() <= -2049) ||
   5287               (2048 <= -rt.immediate() && -rt.immediate() <= 4094)) {
   5288      addiw(rd, rs, -rt.immediate() / 2);
   5289      addiw(rd, rd, -rt.immediate() - (-rt.immediate() / 2));
   5290    } else {
   5291      // li handles the relocation.
   5292      UseScratchRegisterScope temps(this);
   5293      Register scratch = temps.Acquire();
   5294      ma_li(scratch, rt.immediate());
   5295      subw(rd, rs, scratch);
   5296    }
   5297  } else {
   5298    MOZ_ASSERT(rt.is_reg());
   5299    subw(rd, rs, rt.rm());
   5300  }
   5301 }
   5302 
   5303 void MacroAssemblerRiscv64::ma_sub64(Register rd, Register rs, Operand rt) {
   5304  if (rt.is_imm()) {
   5305    if (is_int12(-rt.immediate())) {
   5306      addi(rd, rs,
   5307           static_cast<int32_t>(
   5308               -rt.immediate()));  // No subi instr, use addi(x, y, -imm).
   5309    } else if ((-4096 <= -rt.immediate() && -rt.immediate() <= -2049) ||
   5310               (2048 <= -rt.immediate() && -rt.immediate() <= 4094)) {
   5311      addi(rd, rs, -rt.immediate() / 2);
   5312      addi(rd, rd, -rt.immediate() - (-rt.immediate() / 2));
   5313    } else {
   5314      // li handles the relocation.
   5315      UseScratchRegisterScope temps(this);
   5316      Register scratch = temps.Acquire();
   5317      ma_li(scratch, rt.immediate());
   5318      sub(rd, rs, scratch);
   5319    }
   5320  } else {
   5321    MOZ_ASSERT(rt.is_reg());
   5322    sub(rd, rs, rt.rm());
   5323  }
   5324 }
   5325 
   5326 void MacroAssemblerRiscv64::ma_and(Register rd, Register rs, Operand rt) {
   5327  if (rt.is_imm()) {
   5328    if (is_int12(rt.immediate())) {
   5329      andi(rd, rs, rt.immediate());
   5330    } else {
   5331      UseScratchRegisterScope temps(this);
   5332      Register scratch = temps.Acquire();
   5333      ma_li(scratch, rt.immediate());
   5334      and_(rd, rs, scratch);
   5335    }
   5336  } else {
   5337    MOZ_ASSERT(rt.is_reg());
   5338    and_(rd, rs, rt.rm());
   5339  }
   5340 }
   5341 
   5342 void MacroAssemblerRiscv64::ma_or(Register rd, Register rs, Operand rt) {
   5343  if (rt.is_imm()) {
   5344    if (is_int12(rt.immediate())) {
   5345      ori(rd, rs, rt.immediate());
   5346    } else {
   5347      UseScratchRegisterScope temps(this);
   5348      Register scratch = temps.Acquire();
   5349      ma_li(scratch, rt.immediate());
   5350      or_(rd, rs, scratch);
   5351    }
   5352  } else {
   5353    MOZ_ASSERT(rt.is_reg());
   5354    or_(rd, rs, rt.rm());
   5355  }
   5356 }
   5357 
   5358 void MacroAssemblerRiscv64::ma_xor(Register rd, Register rs, Operand rt) {
   5359  if (rt.is_imm()) {
   5360    if (is_int12(rt.immediate())) {
   5361      xori(rd, rs, rt.immediate());
   5362    } else {
   5363      UseScratchRegisterScope temps(this);
   5364      Register scratch = temps.Acquire();
   5365      ma_li(scratch, rt.immediate());
   5366      xor_(rd, rs, scratch);
   5367    }
   5368  } else {
   5369    MOZ_ASSERT(rt.is_reg());
   5370    xor_(rd, rs, rt.rm());
   5371  }
   5372 }
   5373 
   5374 void MacroAssemblerRiscv64::ma_nor(Register rd, Register rs, Operand rt) {
   5375  if (rt.is_imm()) {
   5376    UseScratchRegisterScope temps(this);
   5377    Register scratch = temps.Acquire();
   5378    ma_li(scratch, rt.immediate());
   5379    nor(rd, rs, scratch);
   5380  } else {
   5381    MOZ_ASSERT(rt.is_reg());
   5382    nor(rd, rs, rt.rm());
   5383  }
   5384 }
   5385 
   5386 void MacroAssemblerRiscv64::ma_div32(Register rd, Register rs, Operand rt) {
   5387  if (rt.is_imm()) {
   5388    UseScratchRegisterScope temps(this);
   5389    Register scratch = temps.Acquire();
   5390    ma_li(scratch, rt.immediate());
   5391    divw(rd, rs, scratch);
   5392  } else {
   5393    MOZ_ASSERT(rt.is_reg());
   5394    divw(rd, rs, rt.rm());
   5395  }
   5396 }
   5397 
   5398 void MacroAssemblerRiscv64::ma_divu32(Register rd, Register rs, Operand rt) {
   5399  if (rt.is_imm()) {
   5400    UseScratchRegisterScope temps(this);
   5401    Register scratch = temps.Acquire();
   5402    ma_li(scratch, rt.immediate());
   5403    divuw(rd, rs, scratch);
   5404  } else {
   5405    MOZ_ASSERT(rt.is_reg());
   5406    divuw(rd, rs, rt.rm());
   5407  }
   5408 }
   5409 
   5410 void MacroAssemblerRiscv64::ma_div64(Register rd, Register rs, Operand rt) {
   5411  if (rt.is_imm()) {
   5412    UseScratchRegisterScope temps(this);
   5413    Register scratch = temps.Acquire();
   5414    ma_li(scratch, rt.immediate());
   5415    div(rd, rs, scratch);
   5416  } else {
   5417    MOZ_ASSERT(rt.is_reg());
   5418    div(rd, rs, rt.rm());
   5419  }
   5420 }
   5421 
   5422 void MacroAssemblerRiscv64::ma_divu64(Register rd, Register rs, Operand rt) {
   5423  if (rt.is_imm()) {
   5424    UseScratchRegisterScope temps(this);
   5425    Register scratch = temps.Acquire();
   5426    ma_li(scratch, rt.immediate());
   5427    divu(rd, rs, scratch);
   5428  } else {
   5429    MOZ_ASSERT(rt.is_reg());
   5430    divu(rd, rs, rt.rm());
   5431  }
   5432 }
   5433 
   5434 void MacroAssemblerRiscv64::ma_mod32(Register rd, Register rs, Operand rt) {
   5435  if (rt.is_imm()) {
   5436    UseScratchRegisterScope temps(this);
   5437    Register scratch = temps.Acquire();
   5438    ma_li(scratch, rt.immediate());
   5439    remw(rd, rs, scratch);
   5440  } else {
   5441    MOZ_ASSERT(rt.is_reg());
   5442    remw(rd, rs, rt.rm());
   5443  }
   5444 }
   5445 
   5446 void MacroAssemblerRiscv64::ma_modu32(Register rd, Register rs, Operand rt) {
   5447  if (rt.is_imm()) {
   5448    UseScratchRegisterScope temps(this);
   5449    Register scratch = temps.Acquire();
   5450    ma_li(scratch, rt.immediate());
   5451    remuw(rd, rs, scratch);
   5452  } else {
   5453    MOZ_ASSERT(rt.is_reg());
   5454    remuw(rd, rs, rt.rm());
   5455  }
   5456 }
   5457 
   5458 void MacroAssemblerRiscv64::ma_mod64(Register rd, Register rs, Operand rt) {
   5459  if (rt.is_imm()) {
   5460    UseScratchRegisterScope temps(this);
   5461    Register scratch = temps.Acquire();
   5462    ma_li(scratch, rt.immediate());
   5463    rem(rd, rs, scratch);
   5464  } else {
   5465    MOZ_ASSERT(rt.is_reg());
   5466    rem(rd, rs, rt.rm());
   5467  }
   5468 }
   5469 
   5470 void MacroAssemblerRiscv64::ma_modu64(Register rd, Register rs, Operand rt) {
   5471  if (rt.is_imm()) {
   5472    UseScratchRegisterScope temps(this);
   5473    Register scratch = temps.Acquire();
   5474    ma_li(scratch, rt.immediate());
   5475    remu(rd, rs, scratch);
   5476  } else {
   5477    MOZ_ASSERT(rt.is_reg());
   5478    remu(rd, rs, rt.rm());
   5479  }
   5480 }
   5481 
   5482 void MacroAssemblerRiscv64::ma_mul32(Register rd, Register rs, Operand rt) {
   5483  if (rt.is_imm()) {
   5484    UseScratchRegisterScope temps(this);
   5485    Register scratch = temps.Acquire();
   5486    ma_li(scratch, rt.immediate());
   5487    mulw(rd, rs, scratch);
   5488  } else {
   5489    MOZ_ASSERT(rt.is_reg());
   5490    mulw(rd, rs, rt.rm());
   5491  }
   5492 }
   5493 
   5494 void MacroAssemblerRiscv64::ma_mulh32(Register rd, Register rs, Operand rt) {
   5495  if (rt.is_imm()) {
   5496    UseScratchRegisterScope temps(this);
   5497    Register scratch = temps.Acquire();
   5498    ma_li(scratch, rt.immediate());
   5499    mul(rd, rs, scratch);
   5500  } else {
   5501    MOZ_ASSERT(rt.is_reg());
   5502    mul(rd, rs, rt.rm());
   5503  }
   5504  srai(rd, rd, 32);
   5505 }
   5506 
   5507 void MacroAssemblerRiscv64::ma_mul64(Register rd, Register rs, Operand rt) {
   5508  if (rt.is_imm()) {
   5509    UseScratchRegisterScope temps(this);
   5510    Register scratch = temps.Acquire();
   5511    ma_li(scratch, rt.immediate());
   5512    mul(rd, rs, scratch);
   5513  } else {
   5514    MOZ_ASSERT(rt.is_reg());
   5515    mul(rd, rs, rt.rm());
   5516  }
   5517 }
   5518 
   5519 void MacroAssemblerRiscv64::ma_mulh64(Register rd, Register rs, Operand rt) {
   5520  if (rt.is_imm()) {
   5521    UseScratchRegisterScope temps(this);
   5522    Register scratch = temps.Acquire();
   5523    ma_li(scratch, rt.immediate());
   5524    mulh(rd, rs, scratch);
   5525  } else {
   5526    MOZ_ASSERT(rt.is_reg());
   5527    mulh(rd, rs, rt.rm());
   5528  }
   5529 }
   5530 
   5531 void MacroAssemblerRiscv64::ma_sll64(Register rd, Register rs, Operand rt) {
   5532  if (rt.is_reg()) {
   5533    sll(rd, rs, rt.rm());
   5534  } else {
   5535    MOZ_ASSERT(rt.is_imm());
   5536    uint8_t shamt = static_cast<uint8_t>(rt.immediate());
   5537    slli(rd, rs, shamt);
   5538  }
   5539 }
   5540 
   5541 void MacroAssemblerRiscv64::ma_sll32(Register rd, Register rs, Operand rt) {
   5542  if (rt.is_reg()) {
   5543    sllw(rd, rs, rt.rm());
   5544  } else {
   5545    MOZ_ASSERT(rt.is_imm());
   5546    uint8_t shamt = static_cast<uint8_t>(rt.immediate());
   5547    slliw(rd, rs, shamt);
   5548  }
   5549 }
   5550 
   5551 void MacroAssemblerRiscv64::ma_sra64(Register rd, Register rs, Operand rt) {
   5552  if (rt.is_reg()) {
   5553    sra(rd, rs, rt.rm());
   5554  } else {
   5555    MOZ_ASSERT(rt.is_imm());
   5556    uint8_t shamt = static_cast<uint8_t>(rt.immediate());
   5557    srai(rd, rs, shamt);
   5558  }
   5559 }
   5560 
   5561 void MacroAssemblerRiscv64::ma_sra32(Register rd, Register rs, Operand rt) {
   5562  if (rt.is_reg()) {
   5563    sraw(rd, rs, rt.rm());
   5564  } else {
   5565    MOZ_ASSERT(rt.is_imm());
   5566    uint8_t shamt = static_cast<uint8_t>(rt.immediate());
   5567    sraiw(rd, rs, shamt);
   5568  }
   5569 }
   5570 
   5571 void MacroAssemblerRiscv64::ma_srl64(Register rd, Register rs, Operand rt) {
   5572  if (rt.is_reg()) {
   5573    srl(rd, rs, rt.rm());
   5574  } else {
   5575    MOZ_ASSERT(rt.is_imm());
   5576    uint8_t shamt = static_cast<uint8_t>(rt.immediate());
   5577    srli(rd, rs, shamt);
   5578  }
   5579 }
   5580 
   5581 void MacroAssemblerRiscv64::ma_srl32(Register rd, Register rs, Operand rt) {
   5582  if (rt.is_reg()) {
   5583    srlw(rd, rs, rt.rm());
   5584  } else {
   5585    MOZ_ASSERT(rt.is_imm());
   5586    uint8_t shamt = static_cast<uint8_t>(rt.immediate());
   5587    srliw(rd, rs, shamt);
   5588  }
   5589 }
   5590 
   5591 void MacroAssemblerRiscv64::ma_slt(Register rd, Register rs, Operand rt) {
   5592  if (rt.is_reg()) {
   5593    slt(rd, rs, rt.rm());
   5594  } else {
   5595    MOZ_ASSERT(rt.is_imm());
   5596    if (is_int12(rt.immediate())) {
   5597      slti(rd, rs, static_cast<int32_t>(rt.immediate()));
   5598    } else {
   5599      // li handles the relocation.
   5600      UseScratchRegisterScope temps(this);
   5601      Register scratch = temps.Acquire();
   5602      BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5603      ma_li(scratch, rt.immediate());
   5604      slt(rd, rs, scratch);
   5605    }
   5606  }
   5607 }
   5608 
   5609 void MacroAssemblerRiscv64::ma_sltu(Register rd, Register rs, Operand rt) {
   5610  if (rt.is_reg()) {
   5611    sltu(rd, rs, rt.rm());
   5612  } else {
   5613    MOZ_ASSERT(rt.is_imm());
   5614    if (is_int12(rt.immediate())) {
   5615      sltiu(rd, rs, static_cast<int32_t>(rt.immediate()));
   5616    } else {
   5617      // li handles the relocation.
   5618      UseScratchRegisterScope temps(this);
   5619      Register scratch = temps.Acquire();
   5620      BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5621      ma_li(scratch, rt.immediate());
   5622      sltu(rd, rs, scratch);
   5623    }
   5624  }
   5625 }
   5626 
   5627 void MacroAssemblerRiscv64::ma_sle(Register rd, Register rs, Operand rt) {
   5628  if (rt.is_reg()) {
   5629    slt(rd, rt.rm(), rs);
   5630  } else {
   5631    MOZ_ASSERT(rt.is_imm());
   5632    // li handles the relocation.
   5633    UseScratchRegisterScope temps(this);
   5634    Register scratch = temps.Acquire();
   5635    BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5636    ma_li(scratch, rt.immediate());
   5637    slt(rd, scratch, rs);
   5638  }
   5639  xori(rd, rd, 1);
   5640 }
   5641 
   5642 void MacroAssemblerRiscv64::ma_sleu(Register rd, Register rs, Operand rt) {
   5643  if (rt.is_reg()) {
   5644    sltu(rd, rt.rm(), rs);
   5645  } else {
   5646    MOZ_ASSERT(rt.is_imm());
   5647    // li handles the relocation.
   5648    UseScratchRegisterScope temps(this);
   5649    Register scratch = temps.Acquire();
   5650    BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5651    ma_li(scratch, rt.immediate());
   5652    sltu(rd, scratch, rs);
   5653  }
   5654  xori(rd, rd, 1);
   5655 }
   5656 
   5657 void MacroAssemblerRiscv64::ma_sgt(Register rd, Register rs, Operand rt) {
   5658  if (rt.is_reg()) {
   5659    slt(rd, rt.rm(), rs);
   5660  } else {
   5661    MOZ_ASSERT(rt.is_imm());
   5662    // li handles the relocation.
   5663    UseScratchRegisterScope temps(this);
   5664    Register scratch = temps.Acquire();
   5665    BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5666    ma_li(scratch, rt.immediate());
   5667    slt(rd, scratch, rs);
   5668  }
   5669 }
   5670 
   5671 void MacroAssemblerRiscv64::ma_sgtu(Register rd, Register rs, Operand rt) {
   5672  if (rt.is_reg()) {
   5673    sltu(rd, rt.rm(), rs);
   5674  } else {
   5675    MOZ_ASSERT(rt.is_imm());
   5676    // li handles the relocation.
   5677    UseScratchRegisterScope temps(this);
   5678    Register scratch = temps.Acquire();
   5679    BlockTrampolinePoolScope block_trampoline_pool(this, 9);
   5680    ma_li(scratch, rt.immediate());
   5681    sltu(rd, scratch, rs);
   5682  }
   5683 }
   5684 
   5685 void MacroAssemblerRiscv64::ma_sge(Register rd, Register rs, Operand rt) {
   5686  ma_slt(rd, rs, rt);
   5687  xori(rd, rd, 1);
   5688 }
   5689 
   5690 void MacroAssemblerRiscv64::ma_sgeu(Register rd, Register rs, Operand rt) {
   5691  ma_sltu(rd, rs, rt);
   5692  xori(rd, rd, 1);
   5693 }
   5694 
   5695 static inline bool IsZero(const Operand& rt) {
   5696  if (rt.is_reg()) {
   5697    return rt.rm() == zero_reg;
   5698  } else {
   5699    MOZ_ASSERT(rt.is_imm());
   5700    return rt.immediate() == 0;
   5701  }
   5702 }
   5703 
   5704 void MacroAssemblerRiscv64::ma_seq(Register rd, Register rs, Operand rt) {
   5705  if (rs == zero_reg) {
   5706    ma_seqz(rd, rt);
   5707  } else if (IsZero(rt)) {
   5708    seqz(rd, rs);
   5709  } else {
   5710    ma_sub64(rd, rs, rt);
   5711    seqz(rd, rd);
   5712  }
   5713 }
   5714 
   5715 void MacroAssemblerRiscv64::ma_sne(Register rd, Register rs, Operand rt) {
   5716  if (rs == zero_reg) {
   5717    ma_snez(rd, rt);
   5718  } else if (IsZero(rt)) {
   5719    snez(rd, rs);
   5720  } else {
   5721    ma_sub64(rd, rs, rt);
   5722    snez(rd, rd);
   5723  }
   5724 }
   5725 
   5726 void MacroAssemblerRiscv64::ma_seqz(Register rd, const Operand& rt) {
   5727  if (rt.is_reg()) {
   5728    seqz(rd, rt.rm());
   5729  } else {
   5730    ma_li(rd, rt.immediate() == 0);
   5731  }
   5732 }
   5733 
   5734 void MacroAssemblerRiscv64::ma_snez(Register rd, const Operand& rt) {
   5735  if (rt.is_reg()) {
   5736    snez(rd, rt.rm());
   5737  } else {
   5738    ma_li(rd, rt.immediate() != 0);
   5739  }
   5740 }
   5741 
   5742 void MacroAssemblerRiscv64::ma_neg(Register rd, const Operand& rt) {
   5743  MOZ_ASSERT(rt.is_reg());
   5744  neg(rd, rt.rm());
   5745 }
   5746 
   5747 void MacroAssemblerRiscv64::ma_jump(ImmPtr dest) {
   5748  DEBUG_PRINTF("[ %s\n", __FUNCTION__);
   5749  BlockTrampolinePoolScope block_trampoline_pool(this, 8);
   5750  UseScratchRegisterScope temps(this);
   5751  Register scratch = temps.Acquire();
   5752  ma_liPatchable(scratch, dest);
   5753  jr(scratch, 0);
   5754  DEBUG_PRINTF("]\n");
   5755 }
   5756 // fp instructions
   5757 void MacroAssemblerRiscv64::ma_lid(FloatRegister dest, double value) {
   5758  ImmWord imm(mozilla::BitwiseCast<uint64_t>(value));
   5759 
   5760  if (imm.value != 0) {
   5761    UseScratchRegisterScope temps(this);
   5762    Register scratch = temps.Acquire();
   5763    ma_li(scratch, imm);
   5764    fmv_d_x(dest, scratch);
   5765  } else {
   5766    fmv_d_x(dest, zero);
   5767  }
   5768 }
   5769 // fp instructions
   5770 void MacroAssemblerRiscv64::ma_lis(FloatRegister dest, float value) {
   5771  Imm32 imm(mozilla::BitwiseCast<uint32_t>(value));
   5772 
   5773  if (imm.value != 0) {
   5774    UseScratchRegisterScope temps(this);
   5775    Register scratch = temps.Acquire();
   5776    ma_li(scratch, imm);
   5777    fmv_w_x(dest, scratch);
   5778  } else {
   5779    fmv_w_x(dest, zero);
   5780  }
   5781 }
   5782 
   5783 void MacroAssemblerRiscv64::ma_sub32TestOverflow(Register rd, Register rj,
   5784                                                 Register rk, Label* overflow) {
   5785  UseScratchRegisterScope temps(this);
   5786  Register scratch = temps.Acquire();
   5787  sub(scratch, rj, rk);
   5788  subw(rd, rj, rk);
   5789  ma_b(rd, Register(scratch), overflow, Assembler::NotEqual);
   5790 }
   5791 
   5792 void MacroAssemblerRiscv64::ma_sub32TestOverflow(Register rd, Register rj,
   5793                                                 Imm32 imm, Label* overflow) {
   5794  if (imm.value != INT32_MIN) {
   5795    ma_add32TestOverflow(rd, rj, Imm32(-imm.value), overflow);
   5796  } else {
   5797    UseScratchRegisterScope temps(this);
   5798    Register scratch = temps.Acquire();
   5799    MOZ_ASSERT(rj != scratch);
   5800    ma_li(scratch, Imm32(imm.value));
   5801    ma_sub32TestOverflow(rd, rj, scratch, overflow);
   5802  }
   5803 }
   5804 
   5805 void MacroAssemblerRiscv64::ma_add32TestOverflow(Register rd, Register rj,
   5806                                                 Register rk, Label* overflow) {
   5807  UseScratchRegisterScope temps(this);
   5808  Register scratch = temps.Acquire();
   5809  add(scratch, rj, rk);
   5810  addw(rd, rj, rk);
   5811  ma_b(rd, Register(scratch), overflow, Assembler::NotEqual);
   5812 }
   5813 
   5814 void MacroAssemblerRiscv64::ma_add32TestOverflow(Register rd, Register rj,
   5815                                                 Imm32 imm, Label* overflow) {
   5816  // Check for signed range because of addi
   5817  if (is_intn(imm.value, 12)) {
   5818    UseScratchRegisterScope temps(this);
   5819    Register scratch = temps.Acquire();
   5820    addi(scratch, rj, imm.value);
   5821    addiw(rd, rj, imm.value);
   5822    ma_b(rd, scratch, overflow, Assembler::NotEqual);
   5823  } else {
   5824    UseScratchRegisterScope temps(this);
   5825    Register scratch2 = temps.Acquire();
   5826    ma_li(scratch2, imm);
   5827    ma_add32TestOverflow(rd, rj, scratch2, overflow);
   5828  }
   5829 }
   5830 
   5831 void MacroAssemblerRiscv64::ma_subPtrTestOverflow(Register rd, Register rj,
   5832                                                  Register rk,
   5833                                                  Label* overflow) {
   5834  UseScratchRegisterScope temps(this);
   5835  Register scratch2 = temps.Acquire();
   5836  MOZ_ASSERT_IF(rj == rd, rj != rk);
   5837  MOZ_ASSERT(rj != scratch2);
   5838  MOZ_ASSERT(rk != scratch2);
   5839  MOZ_ASSERT(rd != scratch2);
   5840 
   5841  Register rj_copy = rj;
   5842 
   5843  if (rj == rd) {
   5844    ma_or(scratch2, rj, zero);
   5845    rj_copy = scratch2;
   5846  }
   5847 
   5848  {
   5849    Register scratch = temps.Acquire();
   5850    MOZ_ASSERT(rd != scratch);
   5851 
   5852    sub(rd, rj, rk);
   5853    // If the sign of rj and rk are the same, no overflow
   5854    ma_xor(scratch, rj_copy, rk);
   5855    // Check if the sign of rd and rj are the same
   5856    ma_xor(scratch2, rd, rj_copy);
   5857    ma_and(scratch2, scratch2, scratch);
   5858  }
   5859 
   5860  ma_b(scratch2, zero, overflow, Assembler::LessThan);
   5861 }
   5862 
   5863 void MacroAssemblerRiscv64::ma_addPtrTestOverflow(Register rd, Register rj,
   5864                                                  Register rk,
   5865                                                  Label* overflow) {
   5866  UseScratchRegisterScope temps(this);
   5867  Register scratch = temps.Acquire();
   5868  MOZ_ASSERT(rd != scratch);
   5869 
   5870  if (rj == rk) {
   5871    if (rj == rd) {
   5872      ma_or(scratch, rj, zero);
   5873      rj = scratch;
   5874    }
   5875 
   5876    add(rd, rj, rj);
   5877    ma_xor(scratch, rj, rd);
   5878    ma_b(scratch, zero, overflow, Assembler::LessThan);
   5879  } else {
   5880    UseScratchRegisterScope temps(this);
   5881    Register scratch2 = temps.Acquire();
   5882    MOZ_ASSERT(rj != scratch);
   5883    MOZ_ASSERT(rd != scratch2);
   5884 
   5885    if (rj == rd) {
   5886      ma_or(scratch2, rj, zero);
   5887      rj = scratch2;
   5888    }
   5889 
   5890    add(rd, rj, rk);
   5891    slti(scratch, rj, 0);
   5892    slt(scratch2, rd, rk);
   5893    ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
   5894  }
   5895 }
   5896 
   5897 void MacroAssemblerRiscv64::ma_addPtrTestOverflow(Register rd, Register rj,
   5898                                                  Imm32 imm, Label* overflow) {
   5899  UseScratchRegisterScope temps(this);
   5900  Register scratch2 = temps.Acquire();
   5901 
   5902  if (imm.value == 0) {
   5903    ori(rd, rj, 0);
   5904    return;
   5905  }
   5906 
   5907  if (rj == rd) {
   5908    ori(scratch2, rj, 0);
   5909    rj = scratch2;
   5910  }
   5911 
   5912  ma_add64(rd, rj, imm);
   5913 
   5914  if (imm.value > 0) {
   5915    ma_b(rd, rj, overflow, Assembler::LessThan);
   5916  } else {
   5917    MOZ_ASSERT(imm.value < 0);
   5918    ma_b(rd, rj, overflow, Assembler::GreaterThan);
   5919  }
   5920 }
   5921 
   5922 void MacroAssemblerRiscv64::ma_addPtrTestOverflow(Register rd, Register rj,
   5923                                                  ImmWord imm,
   5924                                                  Label* overflow) {
   5925  UseScratchRegisterScope temps(this);
   5926  Register scratch2 = temps.Acquire();
   5927 
   5928  if (imm.value == 0) {
   5929    ori(rd, rj, 0);
   5930    return;
   5931  }
   5932 
   5933  if (rj == rd) {
   5934    MOZ_ASSERT(rj != scratch2);
   5935    ori(scratch2, rj, 0);
   5936    rj = scratch2;
   5937  }
   5938 
   5939  ma_li(rd, imm);
   5940  add(rd, rj, rd);
   5941 
   5942  if (imm.value > 0) {
   5943    ma_b(rd, rj, overflow, Assembler::LessThan);
   5944  } else {
   5945    MOZ_ASSERT(imm.value < 0);
   5946    ma_b(rd, rj, overflow, Assembler::GreaterThan);
   5947  }
   5948 }
   5949 
   5950 void MacroAssemblerRiscv64::ma_add32TestCarry(Condition cond, Register rd,
   5951                                              Register rj, Register rk,
   5952                                              Label* overflow) {
   5953  MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
   5954  MOZ_ASSERT_IF(rd == rj, rk != rd);
   5955  UseScratchRegisterScope temps(this);
   5956  Register scratch = temps.Acquire();
   5957  addw(rd, rj, rk);
   5958  sltu(scratch, rd, rd == rj ? rk : rj);
   5959  ma_b(Register(scratch), Register(scratch), overflow,
   5960       cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
   5961 }
   5962 
   5963 void MacroAssemblerRiscv64::ma_add32TestCarry(Condition cond, Register rd,
   5964                                              Register rj, Imm32 imm,
   5965                                              Label* overflow) {
   5966  UseScratchRegisterScope temps(this);
   5967  Register scratch2 = temps.Acquire();
   5968  MOZ_ASSERT(rj != scratch2);
   5969  ma_li(scratch2, imm);
   5970  ma_add32TestCarry(cond, rd, rj, scratch2, overflow);
   5971 }
   5972 
   5973 void MacroAssemblerRiscv64::ma_subPtrTestOverflow(Register rd, Register rj,
   5974                                                  Imm32 imm, Label* overflow) {
   5975  // TODO(riscv): Check subPtrTestOverflow
   5976  MOZ_ASSERT(imm.value != INT32_MIN);
   5977  ma_addPtrTestOverflow(rd, rj, Imm32(-imm.value), overflow);
   5978 }
   5979 
   5980 void MacroAssemblerRiscv64::ma_addPtrTestCarry(Condition cond, Register rd,
   5981                                               Register rj, Register rk,
   5982                                               Label* label) {
   5983  UseScratchRegisterScope temps(this);
   5984  Register scratch = temps.Acquire();
   5985  MOZ_ASSERT(rd != rk);
   5986  MOZ_ASSERT(rd != scratch);
   5987  add(rd, rj, rk);
   5988  sltu(scratch, rd, rk);
   5989  ma_b(scratch, Register(scratch), label,
   5990       cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
   5991 }
   5992 
   5993 void MacroAssemblerRiscv64::ma_addPtrTestCarry(Condition cond, Register rd,
   5994                                               Register rj, Imm32 imm,
   5995                                               Label* label) {
   5996  UseScratchRegisterScope temps(this);
   5997  Register scratch2 = temps.Acquire();
   5998 
   5999  // Check for signed range because of addi
   6000  if (is_intn(imm.value, 12)) {
   6001    addi(rd, rj, imm.value);
   6002    sltiu(scratch2, rd, imm.value);
   6003    ma_b(scratch2, scratch2, label,
   6004         cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
   6005  } else {
   6006    ma_li(scratch2, imm);
   6007    ma_addPtrTestCarry(cond, rd, rj, scratch2, label);
   6008  }
   6009 }
   6010 
   6011 void MacroAssemblerRiscv64::ma_addPtrTestCarry(Condition cond, Register rd,
   6012                                               Register rj, ImmWord imm,
   6013                                               Label* label) {
   6014  UseScratchRegisterScope temps(this);
   6015  Register scratch2 = temps.Acquire();
   6016 
   6017  // Check for signed range because of addi_d
   6018  if (is_intn(imm.value, 12)) {
   6019    uint32_t value = imm.value;
   6020    addi(rd, rj, value);
   6021    ma_sltu(scratch2, rd, Operand(value));
   6022    ma_b(scratch2, scratch2, label,
   6023         cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
   6024  } else {
   6025    ma_li(scratch2, imm);
   6026    ma_addPtrTestCarry(cond, rd, rj, scratch2, label);
   6027  }
   6028 }
   6029 
   6030 void MacroAssemblerRiscv64::ma_addPtrTestSigned(Condition cond, Register rd,
   6031                                                Register rj, Register rk,
   6032                                                Label* taken) {
   6033  MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned);
   6034 
   6035  ma_add64(rd, rj, rk);
   6036  ma_b(rd, rd, taken, cond);
   6037 }
   6038 
   6039 void MacroAssemblerRiscv64::ma_addPtrTestSigned(Condition cond, Register rd,
   6040                                                Register rj, Imm32 imm,
   6041                                                Label* taken) {
   6042  MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned);
   6043 
   6044  ma_add64(rd, rj, imm);
   6045  ma_b(rd, rd, taken, cond);
   6046 }
   6047 
   6048 void MacroAssemblerRiscv64::ma_addPtrTestSigned(Condition cond, Register rd,
   6049                                                Register rj, ImmWord imm,
   6050                                                Label* taken) {
   6051  MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned);
   6052 
   6053  ma_add64(rd, rj, Operand(imm.value));
   6054  ma_b(rd, rd, taken, cond);
   6055 }
   6056 
   6057 FaultingCodeOffset MacroAssemblerRiscv64::ma_load(
   6058    Register dest, const BaseIndex& src, LoadStoreSize size,
   6059    LoadStoreExtension extension) {
   6060  UseScratchRegisterScope temps(this);
   6061  Register scratch2 = temps.Acquire();
   6062  computeScaledAddress(src, scratch2);
   6063  return ma_load(dest, Address(scratch2, src.offset), size, extension);
   6064 }
   6065 void MacroAssemblerRiscv64::ma_pop(FloatRegister f) {
   6066  if (f.isDouble()) {
   6067    fld(f, StackPointer, 0);
   6068  } else {
   6069    MOZ_ASSERT(f.isSingle(), "simd128 is not supported");
   6070    flw(f, StackPointer, 0);
   6071  }
   6072  // See also MacroAssemblerRiscv64::ma_push -- Free space for double even when
   6073  // storing a float.
   6074  addi(StackPointer, StackPointer, sizeof(double));
   6075 }
   6076 
   6077 void MacroAssemblerRiscv64::ma_push(FloatRegister f) {
   6078  // We allocate space for double even when storing a float.
   6079  addi(StackPointer, StackPointer, (int32_t)-sizeof(double));
   6080  if (f.isDouble()) {
   6081    fsd(f, StackPointer, 0);
   6082  } else {
   6083    MOZ_ASSERT(f.isSingle(), "simd128 is not supported");
   6084    fsw(f, StackPointer, 0);
   6085  }
   6086 }
   6087 
   6088 FaultingCodeOffset MacroAssemblerRiscv64::ma_fld_s(FloatRegister ft,
   6089                                                   Address address) {
   6090  int32_t offset = address.offset;
   6091  Register base = address.base;
   6092 
   6093  FaultingCodeOffset fco;
   6094  if (is_intn(offset, 12)) {
   6095    fco = FaultingCodeOffset(currentOffset());
   6096    flw(ft, base, offset);
   6097  } else {
   6098    UseScratchRegisterScope temps(this);
   6099    Register scratch = temps.Acquire();
   6100    MOZ_ASSERT(base != scratch);
   6101    ma_li(scratch, Imm32(offset));
   6102    ma_add64(scratch, base, scratch);
   6103    fco = FaultingCodeOffset(currentOffset());
   6104    flw(ft, scratch, 0);
   6105  }
   6106  return fco;
   6107 }
   6108 FaultingCodeOffset MacroAssemblerRiscv64::ma_fld_d(FloatRegister ft,
   6109                                                   Address address) {
   6110  int32_t offset = address.offset;
   6111  Register base = address.base;
   6112 
   6113  FaultingCodeOffset fco;
   6114  if (is_intn(offset, 12)) {
   6115    fco = FaultingCodeOffset(currentOffset());
   6116    fld(ft, base, offset);
   6117  } else {
   6118    UseScratchRegisterScope temps(this);
   6119    Register scratch = temps.Acquire();
   6120    MOZ_ASSERT(base != scratch);
   6121    ma_li(scratch, Imm32(offset));
   6122    ma_add64(scratch, base, scratch);
   6123    fco = FaultingCodeOffset(currentOffset());
   6124    fld(ft, scratch, 0);
   6125  }
   6126  return fco;
   6127 }
   6128 FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_d(FloatRegister ft,
   6129                                                   Address address) {
   6130  int32_t offset = address.offset;
   6131  Register base = address.base;
   6132 
   6133  FaultingCodeOffset fco;
   6134  if (is_intn(offset, 12)) {
   6135    fco = FaultingCodeOffset(currentOffset());
   6136    fsd(ft, base, offset);
   6137  } else {
   6138    UseScratchRegisterScope temps(this);
   6139    Register scratch = temps.Acquire();
   6140    MOZ_ASSERT(base != scratch);
   6141    ma_li(scratch, Imm32(offset));
   6142    ma_add64(scratch, base, scratch);
   6143    fco = FaultingCodeOffset(currentOffset());
   6144    fsd(ft, scratch, 0);
   6145  }
   6146  return fco;
   6147 }
   6148 FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_s(FloatRegister ft,
   6149                                                   Address address) {
   6150  int32_t offset = address.offset;
   6151  Register base = address.base;
   6152  FaultingCodeOffset fco;
   6153  if (is_intn(offset, 12)) {
   6154    fco = FaultingCodeOffset(currentOffset());
   6155    fsw(ft, base, offset);
   6156  } else {
   6157    UseScratchRegisterScope temps(this);
   6158    Register scratch = temps.Acquire();
   6159    MOZ_ASSERT(base != scratch);
   6160    ma_li(scratch, Imm32(offset));
   6161    ma_add64(scratch, base, scratch);
   6162    fco = FaultingCodeOffset(currentOffset());
   6163    fsw(ft, scratch, 0);
   6164  }
   6165  return fco;
   6166 }
   6167 
   6168 FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_d(FloatRegister ft,
   6169                                                   BaseIndex address) {
   6170  UseScratchRegisterScope temps(this);
   6171  Register scratch = temps.Acquire();
   6172  computeScaledAddress(address, scratch);
   6173  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   6174  ma_fst_d(ft, Address(scratch, address.offset));
   6175  return fco;
   6176 }
   6177 
   6178 FaultingCodeOffset MacroAssemblerRiscv64::ma_fst_s(FloatRegister ft,
   6179                                                   BaseIndex address) {
   6180  UseScratchRegisterScope temps(this);
   6181  Register scratch = temps.Acquire();
   6182  computeScaledAddress(address, scratch);
   6183  FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
   6184  ma_fst_s(ft, Address(scratch, address.offset));
   6185  return fco;
   6186 }
   6187 
   6188 void MacroAssemblerRiscv64::ma_fld_d(FloatRegister ft, const BaseIndex& src) {
   6189  UseScratchRegisterScope temps(this);
   6190  Register scratch = temps.Acquire();
   6191  computeScaledAddress(src, scratch);
   6192  ma_fld_d(ft, Address(scratch, src.offset));
   6193 }
   6194 
   6195 void MacroAssemblerRiscv64::ma_fld_s(FloatRegister ft, const BaseIndex& src) {
   6196  UseScratchRegisterScope temps(this);
   6197  Register scratch = temps.Acquire();
   6198  computeScaledAddress(src, scratch);
   6199  ma_fld_s(ft, Address(scratch, src.offset));
   6200 }
   6201 
   6202 void MacroAssemblerRiscv64::ma_call(ImmPtr dest) {
   6203  DEBUG_PRINTF("[ %s\n", __FUNCTION__);
   6204  BlockTrampolinePoolScope block_trampoline_pool(this, 8);
   6205  UseScratchRegisterScope temps(this);
   6206  temps.Exclude(GeneralRegisterSet(1 << CallReg.code()));
   6207  ma_liPatchable(CallReg, dest);
   6208  jalr(CallReg, 0);
   6209  DEBUG_PRINTF("]\n");
   6210 }
   6211 
   6212 void MacroAssemblerRiscv64::CompareIsNotNanF32(Register rd, FPURegister cmp1,
   6213                                               FPURegister cmp2) {
   6214  feq_s(rd, cmp1, cmp1);  // rd <- !isNan(cmp1)
   6215  if (cmp1 != cmp2) {
   6216    UseScratchRegisterScope temps(this);
   6217    Register scratch = temps.Acquire();
   6218 
   6219    feq_s(scratch, cmp2, cmp2);  // scratch <- !isNaN(cmp2)
   6220    ma_and(rd, rd, scratch);     // rd <- !isNan(cmp1) && !isNan(cmp2)
   6221  }
   6222 }
   6223 
   6224 void MacroAssemblerRiscv64::CompareIsNotNanF64(Register rd, FPURegister cmp1,
   6225                                               FPURegister cmp2) {
   6226  feq_d(rd, cmp1, cmp1);  // rd <- !isNan(cmp1)
   6227  if (cmp1 != cmp2) {
   6228    UseScratchRegisterScope temps(this);
   6229    Register scratch = temps.Acquire();
   6230 
   6231    feq_d(scratch, cmp2, cmp2);  // scratch <- !isNaN(cmp2)
   6232    ma_and(rd, rd, scratch);     // rd <- !isNan(cmp1) && !isNan(cmp2)
   6233  }
   6234 }
   6235 
   6236 void MacroAssemblerRiscv64::CompareIsNanF32(Register rd, FPURegister cmp1,
   6237                                            FPURegister cmp2) {
   6238  CompareIsNotNanF32(rd, cmp1, cmp2);  // rd <- !isNan(cmp1) && !isNan(cmp2)
   6239  ma_xor(rd, rd, Operand(1));          // rd <- isNan(cmp1) || isNan(cmp2)
   6240 }
   6241 
   6242 void MacroAssemblerRiscv64::CompareIsNanF64(Register rd, FPURegister cmp1,
   6243                                            FPURegister cmp2) {
   6244  CompareIsNotNanF64(rd, cmp1, cmp2);  // rd <- !isNan(cmp1) && !isNan(cmp2)
   6245  ma_xor(rd, rd, Operand(1));          // rd <- isNan(cmp1) || isNan(cmp2)
   6246 }
   6247 
   6248 void MacroAssemblerRiscv64::BranchFloat32(DoubleCondition cc,
   6249                                          FloatRegister frs1,
   6250                                          FloatRegister frs2, Label* L,
   6251                                          JumpKind jumpKind) {
   6252  UseScratchRegisterScope temps(this);
   6253  Register scratch = temps.Acquire();
   6254  ma_compareF32(scratch, cc, frs1, frs2);
   6255  ma_b(scratch, Imm32(0), L, NotEqual, jumpKind);
   6256 }
   6257 
   6258 void MacroAssemblerRiscv64::BranchFloat64(DoubleCondition cc,
   6259                                          FloatRegister frs1,
   6260                                          FloatRegister frs2, Label* L,
   6261                                          JumpKind jumpKind) {
   6262  UseScratchRegisterScope temps(this);
   6263  Register scratch = temps.Acquire();
   6264  ma_compareF64(scratch, cc, frs1, frs2);
   6265  ma_b(scratch, Imm32(0), L, NotEqual, jumpKind);
   6266 }
   6267 
   6268 void MacroAssemblerRiscv64::Clz32(Register rd, Register xx) {
   6269  if (HasZbbExtension()) {
   6270 #if JS_CODEGEN_RISCV64
   6271    clzw(rd, xx);
   6272 #else
   6273    clz(rd, xx);
   6274 #endif
   6275    return;
   6276  }
   6277 
   6278  // 32 bit unsigned in lower word: count number of leading zeros.
   6279  //  int n = 32;
   6280  //  unsigned y;
   6281 
   6282  //  y = x >>16; if (y != 0) { n = n -16; x = y; }
   6283  //  y = x >> 8; if (y != 0) { n = n - 8; x = y; }
   6284  //  y = x >> 4; if (y != 0) { n = n - 4; x = y; }
   6285  //  y = x >> 2; if (y != 0) { n = n - 2; x = y; }
   6286  //  y = x >> 1; if (y != 0) {rd = n - 2; return;}
   6287  //  rd = n - x;
   6288 
   6289  Label L0, L1, L2, L3, L4;
   6290  UseScratchRegisterScope temps(this);
   6291  Register x = rd;
   6292  Register y = temps.Acquire();
   6293  Register n = temps.Acquire();
   6294  MOZ_ASSERT(xx != y && xx != n);
   6295  mv(x, xx);
   6296  ma_li(n, Imm32(32));
   6297 #if JS_CODEGEN_RISCV64
   6298  srliw(y, x, 16);
   6299  ma_branch(&L0, Equal, y, Operand(zero_reg));
   6300  mv(x, y);
   6301  addiw(n, n, -16);
   6302  bind(&L0);
   6303  srliw(y, x, 8);
   6304  ma_branch(&L1, Equal, y, Operand(zero_reg));
   6305  addiw(n, n, -8);
   6306  mv(x, y);
   6307  bind(&L1);
   6308  srliw(y, x, 4);
   6309  ma_branch(&L2, Equal, y, Operand(zero_reg));
   6310  addiw(n, n, -4);
   6311  mv(x, y);
   6312  bind(&L2);
   6313  srliw(y, x, 2);
   6314  ma_branch(&L3, Equal, y, Operand(zero_reg));
   6315  addiw(n, n, -2);
   6316  mv(x, y);
   6317  bind(&L3);
   6318  srliw(y, x, 1);
   6319  subw(rd, n, x);
   6320  ma_branch(&L4, Equal, y, Operand(zero_reg));
   6321  addiw(rd, n, -2);
   6322  bind(&L4);
   6323 #elif JS_CODEGEN_RISCV32
   6324  srli(y, x, 16);
   6325  ma_branch(&L0, Equal, y, Operand(zero_reg));
   6326  mv(x, y);
   6327  addi(n, n, -16);
   6328  bind(&L0);
   6329  srli(y, x, 8);
   6330  ma_branch(&L1, Equal, y, Operand(zero_reg));
   6331  addi(n, n, -8);
   6332  mv(x, y);
   6333  bind(&L1);
   6334  srli(y, x, 4);
   6335  ma_branch(&L2, Equal, y, Operand(zero_reg));
   6336  addi(n, n, -4);
   6337  mv(x, y);
   6338  bind(&L2);
   6339  srli(y, x, 2);
   6340  ma_branch(&L3, Equal, y, Operand(zero_reg));
   6341  addi(n, n, -2);
   6342  mv(x, y);
   6343  bind(&L3);
   6344  srli(y, x, 1);
   6345  sub(rd, n, x);
   6346  ma_branch(&L4, Equal, y, Operand(zero_reg));
   6347  addi(rd, n, -2);
   6348  bind(&L4);
   6349 #endif
   6350 }
   6351 
   6352 #if JS_CODEGEN_RISCV64
   6353 void MacroAssemblerRiscv64::Clz64(Register rd, Register xx) {
   6354  // 64 bit: count number of leading zeros.
   6355  //  int n = 64;
   6356  //  unsigned y;
   6357 
   6358  //  y = x >>32; if (y != 0) { n = n - 32; x = y; }
   6359  //  y = x >>16; if (y != 0) { n = n - 16; x = y; }
   6360  //  y = x >> 8; if (y != 0) { n = n - 8; x = y; }
   6361  //  y = x >> 4; if (y != 0) { n = n - 4; x = y; }
   6362  //  y = x >> 2; if (y != 0) { n = n - 2; x = y; }
   6363  //  y = x >> 1; if (y != 0) {rd = n - 2; return;}
   6364  //  rd = n - x;
   6365 
   6366  Label L0, L1, L2, L3, L4, L5;
   6367  UseScratchRegisterScope temps(this);
   6368  Register x = rd;
   6369  Register y = temps.Acquire();
   6370  Register n = temps.Acquire();
   6371  MOZ_ASSERT(xx != y && xx != n);
   6372  mv(x, xx);
   6373  ma_li(n, Imm32(64));
   6374  srli(y, x, 32);
   6375  ma_branch(&L0, Equal, y, Operand(zero_reg));
   6376  addiw(n, n, -32);
   6377  mv(x, y);
   6378  bind(&L0);
   6379  srli(y, x, 16);
   6380  ma_branch(&L1, Equal, y, Operand(zero_reg));
   6381  addiw(n, n, -16);
   6382  mv(x, y);
   6383  bind(&L1);
   6384  srli(y, x, 8);
   6385  ma_branch(&L2, Equal, y, Operand(zero_reg));
   6386  addiw(n, n, -8);
   6387  mv(x, y);
   6388  bind(&L2);
   6389  srli(y, x, 4);
   6390  ma_branch(&L3, Equal, y, Operand(zero_reg));
   6391  addiw(n, n, -4);
   6392  mv(x, y);
   6393  bind(&L3);
   6394  srli(y, x, 2);
   6395  ma_branch(&L4, Equal, y, Operand(zero_reg));
   6396  addiw(n, n, -2);
   6397  mv(x, y);
   6398  bind(&L4);
   6399  srli(y, x, 1);
   6400  subw(rd, n, x);
   6401  ma_branch(&L5, Equal, y, Operand(zero_reg));
   6402  addiw(rd, n, -2);
   6403  bind(&L5);
   6404 }
   6405 #endif
   6406 void MacroAssemblerRiscv64::Ctz32(Register rd, Register rs) {
   6407  if (HasZbbExtension()) {
   6408 #if JS_CODEGEN_RISCV64
   6409    ctzw(rd, rs);
   6410 #else
   6411    ctz(rd, rs);
   6412 #endif
   6413    return;
   6414  }
   6415 
   6416  // Convert trailing zeroes to trailing ones, and bits to their left
   6417  // to zeroes.
   6418 
   6419  {
   6420    UseScratchRegisterScope temps(this);
   6421    Register scratch = temps.Acquire();
   6422    ma_add64(scratch, rs, Operand(-1));
   6423    ma_xor(rd, scratch, rs);
   6424    ma_and(rd, rd, scratch);
   6425    // Count number of leading zeroes.
   6426  }
   6427  Clz32(rd, rd);
   6428  {
   6429    // Subtract number of leading zeroes from 32 to get number of trailing
   6430    // ones. Remember that the trailing ones were formerly trailing zeroes.
   6431    UseScratchRegisterScope temps(this);
   6432    Register scratch = temps.Acquire();
   6433    ma_li(scratch, Imm32(32));
   6434    ma_sub32(rd, scratch, rd);
   6435  }
   6436 }
   6437 #if JS_CODEGEN_RISCV64
   6438 void MacroAssemblerRiscv64::Ctz64(Register rd, Register rs) {
   6439  if (HasZbbExtension()) {
   6440    ctz(rd, rs);
   6441    return;
   6442  }
   6443 
   6444  // Convert trailing zeroes to trailing ones, and bits to their left
   6445  // to zeroes.
   6446  {
   6447    UseScratchRegisterScope temps(this);
   6448    Register scratch = temps.Acquire();
   6449    ma_add64(scratch, rs, Operand(-1));
   6450    ma_xor(rd, scratch, rs);
   6451    ma_and(rd, rd, scratch);
   6452    // Count number of leading zeroes.
   6453  }
   6454  Clz64(rd, rd);
   6455  {
   6456    // Subtract number of leading zeroes from 64 to get number of trailing
   6457    // ones. Remember that the trailing ones were formerly trailing zeroes.
   6458    UseScratchRegisterScope temps(this);
   6459    Register scratch = temps.Acquire();
   6460    ma_li(scratch, 64);
   6461    ma_sub64(rd, scratch, rd);
   6462  }
   6463 }
   6464 #endif
   6465 void MacroAssemblerRiscv64::Popcnt32(Register rd, Register rs,
   6466                                     Register scratch) {
   6467  if (HasZbbExtension()) {
   6468 #if JS_CODEGEN_RISCV64
   6469    cpopw(rd, rs);
   6470 #else
   6471    cpop(rd, rs);
   6472 #endif
   6473    return;
   6474  }
   6475 
   6476  MOZ_ASSERT(scratch != rs);
   6477  MOZ_ASSERT(scratch != rd);
   6478  // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
   6479  //
   6480  // A generalization of the best bit counting method to integers of
   6481  // bit-widths up to 128 (parameterized by type T) is this:
   6482  //
   6483  // v = v - ((v >> 1) & (T)~(T)0/3);                           // temp
   6484  // v = (v & (T)~(T)0/15*3) + ((v >> 2) & (T)~(T)0/15*3);      // temp
   6485  // v = (v + (v >> 4)) & (T)~(T)0/255*15;                      // temp
   6486  // c = (T)(v * ((T)~(T)0/255)) >> (sizeof(T) - 1) * BITS_PER_BYTE; //count
   6487  //
   6488  // There are algorithms which are faster in the cases where very few
   6489  // bits are set but the algorithm here attempts to minimize the total
   6490  // number of instructions executed even when a large number of bits
   6491  // are set.
   6492  // The number of instruction is 20.
   6493  // uint32_t B0 = 0x55555555;     // (T)~(T)0/3
   6494  // uint32_t B1 = 0x33333333;     // (T)~(T)0/15*3
   6495  // uint32_t B2 = 0x0F0F0F0F;     // (T)~(T)0/255*15
   6496  // uint32_t value = 0x01010101;  // (T)~(T)0/255
   6497 
   6498  uint32_t shift = 24;
   6499  UseScratchRegisterScope temps(this);
   6500  Register scratch2 = temps.Acquire();
   6501  Register value = temps.Acquire();
   6502  MOZ_ASSERT((rd != value) && (rs != value));
   6503  ma_li(value, 0x01010101);     // value = 0x01010101;
   6504  ma_li(scratch2, 0x55555555);  // B0 = 0x55555555;
   6505  ma_srl32(scratch, rs, Operand(1));
   6506  ma_and(scratch, scratch, scratch2);
   6507  ma_sub32(scratch, rs, scratch);
   6508  ma_li(scratch2, 0x33333333);  // B1 = 0x33333333;
   6509  slli(rd, scratch2, 4);
   6510  or_(scratch2, scratch2, rd);
   6511  ma_and(rd, scratch, scratch2);
   6512  ma_srl32(scratch, scratch, Operand(2));
   6513  ma_and(scratch, scratch, scratch2);
   6514  ma_add32(scratch, rd, scratch);
   6515  ma_srl32(rd, scratch, Operand(4));
   6516  ma_add32(rd, rd, scratch);
   6517  ma_li(scratch2, 0xF);
   6518  ma_mul32(scratch2, value, scratch2);  // B2 = 0x0F0F0F0F;
   6519  ma_and(rd, rd, scratch2);
   6520  ma_mul32(rd, rd, value);
   6521  ma_srl32(rd, rd, Operand(shift));
   6522 }
   6523 
   6524 #if JS_CODEGEN_RISCV64
   6525 void MacroAssemblerRiscv64::Popcnt64(Register rd, Register rs,
   6526                                     Register scratch) {
   6527  if (HasZbbExtension()) {
   6528    cpop(rd, rs);
   6529    return;
   6530  }
   6531 
   6532  MOZ_ASSERT(scratch != rs);
   6533  MOZ_ASSERT(scratch != rd);
   6534  // uint64_t B0 = 0x5555555555555555l;     // (T)~(T)0/3
   6535  // uint64_t B1 = 0x3333333333333333l;     // (T)~(T)0/15*3
   6536  // uint64_t B2 = 0x0F0F0F0F0F0F0F0Fl;     // (T)~(T)0/255*15
   6537  // uint64_t value = 0x0101010101010101l;  // (T)~(T)0/255
   6538  // uint64_t shift = 24;                   // (sizeof(T) - 1) * BITS_PER_BYTE
   6539  uint64_t shift = 24;
   6540  UseScratchRegisterScope temps(this);
   6541  Register scratch2 = temps.Acquire();
   6542  Register value = temps.Acquire();
   6543  MOZ_ASSERT((rd != value) && (rs != value));
   6544  ma_li(value, 0x1111111111111111l);  // value = 0x1111111111111111l;
   6545  ma_li(scratch2, 5);
   6546  ma_mul64(scratch2, value, scratch2);  // B0 = 0x5555555555555555l;
   6547  ma_srl64(scratch, rs, Operand(1));
   6548  ma_and(scratch, scratch, scratch2);
   6549  ma_sub64(scratch, rs, scratch);
   6550  ma_li(scratch2, 3);
   6551  ma_mul64(scratch2, value, scratch2);  // B1 = 0x3333333333333333l;
   6552  ma_and(rd, scratch, scratch2);
   6553  ma_srl64(scratch, scratch, Operand(2));
   6554  ma_and(scratch, scratch, scratch2);
   6555  ma_add64(scratch, rd, scratch);
   6556  ma_srl64(rd, scratch, Operand(4));
   6557  ma_add64(rd, rd, scratch);
   6558  ma_li(scratch2, 0xF);
   6559  ma_li(value, 0x0101010101010101l);    // value = 0x0101010101010101l;
   6560  ma_mul64(scratch2, value, scratch2);  // B2 = 0x0F0F0F0F0F0F0F0Fl;
   6561  ma_and(rd, rd, scratch2);
   6562  ma_mul64(rd, rd, value);
   6563  srli(rd, rd, 32 + shift);
   6564 }
   6565 #endif
   6566 
   6567 void MacroAssemblerRiscv64::ma_mod_mask(Register src, Register dest,
   6568                                        Register hold, Register remain,
   6569                                        int32_t shift, Label* negZero) {
   6570  // MATH:
   6571  // We wish to compute x % (1<<y) - 1 for a known constant, y.
   6572  // First, let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit
   6573  // dividend as a number in base b, namely
   6574  // c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
   6575  // now, since both addition and multiplication commute with modulus,
   6576  // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
   6577  // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
   6578  // now, since b == C + 1, b % C == 1, and b^n % C == 1
   6579  // this means that the whole thing simplifies to:
   6580  // c_0 + c_1 + c_2 ... c_n % C
   6581  // each c_n can easily be computed by a shift/bitextract, and the modulus
   6582  // can be maintained by simply subtracting by C whenever the number gets
   6583  // over C.
   6584  int32_t mask = (1 << shift) - 1;
   6585  Label head, negative, sumSigned, done;
   6586 
   6587  // hold holds -1 if the value was negative, 1 otherwise.
   6588  // remain holds the remaining bits that have not been processed
   6589  // scratch2 serves as a temporary location to store extracted bits
   6590  // into as well as holding the trial subtraction as a temp value dest is
   6591  // the accumulator (and holds the final result)
   6592 
   6593  // move the whole value into the remain.
   6594  or_(remain, src, zero);
   6595  // Zero out the dest.
   6596  ma_li(dest, Imm32(0));
   6597  // Set the hold appropriately.
   6598  ma_b(remain, remain, &negative, Signed, ShortJump);
   6599  ma_li(hold, Imm32(1));
   6600  ma_branch(&head, ShortJump);
   6601 
   6602  bind(&negative);
   6603  ma_li(hold, Imm32(-1));
   6604  negw(remain, remain);
   6605 
   6606  // Begin the main loop.
   6607  bind(&head);
   6608 
   6609  UseScratchRegisterScope temps(this);
   6610  Register scratch2 = temps.Acquire();
   6611  // Extract the bottom bits into scratch2.
   6612  ma_and(scratch2, remain, Imm32(mask));
   6613  // Add those bits to the accumulator.
   6614  addw(dest, dest, scratch2);
   6615  // Do a trial subtraction
   6616  ma_sub32(scratch2, dest, Imm32(mask));
   6617  // If (sum - C) > 0, store sum - C back into sum, thus performing a
   6618  // modulus.
   6619  ma_b(scratch2, Register(scratch2), &sumSigned, Signed, ShortJump);
   6620  or_(dest, scratch2, zero);
   6621  bind(&sumSigned);
   6622  // Get rid of the bits that we extracted before.
   6623  srliw(remain, remain, shift);
   6624  // If the shift produced zero, finish, otherwise, continue in the loop.
   6625  ma_b(remain, remain, &head, NonZero, ShortJump);
   6626  // Check the hold to see if we need to negate the result.
   6627  ma_b(hold, hold, &done, NotSigned, ShortJump);
   6628 
   6629  if (negZero != nullptr) {
   6630    // Jump out in case of negative zero.
   6631    ma_b(dest, dest, negZero, Zero);
   6632  }
   6633  // If the hold was non-zero, negate the result to be in line with
   6634  // what JS wants
   6635  negw(dest, dest);
   6636 
   6637  bind(&done);
   6638 }
   6639 
   6640 void MacroAssemblerRiscv64::ma_fmovz(FloatFormat fmt, FloatRegister fd,
   6641                                     FloatRegister fj, Register rk) {
   6642  Label done;
   6643  ma_b(rk, zero, &done, Assembler::NotEqual);
   6644  if (fmt == SingleFloat) {
   6645    fmv_s(fd, fj);
   6646  } else {
   6647    fmv_d(fd, fj);
   6648  }
   6649  bind(&done);
   6650 }
   6651 
   6652 void MacroAssemblerRiscv64::ByteSwap(Register rd, Register rs, int operand_size,
   6653                                     Register scratch) {
   6654  MOZ_ASSERT(operand_size == 4 || operand_size == 8);
   6655 #if JS_CODEGEN_RISCV64
   6656  if (HasZbbExtension()) {
   6657    rev8(rd, rs);
   6658    if (operand_size == 4) {
   6659      srai(rd, rd, 32);
   6660    }
   6661    return;
   6662  }
   6663 #endif
   6664 
   6665  MOZ_ASSERT(scratch != rs);
   6666  MOZ_ASSERT(scratch != rd);
   6667  if (operand_size == 4) {
   6668    // Uint32_t x1 = 0x00FF00FF;
   6669    // x0 = (x0 << 16 | x0 >> 16);
   6670    // x0 = (((x0 & x1) << 8)  | ((x0 & (x1 << 8)) >> 8));
   6671    UseScratchRegisterScope temps(this);
   6672    BlockTrampolinePoolScope block_trampoline_pool(this, 17);
   6673    MOZ_ASSERT((rd != t6) && (rs != t6));
   6674    Register x0 = temps.Acquire();
   6675    Register x1 = temps.Acquire();
   6676    Register x2 = scratch;
   6677    RV_li(x1, 0x00FF00FF);
   6678    slliw(x0, rs, 16);
   6679    srliw(rd, rs, 16);
   6680    or_(x0, rd, x0);   // x0 <- x0 << 16 | x0 >> 16
   6681    and_(x2, x0, x1);  // x2 <- x0 & 0x00FF00FF
   6682    slliw(x2, x2, 8);  // x2 <- (x0 & x1) << 8
   6683    slliw(x1, x1, 8);  // x1 <- 0xFF00FF00
   6684    and_(rd, x0, x1);  // x0 & 0xFF00FF00
   6685    srliw(rd, rd, 8);
   6686    or_(rd, rd, x2);  // (((x0 & x1) << 8)  | ((x0 & (x1 << 8)) >> 8))
   6687  } else {
   6688    // uinx24_t x1 = 0x0000FFFF0000FFFFl;
   6689    // uinx24_t x1 = 0x00FF00FF00FF00FFl;
   6690    // x0 = (x0 << 32 | x0 >> 32);
   6691    // x0 = (x0 & x1) << 16 | (x0 & (x1 << 16)) >> 16;
   6692    // x0 = (x0 & x1) << 8  | (x0 & (x1 << 8)) >> 8;
   6693    UseScratchRegisterScope temps(this);
   6694    BlockTrampolinePoolScope block_trampoline_pool(this, 30);
   6695    MOZ_ASSERT((rd != t6) && (rs != t6));
   6696    Register x0 = temps.Acquire();
   6697    Register x1 = temps.Acquire();
   6698    Register x2 = scratch;
   6699    RV_li(x1, 0x0000FFFF0000FFFFl);
   6700    slli(x0, rs, 32);
   6701    srli(rd, rs, 32);
   6702    or_(x0, rd, x0);   // x0 <- x0 << 32 | x0 >> 32
   6703    and_(x2, x0, x1);  // x2 <- x0 & 0x0000FFFF0000FFFF
   6704    slli(x2, x2, 16);  // x2 <- (x0 & 0x0000FFFF0000FFFF) << 16
   6705    slli(x1, x1, 16);  // x1 <- 0xFFFF0000FFFF0000
   6706    and_(rd, x0, x1);  // rd <- x0 & 0xFFFF0000FFFF0000
   6707    srli(rd, rd, 16);  // rd <- x0 & (x1 << 16)) >> 16
   6708    or_(x0, rd, x2);   // (x0 & x1) << 16 | (x0 & (x1 << 16)) >> 16;
   6709    RV_li(x1, 0x00FF00FF00FF00FFl);
   6710    and_(x2, x0, x1);  // x2 <- x0 & 0x00FF00FF00FF00FF
   6711    slli(x2, x2, 8);   // x2 <- (x0 & x1) << 8
   6712    slli(x1, x1, 8);   // x1 <- 0xFF00FF00FF00FF00
   6713    and_(rd, x0, x1);
   6714    srli(rd, rd, 8);  // rd <- (x0 & (x1 << 8)) >> 8
   6715    or_(rd, rd, x2);  // (((x0 & x1) << 8)  | ((x0 & (x1 << 8)) >> 8))
   6716  }
   6717 }
   6718 
   6719 template <typename F_TYPE>
   6720 void MacroAssemblerRiscv64::FloatMinMaxHelper(FPURegister dst, FPURegister src1,
   6721                                              FPURegister src2,
   6722                                              MaxMinKind kind) {
   6723  MOZ_ASSERT((std::is_same<F_TYPE, float>::value) ||
   6724             (std::is_same<F_TYPE, double>::value));
   6725 
   6726  if (src1 == src2) {
   6727    if (dst != src1) {
   6728      if (std::is_same<float, F_TYPE>::value) {
   6729        fmv_s(dst, src1);
   6730      } else {
   6731        fmv_d(dst, src1);
   6732      }
   6733    }
   6734    return;
   6735  }
   6736 
   6737  Label done, nan;
   6738 
   6739  // For RISCV, fmin_s returns the other non-NaN operand as result if only one
   6740  // operand is NaN; but for JS, if any operand is NaN, result is Nan. The
   6741  // following handles the discrepency between handling of NaN between ISA and
   6742  // JS semantics
   6743  if (std::is_same<float, F_TYPE>::value) {
   6744    BranchFloat32(Assembler::DoubleUnordered, src1, src2, &nan, ShortJump);
   6745  } else {
   6746    BranchFloat64(Assembler::DoubleUnordered, src1, src2, &nan, ShortJump);
   6747  }
   6748 
   6749  if (kind == MaxMinKind::kMax) {
   6750    if (std::is_same<float, F_TYPE>::value) {
   6751      fmax_s(dst, src1, src2);
   6752    } else {
   6753      fmax_d(dst, src1, src2);
   6754    }
   6755  } else {
   6756    if (std::is_same<float, F_TYPE>::value) {
   6757      fmin_s(dst, src1, src2);
   6758    } else {
   6759      fmin_d(dst, src1, src2);
   6760    }
   6761  }
   6762  jump(&done);
   6763 
   6764  bind(&nan);
   6765  // if any operand is NaN, return NaN (fadd returns NaN if any operand is NaN)
   6766  if (std::is_same<float, F_TYPE>::value) {
   6767    fadd_s(dst, src1, src2);
   6768  } else {
   6769    fadd_d(dst, src1, src2);
   6770  }
   6771 
   6772  bind(&done);
   6773 }
   6774 
   6775 void MacroAssemblerRiscv64::Float32Max(FPURegister dst, FPURegister src1,
   6776                                       FPURegister src2) {
   6777  comment(__FUNCTION__);
   6778  FloatMinMaxHelper<float>(dst, src1, src2, MaxMinKind::kMax);
   6779 }
   6780 
   6781 void MacroAssemblerRiscv64::Float32Min(FPURegister dst, FPURegister src1,
   6782                                       FPURegister src2) {
   6783  comment(__FUNCTION__);
   6784  FloatMinMaxHelper<float>(dst, src1, src2, MaxMinKind::kMin);
   6785 }
   6786 
   6787 void MacroAssemblerRiscv64::Float64Max(FPURegister dst, FPURegister src1,
   6788                                       FPURegister src2) {
   6789  comment(__FUNCTION__);
   6790  FloatMinMaxHelper<double>(dst, src1, src2, MaxMinKind::kMax);
   6791 }
   6792 
   6793 void MacroAssemblerRiscv64::Float64Min(FPURegister dst, FPURegister src1,
   6794                                       FPURegister src2) {
   6795  comment(__FUNCTION__);
   6796  FloatMinMaxHelper<double>(dst, src1, src2, MaxMinKind::kMin);
   6797 }
   6798 
   6799 void MacroAssemblerRiscv64::Rol(Register rd, Register rs, const Operand& rt) {
   6800  if (rt.is_reg()) {
   6801    UseScratchRegisterScope temps(this);
   6802    Register scratch = temps.Acquire();
   6803 
   6804    negw(scratch, rt.rm());
   6805    srlw(scratch, rs, scratch);
   6806    sllw(rd, rs, rt.rm());
   6807    or_(rd, scratch, rd);
   6808    sext_w(rd, rd);
   6809  } else {
   6810    Ror(rd, rs, Operand(32 - (rt.immediate() & 0x1f)));
   6811  }
   6812 }
   6813 
   6814 void MacroAssemblerRiscv64::Ror(Register rd, Register rs, const Operand& rt) {
   6815  if (HasZbbExtension()) {
   6816    if (rt.is_reg()) {
   6817      rorw(rd, rs, rt.rm());
   6818    } else {
   6819      int64_t ror_value = rt.immediate() % 32;
   6820      if (ror_value < 0) {
   6821        ror_value += 32;
   6822      }
   6823      roriw(rd, rs, ror_value);
   6824    }
   6825    return;
   6826  }
   6827  UseScratchRegisterScope temps(this);
   6828  Register scratch = temps.Acquire();
   6829  if (rt.is_reg()) {
   6830    negw(scratch, rt.rm());
   6831    sllw(scratch, rs, scratch);
   6832    srlw(rd, rs, rt.rm());
   6833    or_(rd, scratch, rd);
   6834    sext_w(rd, rd);
   6835  } else {
   6836    int64_t ror_value = rt.immediate() & 0x1f;
   6837    if (ror_value == 0) {
   6838      mv(rd, rs);
   6839      return;
   6840    }
   6841    srliw(scratch, rs, ror_value);
   6842    slliw(rd, rs, 32 - ror_value);
   6843    or_(rd, scratch, rd);
   6844    sext_w(rd, rd);
   6845  }
   6846 }
   6847 
   6848 void MacroAssemblerRiscv64::Drol(Register rd, Register rs, const Operand& rt) {
   6849  if (rt.is_reg()) {
   6850    UseScratchRegisterScope temps(this);
   6851    Register scratch = temps.Acquire();
   6852 
   6853    negw(scratch, rt.rm());
   6854    srl(scratch, rs, scratch);
   6855    sll(rd, rs, rt.rm());
   6856    or_(rd, scratch, rd);
   6857  } else {
   6858    Dror(rd, rs, Operand(64 - (rt.immediate() & 0x3f)));
   6859  }
   6860 }
   6861 
   6862 void MacroAssemblerRiscv64::Dror(Register rd, Register rs, const Operand& rt) {
   6863  if (HasZbbExtension()) {
   6864    if (rt.is_reg()) {
   6865      ror(rd, rs, rt.rm());
   6866    } else {
   6867      int64_t dror_value = rt.immediate() % 64;
   6868      if (dror_value < 0) {
   6869        dror_value += 64;
   6870      }
   6871      rori(rd, rs, dror_value);
   6872    }
   6873    return;
   6874  }
   6875  UseScratchRegisterScope temps(this);
   6876  Register scratch = temps.Acquire();
   6877  if (rt.is_reg()) {
   6878    negw(scratch, rt.rm());
   6879    sll(scratch, rs, scratch);
   6880    srl(rd, rs, rt.rm());
   6881    or_(rd, scratch, rd);
   6882  } else {
   6883    int64_t dror_value = rt.immediate() & 0x3f;
   6884    if (dror_value == 0) {
   6885      mv(rd, rs);
   6886      return;
   6887    }
   6888    srli(scratch, rs, dror_value);
   6889    slli(rd, rs, 64 - dror_value);
   6890    or_(rd, scratch, rd);
   6891  }
   6892 }
   6893 
   6894 void MacroAssemblerRiscv64::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
   6895                                         Register memoryBase, Register ptr,
   6896                                         Register ptrScratch,
   6897                                         AnyRegister output, Register tmp) {
   6898  access.assertOffsetInGuardPages();
   6899  uint32_t offset = access.offset32();
   6900  MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
   6901 
   6902  // Maybe add the offset.
   6903  if (offset) {
   6904    asMasm().addPtr(ImmWord(offset), ptrScratch);
   6905    ptr = ptrScratch;
   6906  }
   6907 
   6908  asMasm().memoryBarrierBefore(access.sync());
   6909  UseScratchRegisterScope temps(this);
   6910  Register scratch = temps.Acquire();
   6911  FaultingCodeOffset fco;
   6912  switch (access.type()) {
   6913    case Scalar::Int8:
   6914      add(scratch, memoryBase, ptr);
   6915      fco = FaultingCodeOffset(currentOffset());
   6916      lb(output.gpr(), scratch, 0);
   6917      break;
   6918    case Scalar::Uint8:
   6919      add(scratch, memoryBase, ptr);
   6920      fco = FaultingCodeOffset(currentOffset());
   6921      lbu(output.gpr(), scratch, 0);
   6922      break;
   6923    case Scalar::Int16:
   6924      add(scratch, memoryBase, ptr);
   6925      fco = FaultingCodeOffset(currentOffset());
   6926      lh(output.gpr(), scratch, 0);
   6927      break;
   6928    case Scalar::Uint16:
   6929      add(scratch, memoryBase, ptr);
   6930      fco = FaultingCodeOffset(currentOffset());
   6931      lhu(output.gpr(), scratch, 0);
   6932      break;
   6933    case Scalar::Int32:
   6934      add(scratch, memoryBase, ptr);
   6935      fco = FaultingCodeOffset(currentOffset());
   6936      lw(output.gpr(), scratch, 0);
   6937      break;
   6938    case Scalar::Uint32:
   6939      add(scratch, memoryBase, ptr);
   6940      fco = FaultingCodeOffset(currentOffset());
   6941      lwu(output.gpr(), scratch, 0);
   6942      break;
   6943    case Scalar::Float64:
   6944      add(scratch, memoryBase, ptr);
   6945      fco = FaultingCodeOffset(currentOffset());
   6946      fld(output.fpu(), scratch, 0);
   6947      break;
   6948    case Scalar::Float32:
   6949      add(scratch, memoryBase, ptr);
   6950      fco = FaultingCodeOffset(currentOffset());
   6951      flw(output.fpu(), scratch, 0);
   6952      break;
   6953    default:
   6954      MOZ_CRASH("unexpected array type");
   6955  }
   6956 
   6957  append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()), fco);
   6958  asMasm().memoryBarrierAfter(access.sync());
   6959 }
   6960 
   6961 void MacroAssemblerRiscv64::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
   6962                                          AnyRegister value,
   6963                                          Register memoryBase, Register ptr,
   6964                                          Register ptrScratch, Register tmp) {
   6965  access.assertOffsetInGuardPages();
   6966  uint32_t offset = access.offset32();
   6967  MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
   6968 
   6969  // Maybe add the offset.
   6970  if (offset) {
   6971    asMasm().addPtr(ImmWord(offset), ptrScratch);
   6972    ptr = ptrScratch;
   6973  }
   6974 
   6975  unsigned byteSize = access.byteSize();
   6976  bool isSigned = Scalar::isSignedIntType(access.type());
   6977  bool isFloat = Scalar::isFloatingType(access.type());
   6978 
   6979  BaseIndex address(memoryBase, ptr, TimesOne);
   6980  asMasm().memoryBarrierBefore(access.sync());
   6981  FaultingCodeOffset fco;
   6982  if (isFloat) {
   6983    if (byteSize == 4) {
   6984      fco = ma_fst_s(value.fpu(), address);
   6985    } else {
   6986      fco = ma_fst_d(value.fpu(), address);
   6987    }
   6988  } else {
   6989    fco =
   6990        ma_store(value.gpr(), address, static_cast<LoadStoreSize>(8 * byteSize),
   6991                 isSigned ? SignExtend : ZeroExtend);
   6992  }
   6993  // Only the last emitted instruction is a memory access.
   6994  append(access, js::wasm::TrapMachineInsnForStore(access.byteSize()), fco);
   6995  asMasm().memoryBarrierAfter(access.sync());
   6996 }
   6997 
   6998 void MacroAssemblerRiscv64::GenPCRelativeJumpAndLink(Register rd,
   6999                                                     int32_t imm32) {
   7000  MOZ_ASSERT(is_int32(imm32 + 0x800));
   7001  int32_t Hi20 = ((imm32 + 0x800) >> 12);
   7002  int32_t Lo12 = imm32 << 20 >> 20;
   7003  auipc(rd, Hi20);  // Read PC + Hi20 into scratch.
   7004  jalr(rd, Lo12);   // jump PC + Hi20 + Lo12
   7005 }
   7006 
   7007 CodeOffset MacroAssemblerRiscv64::BranchAndLinkShortHelper(int32_t offset,
   7008                                                           Label* L) {
   7009  MOZ_ASSERT(L == nullptr || offset == 0);
   7010  BlockTrampolinePoolScope block_trampoline_pool(this, 2, 1);
   7011  offset = GetOffset(offset, L, OffsetSize::kOffset21);
   7012  return jal(offset);
   7013 }
   7014 
   7015 CodeOffset MacroAssemblerRiscv64::BranchAndLinkShort(int32_t offset) {
   7016  MOZ_ASSERT(is_int21(offset));
   7017  return BranchAndLinkShortHelper(offset, nullptr);
   7018 }
   7019 
   7020 CodeOffset MacroAssemblerRiscv64::BranchAndLinkShort(Label* L) {
   7021  return BranchAndLinkShortHelper(0, L);
   7022 }
   7023 
   7024 CodeOffset MacroAssemblerRiscv64::BranchAndLink(Label* L) {
   7025  if (L->bound() && !is_near(L)) {
   7026    return BranchAndLinkLong(L);
   7027  }
   7028  return BranchAndLinkShort(L);
   7029 }
   7030 
   7031 void MacroAssemblerRiscv64::ma_fmv_d(FloatRegister src, ValueOperand dest) {
   7032  fmv_x_d(dest.valueReg(), src);
   7033 }
   7034 
   7035 void MacroAssemblerRiscv64::ma_fmv_d(ValueOperand src, FloatRegister dest) {
   7036  fmv_d_x(dest, src.valueReg());
   7037 }
   7038 
   7039 void MacroAssemblerRiscv64::ma_fmv_w(FloatRegister src, ValueOperand dest) {
   7040  fmv_x_w(dest.valueReg(), src);
   7041 }
   7042 
   7043 void MacroAssemblerRiscv64::ma_fmv_w(ValueOperand src, FloatRegister dest) {
   7044  fmv_w_x(dest, src.valueReg());
   7045 }
   7046 
   7047 }  // namespace jit
   7048 }  // namespace js
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE