MacroAssembler-loong64.cpp (197991B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "jit/loong64/MacroAssembler-loong64.h" 8 9 #include "jsmath.h" 10 11 #include "jit/Bailouts.h" 12 #include "jit/BaselineFrame.h" 13 #include "jit/JitFrames.h" 14 #include "jit/JitRuntime.h" 15 #include "jit/loong64/SharedICRegisters-loong64.h" 16 #include "jit/MacroAssembler.h" 17 #include "jit/MoveEmitter.h" 18 #include "util/Memory.h" 19 #include "vm/JitActivation.h" // js::jit::JitActivation 20 #include "vm/JSContext.h" 21 #include "wasm/WasmStubs.h" 22 23 #include "jit/MacroAssembler-inl.h" 24 25 namespace js { 26 namespace jit { 27 28 void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) { 29 UseScratchRegisterScope temps(asMasm()); 30 Register scratch = temps.Acquire(); 31 ScratchDoubleScope fpscratch(asMasm()); 32 as_ftintrne_l_d(fpscratch, input); 33 as_movfr2gr_d(output, fpscratch); 34 // if (res < 0); res = 0; 35 as_slt(scratch, output, zero); 36 as_masknez(output, output, scratch); 37 // if res > 255; res = 255; 38 as_sltui(scratch, output, 255); 39 as_addi_d(output, output, -255); 40 as_maskeqz(output, output, scratch); 41 as_addi_d(output, output, 255); 42 } 43 44 bool MacroAssemblerLOONG64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) { 45 asMasm().Push(FrameDescriptor(FrameType::IonJS)); // descriptor_ 46 asMasm().Push(ImmPtr(fakeReturnAddr)); 47 asMasm().Push(FramePointer); 48 return true; 49 } 50 51 void MacroAssemblerLOONG64Compat::convertUInt32ToDouble(Register src, 52 FloatRegister dest) { 53 UseScratchRegisterScope temps(asMasm()); 54 Register scratch = temps.Acquire(); 55 as_bstrpick_d(scratch, src, 31, 0); 56 asMasm().convertInt64ToDouble(Register64(scratch), dest); 57 } 58 59 void MacroAssemblerLOONG64Compat::convertUInt64ToDouble(Register src, 60 FloatRegister dest) { 61 Label positive, done; 62 ma_b(src, src, &positive, NotSigned, ShortJump); 63 UseScratchRegisterScope temp(asMasm()); 64 Register scratch = temp.Acquire(); 65 Register scratch2 = temp.Acquire(); 66 67 MOZ_ASSERT(src != scratch); 68 MOZ_ASSERT(src != scratch2); 69 70 ma_and(scratch, src, Imm32(1)); 71 as_srli_d(scratch2, src, 1); 72 as_or(scratch, scratch, scratch2); 73 as_movgr2fr_d(dest, scratch); 74 as_ffint_d_l(dest, dest); 75 asMasm().addDouble(dest, dest); 76 ma_b(&done, ShortJump); 77 78 bind(&positive); 79 as_movgr2fr_d(dest, src); 80 as_ffint_d_l(dest, dest); 81 82 bind(&done); 83 } 84 85 void MacroAssemblerLOONG64Compat::convertUInt32ToFloat32(Register src, 86 FloatRegister dest) { 87 UseScratchRegisterScope temps(asMasm()); 88 Register scratch = temps.Acquire(); 89 as_bstrpick_d(scratch, src, 31, 0); 90 asMasm().convertInt64ToFloat32(Register64(scratch), dest); 91 } 92 93 void MacroAssemblerLOONG64Compat::convertDoubleToFloat32(FloatRegister src, 94 FloatRegister dest) { 95 as_fcvt_s_d(dest, src); 96 } 97 98 const int CauseBitPos = int(Assembler::CauseI); 99 const int CauseBitCount = 1 + int(Assembler::CauseV) - int(Assembler::CauseI); 100 const int CauseIOrVMask = ((1 << int(Assembler::CauseI)) | 101 (1 << int(Assembler::CauseV))) >> 102 int(Assembler::CauseI); 103 104 // Checks whether a double is representable as a 32-bit integer. If so, the 105 // integer is written to the output register. Otherwise, a bailout is taken to 106 // the given snapshot. This function overwrites the scratch float register. 107 void MacroAssemblerLOONG64Compat::convertDoubleToInt32(FloatRegister src, 108 Register dest, 109 Label* fail, 110 bool negativeZeroCheck) { 111 if (negativeZeroCheck) { 112 moveFromDouble(src, dest); 113 as_rotri_d(dest, dest, 63); 114 ma_b(dest, Imm32(1), fail, Assembler::Equal); 115 } 116 117 UseScratchRegisterScope temps(asMasm()); 118 Register scratch = temps.Acquire(); 119 ScratchFloat32Scope fpscratch(asMasm()); 120 // Truncate double to int ; if result is inexact or invalid fail. 121 as_ftintrz_w_d(fpscratch, src); 122 as_movfcsr2gr(scratch); 123 moveFromFloat32(fpscratch, dest); 124 as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); 125 as_andi(scratch, scratch, 126 CauseIOrVMask); // masking for Inexact and Invalid flag. 127 ma_b(scratch, zero, fail, Assembler::NotEqual); 128 } 129 130 void MacroAssemblerLOONG64Compat::convertDoubleToPtr(FloatRegister src, 131 Register dest, Label* fail, 132 bool negativeZeroCheck) { 133 if (negativeZeroCheck) { 134 moveFromDouble(src, dest); 135 as_rotri_d(dest, dest, 63); 136 ma_b(dest, Imm32(1), fail, Assembler::Equal); 137 } 138 139 UseScratchRegisterScope temps(asMasm()); 140 Register scratch = temps.Acquire(); 141 ScratchDoubleScope fpscratch(asMasm()); 142 // Truncate double to int64 ; if result is inexact or invalid fail. 143 as_ftintrz_l_d(fpscratch, src); 144 as_movfcsr2gr(scratch); 145 moveFromDouble(fpscratch, dest); 146 as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); 147 as_andi(scratch, scratch, 148 CauseIOrVMask); // masking for Inexact and Invalid flag. 149 ma_b(scratch, zero, fail, Assembler::NotEqual); 150 } 151 152 // Checks whether a float32 is representable as a 32-bit integer. If so, the 153 // integer is written to the output register. Otherwise, a bailout is taken to 154 // the given snapshot. This function overwrites the scratch float register. 155 void MacroAssemblerLOONG64Compat::convertFloat32ToInt32( 156 FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck) { 157 if (negativeZeroCheck) { 158 moveFromFloat32(src, dest); 159 ma_b(dest, Imm32(INT32_MIN), fail, Assembler::Equal); 160 } 161 162 UseScratchRegisterScope temps(asMasm()); 163 Register scratch = temps.Acquire(); 164 ScratchFloat32Scope fpscratch(asMasm()); 165 as_ftintrz_w_s(fpscratch, src); 166 as_movfcsr2gr(scratch); 167 moveFromFloat32(fpscratch, dest); 168 MOZ_ASSERT(CauseBitPos + CauseBitCount < 33); 169 MOZ_ASSERT(CauseBitPos < 32); 170 as_bstrpick_w(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos); 171 as_andi(scratch, scratch, CauseIOrVMask); 172 ma_b(scratch, zero, fail, Assembler::NotEqual); 173 } 174 175 void MacroAssemblerLOONG64Compat::convertFloat32ToDouble(FloatRegister src, 176 FloatRegister dest) { 177 as_fcvt_d_s(dest, src); 178 } 179 180 void MacroAssemblerLOONG64Compat::convertInt32ToFloat32(Register src, 181 FloatRegister dest) { 182 as_movgr2fr_w(dest, src); 183 as_ffint_s_w(dest, dest); 184 } 185 186 void MacroAssemblerLOONG64Compat::convertInt32ToFloat32(const Address& src, 187 FloatRegister dest) { 188 ma_fld_s(dest, src); 189 as_ffint_s_w(dest, dest); 190 } 191 192 void MacroAssemblerLOONG64::ma_li(Register dest, CodeLabel* label) { 193 BufferOffset bo = m_buffer.nextOffset(); 194 ma_liPatchable(dest, ImmWord(/* placeholder */ 0)); 195 label->patchAt()->bind(bo.getOffset()); 196 label->setLinkMode(CodeLabel::MoveImmediate); 197 } 198 199 void MacroAssemblerLOONG64::ma_li(Register dest, ImmWord imm) { 200 int64_t value = imm.value; 201 202 if (-1 == (value >> 11) || 0 == (value >> 11)) { 203 as_addi_w(dest, zero, value); 204 return; 205 } 206 207 if (0 == (value >> 12)) { 208 as_ori(dest, zero, value); 209 return; 210 } 211 212 if (-1 == (value >> 31) || 0 == (value >> 31)) { 213 as_lu12i_w(dest, (value >> 12) & 0xfffff); 214 } else if (0 == (value >> 32)) { 215 as_lu12i_w(dest, (value >> 12) & 0xfffff); 216 as_bstrins_d(dest, zero, 63, 32); 217 } else if (-1 == (value >> 51) || 0 == (value >> 51)) { 218 if (is_uintN((value >> 12) & 0xfffff, 20)) { 219 as_lu12i_w(dest, (value >> 12) & 0xfffff); 220 } 221 as_lu32i_d(dest, (value >> 32) & 0xfffff); 222 } else if (0 == (value >> 52)) { 223 if (is_uintN((value >> 12) & 0xfffff, 20)) { 224 as_lu12i_w(dest, (value >> 12) & 0xfffff); 225 } 226 as_lu32i_d(dest, (value >> 32) & 0xfffff); 227 as_bstrins_d(dest, zero, 63, 52); 228 } else { 229 if (is_uintN((value >> 12) & 0xfffff, 20)) { 230 as_lu12i_w(dest, (value >> 12) & 0xfffff); 231 } 232 if (is_uintN((value >> 32) & 0xfffff, 20)) { 233 as_lu32i_d(dest, (value >> 32) & 0xfffff); 234 } 235 as_lu52i_d(dest, dest, (value >> 52) & 0xfff); 236 } 237 238 if (is_uintN(value & 0xfff, 12)) { 239 as_ori(dest, dest, value & 0xfff); 240 } 241 } 242 243 // This method generates lu32i_d, lu12i_w and ori instruction block that can be 244 // modified by UpdateLoad64Value, either during compilation (eg. 245 // Assembler::bind), or during execution (eg. jit::PatchJump). 246 void MacroAssemblerLOONG64::ma_liPatchable(Register dest, ImmPtr imm) { 247 return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value))); 248 } 249 250 void MacroAssemblerLOONG64::ma_liPatchable(Register dest, ImmWord imm, 251 LiFlags flags) { 252 // hi12, hi20, low20, low12 253 if (Li64 == flags) { // Li64: Imm data 254 m_buffer.ensureSpace(4 * sizeof(uint32_t)); 255 as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20 256 as_ori(dest, dest, imm.value & 0xfff); // low12 257 as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20 258 as_lu52i_d(dest, dest, imm.value >> 52 & 0xfff); // hi12 259 } else { // Li48 address 260 m_buffer.ensureSpace(3 * sizeof(uint32_t)); 261 as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20 262 as_ori(dest, dest, imm.value & 0xfff); // low12 263 as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20 264 } 265 } 266 267 // Memory access ops. 268 269 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_b(Register dest, 270 Address address) { 271 int32_t offset = address.offset; 272 Register base = address.base; 273 FaultingCodeOffset fco; 274 275 if (is_intN(offset, 12)) { 276 fco = FaultingCodeOffset(currentOffset()); 277 as_ld_b(dest, base, offset); 278 } else if (base != dest) { 279 ma_li(dest, Imm32(offset)); 280 fco = FaultingCodeOffset(currentOffset()); 281 as_ldx_b(dest, base, dest); 282 } else { 283 UseScratchRegisterScope temps(asMasm()); 284 Register scratch = temps.Acquire(); 285 MOZ_ASSERT(base != scratch); 286 ma_li(scratch, Imm32(offset)); 287 fco = FaultingCodeOffset(currentOffset()); 288 as_ldx_b(dest, base, scratch); 289 } 290 return fco; 291 } 292 293 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_bu(Register dest, 294 Address address) { 295 int32_t offset = address.offset; 296 Register base = address.base; 297 FaultingCodeOffset fco; 298 299 if (is_intN(offset, 12)) { 300 fco = FaultingCodeOffset(currentOffset()); 301 as_ld_bu(dest, base, offset); 302 } else if (base != dest) { 303 ma_li(dest, Imm32(offset)); 304 fco = FaultingCodeOffset(currentOffset()); 305 as_ldx_bu(dest, base, dest); 306 } else { 307 UseScratchRegisterScope temps(asMasm()); 308 Register scratch = temps.Acquire(); 309 MOZ_ASSERT(base != scratch); 310 ma_li(scratch, Imm32(offset)); 311 fco = FaultingCodeOffset(currentOffset()); 312 as_ldx_bu(dest, base, scratch); 313 } 314 return fco; 315 } 316 317 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_h(Register dest, 318 Address address) { 319 int32_t offset = address.offset; 320 Register base = address.base; 321 FaultingCodeOffset fco; 322 323 if (is_intN(offset, 12)) { 324 fco = FaultingCodeOffset(currentOffset()); 325 as_ld_h(dest, base, offset); 326 } else if (base != dest) { 327 ma_li(dest, Imm32(offset)); 328 fco = FaultingCodeOffset(currentOffset()); 329 as_ldx_h(dest, base, dest); 330 } else { 331 UseScratchRegisterScope temps(asMasm()); 332 Register scratch = temps.Acquire(); 333 MOZ_ASSERT(base != scratch); 334 ma_li(scratch, Imm32(offset)); 335 fco = FaultingCodeOffset(currentOffset()); 336 as_ldx_h(dest, base, scratch); 337 } 338 return fco; 339 } 340 341 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_hu(Register dest, 342 Address address) { 343 int32_t offset = address.offset; 344 Register base = address.base; 345 FaultingCodeOffset fco; 346 347 if (is_intN(offset, 12)) { 348 fco = FaultingCodeOffset(currentOffset()); 349 as_ld_hu(dest, base, offset); 350 } else if (base != dest) { 351 ma_li(dest, Imm32(offset)); 352 fco = FaultingCodeOffset(currentOffset()); 353 as_ldx_hu(dest, base, dest); 354 } else { 355 UseScratchRegisterScope temps(asMasm()); 356 Register scratch = temps.Acquire(); 357 MOZ_ASSERT(base != scratch); 358 ma_li(scratch, Imm32(offset)); 359 fco = FaultingCodeOffset(currentOffset()); 360 as_ldx_hu(dest, base, scratch); 361 } 362 return fco; 363 } 364 365 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_w(Register dest, 366 Address address) { 367 int32_t offset = address.offset; 368 Register base = address.base; 369 FaultingCodeOffset fco; 370 371 if (is_intN(offset, 12)) { 372 fco = FaultingCodeOffset(currentOffset()); 373 as_ld_w(dest, base, offset); 374 } else if (base != dest) { 375 ma_li(dest, Imm32(offset)); 376 fco = FaultingCodeOffset(currentOffset()); 377 as_ldx_w(dest, base, dest); 378 } else { 379 UseScratchRegisterScope temps(asMasm()); 380 Register scratch = temps.Acquire(); 381 MOZ_ASSERT(base != scratch); 382 ma_li(scratch, Imm32(offset)); 383 fco = FaultingCodeOffset(currentOffset()); 384 as_ldx_w(dest, base, scratch); 385 } 386 return fco; 387 } 388 389 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_wu(Register dest, 390 Address address) { 391 int32_t offset = address.offset; 392 Register base = address.base; 393 FaultingCodeOffset fco; 394 395 if (is_intN(offset, 12)) { 396 fco = FaultingCodeOffset(currentOffset()); 397 as_ld_wu(dest, base, offset); 398 } else if (base != dest) { 399 ma_li(dest, Imm32(offset)); 400 fco = FaultingCodeOffset(currentOffset()); 401 as_ldx_wu(dest, base, dest); 402 } else { 403 UseScratchRegisterScope temps(asMasm()); 404 Register scratch = temps.Acquire(); 405 MOZ_ASSERT(base != scratch); 406 ma_li(scratch, Imm32(offset)); 407 fco = FaultingCodeOffset(currentOffset()); 408 as_ldx_wu(dest, base, scratch); 409 } 410 return fco; 411 } 412 413 FaultingCodeOffset MacroAssemblerLOONG64::ma_ld_d(Register dest, 414 Address address) { 415 int32_t offset = address.offset; 416 Register base = address.base; 417 FaultingCodeOffset fco; 418 419 if (is_intN(offset, 12)) { 420 fco = FaultingCodeOffset(currentOffset()); 421 as_ld_d(dest, base, offset); 422 } else if (base != dest) { 423 ma_li(dest, Imm32(offset)); 424 fco = FaultingCodeOffset(currentOffset()); 425 as_ldx_d(dest, base, dest); 426 } else { 427 UseScratchRegisterScope temps(asMasm()); 428 Register scratch = temps.Acquire(); 429 MOZ_ASSERT(base != scratch); 430 ma_li(scratch, Imm32(offset)); 431 fco = FaultingCodeOffset(currentOffset()); 432 as_ldx_d(dest, base, scratch); 433 } 434 return fco; 435 } 436 437 FaultingCodeOffset MacroAssemblerLOONG64::ma_st_b(Register src, 438 Address address) { 439 int32_t offset = address.offset; 440 Register base = address.base; 441 FaultingCodeOffset fco; 442 443 if (is_intN(offset, 12)) { 444 fco = FaultingCodeOffset(currentOffset()); 445 as_st_b(src, base, offset); 446 } else { 447 UseScratchRegisterScope temps(asMasm()); 448 Register scratch = temps.Acquire(); 449 MOZ_ASSERT(src != scratch); 450 MOZ_ASSERT(base != scratch); 451 ma_li(scratch, Imm32(offset)); 452 fco = FaultingCodeOffset(currentOffset()); 453 as_stx_b(src, base, scratch); 454 } 455 return fco; 456 } 457 458 FaultingCodeOffset MacroAssemblerLOONG64::ma_st_h(Register src, 459 Address address) { 460 int32_t offset = address.offset; 461 Register base = address.base; 462 FaultingCodeOffset fco; 463 464 if (is_intN(offset, 12)) { 465 fco = FaultingCodeOffset(currentOffset()); 466 as_st_h(src, base, offset); 467 } else { 468 UseScratchRegisterScope temps(asMasm()); 469 Register scratch = temps.Acquire(); 470 MOZ_ASSERT(src != scratch); 471 MOZ_ASSERT(base != scratch); 472 ma_li(scratch, Imm32(offset)); 473 fco = FaultingCodeOffset(currentOffset()); 474 as_stx_h(src, base, scratch); 475 } 476 return fco; 477 } 478 479 FaultingCodeOffset MacroAssemblerLOONG64::ma_st_w(Register src, 480 Address address) { 481 int32_t offset = address.offset; 482 Register base = address.base; 483 FaultingCodeOffset fco; 484 485 if (is_intN(offset, 12)) { 486 fco = FaultingCodeOffset(currentOffset()); 487 as_st_w(src, base, offset); 488 } else { 489 UseScratchRegisterScope temps(asMasm()); 490 Register scratch = temps.Acquire(); 491 MOZ_ASSERT(src != scratch); 492 MOZ_ASSERT(base != scratch); 493 ma_li(scratch, Imm32(offset)); 494 fco = FaultingCodeOffset(currentOffset()); 495 as_stx_w(src, base, scratch); 496 } 497 return fco; 498 } 499 500 FaultingCodeOffset MacroAssemblerLOONG64::ma_st_d(Register src, 501 Address address) { 502 int32_t offset = address.offset; 503 Register base = address.base; 504 FaultingCodeOffset fco; 505 506 if (is_intN(offset, 12)) { 507 fco = FaultingCodeOffset(currentOffset()); 508 as_st_d(src, base, offset); 509 } else { 510 UseScratchRegisterScope temps(asMasm()); 511 Register scratch = temps.Acquire(); 512 MOZ_ASSERT(src != scratch); 513 MOZ_ASSERT(base != scratch); 514 ma_li(scratch, Imm32(offset)); 515 fco = FaultingCodeOffset(currentOffset()); 516 as_stx_d(src, base, scratch); 517 } 518 return fco; 519 } 520 521 // Arithmetic-based ops. 522 523 // Add. 524 void MacroAssemblerLOONG64::ma_add_d(Register rd, Register rj, Imm32 imm) { 525 if (is_intN(imm.value, 12)) { 526 as_addi_d(rd, rj, imm.value); 527 } else if (rd != rj) { 528 ma_li(rd, imm); 529 as_add_d(rd, rj, rd); 530 } else { 531 UseScratchRegisterScope temps(asMasm()); 532 Register scratch = temps.Acquire(); 533 MOZ_ASSERT(rj != scratch); 534 ma_li(scratch, imm); 535 as_add_d(rd, rj, scratch); 536 } 537 } 538 539 void MacroAssemblerLOONG64::ma_add_d(Register rd, Register rj, ImmWord imm) { 540 if (is_intN(imm.value, 12)) { 541 as_addi_d(rd, rj, imm.value); 542 } else if (rd != rj) { 543 ma_li(rd, imm); 544 as_add_d(rd, rj, rd); 545 } else { 546 UseScratchRegisterScope temps(asMasm()); 547 Register scratch = temps.Acquire(); 548 MOZ_ASSERT(rj != scratch); 549 ma_li(scratch, imm); 550 as_add_d(rd, rj, scratch); 551 } 552 } 553 554 void MacroAssemblerLOONG64::ma_add32TestOverflow(Register rd, Register rj, 555 Register rk, Label* overflow) { 556 UseScratchRegisterScope temps(asMasm()); 557 Register scratch = temps.Acquire(); 558 as_add_d(scratch, rj, rk); 559 as_add_w(rd, rj, rk); 560 ma_b(rd, Register(scratch), overflow, Assembler::NotEqual); 561 } 562 563 void MacroAssemblerLOONG64::ma_add32TestOverflow(Register rd, Register rj, 564 Imm32 imm, Label* overflow) { 565 UseScratchRegisterScope temps(asMasm()); 566 Register scratch = temps.Acquire(); 567 // Check for signed range because of as_addi_d 568 if (is_intN(imm.value, 12)) { 569 as_addi_d(scratch, rj, imm.value); 570 as_addi_w(rd, rj, imm.value); 571 ma_b(rd, scratch, overflow, Assembler::NotEqual); 572 } else { 573 ma_li(scratch, imm); 574 ma_add32TestOverflow(rd, rj, scratch, overflow); 575 } 576 } 577 578 void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj, 579 Register rk, 580 Label* overflow) { 581 UseScratchRegisterScope temps(asMasm()); 582 Register scratch = temps.Acquire(); 583 MOZ_ASSERT(rd != scratch); 584 585 if (rj == rk) { 586 if (rj == rd) { 587 as_or(scratch, rj, zero); 588 rj = scratch; 589 } 590 591 as_add_d(rd, rj, rj); 592 as_xor(scratch, rj, rd); 593 ma_b(scratch, zero, overflow, Assembler::LessThan); 594 } else { 595 Register scratch2 = temps.Acquire(); 596 MOZ_ASSERT(rj != scratch); 597 MOZ_ASSERT(rd != scratch2); 598 599 if (rj == rd) { 600 as_or(scratch2, rj, zero); 601 rj = scratch2; 602 } 603 604 as_add_d(rd, rj, rk); 605 // rd = rj + rk overflow conditions: 606 // 1. rj < 0 and rd >= rk 607 // 2. rj >= 0 and rd < rk 608 as_slti(scratch, rj, 0); 609 as_slt(scratch2, rd, rk); 610 ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); 611 } 612 } 613 614 void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj, 615 Imm32 imm, Label* overflow) { 616 UseScratchRegisterScope temps(asMasm()); 617 Register scratch = temps.Acquire(); 618 619 if (imm.value == 0) { 620 as_ori(rd, rj, 0); 621 return; 622 } 623 624 if (rj == rd) { 625 as_ori(scratch, rj, 0); 626 rj = scratch; 627 } 628 629 ma_add_d(rd, rj, imm); 630 631 if (imm.value > 0) { 632 ma_b(rd, rj, overflow, Assembler::LessThan); 633 } else { 634 MOZ_ASSERT(imm.value < 0); 635 ma_b(rd, rj, overflow, Assembler::GreaterThan); 636 } 637 } 638 639 void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj, 640 ImmWord imm, 641 Label* overflow) { 642 UseScratchRegisterScope temps(asMasm()); 643 Register scratch = temps.Acquire(); 644 645 if (imm.value == 0) { 646 as_ori(rd, rj, 0); 647 return; 648 } 649 650 if (rj == rd) { 651 MOZ_ASSERT(rj != scratch); 652 as_ori(scratch, rj, 0); 653 rj = scratch; 654 } 655 656 ma_li(rd, imm); 657 as_add_d(rd, rj, rd); 658 659 if (imm.value > 0) { 660 ma_b(rd, rj, overflow, Assembler::LessThan); 661 } else { 662 MOZ_ASSERT(imm.value < 0); 663 ma_b(rd, rj, overflow, Assembler::GreaterThan); 664 } 665 } 666 667 void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd, 668 Register rj, Register rk, 669 Label* label) { 670 UseScratchRegisterScope temps(asMasm()); 671 Register scratch = temps.Acquire(); 672 MOZ_ASSERT(rd != rk); 673 MOZ_ASSERT(rd != scratch); 674 as_add_d(rd, rj, rk); 675 as_sltu(scratch, rd, rk); 676 ma_b(scratch, Register(scratch), label, 677 cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); 678 } 679 680 void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd, 681 Register rj, Imm32 imm, 682 Label* label) { 683 UseScratchRegisterScope temps(asMasm()); 684 Register scratch = temps.Acquire(); 685 686 // Check for signed range because of as_addi_d 687 if (is_intN(imm.value, 12)) { 688 as_addi_d(rd, rj, imm.value); 689 as_sltui(scratch, rd, imm.value); 690 ma_b(scratch, scratch, label, 691 cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); 692 } else { 693 ma_li(scratch, imm); 694 ma_addPtrTestCarry(cond, rd, rj, scratch, label); 695 } 696 } 697 698 void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd, 699 Register rj, ImmWord imm, 700 Label* label) { 701 UseScratchRegisterScope temps(asMasm()); 702 Register scratch = temps.Acquire(); 703 704 // Check for signed range because of as_addi_d 705 if (is_intN(imm.value, 12)) { 706 as_addi_d(rd, rj, imm.value); 707 as_sltui(scratch, rd, imm.value); 708 ma_b(scratch, scratch, label, 709 cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); 710 } else { 711 ma_li(scratch, imm); 712 ma_addPtrTestCarry(cond, rd, rj, scratch, label); 713 } 714 } 715 716 void MacroAssemblerLOONG64::ma_addPtrTestSigned(Condition cond, Register rd, 717 Register rj, Register rk, 718 Label* taken) { 719 MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned); 720 721 as_add_d(rd, rj, rk); 722 ma_b(rd, rd, taken, cond); 723 } 724 725 void MacroAssemblerLOONG64::ma_addPtrTestSigned(Condition cond, Register rd, 726 Register rj, Imm32 imm, 727 Label* taken) { 728 MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned); 729 730 ma_add_d(rd, rj, imm); 731 ma_b(rd, rd, taken, cond); 732 } 733 734 void MacroAssemblerLOONG64::ma_addPtrTestSigned(Condition cond, Register rd, 735 Register rj, ImmWord imm, 736 Label* taken) { 737 MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned); 738 739 UseScratchRegisterScope temps(asMasm()); 740 Register scratch = temps.Acquire(); 741 ma_li(scratch, imm); 742 ma_addPtrTestSigned(cond, rd, rj, scratch, taken); 743 } 744 745 // Subtract. 746 void MacroAssemblerLOONG64::ma_sub_d(Register rd, Register rj, Imm32 imm) { 747 if (is_intN(-imm.value, 12)) { 748 as_addi_d(rd, rj, -imm.value); 749 } else { 750 UseScratchRegisterScope temps(asMasm()); 751 Register scratch = temps.Acquire(); 752 ma_li(scratch, imm); 753 as_sub_d(rd, rj, scratch); 754 } 755 } 756 757 void MacroAssemblerLOONG64::ma_sub_d(Register rd, Register rj, ImmWord imm) { 758 if (is_intN(-imm.value, 12)) { 759 as_addi_d(rd, rj, -imm.value); 760 } else { 761 UseScratchRegisterScope temps(asMasm()); 762 Register scratch = temps.Acquire(); 763 ma_li(scratch, imm); 764 as_sub_d(rd, rj, scratch); 765 } 766 } 767 768 void MacroAssemblerLOONG64::ma_sub32TestOverflow(Register rd, Register rj, 769 Register rk, Label* overflow) { 770 UseScratchRegisterScope temps(asMasm()); 771 Register scratch = temps.Acquire(); 772 as_sub_d(scratch, rj, rk); 773 as_sub_w(rd, rj, rk); 774 ma_b(rd, Register(scratch), overflow, Assembler::NotEqual); 775 } 776 777 void MacroAssemblerLOONG64::ma_subPtrTestOverflow(Register rd, Register rj, 778 Register rk, 779 Label* overflow) { 780 UseScratchRegisterScope temps2(asMasm()); 781 Register scratch2 = temps2.Acquire(); 782 MOZ_ASSERT_IF(rj == rd, rj != rk); 783 MOZ_ASSERT(rj != scratch2); 784 MOZ_ASSERT(rk != scratch2); 785 MOZ_ASSERT(rd != scratch2); 786 787 Register rj_copy = rj; 788 789 if (rj == rd) { 790 as_or(scratch2, rj, zero); 791 rj_copy = scratch2; 792 } 793 794 { 795 UseScratchRegisterScope temps(asMasm()); 796 Register scratch = temps.Acquire(); 797 MOZ_ASSERT(rd != scratch); 798 799 as_sub_d(rd, rj, rk); 800 // If the sign of rj and rk are the same, no overflow 801 as_xor(scratch, rj_copy, rk); 802 // Check if the sign of rd and rj are the same 803 as_xor(scratch2, rd, rj_copy); 804 as_and(scratch2, scratch2, scratch); 805 } 806 807 ma_b(scratch2, zero, overflow, Assembler::LessThan); 808 } 809 810 void MacroAssemblerLOONG64::ma_subPtrTestOverflow(Register rd, Register rj, 811 Imm32 imm, Label* overflow) { 812 // TODO(loong64): Check subPtrTestOverflow 813 MOZ_ASSERT(imm.value != INT32_MIN); 814 ma_addPtrTestOverflow(rd, rj, Imm32(-imm.value), overflow); 815 } 816 817 void MacroAssemblerLOONG64::ma_mul_d(Register rd, Register rj, Imm32 imm) { 818 // li handles the relocation. 819 UseScratchRegisterScope temps(asMasm()); 820 Register scratch = temps.Acquire(); 821 MOZ_ASSERT(rj != scratch); 822 ma_li(scratch, imm); 823 as_mul_d(rd, rj, scratch); 824 } 825 826 void MacroAssemblerLOONG64::ma_mul_d(Register rd, Register rj, ImmWord imm) { 827 // li handles the relocation. 828 UseScratchRegisterScope temps(asMasm()); 829 Register scratch = temps.Acquire(); 830 MOZ_ASSERT(rj != scratch); 831 ma_li(scratch, imm); 832 as_mul_d(rd, rj, scratch); 833 } 834 835 void MacroAssemblerLOONG64::ma_mulh_d(Register rd, Register rj, Imm32 imm) { 836 // li handles the relocation. 837 UseScratchRegisterScope temps(asMasm()); 838 Register scratch = temps.Acquire(); 839 MOZ_ASSERT(rj != scratch); 840 ma_li(scratch, imm); 841 as_mulh_d(rd, rj, scratch); 842 } 843 844 void MacroAssemblerLOONG64::ma_mulPtrTestOverflow(Register rd, Register rj, 845 Register rk, 846 Label* overflow) { 847 UseScratchRegisterScope temps(asMasm()); 848 Register scratch = temps.Acquire(); 849 Register scratch2 = temps.Acquire(); 850 MOZ_ASSERT(rd != scratch); 851 852 if (rd == rj) { 853 as_or(scratch, rj, zero); 854 rj = scratch; 855 rk = (rd == rk) ? rj : rk; 856 } else if (rd == rk) { 857 as_or(scratch, rk, zero); 858 rk = scratch; 859 } 860 861 as_mul_d(rd, rj, rk); 862 as_mulh_d(scratch, rj, rk); 863 as_srai_d(scratch2, rd, 63); 864 ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual); 865 } 866 867 // Memory. 868 869 FaultingCodeOffset MacroAssemblerLOONG64::ma_load( 870 Register dest, Address address, LoadStoreSize size, 871 LoadStoreExtension extension) { 872 int32_t encodedOffset; 873 Register base; 874 FaultingCodeOffset fco; 875 876 // TODO: use as_ldx_b/h/w/d, could decrease as_add_d instr. 877 UseScratchRegisterScope temps(*this); 878 switch (size) { 879 case SizeByte: 880 case SizeHalfWord: 881 if (!is_intN(address.offset, 12)) { 882 Register scratch = temps.Acquire(); 883 ma_li(scratch, Imm32(address.offset)); 884 as_add_d(scratch, address.base, scratch); 885 base = scratch; 886 encodedOffset = 0; 887 } else { 888 encodedOffset = address.offset; 889 base = address.base; 890 } 891 892 fco = FaultingCodeOffset(currentOffset()); 893 if (size == SizeByte) { 894 if (ZeroExtend == extension) { 895 as_ld_bu(dest, base, encodedOffset); 896 } else { 897 as_ld_b(dest, base, encodedOffset); 898 } 899 } else { 900 if (ZeroExtend == extension) { 901 as_ld_hu(dest, base, encodedOffset); 902 } else { 903 as_ld_h(dest, base, encodedOffset); 904 } 905 } 906 break; 907 case SizeWord: 908 case SizeDouble: 909 if ((address.offset & 0x3) == 0 && 910 (size == SizeDouble || 911 (size == SizeWord && SignExtend == extension))) { 912 if (!Imm16::IsInSignedRange(address.offset)) { 913 Register scratch = temps.Acquire(); 914 ma_li(scratch, Imm32(address.offset)); 915 as_add_d(scratch, address.base, scratch); 916 base = scratch; 917 encodedOffset = 0; 918 } else { 919 encodedOffset = address.offset; 920 base = address.base; 921 } 922 923 fco = FaultingCodeOffset(currentOffset()); 924 if (size == SizeWord) { 925 as_ldptr_w(dest, base, encodedOffset); 926 } else { 927 as_ldptr_d(dest, base, encodedOffset); 928 } 929 } else { 930 if (!is_intN(address.offset, 12)) { 931 Register scratch = temps.Acquire(); 932 ma_li(scratch, Imm32(address.offset)); 933 as_add_d(scratch, address.base, scratch); 934 base = scratch; 935 encodedOffset = 0; 936 } else { 937 encodedOffset = address.offset; 938 base = address.base; 939 } 940 941 fco = FaultingCodeOffset(currentOffset()); 942 if (size == SizeWord) { 943 if (ZeroExtend == extension) { 944 as_ld_wu(dest, base, encodedOffset); 945 } else { 946 as_ld_w(dest, base, encodedOffset); 947 } 948 } else { 949 as_ld_d(dest, base, encodedOffset); 950 } 951 } 952 break; 953 default: 954 MOZ_CRASH("Invalid argument for ma_load"); 955 } 956 return fco; 957 } 958 959 FaultingCodeOffset MacroAssemblerLOONG64::ma_store( 960 Register data, Address address, LoadStoreSize size, 961 LoadStoreExtension extension) { 962 int32_t encodedOffset; 963 Register base; 964 FaultingCodeOffset fco; 965 966 // TODO: use as_stx_b/h/w/d, could decrease as_add_d instr. 967 UseScratchRegisterScope temps(*this); 968 switch (size) { 969 case SizeByte: 970 case SizeHalfWord: 971 if (!is_intN(address.offset, 12)) { 972 Register scratch = temps.Acquire(); 973 ma_li(scratch, Imm32(address.offset)); 974 as_add_d(scratch, address.base, scratch); 975 base = scratch; 976 encodedOffset = 0; 977 } else { 978 encodedOffset = address.offset; 979 base = address.base; 980 } 981 982 fco = FaultingCodeOffset(currentOffset()); 983 if (size == SizeByte) { 984 as_st_b(data, base, encodedOffset); 985 } else { 986 as_st_h(data, base, encodedOffset); 987 } 988 break; 989 case SizeWord: 990 case SizeDouble: 991 if ((address.offset & 0x3) == 0) { 992 if (!Imm16::IsInSignedRange(address.offset)) { 993 Register scratch = temps.Acquire(); 994 ma_li(scratch, Imm32(address.offset)); 995 as_add_d(scratch, address.base, scratch); 996 base = scratch; 997 encodedOffset = 0; 998 } else { 999 encodedOffset = address.offset; 1000 base = address.base; 1001 } 1002 1003 fco = FaultingCodeOffset(currentOffset()); 1004 if (size == SizeWord) { 1005 as_stptr_w(data, base, encodedOffset); 1006 } else { 1007 as_stptr_d(data, base, encodedOffset); 1008 } 1009 } else { 1010 if (!is_intN(address.offset, 12)) { 1011 Register scratch = temps.Acquire(); 1012 ma_li(scratch, Imm32(address.offset)); 1013 as_add_d(scratch, address.base, scratch); 1014 base = scratch; 1015 encodedOffset = 0; 1016 } else { 1017 encodedOffset = address.offset; 1018 base = address.base; 1019 } 1020 1021 fco = FaultingCodeOffset(currentOffset()); 1022 if (size == SizeWord) { 1023 as_st_w(data, base, encodedOffset); 1024 } else { 1025 as_st_d(data, base, encodedOffset); 1026 } 1027 } 1028 break; 1029 default: 1030 MOZ_CRASH("Invalid argument for ma_store"); 1031 } 1032 return fco; 1033 } 1034 1035 void MacroAssemblerLOONG64Compat::computeScaledAddress(const BaseIndex& address, 1036 Register dest) { 1037 Register base = address.base; 1038 Register index = address.index; 1039 int32_t shift = Imm32::ShiftOf(address.scale).value; 1040 1041 if (shift) { 1042 MOZ_ASSERT(shift <= 4); 1043 as_alsl_d(dest, index, base, shift - 1); 1044 } else { 1045 as_add_d(dest, base, index); 1046 } 1047 } 1048 1049 void MacroAssemblerLOONG64Compat::computeScaledAddress32( 1050 const BaseIndex& address, Register dest) { 1051 Register base = address.base; 1052 Register index = address.index; 1053 int32_t shift = Imm32::ShiftOf(address.scale).value; 1054 1055 if (shift) { 1056 MOZ_ASSERT(shift <= 4); 1057 as_alsl_w(dest, index, base, shift - 1); 1058 } else { 1059 as_add_w(dest, base, index); 1060 } 1061 } 1062 1063 void MacroAssemblerLOONG64::ma_pop(Register r) { 1064 MOZ_ASSERT(r != StackPointer); 1065 as_ld_d(r, StackPointer, 0); 1066 as_addi_d(StackPointer, StackPointer, sizeof(intptr_t)); 1067 } 1068 1069 void MacroAssemblerLOONG64::ma_push(Register r) { 1070 if (r == StackPointer) { 1071 UseScratchRegisterScope temps(asMasm()); 1072 Register scratch = temps.Acquire(); 1073 as_or(scratch, r, zero); 1074 as_addi_d(StackPointer, StackPointer, -int32_t(sizeof(intptr_t))); 1075 as_st_d(scratch, StackPointer, 0); 1076 } else { 1077 as_addi_d(StackPointer, StackPointer, -int32_t(sizeof(intptr_t))); 1078 as_st_d(r, StackPointer, 0); 1079 } 1080 } 1081 1082 // Branches when done from within loongarch-specific code. 1083 void MacroAssemblerLOONG64::ma_b(Register lhs, ImmWord imm, Label* label, 1084 Condition c, JumpKind jumpKind) { 1085 if (imm.value <= INT32_MAX) { 1086 ma_b(lhs, Imm32(uint32_t(imm.value)), label, c, jumpKind); 1087 } else { 1088 UseScratchRegisterScope temps(asMasm()); 1089 Register scratch = temps.Acquire(); 1090 MOZ_ASSERT(lhs != scratch); 1091 ma_li(scratch, imm); 1092 ma_b(lhs, Register(scratch), label, c, jumpKind, scratch); 1093 } 1094 } 1095 1096 void MacroAssemblerLOONG64::ma_b(Register lhs, Address addr, Label* label, 1097 Condition c, JumpKind jumpKind) { 1098 UseScratchRegisterScope temps(asMasm()); 1099 Register scratch = temps.Acquire(); 1100 MOZ_ASSERT(lhs != scratch); 1101 ma_ld_d(scratch, addr); 1102 ma_b(lhs, Register(scratch), label, c, jumpKind, scratch); 1103 } 1104 1105 void MacroAssemblerLOONG64::ma_b(Address addr, Imm32 imm, Label* label, 1106 Condition c, JumpKind jumpKind) { 1107 UseScratchRegisterScope temps(asMasm()); 1108 Register scratch = temps.Acquire(); 1109 ma_ld_d(scratch, addr); 1110 ma_b(Register(scratch), imm, label, c, jumpKind); 1111 } 1112 1113 void MacroAssemblerLOONG64::ma_b(Address addr, ImmGCPtr imm, Label* label, 1114 Condition c, JumpKind jumpKind) { 1115 UseScratchRegisterScope temps(asMasm()); 1116 Register scratch = temps.Acquire(); 1117 ma_ld_d(scratch, addr); 1118 ma_b(Register(scratch), imm, label, c, jumpKind); 1119 } 1120 1121 void MacroAssemblerLOONG64::ma_bl(Label* label) { 1122 spew("branch .Llabel %p\n", label); 1123 if (label->bound()) { 1124 // Generate the long jump for calls because return address has to be 1125 // the address after the reserved block. 1126 addLongJump(nextOffset(), BufferOffset(label->offset())); 1127 UseScratchRegisterScope temps(asMasm()); 1128 Register scratch = temps.Acquire(); 1129 ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); 1130 as_jirl(ra, scratch, BOffImm16(0)); 1131 return; 1132 } 1133 1134 // Second word holds a pointer to the next branch in label's chain. 1135 uint32_t nextInChain = 1136 label->used() ? label->offset() : LabelBase::INVALID_OFFSET; 1137 1138 // Make the whole branch continous in the buffer. The '5' 1139 // instructions are writing at below. 1140 m_buffer.ensureSpace(5 * sizeof(uint32_t)); 1141 1142 spew("bal .Llabel %p\n", label); 1143 BufferOffset bo = writeInst(getBranchCode(BranchIsCall).encode()); 1144 writeInst(nextInChain); 1145 if (!oom()) { 1146 label->use(bo.getOffset()); 1147 } 1148 // Leave space for long jump. 1149 as_nop(); 1150 as_nop(); 1151 as_nop(); 1152 } 1153 1154 void MacroAssemblerLOONG64::branchWithCode(InstImm code, Label* label, 1155 JumpKind jumpKind, 1156 Register scratch) { 1157 // simply output the pointer of one label as its id, 1158 // notice that after one label destructor, the pointer will be reused. 1159 spew("branch .Llabel %p", label); 1160 MOZ_ASSERT(code.encode() != 1161 InstImm(op_jirl, BOffImm16(0), zero, ra).encode()); 1162 InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero); 1163 1164 if (label->bound()) { 1165 int32_t offset = label->offset() - m_buffer.nextOffset().getOffset(); 1166 1167 if (BOffImm16::IsInRange(offset)) { 1168 jumpKind = ShortJump; 1169 } 1170 1171 // ShortJump 1172 if (jumpKind == ShortJump) { 1173 MOZ_ASSERT(BOffImm16::IsInRange(offset)); 1174 1175 if (code.extractBitField(31, 26) == ((uint32_t)op_bcz >> 26)) { 1176 code.setImm21(offset); 1177 } else { 1178 code.setBOffImm16(BOffImm16(offset)); 1179 } 1180 #ifdef JS_JITSPEW 1181 decodeBranchInstAndSpew(code); 1182 #endif 1183 writeInst(code.encode()); 1184 return; 1185 } 1186 1187 // LongJump 1188 if (code.encode() == inst_beq.encode()) { 1189 // Handle long jump 1190 addLongJump(nextOffset(), BufferOffset(label->offset())); 1191 if (scratch == Register::Invalid()) { 1192 UseScratchRegisterScope temps(asMasm()); 1193 Register scratch = temps.Acquire(); 1194 ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); 1195 as_jirl(zero, scratch, BOffImm16(0)); // jr scratch 1196 } else { 1197 ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); 1198 as_jirl(zero, scratch, BOffImm16(0)); // jr scratch 1199 } 1200 as_nop(); 1201 return; 1202 } 1203 1204 // OpenLongJump 1205 // Handle long conditional branch, the target offset is based on self, 1206 // point to next instruction of nop at below. 1207 spew("invert branch .Llabel %p", label); 1208 InstImm code_r = invertBranch(code, BOffImm16(5 * sizeof(uint32_t))); 1209 #ifdef JS_JITSPEW 1210 decodeBranchInstAndSpew(code_r); 1211 #endif 1212 writeInst(code_r.encode()); 1213 addLongJump(nextOffset(), BufferOffset(label->offset())); 1214 if (scratch == Register::Invalid()) { 1215 UseScratchRegisterScope temps(asMasm()); 1216 Register scratch = temps.Acquire(); 1217 ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); 1218 as_jirl(zero, scratch, BOffImm16(0)); // jr scratch 1219 } else { 1220 ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET)); 1221 as_jirl(zero, scratch, BOffImm16(0)); // jr scratch 1222 } 1223 as_nop(); 1224 return; 1225 } 1226 1227 // Generate open jump and link it to a label. 1228 1229 // Second word holds a pointer to the next branch in label's chain. 1230 uint32_t nextInChain = 1231 label->used() ? label->offset() : LabelBase::INVALID_OFFSET; 1232 1233 if (jumpKind == ShortJump) { 1234 // Make the whole branch continous in the buffer. 1235 m_buffer.ensureSpace(2 * sizeof(uint32_t)); 1236 1237 // Indicate that this is short jump with offset 4. 1238 code.setBOffImm16(BOffImm16(4)); 1239 #ifdef JS_JITSPEW 1240 decodeBranchInstAndSpew(code); 1241 #endif 1242 BufferOffset bo = writeInst(code.encode()); 1243 writeInst(nextInChain); 1244 if (!oom()) { 1245 label->use(bo.getOffset()); 1246 } 1247 return; 1248 } 1249 1250 bool conditional = code.encode() != inst_beq.encode(); 1251 1252 // Make the whole branch continous in the buffer. The '5' 1253 // instructions are writing at below (contain conditional nop). 1254 m_buffer.ensureSpace(5 * sizeof(uint32_t)); 1255 1256 #ifdef JS_JITSPEW 1257 decodeBranchInstAndSpew(code); 1258 #endif 1259 BufferOffset bo = writeInst(code.encode()); // invert 1260 writeInst(nextInChain); 1261 if (!oom()) { 1262 label->use(bo.getOffset()); 1263 } 1264 // Leave space for potential long jump. 1265 as_nop(); 1266 as_nop(); 1267 if (conditional) { 1268 as_nop(); 1269 } 1270 } 1271 1272 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmWord imm, 1273 Condition c) { 1274 if (imm.value <= INT32_MAX) { 1275 ma_cmp_set(rd, rj, Imm32(uint32_t(imm.value)), c); 1276 } else { 1277 UseScratchRegisterScope temps(asMasm()); 1278 Register scratch = temps.Acquire(); 1279 ma_li(scratch, imm); 1280 ma_cmp_set(rd, rj, scratch, c); 1281 } 1282 } 1283 1284 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmPtr imm, 1285 Condition c) { 1286 ma_cmp_set(rd, rj, ImmWord(uintptr_t(imm.value)), c); 1287 } 1288 1289 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmGCPtr imm, 1290 Condition c) { 1291 UseScratchRegisterScope temps(asMasm()); 1292 Register scratch = temps.Acquire(); 1293 ma_li(scratch, imm); 1294 ma_cmp_set(rd, rj, scratch, c); 1295 } 1296 1297 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Address address, Imm32 imm, 1298 Condition c) { 1299 // TODO(loong64): 32-bit ma_cmp_set? 1300 UseScratchRegisterScope temps(asMasm()); 1301 Register scratch = temps.Acquire(); 1302 ma_ld_w(scratch, address); 1303 ma_cmp_set(rd, Register(scratch), imm, c); 1304 } 1305 1306 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Address address, 1307 Register rk, Condition c) { 1308 UseScratchRegisterScope temps(asMasm()); 1309 Register scratch = temps.Acquire(); 1310 ma_ld_d(scratch, address); 1311 ma_cmp_set(rd, scratch, rk, c); 1312 } 1313 1314 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Address address, 1315 ImmWord imm, Condition c) { 1316 UseScratchRegisterScope temps(asMasm()); 1317 Register scratch = temps.Acquire(); 1318 ma_ld_d(scratch, address); 1319 ma_cmp_set(rd, Register(scratch), imm, c); 1320 } 1321 1322 // fp instructions 1323 void MacroAssemblerLOONG64::ma_lid(FloatRegister dest, double value) { 1324 ImmWord imm(mozilla::BitwiseCast<uint64_t>(value)); 1325 1326 if (imm.value != 0) { 1327 UseScratchRegisterScope temps(asMasm()); 1328 Register scratch = temps.Acquire(); 1329 ma_li(scratch, imm); 1330 moveToDouble(scratch, dest); 1331 } else { 1332 moveToDouble(zero, dest); 1333 } 1334 } 1335 1336 void MacroAssemblerLOONG64::ma_mv(FloatRegister src, ValueOperand dest) { 1337 as_movfr2gr_d(dest.valueReg(), src); 1338 } 1339 1340 void MacroAssemblerLOONG64::ma_mv(ValueOperand src, FloatRegister dest) { 1341 as_movgr2fr_d(dest, src.valueReg()); 1342 } 1343 1344 FaultingCodeOffset MacroAssemblerLOONG64::ma_fld_s(FloatRegister dest, 1345 Address address) { 1346 int32_t offset = address.offset; 1347 Register base = address.base; 1348 js::wasm::FaultingCodeOffset fco; 1349 1350 if (is_intN(offset, 12)) { 1351 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1352 as_fld_s(dest, base, offset); 1353 } else { 1354 UseScratchRegisterScope temps(asMasm()); 1355 Register scratch = temps.Acquire(); 1356 MOZ_ASSERT(base != scratch); 1357 ma_li(scratch, Imm32(offset)); 1358 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1359 as_fldx_s(dest, base, scratch); 1360 } 1361 return fco; 1362 } 1363 1364 FaultingCodeOffset MacroAssemblerLOONG64::ma_fld_d(FloatRegister dest, 1365 Address address) { 1366 int32_t offset = address.offset; 1367 Register base = address.base; 1368 js::wasm::FaultingCodeOffset fco; 1369 1370 if (is_intN(offset, 12)) { 1371 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1372 as_fld_d(dest, base, offset); 1373 } else { 1374 UseScratchRegisterScope temps(asMasm()); 1375 Register scratch = temps.Acquire(); 1376 MOZ_ASSERT(base != scratch); 1377 ma_li(scratch, Imm32(offset)); 1378 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1379 as_fldx_d(dest, base, scratch); 1380 } 1381 return fco; 1382 } 1383 1384 FaultingCodeOffset MacroAssemblerLOONG64::ma_fst_s(FloatRegister src, 1385 Address address) { 1386 int32_t offset = address.offset; 1387 Register base = address.base; 1388 js::wasm::FaultingCodeOffset fco; 1389 1390 if (is_intN(offset, 12)) { 1391 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1392 as_fst_s(src, base, offset); 1393 } else { 1394 UseScratchRegisterScope temps(asMasm()); 1395 Register scratch = temps.Acquire(); 1396 MOZ_ASSERT(base != scratch); 1397 ma_li(scratch, Imm32(offset)); 1398 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1399 as_fstx_s(src, base, scratch); 1400 } 1401 return fco; 1402 } 1403 1404 FaultingCodeOffset MacroAssemblerLOONG64::ma_fst_d(FloatRegister src, 1405 Address address) { 1406 int32_t offset = address.offset; 1407 Register base = address.base; 1408 js::wasm::FaultingCodeOffset fco; 1409 1410 if (is_intN(offset, 12)) { 1411 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1412 as_fst_d(src, base, offset); 1413 } else { 1414 UseScratchRegisterScope temps(asMasm()); 1415 Register scratch = temps.Acquire(); 1416 MOZ_ASSERT(base != scratch); 1417 ma_li(scratch, Imm32(offset)); 1418 fco = js::wasm::FaultingCodeOffset(currentOffset()); 1419 as_fstx_d(src, base, scratch); 1420 } 1421 return fco; 1422 } 1423 1424 void MacroAssemblerLOONG64::ma_pop(FloatRegister f) { 1425 if (f.isDouble()) { 1426 as_fld_d(f, StackPointer, 0); 1427 } else { 1428 MOZ_ASSERT(f.isSingle(), "simd128 is not supported"); 1429 as_fld_s(f, StackPointer, 0); 1430 } 1431 // See also MacroAssemblerLOONG64::ma_push -- Free space for double even when 1432 // storing a float. 1433 as_addi_d(StackPointer, StackPointer, sizeof(double)); 1434 } 1435 1436 void MacroAssemblerLOONG64::ma_push(FloatRegister f) { 1437 // We allocate space for double even when storing a float. 1438 as_addi_d(StackPointer, StackPointer, -int32_t(sizeof(double))); 1439 if (f.isDouble()) { 1440 as_fst_d(f, StackPointer, 0); 1441 } else { 1442 MOZ_ASSERT(f.isSingle(), "simd128 is not supported"); 1443 as_fst_s(f, StackPointer, 0); 1444 } 1445 } 1446 1447 void MacroAssemblerLOONG64::ma_li(Register dest, ImmGCPtr ptr) { 1448 writeDataRelocation(ptr); 1449 asMasm().ma_liPatchable(dest, ImmPtr(ptr.value)); 1450 } 1451 1452 void MacroAssemblerLOONG64::ma_li(Register dest, Imm32 imm) { 1453 if (is_intN(imm.value, 12)) { 1454 as_addi_w(dest, zero, imm.value); 1455 } else if (is_uintN(imm.value, 12)) { 1456 as_ori(dest, zero, imm.value & 0xfff); 1457 } else { 1458 as_lu12i_w(dest, imm.value >> 12 & 0xfffff); 1459 if (imm.value & 0xfff) { 1460 as_ori(dest, dest, imm.value & 0xfff); 1461 } 1462 } 1463 } 1464 1465 // This method generates lu12i_w and ori instruction pair that can be modified 1466 // by UpdateLuiOriValue, either during compilation (eg. Assembler::bind), or 1467 // during execution (eg. jit::PatchJump). 1468 void MacroAssemblerLOONG64::ma_liPatchable(Register dest, Imm32 imm) { 1469 m_buffer.ensureSpace(2 * sizeof(uint32_t)); 1470 as_lu12i_w(dest, imm.value >> 12 & 0xfffff); 1471 as_ori(dest, dest, imm.value & 0xfff); 1472 } 1473 1474 void MacroAssemblerLOONG64::ma_fmovz(FloatFormat fmt, FloatRegister fd, 1475 FloatRegister fj, Register rk) { 1476 Label done; 1477 ma_b(rk, zero, &done, Assembler::NotEqual); 1478 if (fmt == SingleFloat) { 1479 as_fmov_s(fd, fj); 1480 } else { 1481 as_fmov_d(fd, fj); 1482 } 1483 bind(&done); 1484 } 1485 1486 void MacroAssemblerLOONG64::ma_fmovn(FloatFormat fmt, FloatRegister fd, 1487 FloatRegister fj, Register rk) { 1488 Label done; 1489 ma_b(rk, zero, &done, Assembler::Equal); 1490 if (fmt == SingleFloat) { 1491 as_fmov_s(fd, fj); 1492 } else { 1493 as_fmov_d(fd, fj); 1494 } 1495 bind(&done); 1496 } 1497 1498 void MacroAssemblerLOONG64::ma_and(Register rd, Register rj, Imm32 imm) { 1499 if (is_uintN(imm.value, 12)) { 1500 as_andi(rd, rj, imm.value); 1501 } else if (rd != rj) { 1502 ma_li(rd, imm); 1503 as_and(rd, rj, rd); 1504 } else { 1505 UseScratchRegisterScope temps(asMasm()); 1506 Register scratch = temps.Acquire(); 1507 MOZ_ASSERT(rj != scratch); 1508 ma_li(scratch, imm); 1509 as_and(rd, rj, scratch); 1510 } 1511 } 1512 1513 void MacroAssemblerLOONG64::ma_or(Register rd, Register rj, Imm32 imm) { 1514 if (is_uintN(imm.value, 12)) { 1515 as_ori(rd, rj, imm.value); 1516 } else { 1517 UseScratchRegisterScope temps(asMasm()); 1518 Register scratch = temps.Acquire(); 1519 MOZ_ASSERT(rj != scratch); 1520 ma_li(scratch, imm); 1521 as_or(rd, rj, scratch); 1522 } 1523 } 1524 1525 void MacroAssemblerLOONG64::ma_xor(Register rd, Register rj, Imm32 imm) { 1526 if (is_uintN(imm.value, 12)) { 1527 as_xori(rd, rj, imm.value); 1528 } else { 1529 UseScratchRegisterScope temps(asMasm()); 1530 Register scratch = temps.Acquire(); 1531 MOZ_ASSERT(rj != scratch); 1532 ma_li(scratch, imm); 1533 as_xor(rd, rj, scratch); 1534 } 1535 } 1536 1537 // Arithmetic-based ops. 1538 1539 // Add. 1540 void MacroAssemblerLOONG64::ma_add_w(Register rd, Register rj, Imm32 imm) { 1541 if (is_intN(imm.value, 12)) { 1542 as_addi_w(rd, rj, imm.value); 1543 } else { 1544 UseScratchRegisterScope temps(asMasm()); 1545 Register scratch = temps.Acquire(); 1546 MOZ_ASSERT(rj != scratch); 1547 ma_li(scratch, imm); 1548 as_add_w(rd, rj, scratch); 1549 } 1550 } 1551 1552 void MacroAssemblerLOONG64::ma_add32TestCarry(Condition cond, Register rd, 1553 Register rj, Register rk, 1554 Label* overflow) { 1555 MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear); 1556 MOZ_ASSERT_IF(rd == rj, rk != rd); 1557 UseScratchRegisterScope temps(asMasm()); 1558 Register scratch = temps.Acquire(); 1559 as_add_w(rd, rj, rk); 1560 as_sltu(scratch, rd, rd == rj ? rk : rj); 1561 ma_b(Register(scratch), Register(scratch), overflow, 1562 cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero); 1563 } 1564 1565 void MacroAssemblerLOONG64::ma_add32TestCarry(Condition cond, Register rd, 1566 Register rj, Imm32 imm, 1567 Label* overflow) { 1568 UseScratchRegisterScope temps(asMasm()); 1569 Register scratch = temps.Acquire(); 1570 MOZ_ASSERT(rj != scratch); 1571 ma_li(scratch, imm); 1572 ma_add32TestCarry(cond, rd, rj, scratch, overflow); 1573 } 1574 1575 // Subtract. 1576 void MacroAssemblerLOONG64::ma_sub_w(Register rd, Register rj, Imm32 imm) { 1577 if (is_intN(-imm.value, 12)) { 1578 as_addi_w(rd, rj, -imm.value); 1579 } else { 1580 UseScratchRegisterScope temps(asMasm()); 1581 Register scratch = temps.Acquire(); 1582 MOZ_ASSERT(rj != scratch); 1583 ma_li(scratch, imm); 1584 as_sub_w(rd, rj, scratch); 1585 } 1586 } 1587 1588 void MacroAssemblerLOONG64::ma_sub_w(Register rd, Register rj, Register rk) { 1589 as_sub_w(rd, rj, rk); 1590 } 1591 1592 void MacroAssemblerLOONG64::ma_sub32TestOverflow(Register rd, Register rj, 1593 Imm32 imm, Label* overflow) { 1594 if (imm.value != INT32_MIN) { 1595 asMasm().ma_add32TestOverflow(rd, rj, Imm32(-imm.value), overflow); 1596 } else { 1597 UseScratchRegisterScope temps(asMasm()); 1598 Register scratch = temps.Acquire(); 1599 MOZ_ASSERT(rj != scratch); 1600 ma_li(scratch, Imm32(imm.value)); 1601 asMasm().ma_sub32TestOverflow(rd, rj, scratch, overflow); 1602 } 1603 } 1604 1605 void MacroAssemblerLOONG64::ma_mul(Register rd, Register rj, Imm32 imm) { 1606 UseScratchRegisterScope temps(asMasm()); 1607 Register scratch = temps.Acquire(); 1608 MOZ_ASSERT(rj != scratch); 1609 ma_li(scratch, imm); 1610 as_mul_w(rd, rj, scratch); 1611 } 1612 1613 void MacroAssemblerLOONG64::ma_mul32TestOverflow(Register rd, Register rj, 1614 Register rk, Label* overflow) { 1615 UseScratchRegisterScope temps(asMasm()); 1616 Register scratch = temps.Acquire(); 1617 as_mul_d(rd, rj, rk); 1618 as_slli_w(scratch, rd, 0); 1619 ma_b(rd, scratch, overflow, Assembler::NotEqual); 1620 } 1621 1622 void MacroAssemblerLOONG64::ma_mul32TestOverflow(Register rd, Register rj, 1623 Imm32 imm, Label* overflow) { 1624 UseScratchRegisterScope temps(asMasm()); 1625 Register scratch = temps.Acquire(); 1626 ma_li(scratch, imm); 1627 as_mul_d(rd, rj, scratch); 1628 as_slli_w(scratch, rd, 0); 1629 ma_b(rd, scratch, overflow, Assembler::NotEqual); 1630 } 1631 1632 void MacroAssemblerLOONG64::ma_mod_mask(Register src, Register dest, 1633 Register hold, Register remain, 1634 int32_t shift, Label* negZero) { 1635 // MATH: 1636 // We wish to compute x % (1<<y) - 1 for a known constant, y. 1637 // First, let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit 1638 // dividend as a number in base b, namely 1639 // c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n 1640 // now, since both addition and multiplication commute with modulus, 1641 // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C == 1642 // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)... 1643 // now, since b == C + 1, b % C == 1, and b^n % C == 1 1644 // this means that the whole thing simplifies to: 1645 // c_0 + c_1 + c_2 ... c_n % C 1646 // each c_n can easily be computed by a shift/bitextract, and the modulus 1647 // can be maintained by simply subtracting by C whenever the number gets 1648 // over C. 1649 int32_t mask = (1 << shift) - 1; 1650 Label head, negative, sumSigned, done; 1651 1652 // hold holds -1 if the value was negative, 1 otherwise. 1653 // remain holds the remaining bits that have not been processed 1654 // SecondScratchReg serves as a temporary location to store extracted bits 1655 // into as well as holding the trial subtraction as a temp value dest is 1656 // the accumulator (and holds the final result) 1657 1658 // move the whole value into the remain. 1659 as_or(remain, src, zero); 1660 // Zero out the dest. 1661 ma_li(dest, Imm32(0)); 1662 // Set the hold appropriately. 1663 ma_b(remain, remain, &negative, Signed, ShortJump); 1664 ma_li(hold, Imm32(1)); 1665 ma_b(&head, ShortJump); 1666 1667 bind(&negative); 1668 ma_li(hold, Imm32(-1)); 1669 as_sub_w(remain, zero, remain); 1670 1671 // Begin the main loop. 1672 bind(&head); 1673 1674 UseScratchRegisterScope temps(asMasm()); 1675 Register scratch = temps.Acquire(); 1676 // Extract the bottom bits into SecondScratchReg. 1677 ma_and(scratch, remain, Imm32(mask)); 1678 // Add those bits to the accumulator. 1679 as_add_w(dest, dest, scratch); 1680 // Do a trial subtraction 1681 ma_sub_w(scratch, dest, Imm32(mask)); 1682 // If (sum - C) > 0, store sum - C back into sum, thus performing a 1683 // modulus. 1684 ma_b(scratch, Register(scratch), &sumSigned, Signed, ShortJump); 1685 as_or(dest, scratch, zero); 1686 bind(&sumSigned); 1687 // Get rid of the bits that we extracted before. 1688 as_srli_w(remain, remain, shift); 1689 // If the shift produced zero, finish, otherwise, continue in the loop. 1690 ma_b(remain, remain, &head, NonZero, ShortJump); 1691 // Check the hold to see if we need to negate the result. 1692 ma_b(hold, hold, &done, NotSigned, ShortJump); 1693 1694 if (negZero != nullptr) { 1695 // Jump out in case of negative zero. 1696 ma_b(dest, dest, negZero, Zero); 1697 } 1698 // If the hold was non-zero, negate the result to be in line with 1699 // what JS wants 1700 as_sub_w(dest, zero, dest); 1701 1702 bind(&done); 1703 } 1704 1705 // Memory. 1706 1707 FaultingCodeOffset MacroAssemblerLOONG64::ma_load( 1708 Register dest, const BaseIndex& src, LoadStoreSize size, 1709 LoadStoreExtension extension) { 1710 UseScratchRegisterScope temps(asMasm()); 1711 Register scratch = temps.Acquire(); 1712 asMasm().computeScaledAddress(src, scratch); 1713 return asMasm().ma_load(dest, Address(scratch, src.offset), size, extension); 1714 } 1715 1716 FaultingCodeOffset MacroAssemblerLOONG64::ma_store( 1717 Register data, const BaseIndex& dest, LoadStoreSize size, 1718 LoadStoreExtension extension) { 1719 UseScratchRegisterScope temps(asMasm()); 1720 Register scratch = temps.Acquire(); 1721 asMasm().computeScaledAddress(dest, scratch); 1722 ma_add_d(scratch, scratch, Imm32(dest.offset)); 1723 return asMasm().ma_store(data, Address(scratch, 0), size, extension); 1724 } 1725 1726 void MacroAssemblerLOONG64::ma_store(Imm32 imm, const BaseIndex& dest, 1727 LoadStoreSize size, 1728 LoadStoreExtension extension) { 1729 UseScratchRegisterScope temps(asMasm()); 1730 Register scratch2 = temps.Acquire(); 1731 // Make sure that scratch2 contains absolute address so that offset is 0. 1732 asMasm().computeEffectiveAddress(dest, scratch2); 1733 1734 // Scrach register is free now, use it for loading imm value 1735 Register scratch = temps.Acquire(); 1736 ma_li(scratch, imm); 1737 1738 asMasm().ma_store(scratch, Address(scratch2, 0), size, extension); 1739 } 1740 1741 // Branches when done from within loongarch-specific code. 1742 // TODO(loong64) Optimize ma_b 1743 void MacroAssemblerLOONG64::ma_b(Register lhs, Register rhs, Label* label, 1744 Condition c, JumpKind jumpKind, 1745 Register scratch) { 1746 switch (c) { 1747 case Equal: 1748 case NotEqual: 1749 asMasm().branchWithCode(getBranchCode(lhs, rhs, c), label, jumpKind, 1750 scratch); 1751 break; 1752 case Always: 1753 ma_b(label, jumpKind); 1754 break; 1755 case Zero: 1756 case NonZero: 1757 case Signed: 1758 case NotSigned: 1759 MOZ_ASSERT(lhs == rhs); 1760 asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind, scratch); 1761 break; 1762 default: { 1763 UseScratchRegisterScope temps(*this); 1764 Register scratch = temps.Acquire(); 1765 Condition cond = ma_cmp(scratch, lhs, rhs, c); 1766 asMasm().branchWithCode(getBranchCode(scratch, cond), label, jumpKind, 1767 scratch); 1768 break; 1769 } 1770 } 1771 } 1772 1773 void MacroAssemblerLOONG64::ma_b(Register lhs, Imm32 imm, Label* label, 1774 Condition c, JumpKind jumpKind) { 1775 MOZ_ASSERT(c != Overflow); 1776 if (imm.value == 0) { 1777 if (c == Always || c == AboveOrEqual) { 1778 ma_b(label, jumpKind); 1779 } else if (c == Below) { 1780 ; // This condition is always false. No branch required. 1781 } else { 1782 asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind); 1783 } 1784 } else { 1785 UseScratchRegisterScope temps(asMasm()); 1786 Register scratch = temps.Acquire(); 1787 switch (c) { 1788 case Equal: 1789 case NotEqual: 1790 MOZ_ASSERT(lhs != scratch); 1791 ma_li(scratch, imm); 1792 ma_b(lhs, scratch, label, c, jumpKind); 1793 break; 1794 default: 1795 Condition cond = ma_cmp(scratch, lhs, imm, c); 1796 asMasm().branchWithCode(getBranchCode(scratch, cond), label, jumpKind); 1797 } 1798 } 1799 } 1800 1801 void MacroAssemblerLOONG64::ma_b(Register lhs, ImmPtr imm, Label* l, 1802 Condition c, JumpKind jumpKind) { 1803 asMasm().ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind); 1804 } 1805 1806 void MacroAssemblerLOONG64::ma_b(Label* label, JumpKind jumpKind) { 1807 asMasm().branchWithCode(getBranchCode(BranchIsJump), label, jumpKind); 1808 } 1809 1810 Assembler::Condition MacroAssemblerLOONG64::ma_cmp(Register dest, Register lhs, 1811 Register rhs, Condition c) { 1812 switch (c) { 1813 case Above: 1814 // bgtu s,t,label => 1815 // sltu at,t,s 1816 // bne at,$zero,offs 1817 as_sltu(dest, rhs, lhs); 1818 return NotEqual; 1819 case AboveOrEqual: 1820 // bgeu s,t,label => 1821 // sltu at,s,t 1822 // beq at,$zero,offs 1823 as_sltu(dest, lhs, rhs); 1824 return Equal; 1825 case Below: 1826 // bltu s,t,label => 1827 // sltu at,s,t 1828 // bne at,$zero,offs 1829 as_sltu(dest, lhs, rhs); 1830 return NotEqual; 1831 case BelowOrEqual: 1832 // bleu s,t,label => 1833 // sltu at,t,s 1834 // beq at,$zero,offs 1835 as_sltu(dest, rhs, lhs); 1836 return Equal; 1837 case GreaterThan: 1838 // bgt s,t,label => 1839 // slt at,t,s 1840 // bne at,$zero,offs 1841 as_slt(dest, rhs, lhs); 1842 return NotEqual; 1843 case GreaterThanOrEqual: 1844 // bge s,t,label => 1845 // slt at,s,t 1846 // beq at,$zero,offs 1847 as_slt(dest, lhs, rhs); 1848 return Equal; 1849 case LessThan: 1850 // blt s,t,label => 1851 // slt at,s,t 1852 // bne at,$zero,offs 1853 as_slt(dest, lhs, rhs); 1854 return NotEqual; 1855 case LessThanOrEqual: 1856 // ble s,t,label => 1857 // slt at,t,s 1858 // beq at,$zero,offs 1859 as_slt(dest, rhs, lhs); 1860 return Equal; 1861 default: 1862 MOZ_CRASH("Invalid condition."); 1863 } 1864 return Always; 1865 } 1866 1867 Assembler::Condition MacroAssemblerLOONG64::ma_cmp(Register dest, Register lhs, 1868 Imm32 imm, Condition c) { 1869 UseScratchRegisterScope temps(*this); 1870 1871 switch (c) { 1872 case Above: 1873 case BelowOrEqual: 1874 if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12) && 1875 imm.value != -1) { 1876 // lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow 1877 as_sltui(dest, lhs, imm.value + 1); 1878 1879 return (c == BelowOrEqual ? NotEqual : Equal); 1880 } else { 1881 Register scratch = dest == lhs ? temps.Acquire() : dest; 1882 ma_li(scratch, imm); 1883 as_sltu(dest, scratch, lhs); 1884 return (c == BelowOrEqual ? Equal : NotEqual); 1885 } 1886 case AboveOrEqual: 1887 case Below: 1888 if (is_intN(imm.value, 12)) { 1889 as_sltui(dest, lhs, imm.value); 1890 } else { 1891 Register scratch = dest == lhs ? temps.Acquire() : dest; 1892 ma_li(scratch, imm); 1893 as_sltu(dest, lhs, scratch); 1894 } 1895 return (c == AboveOrEqual ? Equal : NotEqual); 1896 case GreaterThan: 1897 case LessThanOrEqual: 1898 if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12)) { 1899 // lhs <= rhs via lhs < rhs + 1. 1900 as_slti(dest, lhs, imm.value + 1); 1901 return (c == LessThanOrEqual ? NotEqual : Equal); 1902 } else { 1903 Register scratch = dest == lhs ? temps.Acquire() : dest; 1904 ma_li(scratch, imm); 1905 as_slt(dest, scratch, lhs); 1906 return (c == LessThanOrEqual ? Equal : NotEqual); 1907 } 1908 case GreaterThanOrEqual: 1909 case LessThan: 1910 if (is_intN(imm.value, 12)) { 1911 as_slti(dest, lhs, imm.value); 1912 } else { 1913 Register scratch = dest == lhs ? temps.Acquire() : dest; 1914 ma_li(scratch, imm); 1915 as_slt(dest, lhs, scratch); 1916 } 1917 return (c == GreaterThanOrEqual ? Equal : NotEqual); 1918 default: 1919 MOZ_CRASH("Invalid condition."); 1920 } 1921 return Always; 1922 } 1923 1924 // fp instructions 1925 void MacroAssemblerLOONG64::ma_lis(FloatRegister dest, float value) { 1926 Imm32 imm(mozilla::BitwiseCast<uint32_t>(value)); 1927 1928 if (imm.value != 0) { 1929 UseScratchRegisterScope temps(asMasm()); 1930 Register scratch = temps.Acquire(); 1931 ma_li(scratch, imm); 1932 moveToFloat32(scratch, dest); 1933 } else { 1934 moveToFloat32(zero, dest); 1935 } 1936 } 1937 1938 FaultingCodeOffset MacroAssemblerLOONG64::ma_fst_d(FloatRegister ft, 1939 BaseIndex address) { 1940 UseScratchRegisterScope temps(asMasm()); 1941 Register scratch = temps.Acquire(); 1942 asMasm().computeScaledAddress(address, scratch); 1943 return asMasm().ma_fst_d(ft, Address(scratch, address.offset)); 1944 } 1945 1946 FaultingCodeOffset MacroAssemblerLOONG64::ma_fst_s(FloatRegister ft, 1947 BaseIndex address) { 1948 UseScratchRegisterScope temps(asMasm()); 1949 Register scratch = temps.Acquire(); 1950 asMasm().computeScaledAddress(address, scratch); 1951 return asMasm().ma_fst_s(ft, Address(scratch, address.offset)); 1952 } 1953 1954 FaultingCodeOffset MacroAssemblerLOONG64::ma_fld_d(FloatRegister ft, 1955 const BaseIndex& src) { 1956 UseScratchRegisterScope temps(asMasm()); 1957 Register scratch = temps.Acquire(); 1958 asMasm().computeScaledAddress(src, scratch); 1959 return asMasm().ma_fld_d(ft, Address(scratch, src.offset)); 1960 } 1961 1962 FaultingCodeOffset MacroAssemblerLOONG64::ma_fld_s(FloatRegister ft, 1963 const BaseIndex& src) { 1964 UseScratchRegisterScope temps(asMasm()); 1965 Register scratch = temps.Acquire(); 1966 asMasm().computeScaledAddress(src, scratch); 1967 return asMasm().ma_fld_s(ft, Address(scratch, src.offset)); 1968 } 1969 1970 void MacroAssemblerLOONG64::ma_bc_s(FloatRegister lhs, FloatRegister rhs, 1971 Label* label, DoubleCondition c, 1972 JumpKind jumpKind, FPConditionBit fcc) { 1973 compareFloatingPoint(SingleFloat, lhs, rhs, c, fcc); 1974 asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind); 1975 } 1976 1977 void MacroAssemblerLOONG64::ma_bc_d(FloatRegister lhs, FloatRegister rhs, 1978 Label* label, DoubleCondition c, 1979 JumpKind jumpKind, FPConditionBit fcc) { 1980 compareFloatingPoint(DoubleFloat, lhs, rhs, c, fcc); 1981 asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind); 1982 } 1983 1984 void MacroAssemblerLOONG64::ma_call(ImmPtr dest) { 1985 asMasm().ma_liPatchable(CallReg, dest); 1986 as_jirl(ra, CallReg, BOffImm16(0)); 1987 } 1988 1989 void MacroAssemblerLOONG64::ma_jump(ImmPtr dest) { 1990 UseScratchRegisterScope temps(asMasm()); 1991 Register scratch = temps.Acquire(); 1992 asMasm().ma_liPatchable(scratch, dest); 1993 as_jirl(zero, scratch, BOffImm16(0)); 1994 } 1995 1996 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, Register rk, 1997 Condition c) { 1998 switch (c) { 1999 case Equal: 2000 // seq d,s,t => 2001 // xor d,s,t 2002 // sltiu d,d,1 2003 as_xor(rd, rj, rk); 2004 as_sltui(rd, rd, 1); 2005 break; 2006 case NotEqual: 2007 // sne d,s,t => 2008 // xor d,s,t 2009 // sltu d,$zero,d 2010 as_xor(rd, rj, rk); 2011 as_sltu(rd, zero, rd); 2012 break; 2013 case Above: 2014 // sgtu d,s,t => 2015 // sltu d,t,s 2016 as_sltu(rd, rk, rj); 2017 break; 2018 case AboveOrEqual: 2019 // sgeu d,s,t => 2020 // sltu d,s,t 2021 // xori d,d,1 2022 as_sltu(rd, rj, rk); 2023 as_xori(rd, rd, 1); 2024 break; 2025 case Below: 2026 // sltu d,s,t 2027 as_sltu(rd, rj, rk); 2028 break; 2029 case BelowOrEqual: 2030 // sleu d,s,t => 2031 // sltu d,t,s 2032 // xori d,d,1 2033 as_sltu(rd, rk, rj); 2034 as_xori(rd, rd, 1); 2035 break; 2036 case GreaterThan: 2037 // sgt d,s,t => 2038 // slt d,t,s 2039 as_slt(rd, rk, rj); 2040 break; 2041 case GreaterThanOrEqual: 2042 // sge d,s,t => 2043 // slt d,s,t 2044 // xori d,d,1 2045 as_slt(rd, rj, rk); 2046 as_xori(rd, rd, 1); 2047 break; 2048 case LessThan: 2049 // slt d,s,t 2050 as_slt(rd, rj, rk); 2051 break; 2052 case LessThanOrEqual: 2053 // sle d,s,t => 2054 // slt d,t,s 2055 // xori d,d,1 2056 as_slt(rd, rk, rj); 2057 as_xori(rd, rd, 1); 2058 break; 2059 case Zero: 2060 MOZ_ASSERT(rj == rk); 2061 // seq d,s,$zero => 2062 // sltiu d,s,1 2063 as_sltui(rd, rj, 1); 2064 break; 2065 case NonZero: 2066 MOZ_ASSERT(rj == rk); 2067 // sne d,s,$zero => 2068 // sltu d,$zero,s 2069 as_sltu(rd, zero, rj); 2070 break; 2071 case Signed: 2072 MOZ_ASSERT(rj == rk); 2073 as_slt(rd, rj, zero); 2074 break; 2075 case NotSigned: 2076 MOZ_ASSERT(rj == rk); 2077 // sge d,s,$zero => 2078 // slt d,s,$zero 2079 // xori d,d,1 2080 as_slt(rd, rj, zero); 2081 as_xori(rd, rd, 1); 2082 break; 2083 default: 2084 MOZ_CRASH("Invalid condition."); 2085 } 2086 } 2087 2088 void MacroAssemblerLOONG64::ma_cmp_set_double(Register dest, FloatRegister lhs, 2089 FloatRegister rhs, 2090 DoubleCondition c) { 2091 compareFloatingPoint(DoubleFloat, lhs, rhs, c); 2092 as_movcf2gr(dest, FCC0); 2093 } 2094 2095 void MacroAssemblerLOONG64::ma_cmp_set_float32(Register dest, FloatRegister lhs, 2096 FloatRegister rhs, 2097 DoubleCondition c) { 2098 compareFloatingPoint(SingleFloat, lhs, rhs, c); 2099 as_movcf2gr(dest, FCC0); 2100 } 2101 2102 void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, Imm32 imm, 2103 Condition c) { 2104 if (imm.value == 0) { 2105 switch (c) { 2106 case Equal: 2107 case BelowOrEqual: 2108 as_sltui(rd, rj, 1); 2109 break; 2110 case NotEqual: 2111 case Above: 2112 as_sltu(rd, zero, rj); 2113 break; 2114 case AboveOrEqual: 2115 case Below: 2116 as_ori(rd, zero, c == AboveOrEqual ? 1 : 0); 2117 break; 2118 case GreaterThan: 2119 case LessThanOrEqual: 2120 as_slt(rd, zero, rj); 2121 if (c == LessThanOrEqual) { 2122 as_xori(rd, rd, 1); 2123 } 2124 break; 2125 case LessThan: 2126 case GreaterThanOrEqual: 2127 as_slt(rd, rj, zero); 2128 if (c == GreaterThanOrEqual) { 2129 as_xori(rd, rd, 1); 2130 } 2131 break; 2132 case Zero: 2133 as_sltui(rd, rj, 1); 2134 break; 2135 case NonZero: 2136 as_sltu(rd, zero, rj); 2137 break; 2138 case Signed: 2139 as_slt(rd, rj, zero); 2140 break; 2141 case NotSigned: 2142 as_slt(rd, rj, zero); 2143 as_xori(rd, rd, 1); 2144 break; 2145 default: 2146 MOZ_CRASH("Invalid condition."); 2147 } 2148 return; 2149 } 2150 2151 switch (c) { 2152 case Equal: 2153 case NotEqual: 2154 ma_xor(rd, rj, imm); 2155 if (c == Equal) { 2156 as_sltui(rd, rd, 1); 2157 } else { 2158 as_sltu(rd, zero, rd); 2159 } 2160 break; 2161 case Zero: 2162 case NonZero: 2163 case Signed: 2164 case NotSigned: 2165 MOZ_CRASH("Invalid condition."); 2166 default: 2167 Condition cond = ma_cmp(rd, rj, imm, c); 2168 MOZ_ASSERT(cond == Equal || cond == NotEqual); 2169 2170 if (cond == Equal) as_xori(rd, rd, 1); 2171 } 2172 } 2173 2174 void MacroAssemblerLOONG64::compareFloatingPoint(FloatFormat fmt, 2175 FloatRegister lhs, 2176 FloatRegister rhs, 2177 DoubleCondition c, 2178 FPConditionBit fcc) { 2179 switch (c) { 2180 case DoubleOrdered: 2181 as_fcmp_cor(fmt, lhs, rhs, fcc); 2182 break; 2183 case DoubleEqual: 2184 as_fcmp_ceq(fmt, lhs, rhs, fcc); 2185 break; 2186 case DoubleNotEqual: 2187 as_fcmp_cne(fmt, lhs, rhs, fcc); 2188 break; 2189 case DoubleGreaterThan: 2190 as_fcmp_clt(fmt, rhs, lhs, fcc); 2191 break; 2192 case DoubleGreaterThanOrEqual: 2193 as_fcmp_cle(fmt, rhs, lhs, fcc); 2194 break; 2195 case DoubleLessThan: 2196 as_fcmp_clt(fmt, lhs, rhs, fcc); 2197 break; 2198 case DoubleLessThanOrEqual: 2199 as_fcmp_cle(fmt, lhs, rhs, fcc); 2200 break; 2201 case DoubleUnordered: 2202 as_fcmp_cun(fmt, lhs, rhs, fcc); 2203 break; 2204 case DoubleEqualOrUnordered: 2205 as_fcmp_cueq(fmt, lhs, rhs, fcc); 2206 break; 2207 case DoubleNotEqualOrUnordered: 2208 as_fcmp_cune(fmt, lhs, rhs, fcc); 2209 break; 2210 case DoubleGreaterThanOrUnordered: 2211 as_fcmp_cult(fmt, rhs, lhs, fcc); 2212 break; 2213 case DoubleGreaterThanOrEqualOrUnordered: 2214 as_fcmp_cule(fmt, rhs, lhs, fcc); 2215 break; 2216 case DoubleLessThanOrUnordered: 2217 as_fcmp_cult(fmt, lhs, rhs, fcc); 2218 break; 2219 case DoubleLessThanOrEqualOrUnordered: 2220 as_fcmp_cule(fmt, lhs, rhs, fcc); 2221 break; 2222 default: 2223 MOZ_CRASH("Invalid DoubleCondition."); 2224 } 2225 } 2226 2227 void MacroAssemblerLOONG64::minMaxPtr(Register lhs, Register rhs, Register dest, 2228 bool isMax) { 2229 UseScratchRegisterScope temps(asMasm()); 2230 Register scratch = temps.Acquire(); 2231 Register scratch2 = temps.Acquire(); 2232 2233 as_slt(scratch, rhs, lhs); 2234 if (isMax) { 2235 as_masknez(scratch2, rhs, scratch); 2236 as_maskeqz(dest, lhs, scratch); 2237 } else { 2238 as_masknez(scratch2, lhs, scratch); 2239 as_maskeqz(dest, rhs, scratch); 2240 } 2241 as_or(dest, dest, scratch2); 2242 } 2243 2244 void MacroAssemblerLOONG64::minMaxPtr(Register lhs, ImmWord rhs, Register dest, 2245 bool isMax) { 2246 UseScratchRegisterScope temps(asMasm()); 2247 Register scratch = temps.Acquire(); 2248 Register scratch2 = temps.Acquire(); 2249 2250 ma_li(scratch2, rhs); 2251 2252 as_slt(scratch, scratch2, lhs); 2253 if (isMax) { 2254 as_masknez(scratch2, scratch2, scratch); 2255 as_maskeqz(dest, lhs, scratch); 2256 } else { 2257 as_maskeqz(scratch2, scratch2, scratch); 2258 as_masknez(dest, lhs, scratch); 2259 } 2260 as_or(dest, dest, scratch2); 2261 } 2262 2263 void MacroAssemblerLOONG64::minMaxDouble(FloatRegister srcDest, 2264 FloatRegister second, bool handleNaN, 2265 bool isMax) { 2266 if (srcDest == second) return; 2267 2268 Label nan, done; 2269 2270 // First or second is NaN, result is NaN. 2271 ma_bc_d(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump); 2272 if (isMax) { 2273 as_fmax_d(srcDest, srcDest, second); 2274 } else { 2275 as_fmin_d(srcDest, srcDest, second); 2276 } 2277 ma_b(&done, ShortJump); 2278 2279 bind(&nan); 2280 as_fadd_d(srcDest, srcDest, second); 2281 2282 bind(&done); 2283 } 2284 2285 void MacroAssemblerLOONG64::minMaxFloat32(FloatRegister srcDest, 2286 FloatRegister second, bool handleNaN, 2287 bool isMax) { 2288 if (srcDest == second) return; 2289 2290 Label nan, done; 2291 2292 // First or second is NaN, result is NaN. 2293 ma_bc_s(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump); 2294 if (isMax) { 2295 as_fmax_s(srcDest, srcDest, second); 2296 } else { 2297 as_fmin_s(srcDest, srcDest, second); 2298 } 2299 ma_b(&done, ShortJump); 2300 2301 bind(&nan); 2302 as_fadd_s(srcDest, srcDest, second); 2303 2304 bind(&done); 2305 } 2306 2307 FaultingCodeOffset MacroAssemblerLOONG64::loadDouble(const Address& address, 2308 FloatRegister dest) { 2309 return asMasm().ma_fld_d(dest, address); 2310 } 2311 2312 FaultingCodeOffset MacroAssemblerLOONG64::loadDouble(const BaseIndex& src, 2313 FloatRegister dest) { 2314 return asMasm().ma_fld_d(dest, src); 2315 } 2316 2317 FaultingCodeOffset MacroAssemblerLOONG64::loadFloat32(const Address& address, 2318 FloatRegister dest) { 2319 return asMasm().ma_fld_s(dest, address); 2320 } 2321 2322 FaultingCodeOffset MacroAssemblerLOONG64::loadFloat32(const BaseIndex& src, 2323 FloatRegister dest) { 2324 return asMasm().ma_fld_s(dest, src); 2325 } 2326 2327 void MacroAssemblerLOONG64::wasmLoadImpl(const wasm::MemoryAccessDesc& access, 2328 Register memoryBase, Register ptr, 2329 Register ptrScratch, 2330 AnyRegister output, Register tmp) { 2331 access.assertOffsetInGuardPages(); 2332 uint32_t offset = access.offset32(); 2333 MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); 2334 2335 // Maybe add the offset. 2336 if (offset) { 2337 asMasm().addPtr(ImmWord(offset), ptrScratch); 2338 ptr = ptrScratch; 2339 } 2340 2341 asMasm().memoryBarrierBefore(access.sync()); 2342 append(access, wasm::TrapMachineInsnForLoad(byteSize(access.type())), 2343 FaultingCodeOffset(currentOffset())); 2344 2345 switch (access.type()) { 2346 case Scalar::Int8: 2347 as_ldx_b(output.gpr(), memoryBase, ptr); 2348 break; 2349 case Scalar::Uint8: 2350 as_ldx_bu(output.gpr(), memoryBase, ptr); 2351 break; 2352 case Scalar::Int16: 2353 as_ldx_h(output.gpr(), memoryBase, ptr); 2354 break; 2355 case Scalar::Uint16: 2356 as_ldx_hu(output.gpr(), memoryBase, ptr); 2357 break; 2358 case Scalar::Int32: 2359 case Scalar::Uint32: 2360 as_ldx_w(output.gpr(), memoryBase, ptr); 2361 break; 2362 case Scalar::Float64: 2363 as_fldx_d(output.fpu(), memoryBase, ptr); 2364 break; 2365 case Scalar::Float32: 2366 as_fldx_s(output.fpu(), memoryBase, ptr); 2367 break; 2368 default: 2369 MOZ_CRASH("unexpected array type"); 2370 } 2371 2372 asMasm().memoryBarrierAfter(access.sync()); 2373 } 2374 2375 void MacroAssemblerLOONG64::wasmStoreImpl(const wasm::MemoryAccessDesc& access, 2376 AnyRegister value, 2377 Register memoryBase, Register ptr, 2378 Register ptrScratch, Register tmp) { 2379 access.assertOffsetInGuardPages(); 2380 uint32_t offset = access.offset32(); 2381 MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); 2382 2383 // Maybe add the offset. 2384 if (offset) { 2385 asMasm().addPtr(ImmWord(offset), ptrScratch); 2386 ptr = ptrScratch; 2387 } 2388 2389 asMasm().memoryBarrierBefore(access.sync()); 2390 // The next emitted instruction is a memory access. 2391 append(access, wasm::TrapMachineInsnForStore(byteSize(access.type())), 2392 FaultingCodeOffset(currentOffset())); 2393 2394 switch (access.type()) { 2395 case Scalar::Int8: 2396 case Scalar::Uint8: 2397 as_stx_b(value.gpr(), memoryBase, ptr); 2398 break; 2399 case Scalar::Int16: 2400 case Scalar::Uint16: 2401 as_stx_h(value.gpr(), memoryBase, ptr); 2402 break; 2403 case Scalar::Int32: 2404 case Scalar::Uint32: 2405 as_stx_w(value.gpr(), memoryBase, ptr); 2406 break; 2407 case Scalar::Int64: 2408 as_stx_d(value.gpr(), memoryBase, ptr); 2409 break; 2410 case Scalar::Float64: 2411 as_fstx_d(value.fpu(), memoryBase, ptr); 2412 break; 2413 case Scalar::Float32: 2414 as_fstx_s(value.fpu(), memoryBase, ptr); 2415 break; 2416 default: 2417 MOZ_CRASH("unexpected array type"); 2418 } 2419 2420 asMasm().memoryBarrierAfter(access.sync()); 2421 } 2422 2423 void MacroAssemblerLOONG64Compat::wasmLoadI64Impl( 2424 const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, 2425 Register ptrScratch, Register64 output, Register tmp) { 2426 uint32_t offset = access.offset32(); 2427 MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); 2428 2429 // Maybe add the offset. 2430 if (offset) { 2431 asMasm().addPtr(ImmWord(offset), ptrScratch); 2432 ptr = ptrScratch; 2433 } 2434 2435 asMasm().memoryBarrierBefore(access.sync()); 2436 append(access, wasm::TrapMachineInsnForLoad(byteSize(access.type())), 2437 FaultingCodeOffset(currentOffset())); 2438 2439 switch (access.type()) { 2440 case Scalar::Int8: 2441 as_ldx_b(output.reg, memoryBase, ptr); 2442 break; 2443 case Scalar::Uint8: 2444 as_ldx_bu(output.reg, memoryBase, ptr); 2445 break; 2446 case Scalar::Int16: 2447 as_ldx_h(output.reg, memoryBase, ptr); 2448 break; 2449 case Scalar::Uint16: 2450 as_ldx_hu(output.reg, memoryBase, ptr); 2451 break; 2452 case Scalar::Int32: 2453 as_ldx_w(output.reg, memoryBase, ptr); 2454 break; 2455 case Scalar::Uint32: 2456 // TODO(loong64): Why need zero-extension here? 2457 as_ldx_wu(output.reg, memoryBase, ptr); 2458 break; 2459 case Scalar::Int64: 2460 as_ldx_d(output.reg, memoryBase, ptr); 2461 break; 2462 default: 2463 MOZ_CRASH("unexpected array type"); 2464 } 2465 2466 asMasm().memoryBarrierAfter(access.sync()); 2467 } 2468 2469 void MacroAssemblerLOONG64Compat::wasmStoreI64Impl( 2470 const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase, 2471 Register ptr, Register ptrScratch, Register tmp) { 2472 uint32_t offset = access.offset32(); 2473 MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg); 2474 2475 // Maybe add the offset. 2476 if (offset) { 2477 asMasm().addPtr(ImmWord(offset), ptrScratch); 2478 ptr = ptrScratch; 2479 } 2480 2481 asMasm().memoryBarrierBefore(access.sync()); 2482 append(access, wasm::TrapMachineInsnForStore(byteSize(access.type())), 2483 FaultingCodeOffset(currentOffset())); 2484 2485 switch (access.type()) { 2486 case Scalar::Int8: 2487 case Scalar::Uint8: 2488 as_stx_b(value.reg, memoryBase, ptr); 2489 break; 2490 case Scalar::Int16: 2491 case Scalar::Uint16: 2492 as_stx_h(value.reg, memoryBase, ptr); 2493 break; 2494 case Scalar::Int32: 2495 case Scalar::Uint32: 2496 as_stx_w(value.reg, memoryBase, ptr); 2497 break; 2498 case Scalar::Int64: 2499 as_stx_d(value.reg, memoryBase, ptr); 2500 break; 2501 default: 2502 MOZ_CRASH("unexpected array type"); 2503 } 2504 2505 asMasm().memoryBarrierAfter(access.sync()); 2506 } 2507 2508 void MacroAssemblerLOONG64::outOfLineWasmTruncateToInt32Check( 2509 FloatRegister input, Register output, MIRType fromType, TruncFlags flags, 2510 Label* rejoin, const wasm::TrapSiteDesc& trapSiteDesc) { 2511 bool isUnsigned = flags & TRUNC_UNSIGNED; 2512 bool isSaturating = flags & TRUNC_SATURATING; 2513 2514 if (isSaturating) { 2515 ScratchDoubleScope fpscratch(asMasm()); 2516 if (fromType == MIRType::Double) { 2517 asMasm().loadConstantDouble(0.0, fpscratch); 2518 } else { 2519 asMasm().loadConstantFloat32(0.0f, fpscratch); 2520 } 2521 2522 if (isUnsigned) { 2523 ma_li(output, Imm32(UINT32_MAX)); 2524 2525 compareFloatingPoint( 2526 fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, 2527 fpscratch, Assembler::DoubleLessThanOrUnordered); 2528 2529 UseScratchRegisterScope temps(asMasm()); 2530 Register scratch = temps.Acquire(); 2531 as_movcf2gr(scratch, FCC0); 2532 // FCC0 = 1, output = zero; else not change. 2533 as_masknez(output, output, scratch); 2534 } else { 2535 // Positive overflow is already saturated to INT32_MAX, so we only have 2536 // to handle NaN and negative overflow here. 2537 2538 compareFloatingPoint( 2539 fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input, 2540 Assembler::DoubleLessThanOrUnordered); 2541 2542 UseScratchRegisterScope temps(asMasm()); 2543 Register scratch = temps.Acquire(); 2544 as_movcf2gr(scratch, FCC0); 2545 // FCC0 = 1, output = zero; else not change. 2546 as_masknez(output, output, scratch); 2547 2548 compareFloatingPoint( 2549 fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, 2550 fpscratch, Assembler::DoubleLessThan); 2551 2552 as_movcf2gr(scratch, FCC0); 2553 // FCC0 == 1, move INT32_MIN to output; else not change. 2554 as_slli_w(scratch, scratch, 31); 2555 as_or(output, output, scratch); 2556 } 2557 2558 MOZ_ASSERT(rejoin->bound()); 2559 asMasm().jump(rejoin); 2560 return; 2561 } 2562 2563 Label inputIsNaN; 2564 2565 if (fromType == MIRType::Double) { 2566 asMasm().branchDouble(Assembler::DoubleUnordered, input, input, 2567 &inputIsNaN); 2568 } else if (fromType == MIRType::Float32) { 2569 asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN); 2570 } 2571 2572 asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc); 2573 asMasm().bind(&inputIsNaN); 2574 asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc); 2575 } 2576 2577 void MacroAssemblerLOONG64::outOfLineWasmTruncateToInt64Check( 2578 FloatRegister input, Register64 output_, MIRType fromType, TruncFlags flags, 2579 Label* rejoin, const wasm::TrapSiteDesc& trapSiteDesc) { 2580 bool isUnsigned = flags & TRUNC_UNSIGNED; 2581 bool isSaturating = flags & TRUNC_SATURATING; 2582 2583 if (isSaturating) { 2584 ScratchDoubleScope fpscratch(asMasm()); 2585 Register output = output_.reg; 2586 2587 if (fromType == MIRType::Double) { 2588 asMasm().loadConstantDouble(0.0, fpscratch); 2589 } else { 2590 asMasm().loadConstantFloat32(0.0f, fpscratch); 2591 } 2592 2593 if (isUnsigned) { 2594 asMasm().ma_li(output, ImmWord(UINT64_MAX)); 2595 2596 compareFloatingPoint( 2597 fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, 2598 fpscratch, Assembler::DoubleLessThanOrUnordered); 2599 2600 UseScratchRegisterScope temps(asMasm()); 2601 Register scratch = temps.Acquire(); 2602 as_movcf2gr(scratch, FCC0); 2603 // FCC0 = 1, output = zero; else not change. 2604 as_masknez(output, output, scratch); 2605 } else { 2606 // Positive overflow is already saturated to INT64_MAX, so we only have 2607 // to handle NaN and negative overflow here. 2608 2609 compareFloatingPoint( 2610 fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input, 2611 Assembler::DoubleLessThanOrUnordered); 2612 2613 UseScratchRegisterScope temps(asMasm()); 2614 Register scratch = temps.Acquire(); 2615 as_movcf2gr(scratch, FCC0); 2616 // FCC0 = 1, output = zero; else not change. 2617 as_masknez(output, output, scratch); 2618 2619 compareFloatingPoint( 2620 fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, 2621 fpscratch, Assembler::DoubleLessThan); 2622 2623 as_movcf2gr(scratch, FCC0); 2624 // FCC0 == 1, move INT64_MIN to output; else not change. 2625 as_slli_d(scratch, scratch, 63); 2626 as_or(output, output, scratch); 2627 } 2628 2629 MOZ_ASSERT(rejoin->bound()); 2630 asMasm().jump(rejoin); 2631 return; 2632 } 2633 2634 Label inputIsNaN; 2635 2636 if (fromType == MIRType::Double) { 2637 asMasm().branchDouble(Assembler::DoubleUnordered, input, input, 2638 &inputIsNaN); 2639 } else if (fromType == MIRType::Float32) { 2640 asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN); 2641 } 2642 2643 asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapSiteDesc); 2644 asMasm().bind(&inputIsNaN); 2645 asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapSiteDesc); 2646 } 2647 2648 void MacroAssemblerLOONG64Compat::profilerEnterFrame(Register framePtr, 2649 Register scratch) { 2650 asMasm().loadJSContext(scratch); 2651 loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch); 2652 storePtr(framePtr, 2653 Address(scratch, JitActivation::offsetOfLastProfilingFrame())); 2654 storePtr(ImmPtr(nullptr), 2655 Address(scratch, JitActivation::offsetOfLastProfilingCallSite())); 2656 } 2657 2658 void MacroAssemblerLOONG64Compat::profilerExitFrame() { 2659 jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail()); 2660 } 2661 2662 MacroAssembler& MacroAssemblerLOONG64::asMasm() { 2663 return *static_cast<MacroAssembler*>(this); 2664 } 2665 2666 const MacroAssembler& MacroAssemblerLOONG64::asMasm() const { 2667 return *static_cast<const MacroAssembler*>(this); 2668 } 2669 2670 void MacroAssembler::subFromStackPtr(Imm32 imm32) { 2671 if (imm32.value) { 2672 asMasm().subPtr(imm32, StackPointer); 2673 } 2674 } 2675 2676 //{{{ check_macroassembler_style 2677 // =============================================================== 2678 // MacroAssembler high-level usage. 2679 2680 void MacroAssembler::flush() {} 2681 2682 // =============================================================== 2683 // Stack manipulation functions. 2684 2685 size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) { 2686 return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); 2687 } 2688 2689 void MacroAssembler::PushRegsInMask(LiveRegisterSet set) { 2690 int32_t diff = 2691 set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); 2692 const int32_t reserved = diff; 2693 2694 reserveStack(reserved); 2695 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { 2696 diff -= sizeof(intptr_t); 2697 storePtr(*iter, Address(StackPointer, diff)); 2698 } 2699 2700 #ifdef ENABLE_WASM_SIMD 2701 # error "Needs more careful logic if SIMD is enabled" 2702 #endif 2703 2704 for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); 2705 iter.more(); ++iter) { 2706 diff -= sizeof(double); 2707 storeDouble(*iter, Address(StackPointer, diff)); 2708 } 2709 MOZ_ASSERT(diff == 0); 2710 } 2711 2712 void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set, 2713 LiveRegisterSet ignore) { 2714 int32_t diff = 2715 set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); 2716 const int32_t reserved = diff; 2717 2718 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { 2719 diff -= sizeof(intptr_t); 2720 if (!ignore.has(*iter)) { 2721 loadPtr(Address(StackPointer, diff), *iter); 2722 } 2723 } 2724 2725 #ifdef ENABLE_WASM_SIMD 2726 # error "Needs more careful logic if SIMD is enabled" 2727 #endif 2728 2729 for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); 2730 iter.more(); ++iter) { 2731 diff -= sizeof(double); 2732 if (!ignore.has(*iter)) { 2733 loadDouble(Address(StackPointer, diff), *iter); 2734 } 2735 } 2736 MOZ_ASSERT(diff == 0); 2737 freeStack(reserved); 2738 } 2739 2740 void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, 2741 Register) { 2742 FloatRegisterSet fpuSet(set.fpus().reduceSetForPush()); 2743 mozilla::DebugOnly<unsigned> numFpu = fpuSet.size(); 2744 int32_t diffF = fpuSet.getPushSizeInBytes(); 2745 mozilla::DebugOnly<int32_t> diffG = set.gprs().size() * sizeof(intptr_t); 2746 2747 MOZ_ASSERT(dest.offset >= diffG + diffF); 2748 2749 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { 2750 diffG -= sizeof(intptr_t); 2751 dest.offset -= sizeof(intptr_t); 2752 storePtr(*iter, dest); 2753 } 2754 MOZ_ASSERT(diffG == 0); 2755 2756 #ifdef ENABLE_WASM_SIMD 2757 # error "Needs more careful logic if SIMD is enabled" 2758 #endif 2759 2760 for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) { 2761 FloatRegister reg = *iter; 2762 diffF -= reg.size(); 2763 numFpu -= 1; 2764 dest.offset -= reg.size(); 2765 if (reg.isDouble()) { 2766 storeDouble(reg, dest); 2767 } else if (reg.isSingle()) { 2768 storeFloat32(reg, dest); 2769 } else { 2770 MOZ_CRASH("Unknown register type."); 2771 } 2772 } 2773 MOZ_ASSERT(numFpu == 0); 2774 2775 diffF -= diffF % sizeof(uintptr_t); 2776 MOZ_ASSERT(diffF == 0); 2777 } 2778 2779 void MacroAssembler::Push(Register reg) { 2780 push(reg); 2781 adjustFrame(int32_t(sizeof(intptr_t))); 2782 } 2783 2784 void MacroAssembler::Push(const Imm32 imm) { 2785 push(imm); 2786 adjustFrame(int32_t(sizeof(intptr_t))); 2787 } 2788 2789 void MacroAssembler::Push(const ImmWord imm) { 2790 push(imm); 2791 adjustFrame(int32_t(sizeof(intptr_t))); 2792 } 2793 2794 void MacroAssembler::Push(const ImmPtr imm) { 2795 Push(ImmWord(uintptr_t(imm.value))); 2796 } 2797 2798 void MacroAssembler::Push(const ImmGCPtr ptr) { 2799 push(ptr); 2800 adjustFrame(int32_t(sizeof(intptr_t))); 2801 } 2802 2803 void MacroAssembler::Push(FloatRegister f) { 2804 push(f); 2805 // See MacroAssemblerLOONG64::ma_push(FloatRegister) for why we use 2806 // sizeof(double). 2807 adjustFrame(int32_t(sizeof(double))); 2808 } 2809 2810 void MacroAssembler::PushBoxed(FloatRegister reg) { 2811 subFromStackPtr(Imm32(sizeof(double))); 2812 boxDouble(reg, Address(getStackPointer(), 0)); 2813 adjustFrame(sizeof(double)); 2814 } 2815 2816 void MacroAssembler::Pop(Register reg) { 2817 pop(reg); 2818 adjustFrame(-int32_t(sizeof(intptr_t))); 2819 } 2820 2821 void MacroAssembler::Pop(FloatRegister f) { 2822 pop(f); 2823 // See MacroAssemblerLOONG64::ma_pop(FloatRegister) for why we use 2824 // sizeof(double). 2825 adjustFrame(-int32_t(sizeof(double))); 2826 } 2827 2828 void MacroAssembler::Pop(const ValueOperand& val) { 2829 popValue(val); 2830 adjustFrame(-int32_t(sizeof(Value))); 2831 } 2832 2833 void MacroAssembler::PopStackPtr() { 2834 loadPtr(Address(StackPointer, 0), StackPointer); 2835 adjustFrame(-int32_t(sizeof(intptr_t))); 2836 } 2837 2838 void MacroAssembler::freeStackTo(uint32_t framePushed) { 2839 MOZ_ASSERT(framePushed <= framePushed_); 2840 ma_sub_d(StackPointer, FramePointer, Imm32(framePushed)); 2841 framePushed_ = framePushed; 2842 } 2843 2844 // =============================================================== 2845 // Simple call functions. 2846 2847 CodeOffset MacroAssembler::call(Register reg) { 2848 as_jirl(ra, reg, BOffImm16(0)); 2849 return CodeOffset(currentOffset()); 2850 } 2851 2852 CodeOffset MacroAssembler::call(Label* label) { 2853 ma_bl(label); 2854 return CodeOffset(currentOffset()); 2855 } 2856 2857 CodeOffset MacroAssembler::callWithPatch() { 2858 as_bl(JOffImm26(1 * sizeof(uint32_t))); 2859 return CodeOffset(currentOffset()); 2860 } 2861 2862 void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) { 2863 BufferOffset call(callerOffset - 1 * sizeof(uint32_t)); 2864 2865 JOffImm26 offset = BufferOffset(calleeOffset).diffB<JOffImm26>(call); 2866 if (!offset.isInvalid()) { 2867 InstJump* bal = (InstJump*)editSrc(call); 2868 bal->setJOffImm26(offset); 2869 } else { 2870 uint32_t u32Offset = callerOffset - 4 * sizeof(uint32_t); 2871 uint32_t* u32 = 2872 reinterpret_cast<uint32_t*>(editSrc(BufferOffset(u32Offset))); 2873 *u32 = calleeOffset - callerOffset; 2874 } 2875 } 2876 2877 CodeOffset MacroAssembler::farJumpWithPatch() { 2878 UseScratchRegisterScope temps(asMasm()); 2879 Register scratch = temps.Acquire(); 2880 Register scratch2 = temps.Acquire(); 2881 as_pcaddi(scratch, 4); 2882 as_ld_w(scratch2, scratch, 0); 2883 as_add_d(scratch, scratch, scratch2); 2884 as_jirl(zero, scratch, BOffImm16(0)); 2885 // Allocate space which will be patched by patchFarJump(). 2886 CodeOffset farJump(currentOffset()); 2887 spew(".space 32bit initValue 0xffff ffff"); 2888 writeInst(UINT32_MAX); 2889 return farJump; 2890 } 2891 2892 void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) { 2893 uint32_t* u32 = 2894 reinterpret_cast<uint32_t*>(editSrc(BufferOffset(farJump.offset()))); 2895 MOZ_ASSERT(*u32 == UINT32_MAX); 2896 *u32 = targetOffset - farJump.offset(); 2897 } 2898 2899 void MacroAssembler::patchFarJump(uint8_t* farJump, uint8_t* target) { 2900 uint32_t* u32 = reinterpret_cast<uint32_t*>(farJump); 2901 MOZ_ASSERT(*u32 == UINT32_MAX); 2902 2903 *u32 = (int64_t)target - (int64_t)farJump; 2904 } 2905 2906 CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) { 2907 movePtr(target, CallReg); 2908 return call(CallReg); 2909 } 2910 2911 CodeOffset MacroAssembler::call(const Address& addr) { 2912 loadPtr(addr, CallReg); 2913 return call(CallReg); 2914 } 2915 2916 void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); } 2917 2918 void MacroAssembler::call(ImmPtr target) { 2919 BufferOffset bo = m_buffer.nextOffset(); 2920 addPendingJump(bo, target, RelocationKind::HARDCODED); 2921 ma_call(target); 2922 } 2923 2924 void MacroAssembler::call(JitCode* c) { 2925 UseScratchRegisterScope temps(asMasm()); 2926 Register scratch = temps.Acquire(); 2927 BufferOffset bo = m_buffer.nextOffset(); 2928 addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE); 2929 ma_liPatchable(scratch, ImmPtr(c->raw())); 2930 callJitNoProfiler(scratch); 2931 } 2932 2933 CodeOffset MacroAssembler::nopPatchableToCall() { 2934 // LOONG64 2935 as_nop(); // lu12i_w 2936 as_nop(); // ori 2937 as_nop(); // lu32i_d 2938 as_nop(); // jirl 2939 return CodeOffset(currentOffset()); 2940 } 2941 2942 void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) { 2943 Instruction* inst = (Instruction*)call - 4 /* four nops */; 2944 Assembler::WriteLoad64Instructions(inst, SavedScratchRegister, 2945 (uint64_t)target); 2946 inst[3] = InstImm(op_jirl, BOffImm16(0), SavedScratchRegister, ra); 2947 } 2948 2949 void MacroAssembler::patchCallToNop(uint8_t* call) { 2950 Instruction* inst = (Instruction*)call - 4 /* four nops */; 2951 inst[0].makeNop(); // lu12i_w 2952 inst[1].makeNop(); // ori 2953 inst[2].makeNop(); // lu32i_d 2954 inst[3].makeNop(); // jirl 2955 } 2956 2957 CodeOffset MacroAssembler::move32WithPatch(Register dest) { 2958 CodeOffset offs = CodeOffset(currentOffset()); 2959 ma_liPatchable(dest, Imm32(0)); 2960 return offs; 2961 } 2962 2963 void MacroAssembler::patchMove32(CodeOffset offset, Imm32 n) { 2964 patchSub32FromStackPtr(offset, n); 2965 } 2966 2967 void MacroAssembler::pushReturnAddress() { push(ra); } 2968 2969 void MacroAssembler::popReturnAddress() { pop(ra); } 2970 2971 // =============================================================== 2972 // ABI function calls. 2973 2974 void MacroAssembler::setupUnalignedABICall(Register scratch) { 2975 MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); 2976 setupNativeABICall(); 2977 dynamicAlignment_ = true; 2978 2979 as_or(scratch, StackPointer, zero); 2980 2981 // Force sp to be aligned 2982 asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer); 2983 ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1))); 2984 storePtr(scratch, Address(StackPointer, 0)); 2985 } 2986 2987 void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { 2988 MOZ_ASSERT(inCall_); 2989 uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar(); 2990 2991 // Reserve place for $ra. 2992 stackForCall += sizeof(intptr_t); 2993 2994 if (dynamicAlignment_) { 2995 stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment); 2996 } else { 2997 uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0; 2998 stackForCall += ComputeByteAlignment( 2999 stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment); 3000 } 3001 3002 *stackAdjust = stackForCall; 3003 reserveStack(stackForCall); 3004 3005 // Save $ra because call is going to clobber it. Restore it in 3006 // callWithABIPost. NOTE: This is needed for calls from SharedIC. 3007 // Maybe we can do this differently. 3008 storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t))); 3009 3010 // Position all arguments. 3011 { 3012 enoughMemory_ &= moveResolver_.resolve(); 3013 if (!enoughMemory_) { 3014 return; 3015 } 3016 3017 MoveEmitter emitter(*this); 3018 emitter.emit(moveResolver_); 3019 emitter.finish(); 3020 } 3021 3022 assertStackAlignment(ABIStackAlignment); 3023 } 3024 3025 void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { 3026 // Restore ra value (as stored in callWithABIPre()). 3027 loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra); 3028 3029 if (dynamicAlignment_) { 3030 // Restore sp value from stack (as stored in setupUnalignedABICall()). 3031 loadPtr(Address(StackPointer, stackAdjust), StackPointer); 3032 // Use adjustFrame instead of freeStack because we already restored sp. 3033 adjustFrame(-stackAdjust); 3034 } else { 3035 freeStack(stackAdjust); 3036 } 3037 3038 #ifdef DEBUG 3039 MOZ_ASSERT(inCall_); 3040 inCall_ = false; 3041 #endif 3042 } 3043 3044 void MacroAssembler::callWithABINoProfiler(Register fun, ABIType result) { 3045 UseScratchRegisterScope temps(asMasm()); 3046 Register scratch = temps.Acquire(); 3047 // Load the callee in scratch2, no instruction between the movePtr and 3048 // call should clobber it. Note that we can't use fun because it may be 3049 // one of the IntArg registers clobbered before the call. 3050 movePtr(fun, scratch); 3051 3052 uint32_t stackAdjust; 3053 callWithABIPre(&stackAdjust); 3054 call(scratch); 3055 callWithABIPost(stackAdjust, result); 3056 } 3057 3058 void MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result) { 3059 UseScratchRegisterScope temps(asMasm()); 3060 Register scratch = temps.Acquire(); 3061 // Load the callee in scratch, as above. 3062 loadPtr(fun, scratch); 3063 3064 uint32_t stackAdjust; 3065 callWithABIPre(&stackAdjust); 3066 call(scratch); 3067 callWithABIPost(stackAdjust, result); 3068 } 3069 3070 // =============================================================== 3071 // Jit Frames. 3072 3073 uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) { 3074 CodeLabel cl; 3075 3076 ma_li(scratch, &cl); 3077 Push(scratch); 3078 bind(&cl); 3079 uint32_t retAddr = currentOffset(); 3080 3081 addCodeLabel(cl); 3082 return retAddr; 3083 } 3084 3085 // =============================================================== 3086 // Move instructions 3087 3088 void MacroAssembler::moveValue(const ValueOperand& src, 3089 const ValueOperand& dest) { 3090 if (src == dest) { 3091 return; 3092 } 3093 movePtr(src.valueReg(), dest.valueReg()); 3094 } 3095 3096 void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) { 3097 if (!src.isGCThing()) { 3098 ma_li(dest.valueReg(), ImmWord(src.asRawBits())); 3099 return; 3100 } 3101 3102 writeDataRelocation(src); 3103 movWithPatch(ImmWord(src.asRawBits()), dest.valueReg()); 3104 } 3105 3106 // =============================================================== 3107 // Branch functions 3108 3109 void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) { 3110 ma_and(buffer, ptr, Imm32(int32_t(~gc::ChunkMask))); 3111 loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer); 3112 } 3113 3114 void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, 3115 Register temp, Label* label) { 3116 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); 3117 MOZ_ASSERT(ptr != temp); 3118 MOZ_ASSERT(temp != InvalidReg); 3119 3120 ma_and(temp, ptr, Imm32(int32_t(~gc::ChunkMask))); 3121 branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset), 3122 zero, label); 3123 } 3124 3125 void MacroAssembler::branchValueIsNurseryCell(Condition cond, 3126 const Address& address, 3127 Register temp, Label* label) { 3128 branchValueIsNurseryCellImpl(cond, address, temp, label); 3129 } 3130 3131 void MacroAssembler::branchValueIsNurseryCell(Condition cond, 3132 ValueOperand value, Register temp, 3133 Label* label) { 3134 branchValueIsNurseryCellImpl(cond, value, temp, label); 3135 } 3136 3137 template <typename T> 3138 void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, 3139 const T& value, Register temp, 3140 Label* label) { 3141 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); 3142 MOZ_ASSERT(temp != InvalidReg); 3143 Label done; 3144 branchTestGCThing(Assembler::NotEqual, value, 3145 cond == Assembler::Equal ? &done : label); 3146 3147 getGCThingValueChunk(value, temp); 3148 loadPtr(Address(temp, gc::ChunkStoreBufferOffset), temp); 3149 branchPtr(InvertCondition(cond), temp, zero, label); 3150 3151 bind(&done); 3152 } 3153 3154 void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs, 3155 const Value& rhs, Label* label) { 3156 MOZ_ASSERT(cond == Equal || cond == NotEqual); 3157 MOZ_ASSERT(!rhs.isNaN()); 3158 3159 if (!rhs.isGCThing()) { 3160 ma_b(lhs.valueReg(), ImmWord(rhs.asRawBits()), label, cond); 3161 } else { 3162 UseScratchRegisterScope temps(asMasm()); 3163 Register scratch = temps.Acquire(); 3164 MOZ_ASSERT(lhs.valueReg() != scratch); 3165 moveValue(rhs, ValueOperand(scratch)); 3166 ma_b(lhs.valueReg(), scratch, label, cond); 3167 } 3168 } 3169 3170 void MacroAssembler::branchTestNaNValue(Condition cond, const ValueOperand& val, 3171 Register temp, Label* label) { 3172 MOZ_ASSERT(cond == Equal || cond == NotEqual); 3173 UseScratchRegisterScope temps(asMasm()); 3174 Register scratch = temps.Acquire(); 3175 MOZ_ASSERT(val.valueReg() != scratch); 3176 3177 // When testing for NaN, we want to ignore the sign bit. 3178 // Clear the sign bit by extracting bits 62:0. 3179 as_bstrpick_d(temp, val.valueReg(), 62, 0); 3180 3181 // Compare against a NaN with sign bit 0. 3182 static_assert(JS::detail::CanonicalizedNaNSignBit == 0); 3183 moveValue(DoubleValue(JS::GenericNaN()), ValueOperand(scratch)); 3184 ma_b(temp, scratch, label, cond); 3185 } 3186 3187 // ======================================================================== 3188 // Memory access primitives. 3189 3190 template <typename T> 3191 void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, 3192 MIRType valueType, const T& dest) { 3193 MOZ_ASSERT(valueType < MIRType::Value); 3194 3195 if (valueType == MIRType::Double) { 3196 boxDouble(value.reg().typedReg().fpu(), dest); 3197 return; 3198 } 3199 3200 if (value.constant()) { 3201 storeValue(value.value(), dest); 3202 } else { 3203 storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(), 3204 dest); 3205 } 3206 } 3207 3208 template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, 3209 MIRType valueType, 3210 const Address& dest); 3211 template void MacroAssembler::storeUnboxedValue( 3212 const ConstantOrRegister& value, MIRType valueType, 3213 const BaseObjectElementIndex& dest); 3214 3215 void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); } 3216 3217 // =============================================================== 3218 // WebAssembly 3219 3220 FaultingCodeOffset MacroAssembler::wasmTrapInstruction() { 3221 FaultingCodeOffset fco = FaultingCodeOffset(currentOffset()); 3222 as_break(WASM_TRAP); // TODO: as_teq(zero, zero, WASM_TRAP) 3223 return fco; 3224 } 3225 3226 void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, 3227 Register boundsCheckLimit, 3228 Label* label) { 3229 ma_b(index, boundsCheckLimit, label, cond); 3230 } 3231 3232 void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, 3233 Address boundsCheckLimit, Label* label) { 3234 UseScratchRegisterScope temps(asMasm()); 3235 Register scratch = temps.Acquire(); 3236 load32(boundsCheckLimit, scratch); 3237 ma_b(index, Register(scratch), label, cond); 3238 } 3239 3240 void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, 3241 Register64 boundsCheckLimit, 3242 Label* label) { 3243 ma_b(index.reg, boundsCheckLimit.reg, label, cond); 3244 } 3245 3246 void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index, 3247 Address boundsCheckLimit, Label* label) { 3248 UseScratchRegisterScope temps(asMasm()); 3249 Register scratch = temps.Acquire(); 3250 loadPtr(boundsCheckLimit, scratch); 3251 ma_b(index.reg, scratch, label, cond); 3252 } 3253 3254 // FTINTRZ behaves as follows: 3255 // 3256 // on NaN it produces zero 3257 // on too large it produces INT_MAX (for appropriate type) 3258 // on too small it produces INT_MIN (ditto) 3259 3260 void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, 3261 Register output, 3262 bool isSaturating, 3263 Label* oolEntry) { 3264 UseScratchRegisterScope temps(asMasm()); 3265 Register scratch = temps.Acquire(); 3266 ScratchDoubleScope fpscratch(asMasm()); 3267 if (!isSaturating) { 3268 ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered); 3269 } 3270 as_ftintrz_l_d(fpscratch, input); 3271 moveFromDouble(fpscratch, output); 3272 as_srli_d(scratch, output, 32); 3273 as_slli_w(output, output, 0); 3274 ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); 3275 } 3276 3277 void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, 3278 Register output, 3279 bool isSaturating, 3280 Label* oolEntry) { 3281 UseScratchRegisterScope temps(asMasm()); 3282 Register scratch = temps.Acquire(); 3283 ScratchDoubleScope fpscratch(asMasm()); 3284 if (!isSaturating) { 3285 ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered); 3286 } 3287 as_ftintrz_l_s(fpscratch, input); 3288 moveFromDouble(fpscratch, output); 3289 as_srli_d(scratch, output, 32); 3290 as_slli_w(output, output, 0); 3291 ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); 3292 } 3293 3294 // Assembler::CauseV is a enum,called FCSRBit. Assembler::CauseV == 16 3295 void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input, 3296 Register output, 3297 bool isSaturating, 3298 Label* oolEntry) { 3299 UseScratchRegisterScope temps(asMasm()); 3300 Register scratch = temps.Acquire(); 3301 ScratchFloat32Scope fpscratch(asMasm()); 3302 as_ftintrz_w_d(fpscratch, input); 3303 as_movfcsr2gr(scratch); 3304 moveFromFloat32(fpscratch, output); 3305 MOZ_ASSERT(Assembler::CauseV < 32); 3306 as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); 3307 ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); 3308 } 3309 3310 void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, 3311 Register output, 3312 bool isSaturating, 3313 Label* oolEntry) { 3314 UseScratchRegisterScope temps(asMasm()); 3315 Register scratch = temps.Acquire(); 3316 ScratchFloat32Scope fpscratch(asMasm()); 3317 as_ftintrz_w_s(fpscratch, input); 3318 as_movfcsr2gr(scratch); 3319 moveFromFloat32(fpscratch, output); 3320 MOZ_ASSERT(Assembler::CauseV < 32); 3321 as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV); 3322 ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual); 3323 } 3324 3325 void MacroAssembler::wasmTruncateDoubleToUInt64( 3326 FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, 3327 Label* oolRejoin, FloatRegister tempDouble) { 3328 MOZ_ASSERT(tempDouble.isInvalid()); 3329 ScratchDoubleScope fpscratch(asMasm()); 3330 Register output = output_.reg; 3331 3332 Label done; 3333 3334 if (!isSaturating) { 3335 ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered); 3336 } 3337 as_ftintrz_l_d(fpscratch, input); 3338 moveFromDouble(fpscratch, output); 3339 loadConstantDouble(double(INT64_MAX + 1ULL), fpscratch); 3340 3341 UseScratchRegisterScope temps(asMasm()); 3342 Register scratch2 = temps.Acquire(); 3343 ma_li(scratch2, ImmWord(INT64_MAX)); 3344 // For numbers in -1.[ : ]INT64_MAX range do nothing more 3345 ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); 3346 3347 ma_li(scratch2, ImmWord(INT64_MIN)); 3348 as_fsub_d(fpscratch, input, fpscratch); 3349 as_ftintrz_l_d(fpscratch, fpscratch); 3350 Register scratch = temps.Acquire(); 3351 as_movfcsr2gr(scratch); 3352 moveFromDouble(fpscratch, output); 3353 as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); 3354 as_add_d(output, output, scratch2); 3355 3356 // Guard against negative values that result in 0 due the precision loss. 3357 as_sltui(scratch2, output, 1); 3358 as_or(scratch, scratch, scratch2); 3359 3360 ma_b(scratch, zero, oolEntry, Assembler::NotEqual); 3361 3362 bind(&done); 3363 3364 if (isSaturating) { 3365 bind(oolRejoin); 3366 } 3367 } 3368 3369 void MacroAssembler::wasmTruncateFloat32ToUInt64( 3370 FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry, 3371 Label* oolRejoin, FloatRegister tempFloat) { 3372 MOZ_ASSERT(tempFloat.isInvalid()); 3373 ScratchDoubleScope fpscratch(asMasm()); 3374 Register output = output_.reg; 3375 3376 Label done; 3377 3378 if (!isSaturating) { 3379 ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered); 3380 } 3381 as_ftintrz_l_s(fpscratch, input); 3382 moveFromDouble(fpscratch, output); 3383 loadConstantFloat32(float(INT64_MAX + 1ULL), fpscratch); 3384 3385 UseScratchRegisterScope temps(asMasm()); 3386 Register scratch2 = temps.Acquire(); 3387 ma_li(scratch2, ImmWord(INT64_MAX)); 3388 // For numbers in -1.[ : ]INT64_MAX range do nothing more 3389 ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump); 3390 3391 ma_li(scratch2, ImmWord(INT64_MIN)); 3392 as_fsub_s(fpscratch, input, fpscratch); 3393 as_ftintrz_l_s(fpscratch, fpscratch); 3394 Register scratch = temps.Acquire(); 3395 as_movfcsr2gr(scratch); 3396 moveFromDouble(fpscratch, output); 3397 as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); 3398 as_add_d(output, output, scratch2); 3399 3400 // Guard against negative values that result in 0 due the precision loss. 3401 as_sltui(scratch2, output, 1); 3402 as_or(scratch, scratch, scratch2); 3403 3404 ma_b(scratch, zero, oolEntry, Assembler::NotEqual); 3405 3406 bind(&done); 3407 3408 if (isSaturating) { 3409 bind(oolRejoin); 3410 } 3411 } 3412 3413 void MacroAssembler::wasmTruncateDoubleToInt64( 3414 FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, 3415 Label* oolRejoin, FloatRegister tempDouble) { 3416 MOZ_ASSERT(tempDouble.isInvalid()); 3417 UseScratchRegisterScope temps(asMasm()); 3418 Register scratch = temps.Acquire(); 3419 ScratchDoubleScope fpscratch(asMasm()); 3420 3421 as_ftintrz_l_d(fpscratch, input); 3422 as_movfcsr2gr(scratch); 3423 moveFromDouble(fpscratch, output.reg); 3424 as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); 3425 ma_b(scratch, zero, oolEntry, Assembler::NotEqual); 3426 3427 if (isSaturating) { 3428 bind(oolRejoin); 3429 } 3430 } 3431 3432 void MacroAssembler::wasmTruncateFloat32ToInt64( 3433 FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, 3434 Label* oolRejoin, FloatRegister tempFloat) { 3435 MOZ_ASSERT(tempFloat.isInvalid()); 3436 UseScratchRegisterScope temps(asMasm()); 3437 Register scratch = temps.Acquire(); 3438 ScratchDoubleScope fpscratch(asMasm()); 3439 3440 as_ftintrz_l_s(fpscratch, input); 3441 as_movfcsr2gr(scratch); 3442 moveFromDouble(fpscratch, output.reg); 3443 as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV); 3444 ma_b(scratch, zero, oolEntry, Assembler::NotEqual); 3445 3446 if (isSaturating) { 3447 bind(oolRejoin); 3448 } 3449 } 3450 3451 void MacroAssembler::oolWasmTruncateCheckF32ToI32( 3452 FloatRegister input, Register output, TruncFlags flags, 3453 const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { 3454 outOfLineWasmTruncateToInt32Check(input, output, MIRType::Float32, flags, 3455 rejoin, trapSiteDesc); 3456 } 3457 3458 void MacroAssembler::oolWasmTruncateCheckF64ToI32( 3459 FloatRegister input, Register output, TruncFlags flags, 3460 const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { 3461 outOfLineWasmTruncateToInt32Check(input, output, MIRType::Double, flags, 3462 rejoin, trapSiteDesc); 3463 } 3464 3465 void MacroAssembler::oolWasmTruncateCheckF32ToI64( 3466 FloatRegister input, Register64 output, TruncFlags flags, 3467 const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { 3468 outOfLineWasmTruncateToInt64Check(input, output, MIRType::Float32, flags, 3469 rejoin, trapSiteDesc); 3470 } 3471 3472 void MacroAssembler::oolWasmTruncateCheckF64ToI64( 3473 FloatRegister input, Register64 output, TruncFlags flags, 3474 const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { 3475 outOfLineWasmTruncateToInt64Check(input, output, MIRType::Double, flags, 3476 rejoin, trapSiteDesc); 3477 } 3478 3479 void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, 3480 Register memoryBase, Register ptr, 3481 Register ptrScratch, AnyRegister output) { 3482 wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); 3483 } 3484 3485 void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, 3486 Register memoryBase, Register ptr, 3487 Register ptrScratch, Register64 output) { 3488 wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg); 3489 } 3490 3491 void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, 3492 AnyRegister value, Register memoryBase, 3493 Register ptr, Register ptrScratch) { 3494 wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); 3495 } 3496 3497 void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, 3498 Register64 value, Register memoryBase, 3499 Register ptr, Register ptrScratch) { 3500 wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg); 3501 } 3502 3503 void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch, 3504 ExitFrameType type) { 3505 enterFakeExitFrame(cxreg, scratch, type); 3506 } 3507 3508 CodeOffset MacroAssembler::sub32FromMemAndBranchIfNegativeWithPatch( 3509 Address address, Label* label) { 3510 UseScratchRegisterScope temps(asMasm()); 3511 Register scratch = temps.Acquire(); 3512 MOZ_ASSERT(scratch != address.base); 3513 ma_ld_w(scratch, address); 3514 // LoongArch doesn't have subtraction instr that support immediate operand, 3515 // and use 'addi.w rd, rj, -imm' instead to achieve same function. 3516 // 128 is arbitrary, but makes `*address` count upwards, which may help 3517 // to identify cases where the subsequent ::patch..() call was forgotten. 3518 as_addi_w(scratch, scratch, 128); 3519 // Points immediately after the insn to patch 3520 CodeOffset patchPoint = CodeOffset(currentOffset()); 3521 ma_st_w(scratch, address); 3522 ma_b(scratch, Register(scratch), label, Assembler::Signed); 3523 return patchPoint; 3524 } 3525 3526 void MacroAssembler::patchSub32FromMemAndBranchIfNegative(CodeOffset offset, 3527 Imm32 imm) { 3528 int32_t val = imm.value; 3529 // Patching it to zero would make the insn pointless 3530 MOZ_RELEASE_ASSERT(val >= 1 && val <= 127); 3531 Instruction* instrPtr = 3532 (Instruction*)m_buffer.getInst(BufferOffset(offset.offset() - 4)); 3533 // LoongArch doesn't have subtraction instr that support immediate operand, 3534 // and use 'addi.w rd, rj, -imm' instead to achieve same function. 3535 // 31 27 23 21 9 4 3536 // | | | | | | 3537 // 0000 0010 10 si12 rj rd = addi.w rd, rj, #si12 3538 InstImm* inst = (InstImm*)instrPtr; 3539 MOZ_ASSERT(inst->extractBitField(31, 22) == ((uint32_t)op_addi_w >> 22)); 3540 3541 *inst = InstImm(op_addi_w, (int32_t)(-val & 0xfff), 3542 Register::FromCode(inst->extractRJ()), 3543 Register::FromCode(inst->extractRD()), 12); 3544 } 3545 3546 // TODO(loong64): widenInt32 should be nop? 3547 void MacroAssembler::widenInt32(Register r) { 3548 move32To64SignExtend(r, Register64(r)); 3549 } 3550 3551 // ======================================================================== 3552 // Convert floating point. 3553 3554 void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest, 3555 Register temp) { 3556 MOZ_ASSERT(temp == Register::Invalid()); 3557 UseScratchRegisterScope temps(asMasm()); 3558 Register scratch = temps.Acquire(); 3559 Register scratch2 = temps.Acquire(); 3560 3561 Register src = src_.reg; 3562 Label positive, done; 3563 ma_b(src, src, &positive, NotSigned, ShortJump); 3564 3565 MOZ_ASSERT(src != scratch); 3566 MOZ_ASSERT(src != scratch2); 3567 3568 ma_and(scratch, src, Imm32(1)); 3569 as_srli_d(scratch2, src, 1); 3570 as_or(scratch, scratch, scratch2); 3571 as_movgr2fr_d(dest, scratch); 3572 as_ffint_s_l(dest, dest); 3573 addFloat32(dest, dest); 3574 ma_b(&done, ShortJump); 3575 3576 bind(&positive); 3577 as_movgr2fr_d(dest, src); 3578 as_ffint_s_l(dest, dest); 3579 3580 bind(&done); 3581 } 3582 3583 void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) { 3584 as_movgr2fr_d(dest, src.reg); 3585 as_ffint_s_l(dest, dest); 3586 } 3587 3588 bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; } 3589 3590 void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, 3591 Register temp) { 3592 MOZ_ASSERT(temp == Register::Invalid()); 3593 MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest); 3594 } 3595 3596 void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) { 3597 as_movgr2fr_d(dest, src.reg); 3598 as_ffint_d_l(dest, dest); 3599 } 3600 3601 void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) { 3602 convertInt64ToDouble(Register64(src), dest); 3603 } 3604 3605 // ======================================================================== 3606 // Primitive atomic operations. 3607 3608 template <typename T> 3609 static void CompareExchange(MacroAssembler& masm, 3610 const wasm::MemoryAccessDesc* access, 3611 Scalar::Type type, Synchronization sync, 3612 const T& mem, Register oldval, Register newval, 3613 Register valueTemp, Register offsetTemp, 3614 Register maskTemp, Register output) { 3615 UseScratchRegisterScope temps(masm); 3616 bool signExtend = Scalar::isSignedIntType(type); 3617 unsigned nbytes = Scalar::byteSize(type); 3618 3619 switch (nbytes) { 3620 case 1: 3621 case 2: 3622 break; 3623 case 4: 3624 MOZ_ASSERT(valueTemp == InvalidReg); 3625 MOZ_ASSERT(offsetTemp == InvalidReg); 3626 MOZ_ASSERT(maskTemp == InvalidReg); 3627 break; 3628 default: 3629 MOZ_CRASH(); 3630 } 3631 3632 Label again, end; 3633 3634 Register scratch = temps.Acquire(); 3635 masm.computeEffectiveAddress(mem, scratch); 3636 3637 Register scratch2 = temps.Acquire(); 3638 3639 if (nbytes == 4) { 3640 masm.memoryBarrierBefore(sync); 3641 masm.bind(&again); 3642 3643 if (access) { 3644 masm.append(*access, wasm::TrapMachineInsn::Load32, 3645 FaultingCodeOffset(masm.currentOffset())); 3646 } 3647 3648 masm.as_ll_w(output, scratch, 0); 3649 masm.as_slli_w(scratch2, oldval, 0); 3650 masm.ma_b(output, scratch2, &end, Assembler::NotEqual, ShortJump); 3651 masm.as_or(scratch2, newval, zero); 3652 masm.as_sc_w(scratch2, scratch, 0); 3653 masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); 3654 3655 masm.memoryBarrierAfter(sync); 3656 masm.bind(&end); 3657 3658 return; 3659 } 3660 3661 masm.as_andi(offsetTemp, scratch, 3); 3662 masm.subPtr(offsetTemp, scratch); 3663 masm.as_slli_w(offsetTemp, offsetTemp, 3); 3664 masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); 3665 masm.as_sll_w(maskTemp, maskTemp, offsetTemp); 3666 masm.as_nor(maskTemp, zero, maskTemp); 3667 3668 masm.memoryBarrierBefore(sync); 3669 3670 masm.bind(&again); 3671 3672 if (access) { 3673 masm.append(*access, wasm::TrapMachineInsn::Load32, 3674 FaultingCodeOffset(masm.currentOffset())); 3675 } 3676 3677 masm.as_ll_w(scratch2, scratch, 0); 3678 3679 masm.as_srl_w(output, scratch2, offsetTemp); 3680 3681 switch (nbytes) { 3682 case 1: 3683 if (signExtend) { 3684 masm.as_ext_w_b(valueTemp, oldval); 3685 masm.as_ext_w_b(output, output); 3686 } else { 3687 masm.as_andi(valueTemp, oldval, 0xff); 3688 masm.as_andi(output, output, 0xff); 3689 } 3690 break; 3691 case 2: 3692 if (signExtend) { 3693 masm.as_ext_w_h(valueTemp, oldval); 3694 masm.as_ext_w_h(output, output); 3695 } else { 3696 masm.as_bstrpick_d(valueTemp, oldval, 15, 0); 3697 masm.as_bstrpick_d(output, output, 15, 0); 3698 } 3699 break; 3700 } 3701 3702 masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump); 3703 3704 masm.as_sll_w(valueTemp, newval, offsetTemp); 3705 masm.as_andn(valueTemp, valueTemp, maskTemp); 3706 masm.as_and(scratch2, scratch2, maskTemp); 3707 masm.as_or(scratch2, scratch2, valueTemp); 3708 3709 masm.as_sc_w(scratch2, scratch, 0); 3710 3711 masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); 3712 3713 masm.memoryBarrierAfter(sync); 3714 3715 masm.bind(&end); 3716 } 3717 3718 template <typename T> 3719 static void CompareExchange64(MacroAssembler& masm, 3720 const wasm::MemoryAccessDesc* access, 3721 Synchronization sync, const T& mem, 3722 Register64 expect, Register64 replace, 3723 Register64 output) { 3724 MOZ_ASSERT(expect != output && replace != output); 3725 UseScratchRegisterScope temps(masm); 3726 Register scratch = temps.Acquire(); 3727 masm.computeEffectiveAddress(mem, scratch); 3728 3729 Register scratch2 = temps.Acquire(); 3730 3731 Label tryAgain; 3732 Label exit; 3733 3734 masm.memoryBarrierBefore(sync); 3735 3736 masm.bind(&tryAgain); 3737 3738 if (access) { 3739 masm.append(*access, wasm::TrapMachineInsn::Load64, 3740 FaultingCodeOffset(masm.currentOffset())); 3741 } 3742 3743 masm.as_ll_d(output.reg, scratch, 0); 3744 3745 masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump); 3746 masm.movePtr(replace.reg, scratch2); 3747 masm.as_sc_d(scratch2, scratch, 0); 3748 masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, 3749 ShortJump); 3750 3751 masm.memoryBarrierAfter(sync); 3752 3753 masm.bind(&exit); 3754 } 3755 3756 template <typename T> 3757 static void AtomicExchange(MacroAssembler& masm, 3758 const wasm::MemoryAccessDesc* access, 3759 Scalar::Type type, Synchronization sync, 3760 const T& mem, Register value, Register valueTemp, 3761 Register offsetTemp, Register maskTemp, 3762 Register output) { 3763 UseScratchRegisterScope temps(masm); 3764 Register scratch = temps.Acquire(); 3765 Register scratch2 = temps.Acquire(); 3766 bool signExtend = Scalar::isSignedIntType(type); 3767 unsigned nbytes = Scalar::byteSize(type); 3768 3769 switch (nbytes) { 3770 case 1: 3771 case 2: 3772 break; 3773 case 4: 3774 MOZ_ASSERT(valueTemp == InvalidReg); 3775 MOZ_ASSERT(offsetTemp == InvalidReg); 3776 MOZ_ASSERT(maskTemp == InvalidReg); 3777 break; 3778 default: 3779 MOZ_CRASH(); 3780 } 3781 3782 Label again; 3783 3784 Register memTemp = scratch2; 3785 masm.computeEffectiveAddress(mem, memTemp); 3786 3787 if (nbytes == 4) { 3788 masm.memoryBarrierBefore(sync); 3789 masm.bind(&again); 3790 3791 if (access) { 3792 masm.append(*access, wasm::TrapMachineInsn::Load32, 3793 FaultingCodeOffset(masm.currentOffset())); 3794 } 3795 3796 masm.as_ll_w(output, memTemp, 0); 3797 masm.as_or(scratch, value, zero); 3798 masm.as_sc_w(scratch, memTemp, 0); 3799 masm.ma_b(scratch, Register(scratch), &again, Assembler::Zero, ShortJump); 3800 3801 masm.memoryBarrierAfter(sync); 3802 3803 return; 3804 } 3805 3806 masm.as_andi(offsetTemp, memTemp, 3); 3807 masm.subPtr(offsetTemp, memTemp); 3808 masm.as_slli_w(offsetTemp, offsetTemp, 3); 3809 masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); 3810 masm.as_sll_w(maskTemp, maskTemp, offsetTemp); 3811 masm.as_nor(maskTemp, zero, maskTemp); 3812 switch (nbytes) { 3813 case 1: 3814 masm.as_andi(valueTemp, value, 0xff); 3815 break; 3816 case 2: 3817 masm.as_bstrpick_d(valueTemp, value, 15, 0); 3818 break; 3819 } 3820 masm.as_sll_w(valueTemp, valueTemp, offsetTemp); 3821 3822 masm.memoryBarrierBefore(sync); 3823 3824 masm.bind(&again); 3825 3826 if (access) { 3827 masm.append(*access, wasm::TrapMachineInsn::Load32, 3828 FaultingCodeOffset(masm.currentOffset())); 3829 } 3830 3831 masm.as_ll_w(output, memTemp, 0); 3832 masm.as_and(scratch, output, maskTemp); 3833 masm.as_or(scratch, scratch, valueTemp); 3834 3835 masm.as_sc_w(scratch, memTemp, 0); 3836 3837 masm.ma_b(scratch, Register(scratch), &again, Assembler::Zero, ShortJump); 3838 3839 masm.as_srl_w(output, output, offsetTemp); 3840 3841 switch (nbytes) { 3842 case 1: 3843 if (signExtend) { 3844 masm.as_ext_w_b(output, output); 3845 } else { 3846 masm.as_andi(output, output, 0xff); 3847 } 3848 break; 3849 case 2: 3850 if (signExtend) { 3851 masm.as_ext_w_h(output, output); 3852 } else { 3853 masm.as_bstrpick_d(output, output, 15, 0); 3854 } 3855 break; 3856 } 3857 3858 masm.memoryBarrierAfter(sync); 3859 } 3860 3861 template <typename T> 3862 static void AtomicExchange64(MacroAssembler& masm, 3863 const wasm::MemoryAccessDesc* access, 3864 Synchronization sync, const T& mem, 3865 Register64 value, Register64 output) { 3866 MOZ_ASSERT(value != output); 3867 UseScratchRegisterScope temps(masm); 3868 3869 Register scratch = temps.Acquire(); 3870 masm.computeEffectiveAddress(mem, scratch); 3871 3872 Register scratch2 = temps.Acquire(); 3873 3874 Label tryAgain; 3875 3876 masm.memoryBarrierBefore(sync); 3877 3878 masm.bind(&tryAgain); 3879 3880 if (access) { 3881 masm.append(*access, wasm::TrapMachineInsn::Load64, 3882 FaultingCodeOffset(masm.currentOffset())); 3883 } 3884 3885 masm.as_ll_d(output.reg, scratch, 0); 3886 3887 masm.movePtr(value.reg, scratch2); 3888 masm.as_sc_d(scratch2, scratch, 0); 3889 masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero, 3890 ShortJump); 3891 3892 masm.memoryBarrierAfter(sync); 3893 } 3894 3895 template <typename T> 3896 static void AtomicFetchOp(MacroAssembler& masm, 3897 const wasm::MemoryAccessDesc* access, 3898 Scalar::Type type, Synchronization sync, AtomicOp op, 3899 const T& mem, Register value, Register valueTemp, 3900 Register offsetTemp, Register maskTemp, 3901 Register output) { 3902 UseScratchRegisterScope temps(masm); 3903 Register scratch = temps.Acquire(); 3904 Register scratch2 = temps.Acquire(); 3905 bool signExtend = Scalar::isSignedIntType(type); 3906 unsigned nbytes = Scalar::byteSize(type); 3907 3908 switch (nbytes) { 3909 case 1: 3910 case 2: 3911 break; 3912 case 4: 3913 MOZ_ASSERT(valueTemp == InvalidReg); 3914 MOZ_ASSERT(offsetTemp == InvalidReg); 3915 MOZ_ASSERT(maskTemp == InvalidReg); 3916 break; 3917 default: 3918 MOZ_CRASH(); 3919 } 3920 3921 Label again; 3922 3923 Register memTemp = scratch2; 3924 masm.computeEffectiveAddress(mem, memTemp); 3925 3926 if (nbytes == 4) { 3927 masm.memoryBarrierBefore(sync); 3928 masm.bind(&again); 3929 3930 if (access) { 3931 masm.append(*access, wasm::TrapMachineInsn::Load32, 3932 FaultingCodeOffset(masm.currentOffset())); 3933 } 3934 3935 masm.as_ll_w(output, memTemp, 0); 3936 3937 switch (op) { 3938 case AtomicOp::Add: 3939 masm.as_add_w(scratch, output, value); 3940 break; 3941 case AtomicOp::Sub: 3942 masm.as_sub_w(scratch, output, value); 3943 break; 3944 case AtomicOp::And: 3945 masm.as_and(scratch, output, value); 3946 break; 3947 case AtomicOp::Or: 3948 masm.as_or(scratch, output, value); 3949 break; 3950 case AtomicOp::Xor: 3951 masm.as_xor(scratch, output, value); 3952 break; 3953 default: 3954 MOZ_CRASH(); 3955 } 3956 3957 masm.as_sc_w(scratch, memTemp, 0); 3958 masm.ma_b(scratch, Register(scratch), &again, Assembler::Zero, ShortJump); 3959 3960 masm.memoryBarrierAfter(sync); 3961 3962 return; 3963 } 3964 3965 masm.as_andi(offsetTemp, memTemp, 3); 3966 masm.subPtr(offsetTemp, memTemp); 3967 masm.as_slli_w(offsetTemp, offsetTemp, 3); 3968 masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); 3969 masm.as_sll_w(maskTemp, maskTemp, offsetTemp); 3970 masm.as_nor(maskTemp, zero, maskTemp); 3971 3972 masm.memoryBarrierBefore(sync); 3973 3974 masm.bind(&again); 3975 3976 if (access) { 3977 masm.append(*access, wasm::TrapMachineInsn::Load32, 3978 FaultingCodeOffset(masm.currentOffset())); 3979 } 3980 3981 masm.as_ll_w(scratch, memTemp, 0); 3982 masm.as_srl_w(output, scratch, offsetTemp); 3983 3984 switch (op) { 3985 case AtomicOp::Add: 3986 masm.as_add_w(valueTemp, output, value); 3987 break; 3988 case AtomicOp::Sub: 3989 masm.as_sub_w(valueTemp, output, value); 3990 break; 3991 case AtomicOp::And: 3992 masm.as_and(valueTemp, output, value); 3993 break; 3994 case AtomicOp::Or: 3995 masm.as_or(valueTemp, output, value); 3996 break; 3997 case AtomicOp::Xor: 3998 masm.as_xor(valueTemp, output, value); 3999 break; 4000 default: 4001 MOZ_CRASH(); 4002 } 4003 4004 switch (nbytes) { 4005 case 1: 4006 masm.as_andi(valueTemp, valueTemp, 0xff); 4007 break; 4008 case 2: 4009 masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); 4010 break; 4011 } 4012 4013 masm.as_sll_w(valueTemp, valueTemp, offsetTemp); 4014 4015 masm.as_and(scratch, scratch, maskTemp); 4016 masm.as_or(scratch, scratch, valueTemp); 4017 4018 masm.as_sc_w(scratch, memTemp, 0); 4019 4020 masm.ma_b(scratch, Register(scratch), &again, Assembler::Zero, ShortJump); 4021 4022 switch (nbytes) { 4023 case 1: 4024 if (signExtend) { 4025 masm.as_ext_w_b(output, output); 4026 } else { 4027 masm.as_andi(output, output, 0xff); 4028 } 4029 break; 4030 case 2: 4031 if (signExtend) { 4032 masm.as_ext_w_h(output, output); 4033 } else { 4034 masm.as_bstrpick_d(output, output, 15, 0); 4035 } 4036 break; 4037 } 4038 4039 masm.memoryBarrierAfter(sync); 4040 } 4041 4042 template <typename T> 4043 static void AtomicFetchOp64(MacroAssembler& masm, 4044 const wasm::MemoryAccessDesc* access, 4045 Synchronization sync, AtomicOp op, Register64 value, 4046 const T& mem, Register64 temp, Register64 output) { 4047 MOZ_ASSERT(value != output); 4048 MOZ_ASSERT(value != temp); 4049 UseScratchRegisterScope temps(masm); 4050 Register scratch = temps.Acquire(); 4051 masm.computeEffectiveAddress(mem, scratch); 4052 4053 Label tryAgain; 4054 4055 masm.memoryBarrierBefore(sync); 4056 4057 masm.bind(&tryAgain); 4058 4059 if (access) { 4060 masm.append(*access, wasm::TrapMachineInsn::Load64, 4061 FaultingCodeOffset(masm.currentOffset())); 4062 } 4063 4064 masm.as_ll_d(output.reg, scratch, 0); 4065 4066 switch (op) { 4067 case AtomicOp::Add: 4068 masm.as_add_d(temp.reg, output.reg, value.reg); 4069 break; 4070 case AtomicOp::Sub: 4071 masm.as_sub_d(temp.reg, output.reg, value.reg); 4072 break; 4073 case AtomicOp::And: 4074 masm.as_and(temp.reg, output.reg, value.reg); 4075 break; 4076 case AtomicOp::Or: 4077 masm.as_or(temp.reg, output.reg, value.reg); 4078 break; 4079 case AtomicOp::Xor: 4080 masm.as_xor(temp.reg, output.reg, value.reg); 4081 break; 4082 default: 4083 MOZ_CRASH(); 4084 } 4085 4086 masm.as_sc_d(temp.reg, scratch, 0); 4087 masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::Zero, ShortJump); 4088 4089 masm.memoryBarrierAfter(sync); 4090 } 4091 4092 void MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync, 4093 const Address& mem, Register oldval, 4094 Register newval, Register valueTemp, 4095 Register offsetTemp, Register maskTemp, 4096 Register output) { 4097 CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, 4098 offsetTemp, maskTemp, output); 4099 } 4100 4101 void MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync, 4102 const BaseIndex& mem, Register oldval, 4103 Register newval, Register valueTemp, 4104 Register offsetTemp, Register maskTemp, 4105 Register output) { 4106 CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, 4107 offsetTemp, maskTemp, output); 4108 } 4109 4110 void MacroAssembler::compareExchange64(Synchronization sync, const Address& mem, 4111 Register64 expect, Register64 replace, 4112 Register64 output) { 4113 CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); 4114 } 4115 4116 void MacroAssembler::compareExchange64(Synchronization sync, 4117 const BaseIndex& mem, Register64 expect, 4118 Register64 replace, Register64 output) { 4119 CompareExchange64(*this, nullptr, sync, mem, expect, replace, output); 4120 } 4121 4122 void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, 4123 const Address& mem, Register oldval, 4124 Register newval, Register valueTemp, 4125 Register offsetTemp, Register maskTemp, 4126 Register output) { 4127 CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, 4128 newval, valueTemp, offsetTemp, maskTemp, output); 4129 } 4130 4131 void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, 4132 const BaseIndex& mem, Register oldval, 4133 Register newval, Register valueTemp, 4134 Register offsetTemp, Register maskTemp, 4135 Register output) { 4136 CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, 4137 newval, valueTemp, offsetTemp, maskTemp, output); 4138 } 4139 4140 void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, 4141 const Address& mem, 4142 Register64 expect, 4143 Register64 replace, 4144 Register64 output) { 4145 CompareExchange64(*this, &access, access.sync(), mem, expect, replace, 4146 output); 4147 } 4148 4149 void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, 4150 const BaseIndex& mem, 4151 Register64 expect, 4152 Register64 replace, 4153 Register64 output) { 4154 CompareExchange64(*this, &access, access.sync(), mem, expect, replace, 4155 output); 4156 } 4157 4158 void MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync, 4159 const Address& mem, Register value, 4160 Register valueTemp, Register offsetTemp, 4161 Register maskTemp, Register output) { 4162 AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, 4163 maskTemp, output); 4164 } 4165 4166 void MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync, 4167 const BaseIndex& mem, Register value, 4168 Register valueTemp, Register offsetTemp, 4169 Register maskTemp, Register output) { 4170 AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, 4171 maskTemp, output); 4172 } 4173 4174 void MacroAssembler::atomicExchange64(Synchronization sync, const Address& mem, 4175 Register64 value, Register64 output) { 4176 AtomicExchange64(*this, nullptr, sync, mem, value, output); 4177 } 4178 4179 void MacroAssembler::atomicExchange64(Synchronization sync, 4180 const BaseIndex& mem, Register64 value, 4181 Register64 output) { 4182 AtomicExchange64(*this, nullptr, sync, mem, value, output); 4183 } 4184 4185 void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, 4186 const Address& mem, Register value, 4187 Register valueTemp, Register offsetTemp, 4188 Register maskTemp, Register output) { 4189 AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, 4190 valueTemp, offsetTemp, maskTemp, output); 4191 } 4192 4193 void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, 4194 const BaseIndex& mem, Register value, 4195 Register valueTemp, Register offsetTemp, 4196 Register maskTemp, Register output) { 4197 AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, 4198 valueTemp, offsetTemp, maskTemp, output); 4199 } 4200 4201 void MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync, 4202 AtomicOp op, Register value, 4203 const Address& mem, Register valueTemp, 4204 Register offsetTemp, Register maskTemp, 4205 Register output) { 4206 AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, 4207 offsetTemp, maskTemp, output); 4208 } 4209 4210 void MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync, 4211 AtomicOp op, Register value, 4212 const BaseIndex& mem, Register valueTemp, 4213 Register offsetTemp, Register maskTemp, 4214 Register output) { 4215 AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, 4216 offsetTemp, maskTemp, output); 4217 } 4218 4219 void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op, 4220 Register64 value, const Address& mem, 4221 Register64 temp, Register64 output) { 4222 AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); 4223 } 4224 4225 void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op, 4226 Register64 value, const BaseIndex& mem, 4227 Register64 temp, Register64 output) { 4228 AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output); 4229 } 4230 4231 void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op, 4232 Register64 value, const Address& mem, 4233 Register64 temp) { 4234 AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); 4235 } 4236 4237 void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op, 4238 Register64 value, const BaseIndex& mem, 4239 Register64 temp) { 4240 AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp); 4241 } 4242 4243 void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, 4244 AtomicOp op, Register value, 4245 const Address& mem, Register valueTemp, 4246 Register offsetTemp, Register maskTemp, 4247 Register output) { 4248 AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, 4249 valueTemp, offsetTemp, maskTemp, output); 4250 } 4251 4252 void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, 4253 AtomicOp op, Register value, 4254 const BaseIndex& mem, Register valueTemp, 4255 Register offsetTemp, Register maskTemp, 4256 Register output) { 4257 AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, 4258 valueTemp, offsetTemp, maskTemp, output); 4259 } 4260 4261 template <typename T> 4262 static void AtomicEffectOp(MacroAssembler& masm, 4263 const wasm::MemoryAccessDesc* access, 4264 Scalar::Type type, Synchronization sync, AtomicOp op, 4265 const T& mem, Register value, Register valueTemp, 4266 Register offsetTemp, Register maskTemp) { 4267 UseScratchRegisterScope temps(masm); 4268 unsigned nbytes = Scalar::byteSize(type); 4269 4270 switch (nbytes) { 4271 case 1: 4272 case 2: 4273 break; 4274 case 4: 4275 MOZ_ASSERT(valueTemp == InvalidReg); 4276 MOZ_ASSERT(offsetTemp == InvalidReg); 4277 MOZ_ASSERT(maskTemp == InvalidReg); 4278 break; 4279 default: 4280 MOZ_CRASH(); 4281 } 4282 4283 Label again; 4284 4285 Register scratch = temps.Acquire(); 4286 masm.computeEffectiveAddress(mem, scratch); 4287 4288 Register scratch2 = temps.Acquire(); 4289 4290 if (nbytes == 4) { 4291 masm.memoryBarrierBefore(sync); 4292 masm.bind(&again); 4293 4294 if (access) { 4295 masm.append(*access, wasm::TrapMachineInsn::Load32, 4296 FaultingCodeOffset(masm.currentOffset())); 4297 } 4298 4299 masm.as_ll_w(scratch2, scratch, 0); 4300 4301 switch (op) { 4302 case AtomicOp::Add: 4303 masm.as_add_w(scratch2, scratch2, value); 4304 break; 4305 case AtomicOp::Sub: 4306 masm.as_sub_w(scratch2, scratch2, value); 4307 break; 4308 case AtomicOp::And: 4309 masm.as_and(scratch2, scratch2, value); 4310 break; 4311 case AtomicOp::Or: 4312 masm.as_or(scratch2, scratch2, value); 4313 break; 4314 case AtomicOp::Xor: 4315 masm.as_xor(scratch2, scratch2, value); 4316 break; 4317 default: 4318 MOZ_CRASH(); 4319 } 4320 4321 masm.as_sc_w(scratch2, scratch, 0); 4322 masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); 4323 4324 masm.memoryBarrierAfter(sync); 4325 4326 return; 4327 } 4328 4329 masm.as_andi(offsetTemp, scratch, 3); 4330 masm.subPtr(offsetTemp, scratch); 4331 masm.as_slli_w(offsetTemp, offsetTemp, 3); 4332 masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8))); 4333 masm.as_sll_w(maskTemp, maskTemp, offsetTemp); 4334 masm.as_nor(maskTemp, zero, maskTemp); 4335 4336 masm.memoryBarrierBefore(sync); 4337 4338 masm.bind(&again); 4339 4340 if (access) { 4341 masm.append(*access, wasm::TrapMachineInsn::Load32, 4342 FaultingCodeOffset(masm.currentOffset())); 4343 } 4344 4345 masm.as_ll_w(scratch2, scratch, 0); 4346 masm.as_srl_w(valueTemp, scratch2, offsetTemp); 4347 4348 switch (op) { 4349 case AtomicOp::Add: 4350 masm.as_add_w(valueTemp, valueTemp, value); 4351 break; 4352 case AtomicOp::Sub: 4353 masm.as_sub_w(valueTemp, valueTemp, value); 4354 break; 4355 case AtomicOp::And: 4356 masm.as_and(valueTemp, valueTemp, value); 4357 break; 4358 case AtomicOp::Or: 4359 masm.as_or(valueTemp, valueTemp, value); 4360 break; 4361 case AtomicOp::Xor: 4362 masm.as_xor(valueTemp, valueTemp, value); 4363 break; 4364 default: 4365 MOZ_CRASH(); 4366 } 4367 4368 switch (nbytes) { 4369 case 1: 4370 masm.as_andi(valueTemp, valueTemp, 0xff); 4371 break; 4372 case 2: 4373 masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0); 4374 break; 4375 } 4376 4377 masm.as_sll_w(valueTemp, valueTemp, offsetTemp); 4378 4379 masm.as_and(scratch2, scratch2, maskTemp); 4380 masm.as_or(scratch2, scratch2, valueTemp); 4381 4382 masm.as_sc_w(scratch2, scratch, 0); 4383 4384 masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump); 4385 4386 masm.memoryBarrierAfter(sync); 4387 } 4388 4389 void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, 4390 AtomicOp op, Register value, 4391 const Address& mem, Register valueTemp, 4392 Register offsetTemp, 4393 Register maskTemp) { 4394 AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, 4395 valueTemp, offsetTemp, maskTemp); 4396 } 4397 4398 void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, 4399 AtomicOp op, Register value, 4400 const BaseIndex& mem, 4401 Register valueTemp, Register offsetTemp, 4402 Register maskTemp) { 4403 AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value, 4404 valueTemp, offsetTemp, maskTemp); 4405 } 4406 4407 template <typename T> 4408 static void WasmAtomicExchange64(MacroAssembler& masm, 4409 const wasm::MemoryAccessDesc& access, 4410 const T& mem, Register64 value, 4411 Register64 output) { 4412 AtomicExchange64(masm, &access, access.sync(), mem, value, output); 4413 } 4414 4415 void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, 4416 const Address& mem, Register64 src, 4417 Register64 output) { 4418 WasmAtomicExchange64(*this, access, mem, src, output); 4419 } 4420 4421 void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, 4422 const BaseIndex& mem, Register64 src, 4423 Register64 output) { 4424 WasmAtomicExchange64(*this, access, mem, src, output); 4425 } 4426 4427 void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, 4428 AtomicOp op, Register64 value, 4429 const Address& mem, Register64 temp, 4430 Register64 output) { 4431 AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); 4432 } 4433 4434 void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, 4435 AtomicOp op, Register64 value, 4436 const BaseIndex& mem, Register64 temp, 4437 Register64 output) { 4438 AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output); 4439 } 4440 4441 // ======================================================================== 4442 // JS atomic operations. 4443 4444 template <typename T> 4445 static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, 4446 Synchronization sync, const T& mem, 4447 Register oldval, Register newval, 4448 Register valueTemp, Register offsetTemp, 4449 Register maskTemp, Register temp, 4450 AnyRegister output) { 4451 if (arrayType == Scalar::Uint32) { 4452 masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, 4453 offsetTemp, maskTemp, temp); 4454 masm.convertUInt32ToDouble(temp, output.fpu()); 4455 } else { 4456 masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, 4457 offsetTemp, maskTemp, output.gpr()); 4458 } 4459 } 4460 4461 template <typename T> 4462 static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, 4463 Synchronization sync, const T& mem, Register value, 4464 Register valueTemp, Register offsetTemp, 4465 Register maskTemp, Register temp, 4466 AnyRegister output) { 4467 if (arrayType == Scalar::Uint32) { 4468 masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, 4469 maskTemp, temp); 4470 masm.convertUInt32ToDouble(temp, output.fpu()); 4471 } else { 4472 masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, 4473 maskTemp, output.gpr()); 4474 } 4475 } 4476 4477 template <typename T> 4478 static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, 4479 Synchronization sync, AtomicOp op, Register value, 4480 const T& mem, Register valueTemp, 4481 Register offsetTemp, Register maskTemp, 4482 Register temp, AnyRegister output) { 4483 if (arrayType == Scalar::Uint32) { 4484 masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, 4485 maskTemp, temp); 4486 masm.convertUInt32ToDouble(temp, output.fpu()); 4487 } else { 4488 masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, 4489 maskTemp, output.gpr()); 4490 } 4491 } 4492 4493 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, 4494 Synchronization sync, const Address& mem, 4495 Register oldval, Register newval, 4496 Register valueTemp, Register offsetTemp, 4497 Register maskTemp, Register temp, 4498 AnyRegister output) { 4499 CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, 4500 offsetTemp, maskTemp, temp, output); 4501 } 4502 4503 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, 4504 Synchronization sync, 4505 const BaseIndex& mem, Register oldval, 4506 Register newval, Register valueTemp, 4507 Register offsetTemp, Register maskTemp, 4508 Register temp, AnyRegister output) { 4509 CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, 4510 offsetTemp, maskTemp, temp, output); 4511 } 4512 4513 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, 4514 Synchronization sync, const Address& mem, 4515 Register value, Register valueTemp, 4516 Register offsetTemp, Register maskTemp, 4517 Register temp, AnyRegister output) { 4518 AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, 4519 maskTemp, temp, output); 4520 } 4521 4522 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, 4523 Synchronization sync, 4524 const BaseIndex& mem, Register value, 4525 Register valueTemp, Register offsetTemp, 4526 Register maskTemp, Register temp, 4527 AnyRegister output) { 4528 AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, 4529 maskTemp, temp, output); 4530 } 4531 4532 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, 4533 Synchronization sync, AtomicOp op, 4534 Register value, const Address& mem, 4535 Register valueTemp, Register offsetTemp, 4536 Register maskTemp, Register temp, 4537 AnyRegister output) { 4538 AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, 4539 maskTemp, temp, output); 4540 } 4541 4542 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, 4543 Synchronization sync, AtomicOp op, 4544 Register value, const BaseIndex& mem, 4545 Register valueTemp, Register offsetTemp, 4546 Register maskTemp, Register temp, 4547 AnyRegister output) { 4548 AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, 4549 maskTemp, temp, output); 4550 } 4551 4552 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, 4553 Synchronization sync, AtomicOp op, 4554 Register value, const BaseIndex& mem, 4555 Register valueTemp, Register offsetTemp, 4556 Register maskTemp) { 4557 AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, 4558 offsetTemp, maskTemp); 4559 } 4560 4561 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, 4562 Synchronization sync, AtomicOp op, 4563 Register value, const Address& mem, 4564 Register valueTemp, Register offsetTemp, 4565 Register maskTemp) { 4566 AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp, 4567 offsetTemp, maskTemp); 4568 } 4569 4570 void MacroAssembler::atomicPause() { 4571 // LoongArch doesn't have 'pause' or 'yield' instructions like other 4572 // platforms, just use nop here. 4573 nop(); 4574 } 4575 4576 void MacroAssembler::flexibleQuotient32(Register lhs, Register rhs, 4577 Register dest, bool isUnsigned, 4578 const LiveRegisterSet&) { 4579 quotient32(lhs, rhs, dest, isUnsigned); 4580 } 4581 4582 void MacroAssembler::flexibleQuotientPtr(Register lhs, Register rhs, 4583 Register dest, bool isUnsigned, 4584 const LiveRegisterSet&) { 4585 quotient64(lhs, rhs, dest, isUnsigned); 4586 } 4587 4588 void MacroAssembler::flexibleRemainder32(Register lhs, Register rhs, 4589 Register dest, bool isUnsigned, 4590 const LiveRegisterSet&) { 4591 remainder32(lhs, rhs, dest, isUnsigned); 4592 } 4593 4594 void MacroAssembler::flexibleRemainderPtr(Register lhs, Register rhs, 4595 Register dest, bool isUnsigned, 4596 const LiveRegisterSet&) { 4597 remainder64(lhs, rhs, dest, isUnsigned); 4598 } 4599 4600 void MacroAssembler::flexibleDivMod32(Register lhs, Register rhs, 4601 Register divOutput, Register remOutput, 4602 bool isUnsigned, const LiveRegisterSet&) { 4603 MOZ_ASSERT(lhs != divOutput && lhs != remOutput, "lhs is preserved"); 4604 MOZ_ASSERT(rhs != divOutput && rhs != remOutput, "rhs is preserved"); 4605 4606 if (isUnsigned) { 4607 as_div_wu(divOutput, lhs, rhs); 4608 as_mod_wu(remOutput, lhs, rhs); 4609 } else { 4610 as_div_w(divOutput, lhs, rhs); 4611 as_mod_w(remOutput, lhs, rhs); 4612 } 4613 } 4614 4615 CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) { 4616 return movWithPatch(ImmPtr(nullptr), dest); 4617 } 4618 4619 void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc, 4620 CodeLocationLabel target) { 4621 PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr)); 4622 } 4623 4624 // ======================================================================== 4625 // Spectre Mitigations. 4626 4627 void MacroAssembler::speculationBarrier() { MOZ_CRASH(); } 4628 4629 void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest, 4630 Label* fail) { 4631 ScratchFloat32Scope fpscratch(asMasm()); 4632 4633 // Round toward negative infinity. 4634 as_ftintrm_l_s(fpscratch, src); 4635 moveFromDouble(fpscratch, dest); 4636 4637 // Sign extend lower 32 bits to test if the result isn't an Int32. 4638 { 4639 UseScratchRegisterScope temps(*this); 4640 Register scratch = temps.Acquire(); 4641 4642 move32SignExtendToPtr(dest, scratch); 4643 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4644 } 4645 4646 // We have to check for -0 and NaN when the result is zero. 4647 Label notZero; 4648 ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); 4649 { 4650 // If any of the two most significant bits is set, |src| is -0 or NaN. 4651 moveFromFloat32(src, dest); 4652 as_srli_w(dest, dest, 30); 4653 branch32(Assembler::NotEqual, dest, zero, fail); 4654 } 4655 bind(¬Zero); 4656 } 4657 4658 void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest, 4659 Label* fail) { 4660 ScratchDoubleScope fpscratch(asMasm()); 4661 4662 // Round toward negative infinity. 4663 as_ftintrm_l_d(fpscratch, src); 4664 moveFromDouble(fpscratch, dest); 4665 4666 // Sign extend lower 32 bits to test if the result isn't an Int32. 4667 { 4668 UseScratchRegisterScope temps(*this); 4669 Register scratch = temps.Acquire(); 4670 4671 move32SignExtendToPtr(dest, scratch); 4672 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4673 } 4674 4675 // We have to check for -0 and NaN when the result is zero. 4676 Label notZero; 4677 ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); 4678 { 4679 // If any of the two most significant bits is set, |src| is negative or NaN. 4680 moveFromDouble(src, dest); 4681 as_srli_d(dest, dest, 62); 4682 branchPtr(Assembler::NotEqual, dest, zero, fail); 4683 } 4684 bind(¬Zero); 4685 } 4686 4687 void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest, 4688 Label* fail) { 4689 ScratchFloat32Scope fpscratch(asMasm()); 4690 4691 // Round toward positive infinity. 4692 as_ftintrp_l_s(fpscratch, src); 4693 moveFromDouble(fpscratch, dest); 4694 4695 // Sign extend lower 32 bits to test if the result isn't an Int32. 4696 { 4697 UseScratchRegisterScope temps(*this); 4698 Register scratch = temps.Acquire(); 4699 4700 move32SignExtendToPtr(dest, scratch); 4701 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4702 } 4703 4704 // We have to check for (-1, -0] and NaN when the result is zero. 4705 Label notZero; 4706 ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); 4707 { 4708 // If binary value is not zero, the input was not 0, so we bail. 4709 moveFromFloat32(src, dest); 4710 branch32(Assembler::NotEqual, dest, zero, fail); 4711 } 4712 bind(¬Zero); 4713 } 4714 4715 void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest, 4716 Label* fail) { 4717 ScratchDoubleScope fpscratch(asMasm()); 4718 4719 // Round toward positive infinity. 4720 as_ftintrp_l_d(fpscratch, src); 4721 moveFromDouble(fpscratch, dest); 4722 4723 // Sign extend lower 32 bits to test if the result isn't an Int32. 4724 { 4725 UseScratchRegisterScope temps(*this); 4726 Register scratch = temps.Acquire(); 4727 4728 move32SignExtendToPtr(dest, scratch); 4729 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4730 } 4731 4732 // We have to check for (-1, -0] and NaN when the result is zero. 4733 Label notZero; 4734 ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); 4735 { 4736 // If binary value is not zero, the input was not 0, so we bail. 4737 moveFromDouble(src, dest); 4738 branchPtr(Assembler::NotEqual, dest, zero, fail); 4739 } 4740 bind(¬Zero); 4741 } 4742 4743 void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest, 4744 FloatRegister temp, Label* fail) { 4745 ScratchFloat32Scope fpscratch(*this); 4746 4747 Label negative, end, performRound; 4748 4749 // Branch for negative inputs. Doesn't catch NaN or -0. 4750 loadConstantFloat32(0.0f, fpscratch); 4751 ma_bc_s(src, fpscratch, &negative, Assembler::DoubleLessThan, ShortJump); 4752 4753 // If non-negative check for bailout. 4754 ma_bc_s(src, fpscratch, &performRound, Assembler::DoubleNotEqual, ShortJump); 4755 { 4756 // If binary value is not zero, it is NaN or -0, so we bail. 4757 moveFromFloat32(src, dest); 4758 branch32(Assembler::NotEqual, dest, zero, fail); 4759 ma_b(&end, ShortJump); 4760 } 4761 4762 // Input is negative, but isn't -0. 4763 bind(&negative); 4764 { 4765 // Inputs in [-0.5, 0) are rounded to -0. Fail. 4766 loadConstantFloat32(-0.5f, fpscratch); 4767 branchFloat(Assembler::DoubleGreaterThanOrEqual, src, fpscratch, fail); 4768 } 4769 4770 bind(&performRound); 4771 { 4772 // Load biggest number less than 0.5 in the temp register. 4773 loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp); 4774 4775 // Other inputs need the biggest float less than 0.5 added. 4776 as_fadd_s(fpscratch, src, temp); 4777 4778 // Round toward negative infinity. 4779 as_ftintrm_l_s(fpscratch, fpscratch); 4780 moveFromDouble(fpscratch, dest); 4781 4782 // Sign extend lower 32 bits to test if the result isn't an Int32. 4783 { 4784 UseScratchRegisterScope temps(*this); 4785 Register scratch = temps.Acquire(); 4786 4787 move32SignExtendToPtr(dest, scratch); 4788 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4789 } 4790 } 4791 bind(&end); 4792 } 4793 4794 void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest, 4795 FloatRegister temp, Label* fail) { 4796 ScratchDoubleScope fpscratch(*this); 4797 4798 Label negative, end, performRound; 4799 4800 // Branch for negative inputs. Doesn't catch NaN or -0. 4801 loadConstantDouble(0.0, fpscratch); 4802 ma_bc_d(src, fpscratch, &negative, Assembler::DoubleLessThan, ShortJump); 4803 4804 // If non-negative check for bailout. 4805 ma_bc_d(src, fpscratch, &performRound, Assembler::DoubleNotEqual, ShortJump); 4806 { 4807 // If binary value is not zero, it is NaN or -0, so we bail. 4808 moveFromDouble(src, dest); 4809 branchPtr(Assembler::NotEqual, dest, zero, fail); 4810 ma_b(&end, ShortJump); 4811 } 4812 4813 // Input is negative, but isn't -0. 4814 bind(&negative); 4815 { 4816 // Inputs in [-0.5, 0) are rounded to -0. Fail. 4817 loadConstantDouble(-0.5, fpscratch); 4818 branchDouble(Assembler::DoubleGreaterThanOrEqual, src, fpscratch, fail); 4819 } 4820 4821 bind(&performRound); 4822 { 4823 // Load biggest number less than 0.5 in the temp register. 4824 loadConstantDouble(GetBiggestNumberLessThan(0.5), temp); 4825 4826 // Other inputs need the biggest double less than 0.5 added. 4827 as_fadd_d(fpscratch, src, temp); 4828 4829 // Round toward negative infinity. 4830 as_ftintrm_l_d(fpscratch, fpscratch); 4831 moveFromDouble(fpscratch, dest); 4832 4833 // Sign extend lower 32 bits to test if the result isn't an Int32. 4834 { 4835 UseScratchRegisterScope temps(*this); 4836 Register scratch = temps.Acquire(); 4837 4838 move32SignExtendToPtr(dest, scratch); 4839 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4840 } 4841 } 4842 bind(&end); 4843 } 4844 4845 void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest, 4846 Label* fail) { 4847 ScratchFloat32Scope fpscratch(asMasm()); 4848 4849 // Round toward zero. 4850 as_ftintrz_l_s(fpscratch, src); 4851 moveFromDouble(fpscratch, dest); 4852 4853 // Sign extend lower 32 bits to test if the result isn't an Int32. 4854 { 4855 UseScratchRegisterScope temps(*this); 4856 Register scratch = temps.Acquire(); 4857 4858 move32SignExtendToPtr(dest, scratch); 4859 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4860 } 4861 4862 // We have to check for (-1, -0] and NaN when the result is zero. 4863 Label notZero; 4864 ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); 4865 { 4866 // If any of the two most significant bits is set, |src| is negative or NaN. 4867 moveFromFloat32(src, dest); 4868 as_srli_w(dest, dest, 30); 4869 branch32(Assembler::NotEqual, dest, zero, fail); 4870 } 4871 bind(¬Zero); 4872 } 4873 4874 void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest, 4875 Label* fail) { 4876 ScratchDoubleScope fpscratch(asMasm()); 4877 4878 // Round toward zero. 4879 as_ftintrz_l_d(fpscratch, src); 4880 moveFromDouble(fpscratch, dest); 4881 4882 // Sign extend lower 32 bits to test if the result isn't an Int32. 4883 { 4884 UseScratchRegisterScope temps(*this); 4885 Register scratch = temps.Acquire(); 4886 4887 move32SignExtendToPtr(dest, scratch); 4888 branchPtr(Assembler::NotEqual, dest, scratch, fail); 4889 } 4890 4891 // We have to check for (-1, -0] and NaN when the result is zero. 4892 Label notZero; 4893 ma_b(dest, zero, ¬Zero, Assembler::NotEqual, ShortJump); 4894 { 4895 // If any of the two most significant bits is set, |src| is negative or NaN. 4896 moveFromDouble(src, dest); 4897 as_srli_d(dest, dest, 62); 4898 branchPtr(Assembler::NotEqual, dest, zero, fail); 4899 } 4900 bind(¬Zero); 4901 } 4902 4903 void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src, 4904 FloatRegister dest) { 4905 MOZ_CRASH("not supported on this platform"); 4906 } 4907 4908 void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src, 4909 FloatRegister dest) { 4910 MOZ_CRASH("not supported on this platform"); 4911 } 4912 4913 void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs, 4914 FloatRegister output) { 4915 as_fcopysign_d(output, lhs, rhs); 4916 } 4917 4918 void MacroAssembler::copySignFloat32(FloatRegister lhs, FloatRegister rhs, 4919 FloatRegister output) { 4920 as_fcopysign_s(output, lhs, rhs); 4921 } 4922 4923 void MacroAssemblerLOONG64Compat::move32(Imm32 imm, Register dest) { 4924 ma_li(dest, imm); 4925 } 4926 4927 void MacroAssemblerLOONG64Compat::move32(Register src, Register dest) { 4928 as_slli_w(dest, src, 0); 4929 } 4930 4931 void MacroAssemblerLOONG64Compat::movePtr(Register src, Register dest) { 4932 as_or(dest, src, zero); 4933 } 4934 void MacroAssemblerLOONG64Compat::movePtr(ImmWord imm, Register dest) { 4935 ma_li(dest, imm); 4936 } 4937 4938 void MacroAssemblerLOONG64Compat::movePtr(ImmGCPtr imm, Register dest) { 4939 ma_li(dest, imm); 4940 } 4941 4942 void MacroAssemblerLOONG64Compat::movePtr(ImmPtr imm, Register dest) { 4943 movePtr(ImmWord(uintptr_t(imm.value)), dest); 4944 } 4945 4946 void MacroAssemblerLOONG64Compat::movePtr(wasm::SymbolicAddress imm, 4947 Register dest) { 4948 append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm)); 4949 ma_liPatchable(dest, ImmWord(-1)); 4950 } 4951 4952 FaultingCodeOffset MacroAssemblerLOONG64Compat::load8ZeroExtend( 4953 const Address& address, Register dest) { 4954 return ma_load(dest, address, SizeByte, ZeroExtend); 4955 } 4956 4957 FaultingCodeOffset MacroAssemblerLOONG64Compat::load8ZeroExtend( 4958 const BaseIndex& src, Register dest) { 4959 return ma_load(dest, src, SizeByte, ZeroExtend); 4960 } 4961 4962 FaultingCodeOffset MacroAssemblerLOONG64Compat::load8SignExtend( 4963 const Address& address, Register dest) { 4964 return ma_load(dest, address, SizeByte, SignExtend); 4965 } 4966 4967 FaultingCodeOffset MacroAssemblerLOONG64Compat::load8SignExtend( 4968 const BaseIndex& src, Register dest) { 4969 return ma_load(dest, src, SizeByte, SignExtend); 4970 } 4971 4972 FaultingCodeOffset MacroAssemblerLOONG64Compat::load16ZeroExtend( 4973 const Address& address, Register dest) { 4974 return ma_load(dest, address, SizeHalfWord, ZeroExtend); 4975 } 4976 4977 FaultingCodeOffset MacroAssemblerLOONG64Compat::load16ZeroExtend( 4978 const BaseIndex& src, Register dest) { 4979 return ma_load(dest, src, SizeHalfWord, ZeroExtend); 4980 } 4981 4982 FaultingCodeOffset MacroAssemblerLOONG64Compat::load16SignExtend( 4983 const Address& address, Register dest) { 4984 return ma_load(dest, address, SizeHalfWord, SignExtend); 4985 } 4986 4987 FaultingCodeOffset MacroAssemblerLOONG64Compat::load16SignExtend( 4988 const BaseIndex& src, Register dest) { 4989 return ma_load(dest, src, SizeHalfWord, SignExtend); 4990 } 4991 4992 FaultingCodeOffset MacroAssemblerLOONG64Compat::load32(const Address& address, 4993 Register dest) { 4994 return ma_ld_w(dest, address); 4995 } 4996 4997 FaultingCodeOffset MacroAssemblerLOONG64Compat::load32(const BaseIndex& address, 4998 Register dest) { 4999 Register base = address.base; 5000 Register index = address.index; 5001 int32_t offset = address.offset; 5002 uint32_t shift = Imm32::ShiftOf(address.scale).value; 5003 js::wasm::FaultingCodeOffset fco; 5004 5005 UseScratchRegisterScope temps(asMasm()); 5006 if (offset != 0) { 5007 Register scratch = temps.Acquire(); 5008 ma_li(scratch, Imm32(offset)); 5009 if (shift != 0) { 5010 MOZ_ASSERT(shift <= 4); 5011 as_alsl_d(scratch, index, scratch, shift - 1); 5012 } else { 5013 as_add_d(scratch, index, scratch); 5014 } 5015 fco = js::wasm::FaultingCodeOffset(currentOffset()); 5016 as_ldx_w(dest, base, scratch); 5017 } else if (shift != 0) { 5018 Register scratch = temps.Acquire(); 5019 as_slli_d(scratch, index, shift); 5020 fco = js::wasm::FaultingCodeOffset(currentOffset()); 5021 as_ldx_w(dest, base, scratch); 5022 } else { 5023 fco = js::wasm::FaultingCodeOffset(currentOffset()); 5024 as_ldx_w(dest, base, index); 5025 } 5026 return fco; 5027 } 5028 5029 void MacroAssemblerLOONG64Compat::load32(AbsoluteAddress address, 5030 Register dest) { 5031 UseScratchRegisterScope temps(asMasm()); 5032 Register scratch = temps.Acquire(); 5033 movePtr(ImmPtr(address.addr), scratch); 5034 load32(Address(scratch, 0), dest); 5035 } 5036 5037 void MacroAssemblerLOONG64Compat::load32(wasm::SymbolicAddress address, 5038 Register dest) { 5039 UseScratchRegisterScope temps(asMasm()); 5040 Register scratch = temps.Acquire(); 5041 movePtr(address, scratch); 5042 load32(Address(scratch, 0), dest); 5043 } 5044 5045 FaultingCodeOffset MacroAssemblerLOONG64Compat::loadPtr(const Address& address, 5046 Register dest) { 5047 return ma_ld_d(dest, address); 5048 } 5049 5050 FaultingCodeOffset MacroAssemblerLOONG64Compat::loadPtr(const BaseIndex& src, 5051 Register dest) { 5052 Register base = src.base; 5053 Register index = src.index; 5054 int32_t offset = src.offset; 5055 uint32_t shift = Imm32::ShiftOf(src.scale).value; 5056 js::wasm::FaultingCodeOffset fco; 5057 5058 UseScratchRegisterScope temps(asMasm()); 5059 if (offset != 0) { 5060 Register scratch = temps.Acquire(); 5061 ma_li(scratch, Imm32(offset)); 5062 if (shift != 0) { 5063 MOZ_ASSERT(shift <= 4); 5064 as_alsl_d(scratch, index, scratch, shift - 1); 5065 } else { 5066 as_add_d(scratch, index, scratch); 5067 } 5068 fco = js::wasm::FaultingCodeOffset(currentOffset()); 5069 as_ldx_d(dest, base, scratch); 5070 } else if (shift != 0) { 5071 Register scratch = temps.Acquire(); 5072 as_slli_d(scratch, index, shift); 5073 fco = js::wasm::FaultingCodeOffset(currentOffset()); 5074 as_ldx_d(dest, base, scratch); 5075 } else { 5076 fco = js::wasm::FaultingCodeOffset(currentOffset()); 5077 as_ldx_d(dest, base, index); 5078 } 5079 return fco; 5080 } 5081 5082 void MacroAssemblerLOONG64Compat::loadPtr(AbsoluteAddress address, 5083 Register dest) { 5084 UseScratchRegisterScope temps(asMasm()); 5085 Register scratch = temps.Acquire(); 5086 movePtr(ImmPtr(address.addr), scratch); 5087 loadPtr(Address(scratch, 0), dest); 5088 } 5089 5090 void MacroAssemblerLOONG64Compat::loadPtr(wasm::SymbolicAddress address, 5091 Register dest) { 5092 UseScratchRegisterScope temps(asMasm()); 5093 Register scratch = temps.Acquire(); 5094 movePtr(address, scratch); 5095 loadPtr(Address(scratch, 0), dest); 5096 } 5097 5098 void MacroAssemblerLOONG64Compat::loadPrivate(const Address& address, 5099 Register dest) { 5100 loadPtr(address, dest); 5101 } 5102 5103 void MacroAssemblerLOONG64Compat::store8(Imm32 imm, const Address& address) { 5104 UseScratchRegisterScope temps(asMasm()); 5105 Register scratch = temps.Acquire(); 5106 ma_li(scratch, imm); 5107 ma_store(scratch, address, SizeByte); 5108 } 5109 5110 FaultingCodeOffset MacroAssemblerLOONG64Compat::store8(Register src, 5111 const Address& address) { 5112 return ma_store(src, address, SizeByte); 5113 } 5114 5115 void MacroAssemblerLOONG64Compat::store8(Imm32 imm, const BaseIndex& dest) { 5116 ma_store(imm, dest, SizeByte); 5117 } 5118 5119 FaultingCodeOffset MacroAssemblerLOONG64Compat::store8(Register src, 5120 const BaseIndex& dest) { 5121 return ma_store(src, dest, SizeByte); 5122 } 5123 5124 void MacroAssemblerLOONG64Compat::store16(Imm32 imm, const Address& address) { 5125 UseScratchRegisterScope temps(asMasm()); 5126 Register scratch = temps.Acquire(); 5127 ma_li(scratch, imm); 5128 ma_store(scratch, address, SizeHalfWord); 5129 } 5130 5131 FaultingCodeOffset MacroAssemblerLOONG64Compat::store16( 5132 Register src, const Address& address) { 5133 return ma_store(src, address, SizeHalfWord); 5134 } 5135 5136 void MacroAssemblerLOONG64Compat::store16(Imm32 imm, const BaseIndex& dest) { 5137 ma_store(imm, dest, SizeHalfWord); 5138 } 5139 5140 FaultingCodeOffset MacroAssemblerLOONG64Compat::store16( 5141 Register src, const BaseIndex& address) { 5142 return ma_store(src, address, SizeHalfWord); 5143 } 5144 5145 void MacroAssemblerLOONG64Compat::store32(Register src, 5146 AbsoluteAddress address) { 5147 UseScratchRegisterScope temps(asMasm()); 5148 Register scratch = temps.Acquire(); 5149 movePtr(ImmPtr(address.addr), scratch); 5150 store32(src, Address(scratch, 0)); 5151 } 5152 5153 FaultingCodeOffset MacroAssemblerLOONG64Compat::store32( 5154 Register src, const Address& address) { 5155 return ma_store(src, address, SizeWord); 5156 } 5157 5158 void MacroAssemblerLOONG64Compat::store32(Imm32 src, const Address& address) { 5159 UseScratchRegisterScope temps(asMasm()); 5160 Register scratch = temps.Acquire(); 5161 move32(src, scratch); 5162 ma_store(scratch, address, SizeWord); 5163 } 5164 5165 void MacroAssemblerLOONG64Compat::store32(Imm32 imm, const BaseIndex& dest) { 5166 ma_store(imm, dest, SizeWord); 5167 } 5168 5169 FaultingCodeOffset MacroAssemblerLOONG64Compat::store32(Register src, 5170 const BaseIndex& dest) { 5171 return ma_store(src, dest, SizeWord); 5172 } 5173 5174 template <typename T> 5175 void MacroAssemblerLOONG64Compat::storePtr(ImmWord imm, T address) { 5176 UseScratchRegisterScope temps(asMasm()); 5177 Register scratch = temps.Acquire(); 5178 ma_li(scratch, imm); 5179 ma_store(scratch, address, SizeDouble); 5180 } 5181 5182 template void MacroAssemblerLOONG64Compat::storePtr<Address>(ImmWord imm, 5183 Address address); 5184 template void MacroAssemblerLOONG64Compat::storePtr<BaseIndex>( 5185 ImmWord imm, BaseIndex address); 5186 5187 template <typename T> 5188 void MacroAssemblerLOONG64Compat::storePtr(ImmPtr imm, T address) { 5189 storePtr(ImmWord(uintptr_t(imm.value)), address); 5190 } 5191 5192 template void MacroAssemblerLOONG64Compat::storePtr<Address>(ImmPtr imm, 5193 Address address); 5194 template void MacroAssemblerLOONG64Compat::storePtr<BaseIndex>( 5195 ImmPtr imm, BaseIndex address); 5196 5197 template <typename T> 5198 void MacroAssemblerLOONG64Compat::storePtr(ImmGCPtr imm, T address) { 5199 UseScratchRegisterScope temps(asMasm()); 5200 Register scratch = temps.Acquire(); 5201 movePtr(imm, scratch); 5202 storePtr(scratch, address); 5203 } 5204 5205 template void MacroAssemblerLOONG64Compat::storePtr<Address>(ImmGCPtr imm, 5206 Address address); 5207 template void MacroAssemblerLOONG64Compat::storePtr<BaseIndex>( 5208 ImmGCPtr imm, BaseIndex address); 5209 5210 FaultingCodeOffset MacroAssemblerLOONG64Compat::storePtr( 5211 Register src, const Address& address) { 5212 return ma_st_d(src, address); 5213 } 5214 5215 FaultingCodeOffset MacroAssemblerLOONG64Compat::storePtr( 5216 Register src, const BaseIndex& address) { 5217 Register base = address.base; 5218 Register index = address.index; 5219 int32_t offset = address.offset; 5220 int32_t shift = Imm32::ShiftOf(address.scale).value; 5221 FaultingCodeOffset fco; 5222 5223 UseScratchRegisterScope temps(asMasm()); 5224 if ((offset == 0) && (shift == 0)) { 5225 fco = FaultingCodeOffset(currentOffset()); 5226 as_stx_d(src, base, index); 5227 } else if (is_intN(offset, 12)) { 5228 Register scratch = temps.Acquire(); 5229 if (shift == 0) { 5230 as_add_d(scratch, base, index); 5231 } else { 5232 as_alsl_d(scratch, index, base, shift - 1); 5233 } 5234 fco = FaultingCodeOffset(currentOffset()); 5235 as_st_d(src, scratch, offset); 5236 } else { 5237 Register scratch = temps.Acquire(); 5238 ma_li(scratch, Imm32(offset)); 5239 if (shift == 0) { 5240 as_add_d(scratch, scratch, index); 5241 } else { 5242 as_alsl_d(scratch, index, scratch, shift - 1); 5243 } 5244 fco = FaultingCodeOffset(currentOffset()); 5245 as_stx_d(src, base, scratch); 5246 } 5247 return fco; 5248 } 5249 5250 void MacroAssemblerLOONG64Compat::storePtr(Register src, AbsoluteAddress dest) { 5251 UseScratchRegisterScope temps(asMasm()); 5252 Register scratch = temps.Acquire(); 5253 movePtr(ImmPtr(dest.addr), scratch); 5254 storePtr(src, Address(scratch, 0)); 5255 } 5256 5257 void MacroAssemblerLOONG64Compat::testNullSet(Condition cond, 5258 const ValueOperand& value, 5259 Register dest) { 5260 MOZ_ASSERT(cond == Equal || cond == NotEqual); 5261 UseScratchRegisterScope temps(asMasm()); 5262 Register scratch = temps.Acquire(); 5263 splitTag(value, scratch); 5264 ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_NULL), cond); 5265 } 5266 5267 void MacroAssemblerLOONG64Compat::testObjectSet(Condition cond, 5268 const ValueOperand& value, 5269 Register dest) { 5270 MOZ_ASSERT(cond == Equal || cond == NotEqual); 5271 UseScratchRegisterScope temps(asMasm()); 5272 Register scratch = temps.Acquire(); 5273 splitTag(value, scratch); 5274 ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_OBJECT), cond); 5275 } 5276 5277 void MacroAssemblerLOONG64Compat::testUndefinedSet(Condition cond, 5278 const ValueOperand& value, 5279 Register dest) { 5280 MOZ_ASSERT(cond == Equal || cond == NotEqual); 5281 UseScratchRegisterScope temps(asMasm()); 5282 Register scratch = temps.Acquire(); 5283 splitTag(value, scratch); 5284 ma_cmp_set(dest, scratch, ImmTag(JSVAL_TAG_UNDEFINED), cond); 5285 } 5286 5287 void MacroAssemblerLOONG64Compat::unboxInt32(const ValueOperand& operand, 5288 Register dest) { 5289 as_slli_w(dest, operand.valueReg(), 0); 5290 } 5291 5292 void MacroAssemblerLOONG64Compat::unboxInt32(Register src, Register dest) { 5293 as_slli_w(dest, src, 0); 5294 } 5295 5296 void MacroAssemblerLOONG64Compat::unboxInt32(const Address& src, 5297 Register dest) { 5298 load32(Address(src.base, src.offset), dest); 5299 } 5300 5301 void MacroAssemblerLOONG64Compat::unboxInt32(const BaseIndex& src, 5302 Register dest) { 5303 UseScratchRegisterScope temps(asMasm()); 5304 Register scratch = temps.Acquire(); 5305 computeScaledAddress(src, scratch); 5306 load32(Address(scratch, src.offset), dest); 5307 } 5308 5309 void MacroAssemblerLOONG64Compat::unboxBoolean(const ValueOperand& operand, 5310 Register dest) { 5311 as_slli_w(dest, operand.valueReg(), 0); 5312 } 5313 5314 void MacroAssemblerLOONG64Compat::unboxBoolean(Register src, Register dest) { 5315 as_slli_w(dest, src, 0); 5316 } 5317 5318 void MacroAssemblerLOONG64Compat::unboxBoolean(const Address& src, 5319 Register dest) { 5320 ma_ld_w(dest, src); 5321 } 5322 5323 void MacroAssemblerLOONG64Compat::unboxBoolean(const BaseIndex& src, 5324 Register dest) { 5325 UseScratchRegisterScope temps(asMasm()); 5326 Register scratch = temps.Acquire(); 5327 computeScaledAddress(src, scratch); 5328 ma_ld_w(dest, Address(scratch, src.offset)); 5329 } 5330 5331 void MacroAssemblerLOONG64Compat::unboxDouble(const ValueOperand& operand, 5332 FloatRegister dest) { 5333 as_movgr2fr_d(dest, operand.valueReg()); 5334 } 5335 5336 void MacroAssemblerLOONG64Compat::unboxDouble(const Address& src, 5337 FloatRegister dest) { 5338 ma_fld_d(dest, Address(src.base, src.offset)); 5339 } 5340 5341 void MacroAssemblerLOONG64Compat::unboxDouble(const BaseIndex& src, 5342 FloatRegister dest) { 5343 UseScratchRegisterScope temps(asMasm()); 5344 Register scratch = temps.Acquire(); 5345 loadPtr(src, scratch); 5346 unboxDouble(ValueOperand(scratch), dest); 5347 } 5348 5349 void MacroAssemblerLOONG64Compat::unboxString(const ValueOperand& operand, 5350 Register dest) { 5351 unboxNonDouble(operand, dest, JSVAL_TYPE_STRING); 5352 } 5353 5354 void MacroAssemblerLOONG64Compat::unboxString(Register src, Register dest) { 5355 unboxNonDouble(src, dest, JSVAL_TYPE_STRING); 5356 } 5357 5358 void MacroAssemblerLOONG64Compat::unboxString(const Address& src, 5359 Register dest) { 5360 unboxNonDouble(src, dest, JSVAL_TYPE_STRING); 5361 } 5362 5363 void MacroAssemblerLOONG64Compat::unboxSymbol(const ValueOperand& operand, 5364 Register dest) { 5365 unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL); 5366 } 5367 5368 void MacroAssemblerLOONG64Compat::unboxSymbol(Register src, Register dest) { 5369 unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); 5370 } 5371 5372 void MacroAssemblerLOONG64Compat::unboxSymbol(const Address& src, 5373 Register dest) { 5374 unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); 5375 } 5376 5377 void MacroAssemblerLOONG64Compat::unboxBigInt(const ValueOperand& operand, 5378 Register dest) { 5379 unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT); 5380 } 5381 5382 void MacroAssemblerLOONG64Compat::unboxBigInt(Register src, Register dest) { 5383 unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); 5384 } 5385 5386 void MacroAssemblerLOONG64Compat::unboxBigInt(const Address& src, 5387 Register dest) { 5388 unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); 5389 } 5390 5391 void MacroAssemblerLOONG64Compat::unboxObject(const ValueOperand& src, 5392 Register dest) { 5393 unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); 5394 } 5395 5396 void MacroAssemblerLOONG64Compat::unboxObject(Register src, Register dest) { 5397 unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); 5398 } 5399 5400 void MacroAssemblerLOONG64Compat::unboxObject(const Address& src, 5401 Register dest) { 5402 unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); 5403 } 5404 5405 void MacroAssemblerLOONG64Compat::unboxValue(const ValueOperand& src, 5406 AnyRegister dest, 5407 JSValueType type) { 5408 if (dest.isFloat()) { 5409 Label notInt32, end; 5410 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); 5411 convertInt32ToDouble(src.valueReg(), dest.fpu()); 5412 ma_b(&end, ShortJump); 5413 bind(¬Int32); 5414 unboxDouble(src, dest.fpu()); 5415 bind(&end); 5416 } else { 5417 unboxNonDouble(src, dest.gpr(), type); 5418 } 5419 } 5420 5421 void MacroAssemblerLOONG64Compat::boxDouble(FloatRegister src, 5422 const ValueOperand& dest, 5423 FloatRegister) { 5424 as_movfr2gr_d(dest.valueReg(), src); 5425 } 5426 5427 #ifdef DEBUG 5428 static constexpr int32_t PayloadSize(JSValueType type) { 5429 switch (type) { 5430 case JSVAL_TYPE_UNDEFINED: 5431 case JSVAL_TYPE_NULL: 5432 return 0; 5433 case JSVAL_TYPE_BOOLEAN: 5434 return 1; 5435 case JSVAL_TYPE_INT32: 5436 case JSVAL_TYPE_MAGIC: 5437 return 32; 5438 case JSVAL_TYPE_STRING: 5439 case JSVAL_TYPE_SYMBOL: 5440 case JSVAL_TYPE_PRIVATE_GCTHING: 5441 case JSVAL_TYPE_BIGINT: 5442 case JSVAL_TYPE_OBJECT: 5443 return JSVAL_TAG_SHIFT; 5444 case JSVAL_TYPE_DOUBLE: 5445 case JSVAL_TYPE_UNKNOWN: 5446 break; 5447 } 5448 MOZ_CRASH("bad value type"); 5449 } 5450 #endif 5451 5452 static void AssertValidPayload(MacroAssemblerLOONG64Compat& masm, 5453 JSValueType type, Register payload, 5454 Register scratch) { 5455 #ifdef DEBUG 5456 if (type == JSVAL_TYPE_INT32) { 5457 // Ensure the payload is a properly sign-extended int32. 5458 Label signExtended; 5459 masm.as_slli_w(scratch, payload, 0); 5460 masm.ma_b(payload, scratch, &signExtended, Assembler::Equal, ShortJump); 5461 masm.breakpoint(); 5462 masm.bind(&signExtended); 5463 } else { 5464 // All bits above the payload must be zeroed. 5465 Label zeroed; 5466 masm.as_srli_d(scratch, payload, PayloadSize(type)); 5467 masm.ma_b(scratch, scratch, &zeroed, Assembler::Zero, ShortJump); 5468 masm.breakpoint(); 5469 masm.bind(&zeroed); 5470 } 5471 #endif 5472 } 5473 5474 void MacroAssemblerLOONG64Compat::boxValue(JSValueType type, Register src, 5475 Register dest) { 5476 MOZ_ASSERT(src != dest); 5477 MOZ_ASSERT(type != JSVAL_TYPE_UNDEFINED && type != JSVAL_TYPE_NULL); 5478 5479 AssertValidPayload(*this, type, src, dest); 5480 5481 int64_t shifted = int64_t(JSVAL_TYPE_TO_SHIFTED_TAG(type)) >> 32; 5482 MOZ_ASSERT(is_intN(shifted, 20), 5483 "upper 32 bits of shifted tag fit into lu32i.d"); 5484 5485 // Store shifted tag into upper 32 bits. The lower 32 bits are left unchanged. 5486 as_lu32i_d(dest, shifted); 5487 5488 switch (type) { 5489 case JSVAL_TYPE_BOOLEAN: 5490 case JSVAL_TYPE_INT32: 5491 case JSVAL_TYPE_MAGIC: { 5492 // Insert payload into lower 32 bits. 5493 as_bstrins_d(dest, src, 31, 0); 5494 return; 5495 } 5496 case JSVAL_TYPE_STRING: 5497 case JSVAL_TYPE_SYMBOL: 5498 case JSVAL_TYPE_PRIVATE_GCTHING: 5499 case JSVAL_TYPE_BIGINT: 5500 case JSVAL_TYPE_OBJECT: { 5501 // Insert payload into lower 47 bits. 5502 as_bstrins_d(dest, src, JSVAL_TAG_SHIFT - 1, 0); 5503 return; 5504 } 5505 case JSVAL_TYPE_DOUBLE: 5506 case JSVAL_TYPE_UNDEFINED: 5507 case JSVAL_TYPE_NULL: 5508 case JSVAL_TYPE_UNKNOWN: 5509 break; 5510 } 5511 MOZ_CRASH("bad value type"); 5512 } 5513 5514 void MacroAssemblerLOONG64Compat::boxValue(Register type, Register src, 5515 Register dest) { 5516 MOZ_ASSERT(src != dest); 5517 5518 #ifdef DEBUG 5519 Label done, isNullOrUndefined, isBoolean, isInt32OrMagic, isPointerSized; 5520 5521 asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_NULL), 5522 &isNullOrUndefined); 5523 asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_UNDEFINED), 5524 &isNullOrUndefined); 5525 asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_BOOLEAN), 5526 &isBoolean); 5527 asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_INT32), 5528 &isInt32OrMagic); 5529 asMasm().branch32(Assembler::Equal, type, Imm32(JSVAL_TYPE_MAGIC), 5530 &isInt32OrMagic); 5531 // GCThing types aren't currently supported, because as_bstrins_d truncates 5532 // payloads above UINT32_MAX. 5533 breakpoint(); 5534 { 5535 bind(&isNullOrUndefined); 5536 5537 // Ensure no payload for null and undefined. 5538 ma_b(src, src, &done, Assembler::Zero, ShortJump); 5539 breakpoint(); 5540 } 5541 { 5542 bind(&isBoolean); 5543 5544 // Ensure boolean values are either 0 or 1. 5545 ma_b(src, Imm32(1), &done, Assembler::BelowOrEqual, ShortJump); 5546 breakpoint(); 5547 } 5548 { 5549 bind(&isInt32OrMagic); 5550 5551 // Ensure |src| is sign-extended. 5552 UseScratchRegisterScope temps(asMasm()); 5553 Register scratch = temps.Acquire(); 5554 as_slli_w(scratch, src, 0); 5555 ma_b(src, scratch, &done, Assembler::Equal, ShortJump); 5556 breakpoint(); 5557 } 5558 bind(&done); 5559 #endif 5560 5561 ma_or(dest, type, Imm32(JSVAL_TAG_MAX_DOUBLE)); 5562 as_slli_d(dest, dest, JSVAL_TAG_SHIFT); 5563 as_bstrins_d(dest, src, 31, 0); 5564 } 5565 5566 void MacroAssemblerLOONG64Compat::loadConstantFloat32(float f, 5567 FloatRegister dest) { 5568 ma_lis(dest, f); 5569 } 5570 5571 void MacroAssemblerLOONG64Compat::loadInt32OrDouble(const Address& src, 5572 FloatRegister dest) { 5573 UseScratchRegisterScope temps(asMasm()); 5574 Register scratch = temps.Acquire(); 5575 Label end; 5576 5577 // If it's an int, convert it to double. 5578 loadPtr(Address(src.base, src.offset), scratch); 5579 as_movgr2fr_d(dest, scratch); 5580 as_srli_d(scratch, scratch, JSVAL_TAG_SHIFT); 5581 asMasm().branchTestInt32(Assembler::NotEqual, scratch, &end); 5582 as_ffint_d_w(dest, dest); 5583 5584 bind(&end); 5585 } 5586 5587 void MacroAssemblerLOONG64Compat::loadInt32OrDouble(const BaseIndex& addr, 5588 FloatRegister dest) { 5589 UseScratchRegisterScope temps(asMasm()); 5590 Register scratch = temps.Acquire(); 5591 5592 computeScaledAddress(addr, scratch); 5593 loadInt32OrDouble(Address(scratch, addr.offset), dest); 5594 } 5595 5596 void MacroAssemblerLOONG64Compat::loadConstantDouble(double dp, 5597 FloatRegister dest) { 5598 ma_lid(dest, dp); 5599 } 5600 5601 Register MacroAssemblerLOONG64Compat::extractObject(const Address& address, 5602 Register scratch) { 5603 loadPtr(address, scratch); 5604 as_bstrpick_d(scratch, scratch, JSVAL_TAG_SHIFT - 1, 0); 5605 return scratch; 5606 } 5607 5608 Register MacroAssemblerLOONG64Compat::extractTag(const Address& address, 5609 Register scratch) { 5610 loadPtr(address, scratch); 5611 as_bstrpick_d(scratch, scratch, 63, JSVAL_TAG_SHIFT); 5612 return scratch; 5613 } 5614 5615 Register MacroAssemblerLOONG64Compat::extractTag(const BaseIndex& address, 5616 Register scratch) { 5617 computeScaledAddress(address, scratch); 5618 return extractTag(Address(scratch, address.offset), scratch); 5619 } 5620 5621 ///////////////////////////////////////////////////////////////// 5622 // X86/X64-common/ARM/LoongArch interface. 5623 ///////////////////////////////////////////////////////////////// 5624 void MacroAssemblerLOONG64Compat::storeValue(ValueOperand val, 5625 const Address& dest) { 5626 storePtr(val.valueReg(), Address(dest.base, dest.offset)); 5627 } 5628 5629 void MacroAssemblerLOONG64Compat::storeValue(ValueOperand val, 5630 const BaseIndex& dest) { 5631 storePtr(val.valueReg(), dest); 5632 } 5633 5634 void MacroAssemblerLOONG64Compat::storeValue(JSValueType type, Register reg, 5635 Address dest) { 5636 UseScratchRegisterScope temps(asMasm()); 5637 Register scratch = temps.Acquire(); 5638 MOZ_ASSERT(dest.base != scratch); 5639 5640 boxValue(type, reg, scratch); 5641 storePtr(scratch, dest); 5642 } 5643 5644 void MacroAssemblerLOONG64Compat::storeValue(JSValueType type, Register reg, 5645 BaseIndex dest) { 5646 UseScratchRegisterScope temps(asMasm()); 5647 Register scratch = temps.Acquire(); 5648 MOZ_ASSERT(dest.base != scratch); 5649 5650 boxValue(type, reg, scratch); 5651 storePtr(scratch, dest); 5652 } 5653 5654 void MacroAssemblerLOONG64Compat::storeValue(const Value& val, Address dest) { 5655 UseScratchRegisterScope temps(asMasm()); 5656 Register scratch = temps.Acquire(); 5657 MOZ_ASSERT(dest.base != scratch); 5658 5659 if (val.isGCThing()) { 5660 writeDataRelocation(val); 5661 movWithPatch(ImmWord(val.asRawBits()), scratch); 5662 } else { 5663 ma_li(scratch, ImmWord(val.asRawBits())); 5664 } 5665 storePtr(scratch, dest); 5666 } 5667 5668 void MacroAssemblerLOONG64Compat::storeValue(const Value& val, BaseIndex dest) { 5669 UseScratchRegisterScope temps(asMasm()); 5670 Register scratch = temps.Acquire(); 5671 MOZ_ASSERT(dest.base != scratch); 5672 5673 if (val.isGCThing()) { 5674 writeDataRelocation(val); 5675 movWithPatch(ImmWord(val.asRawBits()), scratch); 5676 } else { 5677 ma_li(scratch, ImmWord(val.asRawBits())); 5678 } 5679 storePtr(scratch, dest); 5680 } 5681 5682 void MacroAssemblerLOONG64Compat::loadValue(Address src, ValueOperand val) { 5683 loadPtr(src, val.valueReg()); 5684 } 5685 5686 void MacroAssemblerLOONG64Compat::loadValue(const BaseIndex& src, 5687 ValueOperand val) { 5688 loadPtr(src, val.valueReg()); 5689 } 5690 5691 void MacroAssemblerLOONG64Compat::tagValue(JSValueType type, Register payload, 5692 ValueOperand dest) { 5693 MOZ_ASSERT(type != JSVAL_TYPE_UNDEFINED && type != JSVAL_TYPE_NULL); 5694 5695 if (payload == dest.valueReg()) { 5696 UseScratchRegisterScope temps(asMasm()); 5697 Register scratch = temps.Acquire(); 5698 MOZ_ASSERT(dest.valueReg() != scratch); 5699 5700 AssertValidPayload(*this, type, payload, scratch); 5701 5702 switch (type) { 5703 case JSVAL_TYPE_BOOLEAN: 5704 case JSVAL_TYPE_INT32: 5705 case JSVAL_TYPE_MAGIC: { 5706 int64_t shifted = int64_t(JSVAL_TYPE_TO_SHIFTED_TAG(type)) >> 32; 5707 MOZ_ASSERT(is_intN(shifted, 20), 5708 "upper 32 bit of shifted tag fit into lu32i.d"); 5709 5710 // Store shifted tag into upper 32 bits. The lower 32 bits, containing 5711 // the payload, are left unchanged. 5712 as_lu32i_d(dest.valueReg(), shifted); 5713 return; 5714 } 5715 case JSVAL_TYPE_STRING: 5716 case JSVAL_TYPE_SYMBOL: 5717 case JSVAL_TYPE_PRIVATE_GCTHING: 5718 case JSVAL_TYPE_BIGINT: 5719 case JSVAL_TYPE_OBJECT: { 5720 int64_t signExtendedShiftedTag = 5721 int64_t(JSVAL_TYPE_TO_SHIFTED_TAG(type)) >> JSVAL_TAG_SHIFT; 5722 MOZ_ASSERT(is_intN(signExtendedShiftedTag, 12), 5723 "sign-extended shifted tag can be materialised in a single " 5724 "addi.w instruction"); 5725 5726 // Store sign-extended tag into lower 17 bits of the scratch register. 5727 as_addi_w(scratch, zero, signExtendedShiftedTag); 5728 5729 // Insert tag into the result. 5730 as_bstrins_d(dest.valueReg(), scratch, 63, JSVAL_TAG_SHIFT); 5731 return; 5732 } 5733 case JSVAL_TYPE_DOUBLE: 5734 case JSVAL_TYPE_UNDEFINED: 5735 case JSVAL_TYPE_NULL: 5736 case JSVAL_TYPE_UNKNOWN: 5737 break; 5738 } 5739 MOZ_CRASH("bad value type"); 5740 } else { 5741 boxNonDouble(type, payload, dest); 5742 } 5743 } 5744 5745 void MacroAssemblerLOONG64Compat::pushValue(ValueOperand val) { 5746 push(val.valueReg()); 5747 } 5748 5749 void MacroAssemblerLOONG64Compat::pushValue(const Address& addr) { push(addr); } 5750 5751 void MacroAssemblerLOONG64Compat::popValue(ValueOperand val) { 5752 pop(val.valueReg()); 5753 } 5754 5755 void MacroAssemblerLOONG64Compat::breakpoint(uint32_t value) { 5756 as_break(value); 5757 } 5758 5759 void MacroAssemblerLOONG64Compat::handleFailureWithHandlerTail( 5760 Label* profilerExitTail, Label* bailoutTail, 5761 uint32_t* returnValueCheckOffset) { 5762 // Reserve space for exception information. 5763 int size = (sizeof(ResumeFromException) + ABIStackAlignment) & 5764 ~(ABIStackAlignment - 1); 5765 asMasm().subPtr(Imm32(size), StackPointer); 5766 mov(StackPointer, a0); // Use a0 since it is a first function argument 5767 5768 // Call the handler. 5769 using Fn = void (*)(ResumeFromException* rfe); 5770 asMasm().setupUnalignedABICall(a1); 5771 asMasm().passABIArg(a0); 5772 asMasm().callWithABI<Fn, HandleException>( 5773 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); 5774 5775 *returnValueCheckOffset = asMasm().currentOffset(); 5776 5777 Label entryFrame; 5778 Label catch_; 5779 Label finally; 5780 Label returnBaseline; 5781 Label returnIon; 5782 Label bailout; 5783 Label wasmInterpEntry; 5784 Label wasmCatch; 5785 5786 // Already clobbered a0, so use it... 5787 load32(Address(StackPointer, ResumeFromException::offsetOfKind()), a0); 5788 asMasm().branch32(Assembler::Equal, a0, 5789 Imm32(ExceptionResumeKind::EntryFrame), &entryFrame); 5790 asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Catch), 5791 &catch_); 5792 asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Finally), 5793 &finally); 5794 asMasm().branch32(Assembler::Equal, a0, 5795 Imm32(ExceptionResumeKind::ForcedReturnBaseline), 5796 &returnBaseline); 5797 asMasm().branch32(Assembler::Equal, a0, 5798 Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon); 5799 asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Bailout), 5800 &bailout); 5801 asMasm().branch32(Assembler::Equal, a0, 5802 Imm32(ExceptionResumeKind::WasmInterpEntry), 5803 &wasmInterpEntry); 5804 asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::WasmCatch), 5805 &wasmCatch); 5806 5807 breakpoint(); // Invalid kind. 5808 5809 // No exception handler. Load the error value, restore state and return from 5810 // the entry frame. 5811 bind(&entryFrame); 5812 asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); 5813 loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), 5814 FramePointer); 5815 loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), 5816 StackPointer); 5817 5818 // We're going to be returning by the ion calling convention 5819 ma_pop(ra); 5820 as_jirl(zero, ra, BOffImm16(0)); 5821 5822 // If we found a catch handler, this must be a baseline frame. Restore 5823 // state and jump to the catch block. 5824 bind(&catch_); 5825 loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), a0); 5826 loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), 5827 FramePointer); 5828 loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), 5829 StackPointer); 5830 jump(a0); 5831 5832 // If we found a finally block, this must be a baseline frame. Push three 5833 // values expected by the finally block: the exception, the exception stack, 5834 // and BooleanValue(true). 5835 bind(&finally); 5836 ValueOperand exception = ValueOperand(a1); 5837 loadValue(Address(sp, ResumeFromException::offsetOfException()), exception); 5838 5839 ValueOperand exceptionStack = ValueOperand(a2); 5840 loadValue(Address(sp, ResumeFromException::offsetOfExceptionStack()), 5841 exceptionStack); 5842 5843 loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a0); 5844 loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()), 5845 FramePointer); 5846 loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp); 5847 5848 pushValue(exception); 5849 pushValue(exceptionStack); 5850 pushValue(BooleanValue(true)); 5851 jump(a0); 5852 5853 // Return BaselineFrame->returnValue() to the caller. 5854 // Used in debug mode and for GeneratorReturn. 5855 Label profilingInstrumentation; 5856 bind(&returnBaseline); 5857 loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), 5858 FramePointer); 5859 loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), 5860 StackPointer); 5861 loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()), 5862 JSReturnOperand); 5863 jump(&profilingInstrumentation); 5864 5865 // Return the given value to the caller. 5866 bind(&returnIon); 5867 loadValue(Address(StackPointer, ResumeFromException::offsetOfException()), 5868 JSReturnOperand); 5869 loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), 5870 FramePointer); 5871 loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), 5872 StackPointer); 5873 5874 // If profiling is enabled, then update the lastProfilingFrame to refer to 5875 // caller frame before returning. This code is shared by ForcedReturnIon 5876 // and ForcedReturnBaseline. 5877 bind(&profilingInstrumentation); 5878 { 5879 Label skipProfilingInstrumentation; 5880 // Test if profiler enabled. 5881 AbsoluteAddress addressOfEnabled( 5882 asMasm().runtime()->geckoProfiler().addressOfEnabled()); 5883 asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), 5884 &skipProfilingInstrumentation); 5885 jump(profilerExitTail); 5886 bind(&skipProfilingInstrumentation); 5887 } 5888 5889 as_or(StackPointer, FramePointer, zero); 5890 pop(FramePointer); 5891 ret(); 5892 5893 // If we are bailing out to baseline to handle an exception, jump to 5894 // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success. 5895 bind(&bailout); 5896 loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), a2); 5897 loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), 5898 StackPointer); 5899 ma_li(ReturnReg, Imm32(1)); 5900 jump(bailoutTail); 5901 5902 // Reset SP and FP; SP is pointing to the unwound return address to the wasm 5903 // interpreter entry, so we can just ret(). 5904 bind(&wasmInterpEntry); 5905 loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), 5906 FramePointer); 5907 loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), 5908 StackPointer); 5909 ma_li(InstanceReg, ImmWord(wasm::InterpFailInstanceReg)); 5910 ret(); 5911 5912 // Found a wasm catch handler, restore state and jump to it. 5913 bind(&wasmCatch); 5914 wasm::GenerateJumpToCatchHandler(asMasm(), sp, a1, a2); 5915 } 5916 5917 CodeOffset MacroAssemblerLOONG64Compat::toggledJump(Label* label) { 5918 CodeOffset ret(nextOffset().getOffset()); 5919 ma_b(label); 5920 return ret; 5921 } 5922 5923 CodeOffset MacroAssemblerLOONG64Compat::toggledCall(JitCode* target, 5924 bool enabled) { 5925 UseScratchRegisterScope temps(asMasm()); 5926 Register scratch = temps.Acquire(); 5927 BufferOffset bo = nextOffset(); 5928 CodeOffset offset(bo.getOffset()); // first instruction location,not changed. 5929 addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE); 5930 ma_liPatchable(scratch, ImmPtr(target->raw())); 5931 if (enabled) { 5932 as_jirl(ra, scratch, BOffImm16(0)); 5933 } else { 5934 as_nop(); 5935 } 5936 MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() == 5937 ToggledCallSize(nullptr)); 5938 return offset; // location of first instruction of call instr sequence. 5939 } 5940 5941 void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift, 5942 Register pointer) { 5943 if (IsShiftInScaleRange(shift)) { 5944 computeEffectiveAddress( 5945 BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer); 5946 return; 5947 } 5948 lshift32(Imm32(shift), indexTemp32); 5949 addPtr(indexTemp32, pointer); 5950 } 5951 5952 void MacroAssembler::wasmMarkCallAsSlow() { mov(ra, ra); } 5953 5954 const int32_t SlowCallMarker = 0x03800021; // ori ra, ra, 0 5955 5956 void MacroAssembler::wasmCheckSlowCallsite(Register ra_, Label* notSlow, 5957 Register temp1, Register temp2) { 5958 MOZ_ASSERT(ra_ != temp2); 5959 load32(Address(ra_, 0), temp2); 5960 branch32(Assembler::NotEqual, temp2, Imm32(SlowCallMarker), notSlow); 5961 } 5962 5963 CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc& desc, 5964 const Register reg) { 5965 CodeOffset offset = call(desc, reg); 5966 wasmMarkCallAsSlow(); 5967 return offset; 5968 } 5969 5970 //}}} check_macroassembler_style 5971 5972 } // namespace jit 5973 } // namespace js