AtomicOperations-shared-jit.h (20618B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=4 et sw=4 tw=99: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 /* For overall documentation, see jit/AtomicOperations.h. 8 * 9 * NOTE CAREFULLY: This file is only applicable when we have configured a JIT 10 * and the JIT is for the same architecture that we're compiling the shell for. 11 * Simulators must use a different mechanism. 12 * 13 * See comments before the include nest near the end of jit/AtomicOperations.h 14 * if you didn't understand that. 15 */ 16 17 #ifndef jit_shared_AtomicOperations_shared_jit_h 18 #define jit_shared_AtomicOperations_shared_jit_h 19 20 #include "mozilla/Assertions.h" 21 22 #include <stddef.h> 23 #include <stdint.h> 24 25 #include "jit/AtomicOperationsGenerated.h" 26 #include "vm/Float16.h" 27 #include "vm/Uint8Clamped.h" 28 29 namespace js { 30 namespace jit { 31 32 #ifndef JS_64BIT 33 // `AtomicCompilerFence` erects a reordering boundary for operations on the 34 // current thread. We use it to prevent the compiler from reordering loads and 35 // stores inside larger primitives that are synthesized from cmpxchg. 36 extern void AtomicCompilerFence(); 37 #endif 38 39 // `...MemcpyDown` moves bytes toward lower addresses in memory: dest <= src. 40 // `...MemcpyUp` moves bytes toward higher addresses in memory: dest >= src. 41 extern void AtomicMemcpyDownUnsynchronized(uint8_t* dest, const uint8_t* src, 42 size_t nbytes); 43 extern void AtomicMemcpyUpUnsynchronized(uint8_t* dest, const uint8_t* src, 44 size_t nbytes); 45 46 } // namespace jit 47 } // namespace js 48 49 inline bool js::jit::AtomicOperations::hasAtomic8() { return true; } 50 51 inline bool js::jit::AtomicOperations::isLockfree8() { return true; } 52 53 inline void js::jit::AtomicOperations::fenceSeqCst() { AtomicFenceSeqCst(); } 54 55 inline void js::jit::AtomicOperations::pause() { AtomicPause(); } 56 57 #define JIT_LOADOP(T, U, loadop) \ 58 template <> \ 59 inline T AtomicOperations::loadSeqCst(T* addr) { \ 60 return (T)loadop((U*)addr); \ 61 } 62 63 #ifndef JS_64BIT 64 # define JIT_LOADOP_CAS(T) \ 65 template <> \ 66 inline T AtomicOperations::loadSeqCst(T* addr) { \ 67 AtomicCompilerFence(); \ 68 return (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, 0, 0); \ 69 } 70 #endif // !JS_64BIT 71 72 namespace js { 73 namespace jit { 74 75 JIT_LOADOP(int8_t, uint8_t, AtomicLoad8SeqCst) 76 JIT_LOADOP(uint8_t, uint8_t, AtomicLoad8SeqCst) 77 JIT_LOADOP(int16_t, uint16_t, AtomicLoad16SeqCst) 78 JIT_LOADOP(uint16_t, uint16_t, AtomicLoad16SeqCst) 79 JIT_LOADOP(int32_t, uint32_t, AtomicLoad32SeqCst) 80 JIT_LOADOP(uint32_t, uint32_t, AtomicLoad32SeqCst) 81 82 #ifdef JIT_LOADOP_CAS 83 JIT_LOADOP_CAS(int64_t) 84 JIT_LOADOP_CAS(uint64_t) 85 #else 86 JIT_LOADOP(int64_t, uint64_t, AtomicLoad64SeqCst) 87 JIT_LOADOP(uint64_t, uint64_t, AtomicLoad64SeqCst) 88 #endif 89 90 } // namespace jit 91 } // namespace js 92 93 #undef JIT_LOADOP 94 #undef JIT_LOADOP_CAS 95 96 #define JIT_STOREOP(T, U, storeop) \ 97 template <> \ 98 inline void AtomicOperations::storeSeqCst(T* addr, T val) { \ 99 storeop((U*)addr, val); \ 100 } 101 102 #ifndef JS_64BIT 103 # define JIT_STOREOP_CAS(T) \ 104 template <> \ 105 inline void AtomicOperations::storeSeqCst(T* addr, T val) { \ 106 AtomicCompilerFence(); \ 107 T oldval = *addr; /* good initial approximation */ \ 108 for (;;) { \ 109 T nextval = (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, \ 110 (uint64_t)oldval, (uint64_t)val); \ 111 if (nextval == oldval) { \ 112 break; \ 113 } \ 114 oldval = nextval; \ 115 } \ 116 AtomicCompilerFence(); \ 117 } 118 #endif // !JS_64BIT 119 120 namespace js { 121 namespace jit { 122 123 JIT_STOREOP(int8_t, uint8_t, AtomicStore8SeqCst) 124 JIT_STOREOP(uint8_t, uint8_t, AtomicStore8SeqCst) 125 JIT_STOREOP(int16_t, uint16_t, AtomicStore16SeqCst) 126 JIT_STOREOP(uint16_t, uint16_t, AtomicStore16SeqCst) 127 JIT_STOREOP(int32_t, uint32_t, AtomicStore32SeqCst) 128 JIT_STOREOP(uint32_t, uint32_t, AtomicStore32SeqCst) 129 130 #ifdef JIT_STOREOP_CAS 131 JIT_STOREOP_CAS(int64_t) 132 JIT_STOREOP_CAS(uint64_t) 133 #else 134 JIT_STOREOP(int64_t, uint64_t, AtomicStore64SeqCst) 135 JIT_STOREOP(uint64_t, uint64_t, AtomicStore64SeqCst) 136 #endif 137 138 } // namespace jit 139 } // namespace js 140 141 #undef JIT_STOREOP 142 #undef JIT_STOREOP_CAS 143 144 #define JIT_EXCHANGEOP(T, U, xchgop) \ 145 template <> \ 146 inline T AtomicOperations::exchangeSeqCst(T* addr, T val) { \ 147 return (T)xchgop((U*)addr, (U)val); \ 148 } 149 150 #ifndef JS_64BIT 151 # define JIT_EXCHANGEOP_CAS(T) \ 152 template <> \ 153 inline T AtomicOperations::exchangeSeqCst(T* addr, T val) { \ 154 AtomicCompilerFence(); \ 155 T oldval = *addr; \ 156 for (;;) { \ 157 T nextval = (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, \ 158 (uint64_t)oldval, (uint64_t)val); \ 159 if (nextval == oldval) { \ 160 break; \ 161 } \ 162 oldval = nextval; \ 163 } \ 164 AtomicCompilerFence(); \ 165 return oldval; \ 166 } 167 #endif // !JS_64BIT 168 169 namespace js { 170 namespace jit { 171 172 JIT_EXCHANGEOP(int8_t, uint8_t, AtomicExchange8SeqCst) 173 JIT_EXCHANGEOP(uint8_t, uint8_t, AtomicExchange8SeqCst) 174 JIT_EXCHANGEOP(int16_t, uint16_t, AtomicExchange16SeqCst) 175 JIT_EXCHANGEOP(uint16_t, uint16_t, AtomicExchange16SeqCst) 176 JIT_EXCHANGEOP(int32_t, uint32_t, AtomicExchange32SeqCst) 177 JIT_EXCHANGEOP(uint32_t, uint32_t, AtomicExchange32SeqCst) 178 179 #ifdef JIT_EXCHANGEOP_CAS 180 JIT_EXCHANGEOP_CAS(int64_t) 181 JIT_EXCHANGEOP_CAS(uint64_t) 182 #else 183 JIT_EXCHANGEOP(int64_t, uint64_t, AtomicExchange64SeqCst) 184 JIT_EXCHANGEOP(uint64_t, uint64_t, AtomicExchange64SeqCst) 185 #endif 186 187 } // namespace jit 188 } // namespace js 189 190 #undef JIT_EXCHANGEOP 191 #undef JIT_EXCHANGEOP_CAS 192 193 #define JIT_CAS(T, U, cmpxchg) \ 194 template <> \ 195 inline T AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, \ 196 T newval) { \ 197 return (T)cmpxchg((U*)addr, (U)oldval, (U)newval); \ 198 } 199 200 namespace js { 201 namespace jit { 202 203 JIT_CAS(int8_t, uint8_t, AtomicCmpXchg8SeqCst) 204 JIT_CAS(uint8_t, uint8_t, AtomicCmpXchg8SeqCst) 205 JIT_CAS(int16_t, uint16_t, AtomicCmpXchg16SeqCst) 206 JIT_CAS(uint16_t, uint16_t, AtomicCmpXchg16SeqCst) 207 JIT_CAS(int32_t, uint32_t, AtomicCmpXchg32SeqCst) 208 JIT_CAS(uint32_t, uint32_t, AtomicCmpXchg32SeqCst) 209 JIT_CAS(int64_t, uint64_t, AtomicCmpXchg64SeqCst) 210 JIT_CAS(uint64_t, uint64_t, AtomicCmpXchg64SeqCst) 211 212 } // namespace jit 213 } // namespace js 214 215 #undef JIT_CAS 216 217 #define JIT_FETCHADDOP(T, U, xadd) \ 218 template <> \ 219 inline T AtomicOperations::fetchAddSeqCst(T* addr, T val) { \ 220 return (T)xadd((U*)addr, (U)val); \ 221 } 222 223 #define JIT_FETCHSUBOP(T) \ 224 template <> \ 225 inline T AtomicOperations::fetchSubSeqCst(T* addr, T val) { \ 226 return fetchAddSeqCst(addr, (T)(0 - val)); \ 227 } 228 229 #ifndef JS_64BIT 230 # define JIT_FETCHADDOP_CAS(T) \ 231 template <> \ 232 inline T AtomicOperations::fetchAddSeqCst(T* addr, T val) { \ 233 AtomicCompilerFence(); \ 234 T oldval = *addr; /* Good initial approximation */ \ 235 for (;;) { \ 236 T nextval = (T)AtomicCmpXchg64SeqCst( \ 237 (uint64_t*)addr, (uint64_t)oldval, (uint64_t)(oldval + val)); \ 238 if (nextval == oldval) { \ 239 break; \ 240 } \ 241 oldval = nextval; \ 242 } \ 243 AtomicCompilerFence(); \ 244 return oldval; \ 245 } 246 #endif // !JS_64BIT 247 248 namespace js { 249 namespace jit { 250 251 JIT_FETCHADDOP(int8_t, uint8_t, AtomicAdd8SeqCst) 252 JIT_FETCHADDOP(uint8_t, uint8_t, AtomicAdd8SeqCst) 253 JIT_FETCHADDOP(int16_t, uint16_t, AtomicAdd16SeqCst) 254 JIT_FETCHADDOP(uint16_t, uint16_t, AtomicAdd16SeqCst) 255 JIT_FETCHADDOP(int32_t, uint32_t, AtomicAdd32SeqCst) 256 JIT_FETCHADDOP(uint32_t, uint32_t, AtomicAdd32SeqCst) 257 258 #ifdef JIT_FETCHADDOP_CAS 259 JIT_FETCHADDOP_CAS(int64_t) 260 JIT_FETCHADDOP_CAS(uint64_t) 261 #else 262 JIT_FETCHADDOP(int64_t, uint64_t, AtomicAdd64SeqCst) 263 JIT_FETCHADDOP(uint64_t, uint64_t, AtomicAdd64SeqCst) 264 #endif 265 266 JIT_FETCHSUBOP(int8_t) 267 JIT_FETCHSUBOP(uint8_t) 268 JIT_FETCHSUBOP(int16_t) 269 JIT_FETCHSUBOP(uint16_t) 270 JIT_FETCHSUBOP(int32_t) 271 JIT_FETCHSUBOP(uint32_t) 272 JIT_FETCHSUBOP(int64_t) 273 JIT_FETCHSUBOP(uint64_t) 274 275 } // namespace jit 276 } // namespace js 277 278 #undef JIT_FETCHADDOP 279 #undef JIT_FETCHADDOP_CAS 280 #undef JIT_FETCHSUBOP 281 282 #define JIT_FETCHBITOPX(T, U, name, op) \ 283 template <> \ 284 inline T AtomicOperations::name(T* addr, T val) { \ 285 return (T)op((U*)addr, (U)val); \ 286 } 287 288 #define JIT_FETCHBITOP(T, U, andop, orop, xorop) \ 289 JIT_FETCHBITOPX(T, U, fetchAndSeqCst, andop) \ 290 JIT_FETCHBITOPX(T, U, fetchOrSeqCst, orop) \ 291 JIT_FETCHBITOPX(T, U, fetchXorSeqCst, xorop) 292 293 #ifndef JS_64BIT 294 295 # define AND_OP & 296 # define OR_OP | 297 # define XOR_OP ^ 298 299 # define JIT_FETCHBITOPX_CAS(T, name, OP) \ 300 template <> \ 301 inline T AtomicOperations::name(T* addr, T val) { \ 302 AtomicCompilerFence(); \ 303 T oldval = *addr; \ 304 for (;;) { \ 305 T nextval = (T)AtomicCmpXchg64SeqCst( \ 306 (uint64_t*)addr, (uint64_t)oldval, (uint64_t)(oldval OP val)); \ 307 if (nextval == oldval) { \ 308 break; \ 309 } \ 310 oldval = nextval; \ 311 } \ 312 AtomicCompilerFence(); \ 313 return oldval; \ 314 } 315 316 # define JIT_FETCHBITOP_CAS(T) \ 317 JIT_FETCHBITOPX_CAS(T, fetchAndSeqCst, AND_OP) \ 318 JIT_FETCHBITOPX_CAS(T, fetchOrSeqCst, OR_OP) \ 319 JIT_FETCHBITOPX_CAS(T, fetchXorSeqCst, XOR_OP) 320 321 #endif // !JS_64BIT 322 323 namespace js { 324 namespace jit { 325 326 JIT_FETCHBITOP(int8_t, uint8_t, AtomicAnd8SeqCst, AtomicOr8SeqCst, 327 AtomicXor8SeqCst) 328 JIT_FETCHBITOP(uint8_t, uint8_t, AtomicAnd8SeqCst, AtomicOr8SeqCst, 329 AtomicXor8SeqCst) 330 JIT_FETCHBITOP(int16_t, uint16_t, AtomicAnd16SeqCst, AtomicOr16SeqCst, 331 AtomicXor16SeqCst) 332 JIT_FETCHBITOP(uint16_t, uint16_t, AtomicAnd16SeqCst, AtomicOr16SeqCst, 333 AtomicXor16SeqCst) 334 JIT_FETCHBITOP(int32_t, uint32_t, AtomicAnd32SeqCst, AtomicOr32SeqCst, 335 AtomicXor32SeqCst) 336 JIT_FETCHBITOP(uint32_t, uint32_t, AtomicAnd32SeqCst, AtomicOr32SeqCst, 337 AtomicXor32SeqCst) 338 339 #ifdef JIT_FETCHBITOP_CAS 340 JIT_FETCHBITOP_CAS(int64_t) 341 JIT_FETCHBITOP_CAS(uint64_t) 342 #else 343 JIT_FETCHBITOP(int64_t, uint64_t, AtomicAnd64SeqCst, AtomicOr64SeqCst, 344 AtomicXor64SeqCst) 345 JIT_FETCHBITOP(uint64_t, uint64_t, AtomicAnd64SeqCst, AtomicOr64SeqCst, 346 AtomicXor64SeqCst) 347 #endif 348 349 } // namespace jit 350 } // namespace js 351 352 #undef JIT_FETCHBITOPX_CAS 353 #undef JIT_FETCHBITOPX 354 #undef JIT_FETCHBITOP_CAS 355 #undef JIT_FETCHBITOP 356 357 #define JIT_LOADSAFE(T, U, loadop) \ 358 template <> \ 359 inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { \ 360 union { \ 361 U u; \ 362 T t; \ 363 }; \ 364 u = loadop((U*)addr); \ 365 return t; \ 366 } 367 368 #ifndef JS_64BIT 369 # define JIT_LOADSAFE_TEARING(T) \ 370 template <> \ 371 inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { \ 372 MOZ_ASSERT(sizeof(T) == 8); \ 373 union { \ 374 uint32_t u[2]; \ 375 T t; \ 376 }; \ 377 uint32_t* ptr = (uint32_t*)addr; \ 378 u[0] = AtomicLoad32Unsynchronized(ptr); \ 379 u[1] = AtomicLoad32Unsynchronized(ptr + 1); \ 380 return t; \ 381 } 382 #endif // !JS_64BIT 383 384 namespace js { 385 namespace jit { 386 387 JIT_LOADSAFE(int8_t, uint8_t, AtomicLoad8Unsynchronized) 388 JIT_LOADSAFE(uint8_t, uint8_t, AtomicLoad8Unsynchronized) 389 JIT_LOADSAFE(int16_t, uint16_t, AtomicLoad16Unsynchronized) 390 JIT_LOADSAFE(uint16_t, uint16_t, AtomicLoad16Unsynchronized) 391 JIT_LOADSAFE(int32_t, uint32_t, AtomicLoad32Unsynchronized) 392 JIT_LOADSAFE(uint32_t, uint32_t, AtomicLoad32Unsynchronized) 393 #ifdef JIT_LOADSAFE_TEARING 394 JIT_LOADSAFE_TEARING(int64_t) 395 JIT_LOADSAFE_TEARING(uint64_t) 396 JIT_LOADSAFE_TEARING(double) 397 #else 398 JIT_LOADSAFE(int64_t, uint64_t, AtomicLoad64Unsynchronized) 399 JIT_LOADSAFE(uint64_t, uint64_t, AtomicLoad64Unsynchronized) 400 JIT_LOADSAFE(double, uint64_t, AtomicLoad64Unsynchronized) 401 #endif 402 JIT_LOADSAFE(float, uint32_t, AtomicLoad32Unsynchronized) 403 404 // Clang requires a specialization for uint8_clamped. 405 template <> 406 inline uint8_clamped js::jit::AtomicOperations::loadSafeWhenRacy( 407 uint8_clamped* addr) { 408 return uint8_clamped(loadSafeWhenRacy((uint8_t*)addr)); 409 } 410 411 // Clang requires a specialization for float16. 412 template <> 413 inline float16 js::jit::AtomicOperations::loadSafeWhenRacy(float16* addr) { 414 return float16::fromRawBits(loadSafeWhenRacy((uint16_t*)addr)); 415 } 416 417 } // namespace jit 418 } // namespace js 419 420 #undef JIT_LOADSAFE 421 #undef JIT_LOADSAFE_TEARING 422 423 #define JIT_STORESAFE(T, U, storeop) \ 424 template <> \ 425 inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { \ 426 union { \ 427 U u; \ 428 T t; \ 429 }; \ 430 t = val; \ 431 storeop((U*)addr, u); \ 432 } 433 434 #ifndef JS_64BIT 435 # define JIT_STORESAFE_TEARING(T) \ 436 template <> \ 437 inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { \ 438 union { \ 439 uint32_t u[2]; \ 440 T t; \ 441 }; \ 442 t = val; \ 443 uint32_t* ptr = (uint32_t*)addr; \ 444 AtomicStore32Unsynchronized(ptr, u[0]); \ 445 AtomicStore32Unsynchronized(ptr + 1, u[1]); \ 446 } 447 #endif // !JS_64BIT 448 449 namespace js { 450 namespace jit { 451 452 JIT_STORESAFE(int8_t, uint8_t, AtomicStore8Unsynchronized) 453 JIT_STORESAFE(uint8_t, uint8_t, AtomicStore8Unsynchronized) 454 JIT_STORESAFE(int16_t, uint16_t, AtomicStore16Unsynchronized) 455 JIT_STORESAFE(uint16_t, uint16_t, AtomicStore16Unsynchronized) 456 JIT_STORESAFE(int32_t, uint32_t, AtomicStore32Unsynchronized) 457 JIT_STORESAFE(uint32_t, uint32_t, AtomicStore32Unsynchronized) 458 #ifdef JIT_STORESAFE_TEARING 459 JIT_STORESAFE_TEARING(int64_t) 460 JIT_STORESAFE_TEARING(uint64_t) 461 JIT_STORESAFE_TEARING(double) 462 #else 463 JIT_STORESAFE(int64_t, uint64_t, AtomicStore64Unsynchronized) 464 JIT_STORESAFE(uint64_t, uint64_t, AtomicStore64Unsynchronized) 465 JIT_STORESAFE(double, uint64_t, AtomicStore64Unsynchronized) 466 #endif 467 JIT_STORESAFE(float, uint32_t, AtomicStore32Unsynchronized) 468 469 // Clang requires a specialization for uint8_clamped. 470 template <> 471 inline void js::jit::AtomicOperations::storeSafeWhenRacy(uint8_clamped* addr, 472 uint8_clamped val) { 473 storeSafeWhenRacy((uint8_t*)addr, (uint8_t)val); 474 } 475 476 // Clang requires a specialization for float16. 477 template <> 478 inline void js::jit::AtomicOperations::storeSafeWhenRacy(float16* addr, 479 float16 val) { 480 storeSafeWhenRacy((uint16_t*)addr, val.toRawBits()); 481 } 482 483 } // namespace jit 484 } // namespace js 485 486 #undef JIT_STORESAFE 487 #undef JIT_STORESAFE_TEARING 488 489 void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, const void* src, 490 size_t nbytes) { 491 MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes)); 492 MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes)); 493 AtomicMemcpyDownUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes); 494 } 495 496 inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, 497 const void* src, 498 size_t nbytes) { 499 if ((char*)dest <= (char*)src) { 500 AtomicMemcpyDownUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes); 501 } else { 502 AtomicMemcpyUpUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes); 503 } 504 } 505 506 #endif // jit_shared_AtomicOperations_shared_jit_h