AtomicOperations-feeling-lucky-gcc.h (13169B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 /* For documentation, see jit/AtomicOperations.h, both the comment block at the 8 * beginning and the #ifdef nest near the end. 9 * 10 * This is a common file for tier-3 platforms (including simulators for our 11 * tier-1 platforms) that are not providing hardware-specific implementations of 12 * the atomic operations. Please keep it reasonably platform-independent by 13 * adding #ifdefs at the beginning as much as possible, not throughout the file. 14 * 15 * 16 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 17 * !!!! NOTE !!!! 18 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 19 * 20 * The implementations in this file are NOT SAFE and cannot be safe even in 21 * principle because they rely on C++ undefined behavior. However, they are 22 * frequently good enough for tier-3 platforms. 23 */ 24 25 #ifndef jit_shared_AtomicOperations_feeling_lucky_gcc_h 26 #define jit_shared_AtomicOperations_feeling_lucky_gcc_h 27 28 #include "mozilla/Assertions.h" 29 30 // Explicitly exclude tier-1 platforms. 31 32 #if (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ 33 defined(_M_IX86) || (defined(__arm__) && __ARM_ARCH >= 7) || \ 34 defined(__aarch64__)) 35 # error "Do not use on a tier-1 platform where inline assembly is available" 36 #endif 37 38 #if !(defined(__clang__) || defined(__GNUC__)) 39 # error "This file only for gcc/Clang" 40 #endif 41 42 // 64-bit atomics are not required by the JS spec, and you can compile 43 // SpiderMonkey without them. 64-bit atomics are required for BigInt 44 // support. 45 // 46 // 64-bit lock-free atomics are required for WebAssembly, but gating in the 47 // WebAssembly subsystem ensures that no WebAssembly-supporting platforms need 48 // code in this file. 49 50 #if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \ 51 defined(JS_SIMULATOR_MIPS64) || defined(JS_SIMULATOR_LOONG64) 52 // On some x86 (32-bit) systems this will not work because the compiler does not 53 // open-code 64-bit atomics. If so, try linking with -latomic. If that doesn't 54 // work, you're mostly on your own. 55 # define HAS_64BIT_ATOMICS 56 # define HAS_64BIT_LOCKFREE 57 #endif 58 59 #if defined(__arm__) 60 # define HAS_64BIT_ATOMICS 61 #endif 62 63 #if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ 64 defined(__PPC64LE__) 65 # define HAS_64BIT_ATOMICS 66 # define HAS_64BIT_LOCKFREE 67 #endif 68 69 #if defined(__riscv) && __riscv_xlen == 64 70 # define HAS_64BIT_ATOMICS 71 # define HAS_64BIT_LOCKFREE 72 #endif 73 74 #if defined(__loongarch64) 75 # define HAS_64BIT_ATOMICS 76 # define HAS_64BIT_LOCKFREE 77 #endif 78 79 #ifdef __sparc__ 80 # ifdef __LP64__ 81 # define HAS_64BIT_ATOMICS 82 # define HAS_64BIT_LOCKFREE 83 # endif 84 #endif 85 86 #ifdef JS_CODEGEN_NONE 87 # ifdef JS_64BIT 88 # define HAS_64BIT_ATOMICS 89 # define HAS_64BIT_LOCKFREE 90 # endif 91 #endif 92 93 // The default implementation tactic for gcc/clang is to use the newer __atomic 94 // intrinsics added for use in C++11 <atomic>. Where that isn't available, we 95 // use GCC's older __sync functions instead. 96 // 97 // ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS is kept as a backward compatible 98 // option for older compilers: enable this to use GCC's old __sync functions 99 // instead of the newer __atomic functions. This will be required for GCC 4.6.x 100 // and earlier, and probably for Clang 3.1, should we need to use those 101 // versions. Firefox no longer supports compilers that old. 102 103 // #define ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 104 105 // Sanity check. 106 107 #if defined(HAS_64BIT_LOCKFREE) && !defined(HAS_64BIT_ATOMICS) 108 # error "This combination of features is senseless, please fix" 109 #endif 110 111 // Try to avoid platform #ifdefs below this point. 112 113 // When compiling with Clang on 32-bit linux it will be necessary to link with 114 // -latomic to get the proper 64-bit intrinsics. 115 116 inline bool js::jit::AtomicOperations::hasAtomic8() { 117 #if defined(HAS_64BIT_ATOMICS) 118 return true; 119 #else 120 return false; 121 #endif 122 } 123 124 inline bool js::jit::AtomicOperations::isLockfree8() { 125 #if defined(HAS_64BIT_LOCKFREE) 126 return true; 127 #else 128 return false; 129 #endif 130 } 131 132 inline void js::jit::AtomicOperations::fenceSeqCst() { 133 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 134 __sync_synchronize(); 135 #else 136 __atomic_thread_fence(__ATOMIC_SEQ_CST); 137 #endif 138 } 139 140 inline void js::jit::AtomicOperations::pause() { 141 // No default implementation. 142 } 143 144 template <typename T> 145 inline T js::jit::AtomicOperations::loadSeqCst(T* addr) { 146 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 147 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 148 __sync_synchronize(); 149 T v = *addr; 150 __sync_synchronize(); 151 #else 152 T v; 153 __atomic_load(addr, &v, __ATOMIC_SEQ_CST); 154 #endif 155 return v; 156 } 157 158 #ifndef HAS_64BIT_ATOMICS 159 namespace js { 160 namespace jit { 161 162 template <> 163 inline int64_t AtomicOperations::loadSeqCst(int64_t* addr) { 164 MOZ_CRASH("No 64-bit atomics"); 165 } 166 167 template <> 168 inline uint64_t AtomicOperations::loadSeqCst(uint64_t* addr) { 169 MOZ_CRASH("No 64-bit atomics"); 170 } 171 172 } // namespace jit 173 } // namespace js 174 #endif 175 176 template <typename T> 177 inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) { 178 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 179 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 180 __sync_synchronize(); 181 *addr = val; 182 __sync_synchronize(); 183 #else 184 __atomic_store(addr, &val, __ATOMIC_SEQ_CST); 185 #endif 186 } 187 188 #ifndef HAS_64BIT_ATOMICS 189 namespace js { 190 namespace jit { 191 192 template <> 193 inline void AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) { 194 MOZ_CRASH("No 64-bit atomics"); 195 } 196 197 template <> 198 inline void AtomicOperations::storeSeqCst(uint64_t* addr, uint64_t val) { 199 MOZ_CRASH("No 64-bit atomics"); 200 } 201 202 } // namespace jit 203 } // namespace js 204 #endif 205 206 template <typename T> 207 inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) { 208 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 209 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 210 T v; 211 __sync_synchronize(); 212 do { 213 v = *addr; 214 } while (__sync_val_compare_and_swap(addr, v, val) != v); 215 return v; 216 #else 217 T v; 218 __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST); 219 return v; 220 #endif 221 } 222 223 #ifndef HAS_64BIT_ATOMICS 224 namespace js { 225 namespace jit { 226 227 template <> 228 inline int64_t AtomicOperations::exchangeSeqCst(int64_t* addr, int64_t val) { 229 MOZ_CRASH("No 64-bit atomics"); 230 } 231 232 template <> 233 inline uint64_t AtomicOperations::exchangeSeqCst(uint64_t* addr, uint64_t val) { 234 MOZ_CRASH("No 64-bit atomics"); 235 } 236 237 } // namespace jit 238 } // namespace js 239 #endif 240 241 template <typename T> 242 inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, 243 T newval) { 244 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 245 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 246 return __sync_val_compare_and_swap(addr, oldval, newval); 247 #else 248 __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST, 249 __ATOMIC_SEQ_CST); 250 return oldval; 251 #endif 252 } 253 254 #ifndef HAS_64BIT_ATOMICS 255 namespace js { 256 namespace jit { 257 258 template <> 259 inline int64_t AtomicOperations::compareExchangeSeqCst(int64_t* addr, 260 int64_t oldval, 261 int64_t newval) { 262 MOZ_CRASH("No 64-bit atomics"); 263 } 264 265 template <> 266 inline uint64_t AtomicOperations::compareExchangeSeqCst(uint64_t* addr, 267 uint64_t oldval, 268 uint64_t newval) { 269 MOZ_CRASH("No 64-bit atomics"); 270 } 271 272 } // namespace jit 273 } // namespace js 274 #endif 275 276 template <typename T> 277 inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) { 278 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 279 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 280 return __sync_fetch_and_add(addr, val); 281 #else 282 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST); 283 #endif 284 } 285 286 #ifndef HAS_64BIT_ATOMICS 287 namespace js { 288 namespace jit { 289 290 template <> 291 inline int64_t AtomicOperations::fetchAddSeqCst(int64_t* addr, int64_t val) { 292 MOZ_CRASH("No 64-bit atomics"); 293 } 294 295 template <> 296 inline uint64_t AtomicOperations::fetchAddSeqCst(uint64_t* addr, uint64_t val) { 297 MOZ_CRASH("No 64-bit atomics"); 298 } 299 300 } // namespace jit 301 } // namespace js 302 #endif 303 304 template <typename T> 305 inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) { 306 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 307 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 308 return __sync_fetch_and_sub(addr, val); 309 #else 310 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST); 311 #endif 312 } 313 314 #ifndef HAS_64BIT_ATOMICS 315 namespace js { 316 namespace jit { 317 318 template <> 319 inline int64_t AtomicOperations::fetchSubSeqCst(int64_t* addr, int64_t val) { 320 MOZ_CRASH("No 64-bit atomics"); 321 } 322 323 template <> 324 inline uint64_t AtomicOperations::fetchSubSeqCst(uint64_t* addr, uint64_t val) { 325 MOZ_CRASH("No 64-bit atomics"); 326 } 327 328 } // namespace jit 329 } // namespace js 330 #endif 331 332 template <typename T> 333 inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) { 334 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 335 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 336 return __sync_fetch_and_and(addr, val); 337 #else 338 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST); 339 #endif 340 } 341 342 #ifndef HAS_64BIT_ATOMICS 343 namespace js { 344 namespace jit { 345 346 template <> 347 inline int64_t AtomicOperations::fetchAndSeqCst(int64_t* addr, int64_t val) { 348 MOZ_CRASH("No 64-bit atomics"); 349 } 350 351 template <> 352 inline uint64_t AtomicOperations::fetchAndSeqCst(uint64_t* addr, uint64_t val) { 353 MOZ_CRASH("No 64-bit atomics"); 354 } 355 356 } // namespace jit 357 } // namespace js 358 #endif 359 360 template <typename T> 361 inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) { 362 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 363 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 364 return __sync_fetch_and_or(addr, val); 365 #else 366 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST); 367 #endif 368 } 369 370 #ifndef HAS_64BIT_ATOMICS 371 namespace js { 372 namespace jit { 373 374 template <> 375 inline int64_t AtomicOperations::fetchOrSeqCst(int64_t* addr, int64_t val) { 376 MOZ_CRASH("No 64-bit atomics"); 377 } 378 379 template <> 380 inline uint64_t AtomicOperations::fetchOrSeqCst(uint64_t* addr, uint64_t val) { 381 MOZ_CRASH("No 64-bit atomics"); 382 } 383 384 } // namespace jit 385 } // namespace js 386 #endif 387 388 template <typename T> 389 inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) { 390 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 391 #ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 392 return __sync_fetch_and_xor(addr, val); 393 #else 394 return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST); 395 #endif 396 } 397 398 #ifndef HAS_64BIT_ATOMICS 399 namespace js { 400 namespace jit { 401 402 template <> 403 inline int64_t AtomicOperations::fetchXorSeqCst(int64_t* addr, int64_t val) { 404 MOZ_CRASH("No 64-bit atomics"); 405 } 406 407 template <> 408 inline uint64_t AtomicOperations::fetchXorSeqCst(uint64_t* addr, uint64_t val) { 409 MOZ_CRASH("No 64-bit atomics"); 410 } 411 412 } // namespace jit 413 } // namespace js 414 #endif 415 416 template <typename T> 417 inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { 418 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 419 // This is actually roughly right even on 32-bit platforms since in that 420 // case, double, int64, and uint64 loads need not be access-atomic. 421 // 422 // We could use __atomic_load, but it would be needlessly expensive on 423 // 32-bit platforms that could support it and just plain wrong on others. 424 return *addr; 425 } 426 427 template <typename T> 428 inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { 429 static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); 430 // This is actually roughly right even on 32-bit platforms since in that 431 // case, double, int64, and uint64 loads need not be access-atomic. 432 // 433 // We could use __atomic_store, but it would be needlessly expensive on 434 // 32-bit platforms that could support it and just plain wrong on others. 435 *addr = val; 436 } 437 438 inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, 439 const void* src, 440 size_t nbytes) { 441 MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes)); 442 MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes)); 443 ::memcpy(dest, src, nbytes); 444 } 445 446 inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, 447 const void* src, 448 size_t nbytes) { 449 ::memmove(dest, src, nbytes); 450 } 451 452 #undef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS 453 #undef HAS_64BIT_ATOMICS 454 #undef HAS_64BIT_LOCKFREE 455 456 #endif // jit_shared_AtomicOperations_feeling_lucky_gcc_h