AtomicOperations.h (13069B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_AtomicOperations_h 8 #define jit_AtomicOperations_h 9 10 #include <string.h> 11 12 #include "jit/AtomicOperationsGenerated.h" 13 #include "vm/SharedMem.h" 14 15 namespace js { 16 namespace jit { 17 18 /* 19 * [SMDOC] Atomic Operations 20 * 21 * The atomic operations layer defines types and functions for 22 * JIT-compatible atomic operation. 23 * 24 * The fundamental constraints on the functions are: 25 * 26 * - That their realization here MUST be compatible with code the JIT 27 * generates for its Atomics operations, so that an atomic access 28 * from the interpreter or runtime - from any C++ code - really is 29 * atomic relative to a concurrent, compatible atomic access from 30 * jitted code. That is, these primitives expose JIT-compatible 31 * atomicity functionality to C++. 32 * 33 * - That accesses may race without creating C++ undefined behavior: 34 * atomic accesses (marked "SeqCst") may race with non-atomic 35 * accesses (marked "SafeWhenRacy"); overlapping but non-matching, 36 * and hence incompatible, atomic accesses may race; and non-atomic 37 * accesses may race. The effects of races need not be predictable, 38 * so garbage can be produced by a read or written by a write, but 39 * the effects must be benign: the program must continue to run, and 40 * only the memory in the union of addresses named in the racing 41 * accesses may be affected. 42 * 43 * The compatibility constraint means that if the JIT makes dynamic 44 * decisions about how to implement atomic operations then 45 * corresponding dynamic decisions MUST be made in the implementations 46 * of the functions below. 47 * 48 * The safe-for-races constraint means that by and large, it is hard 49 * to implement these primitives in C++. See "Implementation notes" 50 * below. 51 * 52 * The "SeqCst" suffix on operations means "sequentially consistent" 53 * and means such a function's operation must have "sequentially 54 * consistent" memory ordering. See mfbt/Atomics.h for an explanation 55 * of this memory ordering. 56 * 57 * Note that a "SafeWhenRacy" access does not provide the atomicity of 58 * a "relaxed atomic" access: it can read or write garbage if there's 59 * a race. 60 * 61 * 62 * Implementation notes. 63 * 64 * It's not a requirement that these functions be inlined; performance 65 * is not a great concern. On some platforms these functions may call 66 * functions that use inline assembly. See GenerateAtomicOperations.py. 67 * 68 * In principle these functions will not be written in C++, thus 69 * making races defined behavior if all racy accesses from C++ go via 70 * these functions. (Jitted code will always be safe for races and 71 * provides the same guarantees as these functions.) 72 * 73 * The appropriate implementations will be platform-specific and 74 * there are some obvious implementation strategies to choose 75 * from, sometimes a combination is appropriate: 76 * 77 * - generating the code at run-time with the JIT; 78 * - hand-written assembler (maybe inline); or 79 * - using special compiler intrinsics or directives. 80 * 81 * Trusting the compiler not to generate code that blows up on a 82 * race definitely won't work in the presence of TSan, or even of 83 * optimizing compilers in seemingly-"innocuous" conditions. (See 84 * https://www.usenix.org/legacy/event/hotpar11/tech/final_files/Boehm.pdf 85 * for details.) 86 */ 87 class AtomicOperations { 88 // The following functions are defined for T = int8_t, uint8_t, 89 // int16_t, uint16_t, int32_t, uint32_t, int64_t, and uint64_t. 90 91 // Atomically read *addr. 92 template <typename T> 93 static inline T loadSeqCst(T* addr); 94 95 // Atomically store val in *addr. 96 template <typename T> 97 static inline void storeSeqCst(T* addr, T val); 98 99 // Atomically store val in *addr and return the old value of *addr. 100 template <typename T> 101 static inline T exchangeSeqCst(T* addr, T val); 102 103 // Atomically check that *addr contains oldval and if so replace it 104 // with newval, in any case returning the old contents of *addr. 105 template <typename T> 106 static inline T compareExchangeSeqCst(T* addr, T oldval, T newval); 107 108 // Atomically add, subtract, bitwise-AND, bitwise-OR, or bitwise-XOR 109 // val into *addr and return the old value of *addr. 110 template <typename T> 111 static inline T fetchAddSeqCst(T* addr, T val); 112 113 template <typename T> 114 static inline T fetchSubSeqCst(T* addr, T val); 115 116 template <typename T> 117 static inline T fetchAndSeqCst(T* addr, T val); 118 119 template <typename T> 120 static inline T fetchOrSeqCst(T* addr, T val); 121 122 template <typename T> 123 static inline T fetchXorSeqCst(T* addr, T val); 124 125 // The SafeWhenRacy functions are to be used when C++ code has to access 126 // memory without synchronization and can't guarantee that there won't be a 127 // race on the access. But they are access-atomic for integer data so long 128 // as any racing writes are of the same size and to the same address. 129 130 // Defined for all the integral types as well as for float32 and float64, 131 // but not access-atomic for floats, nor for int64 and uint64 on 32-bit 132 // platforms. 133 template <typename T> 134 static inline T loadSafeWhenRacy(T* addr); 135 136 // Defined for all the integral types as well as for float32 and float64, 137 // but not access-atomic for floats, nor for int64 and uint64 on 32-bit 138 // platforms. 139 template <typename T> 140 static inline void storeSafeWhenRacy(T* addr, T val); 141 142 // Replacement for memcpy(). No access-atomicity guarantees. 143 static inline void memcpySafeWhenRacy(void* dest, const void* src, 144 size_t nbytes); 145 146 // Replacement for memmove(). No access-atomicity guarantees. 147 static inline void memmoveSafeWhenRacy(void* dest, const void* src, 148 size_t nbytes); 149 150 public: 151 // Test lock-freedom for any int32 value. This implements the 152 // Atomics::isLockFree() operation in the ECMAScript Shared Memory and 153 // Atomics specification, as follows: 154 // 155 // 4-byte accesses are always lock free (in the spec). 156 // 1-, 2-, and 8-byte accesses are always lock free (in SpiderMonkey). 157 // 158 // There is no lock-freedom for JS for any other values on any platform. 159 static constexpr inline bool isLockfreeJS(int32_t n); 160 161 // If the return value is true then the templated functions below are 162 // supported for int64_t and uint64_t. If the return value is false then 163 // those functions will MOZ_CRASH. The value of this call does not change 164 // during execution. 165 static inline bool hasAtomic8(); 166 167 // If the return value is true then hasAtomic8() is true and the atomic 168 // operations are indeed lock-free. The value of this call does not change 169 // during execution. 170 static inline bool isLockfree8(); 171 172 // Execute a full memory barrier (LoadLoad+LoadStore+StoreLoad+StoreStore). 173 static inline void fenceSeqCst(); 174 175 // Pause or yield instruction. 176 static inline void pause(); 177 178 // All clients should use the APIs that take SharedMem pointers. 179 // See above for semantics and acceptable types. 180 181 template <typename T> 182 static T loadSeqCst(SharedMem<T*> addr) { 183 return loadSeqCst(addr.unwrap()); 184 } 185 186 template <typename T> 187 static void storeSeqCst(SharedMem<T*> addr, T val) { 188 return storeSeqCst(addr.unwrap(), val); 189 } 190 191 template <typename T> 192 static T exchangeSeqCst(SharedMem<T*> addr, T val) { 193 return exchangeSeqCst(addr.unwrap(), val); 194 } 195 196 template <typename T> 197 static T compareExchangeSeqCst(SharedMem<T*> addr, T oldval, T newval) { 198 return compareExchangeSeqCst(addr.unwrap(), oldval, newval); 199 } 200 201 template <typename T> 202 static T fetchAddSeqCst(SharedMem<T*> addr, T val) { 203 return fetchAddSeqCst(addr.unwrap(), val); 204 } 205 206 template <typename T> 207 static T fetchSubSeqCst(SharedMem<T*> addr, T val) { 208 return fetchSubSeqCst(addr.unwrap(), val); 209 } 210 211 template <typename T> 212 static T fetchAndSeqCst(SharedMem<T*> addr, T val) { 213 return fetchAndSeqCst(addr.unwrap(), val); 214 } 215 216 template <typename T> 217 static T fetchOrSeqCst(SharedMem<T*> addr, T val) { 218 return fetchOrSeqCst(addr.unwrap(), val); 219 } 220 221 template <typename T> 222 static T fetchXorSeqCst(SharedMem<T*> addr, T val) { 223 return fetchXorSeqCst(addr.unwrap(), val); 224 } 225 226 template <typename T> 227 static T loadSafeWhenRacy(SharedMem<T*> addr) { 228 return loadSafeWhenRacy(addr.unwrap()); 229 } 230 231 template <typename T> 232 static void storeSafeWhenRacy(SharedMem<T*> addr, T val) { 233 return storeSafeWhenRacy(addr.unwrap(), val); 234 } 235 236 template <typename T> 237 static void memcpySafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src, 238 size_t nbytes) { 239 memcpySafeWhenRacy(dest.template cast<void*>().unwrap(), 240 src.template cast<void*>().unwrap(), nbytes); 241 } 242 243 template <typename T> 244 static void memcpySafeWhenRacy(SharedMem<T*> dest, T* src, size_t nbytes) { 245 memcpySafeWhenRacy(dest.template cast<void*>().unwrap(), 246 static_cast<void*>(src), nbytes); 247 } 248 249 template <typename T> 250 static void memcpySafeWhenRacy(T* dest, SharedMem<T*> src, size_t nbytes) { 251 memcpySafeWhenRacy(static_cast<void*>(dest), 252 src.template cast<void*>().unwrap(), nbytes); 253 } 254 255 template <typename T> 256 static void memmoveSafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src, 257 size_t nbytes) { 258 memmoveSafeWhenRacy(dest.template cast<void*>().unwrap(), 259 src.template cast<void*>().unwrap(), nbytes); 260 } 261 262 static void memsetSafeWhenRacy(SharedMem<uint8_t*> dest, int value, 263 size_t nbytes) { 264 uint8_t buf[1024]; 265 size_t iterations = nbytes / sizeof(buf); 266 size_t tail = nbytes % sizeof(buf); 267 size_t offs = 0; 268 if (iterations > 0) { 269 memset(buf, value, sizeof(buf)); 270 while (iterations--) { 271 memcpySafeWhenRacy(dest + offs, SharedMem<uint8_t*>::unshared(buf), 272 sizeof(buf)); 273 offs += sizeof(buf); 274 } 275 } else { 276 memset(buf, value, tail); 277 } 278 memcpySafeWhenRacy(dest + offs, SharedMem<uint8_t*>::unshared(buf), tail); 279 } 280 281 template <typename T> 282 static void podCopySafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src, 283 size_t nelem) { 284 memcpySafeWhenRacy(dest, src, nelem * sizeof(T)); 285 } 286 287 template <typename T> 288 static void podMoveSafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src, 289 size_t nelem) { 290 memmoveSafeWhenRacy(dest, src, nelem * sizeof(T)); 291 } 292 }; 293 294 constexpr inline bool AtomicOperations::isLockfreeJS(int32_t size) { 295 // Keep this in sync with atomicIsLockFreeJS() in jit/MacroAssembler.cpp. 296 297 switch (size) { 298 case 1: 299 return true; 300 case 2: 301 return true; 302 case 4: 303 // The spec requires Atomics.isLockFree(4) to return true. 304 return true; 305 case 8: 306 return true; 307 default: 308 return false; 309 } 310 } 311 312 } // namespace jit 313 } // namespace js 314 315 // As explained above, our atomic operations are not portable even in principle, 316 // so we must include platform+compiler specific definitions here. 317 // 318 // x86, x64, arm, and arm64 are maintained by Mozilla. All other platform 319 // setups are by platform maintainers' request and are not maintained by 320 // Mozilla. 321 // 322 // If you are using a platform+compiler combination that causes an error below 323 // (and if the problem isn't just that the compiler uses a different name for a 324 // known architecture), you have basically three options: 325 // 326 // - find an already-supported compiler for the platform and use that instead 327 // 328 // - write your own support code for the platform+compiler and create a new 329 // case below 330 // 331 // - include jit/shared/AtomicOperations-feeling-lucky.h in a case for the 332 // platform below, if you have a gcc-compatible compiler and truly feel 333 // lucky. You may have to add a little code to that file, too. 334 // 335 // Simulators are confusing. These atomic primitives must be compatible with 336 // the code that the JIT emits, but of course for an ARM simulator running on 337 // x86 the primitives here will be for x86, not for ARM, while the JIT emits ARM 338 // code. Our ARM simulator solves that the easy way: by using these primitives 339 // to implement its atomic operations. For other simulators there may need to 340 // be special cases below to provide simulator-compatible primitives, for 341 // example, for our ARM64 simulator the primitives could in principle 342 // participate in the memory exclusivity monitors implemented by the simulator. 343 // Such a solution is likely to be difficult. 344 345 #ifdef JS_HAVE_GENERATED_ATOMIC_OPS 346 # include "jit/shared/AtomicOperations-shared-jit.h" 347 #elif defined(__mips__) 348 # include "jit/mips-shared/AtomicOperations-mips-shared.h" 349 #else 350 # include "jit/shared/AtomicOperations-feeling-lucky.h" 351 #endif 352 353 #endif // jit_AtomicOperations_h