call_once.h (8390B)
1 // Copyright 2017 The Abseil Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // ----------------------------------------------------------------------------- 16 // File: call_once.h 17 // ----------------------------------------------------------------------------- 18 // 19 // This header file provides an Abseil version of `std::call_once` for invoking 20 // a given function at most once, across all threads. This Abseil version is 21 // faster than the C++11 version and incorporates the C++17 argument-passing 22 // fix, so that (for example) non-const references may be passed to the invoked 23 // function. 24 25 #ifndef ABSL_BASE_CALL_ONCE_H_ 26 #define ABSL_BASE_CALL_ONCE_H_ 27 28 #include <algorithm> 29 #include <atomic> 30 #include <cstdint> 31 #include <functional> 32 #include <type_traits> 33 #include <utility> 34 35 #include "absl/base/attributes.h" 36 #include "absl/base/config.h" 37 #include "absl/base/internal/low_level_scheduling.h" 38 #include "absl/base/internal/raw_logging.h" 39 #include "absl/base/internal/scheduling_mode.h" 40 #include "absl/base/internal/spinlock_wait.h" 41 #include "absl/base/macros.h" 42 #include "absl/base/nullability.h" 43 #include "absl/base/optimization.h" 44 #include "absl/base/port.h" 45 46 namespace absl { 47 ABSL_NAMESPACE_BEGIN 48 49 class once_flag; 50 51 namespace base_internal { 52 absl::Nonnull<std::atomic<uint32_t>*> ControlWord( 53 absl::Nonnull<absl::once_flag*> flag); 54 } // namespace base_internal 55 56 // call_once() 57 // 58 // For all invocations using a given `once_flag`, invokes a given `fn` exactly 59 // once across all threads. The first call to `call_once()` with a particular 60 // `once_flag` argument (that does not throw an exception) will run the 61 // specified function with the provided `args`; other calls with the same 62 // `once_flag` argument will not run the function, but will wait 63 // for the provided function to finish running (if it is still running). 64 // 65 // This mechanism provides a safe, simple, and fast mechanism for one-time 66 // initialization in a multi-threaded process. 67 // 68 // Example: 69 // 70 // class MyInitClass { 71 // public: 72 // ... 73 // mutable absl::once_flag once_; 74 // 75 // MyInitClass* init() const { 76 // absl::call_once(once_, &MyInitClass::Init, this); 77 // return ptr_; 78 // } 79 // 80 template <typename Callable, typename... Args> 81 void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args); 82 83 // once_flag 84 // 85 // Objects of this type are used to distinguish calls to `call_once()` and 86 // ensure the provided function is only invoked once across all threads. This 87 // type is not copyable or movable. However, it has a `constexpr` 88 // constructor, and is safe to use as a namespace-scoped global variable. 89 class once_flag { 90 public: 91 constexpr once_flag() : control_(0) {} 92 once_flag(const once_flag&) = delete; 93 once_flag& operator=(const once_flag&) = delete; 94 95 private: 96 friend absl::Nonnull<std::atomic<uint32_t>*> base_internal::ControlWord( 97 absl::Nonnull<once_flag*> flag); 98 std::atomic<uint32_t> control_; 99 }; 100 101 //------------------------------------------------------------------------------ 102 // End of public interfaces. 103 // Implementation details follow. 104 //------------------------------------------------------------------------------ 105 106 namespace base_internal { 107 108 // Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to 109 // initialize entities used by the scheduler implementation. 110 template <typename Callable, typename... Args> 111 void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn, 112 Args&&... args); 113 114 // Disables scheduling while on stack when scheduling mode is non-cooperative. 115 // No effect for cooperative scheduling modes. 116 class SchedulingHelper { 117 public: 118 explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) { 119 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) { 120 guard_result_ = base_internal::SchedulingGuard::DisableRescheduling(); 121 } 122 } 123 124 ~SchedulingHelper() { 125 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) { 126 base_internal::SchedulingGuard::EnableRescheduling(guard_result_); 127 } 128 } 129 130 private: 131 base_internal::SchedulingMode mode_; 132 bool guard_result_ = false; 133 }; 134 135 // Bit patterns for call_once state machine values. Internal implementation 136 // detail, not for use by clients. 137 // 138 // The bit patterns are arbitrarily chosen from unlikely values, to aid in 139 // debugging. However, kOnceInit must be 0, so that a zero-initialized 140 // once_flag will be valid for immediate use. 141 enum { 142 kOnceInit = 0, 143 kOnceRunning = 0x65C2937B, 144 kOnceWaiter = 0x05A308D2, 145 // A very small constant is chosen for kOnceDone so that it fit in a single 146 // compare with immediate instruction for most common ISAs. This is verified 147 // for x86, POWER and ARM. 148 kOnceDone = 221, // Random Number 149 }; 150 151 template <typename Callable, typename... Args> 152 void 153 CallOnceImpl(absl::Nonnull<std::atomic<uint32_t>*> control, 154 base_internal::SchedulingMode scheduling_mode, Callable&& fn, 155 Args&&... args) { 156 #ifndef NDEBUG 157 { 158 uint32_t old_control = control->load(std::memory_order_relaxed); 159 if (old_control != kOnceInit && 160 old_control != kOnceRunning && 161 old_control != kOnceWaiter && 162 old_control != kOnceDone) { 163 ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx", 164 static_cast<unsigned long>(old_control)); // NOLINT 165 } 166 } 167 #endif // NDEBUG 168 static const base_internal::SpinLockWaitTransition trans[] = { 169 {kOnceInit, kOnceRunning, true}, 170 {kOnceRunning, kOnceWaiter, false}, 171 {kOnceDone, kOnceDone, true}}; 172 173 // Must do this before potentially modifying control word's state. 174 base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode); 175 // Short circuit the simplest case to avoid procedure call overhead. 176 // The base_internal::SpinLockWait() call returns either kOnceInit or 177 // kOnceDone. If it returns kOnceDone, it must have loaded the control word 178 // with std::memory_order_acquire and seen a value of kOnceDone. 179 uint32_t old_control = kOnceInit; 180 if (control->compare_exchange_strong(old_control, kOnceRunning, 181 std::memory_order_relaxed) || 182 base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans, 183 scheduling_mode) == kOnceInit) { 184 std::invoke(std::forward<Callable>(fn), std::forward<Args>(args)...); 185 old_control = 186 control->exchange(base_internal::kOnceDone, std::memory_order_release); 187 if (old_control == base_internal::kOnceWaiter) { 188 base_internal::SpinLockWake(control, true); 189 } 190 } // else *control is already kOnceDone 191 } 192 193 inline absl::Nonnull<std::atomic<uint32_t>*> ControlWord( 194 absl::Nonnull<once_flag*> flag) { 195 return &flag->control_; 196 } 197 198 template <typename Callable, typename... Args> 199 void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn, 200 Args&&... args) { 201 std::atomic<uint32_t>* once = base_internal::ControlWord(flag); 202 uint32_t s = once->load(std::memory_order_acquire); 203 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { 204 base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY, 205 std::forward<Callable>(fn), 206 std::forward<Args>(args)...); 207 } 208 } 209 210 } // namespace base_internal 211 212 template <typename Callable, typename... Args> 213 void 214 call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) { 215 std::atomic<uint32_t>* once = base_internal::ControlWord(&flag); 216 uint32_t s = once->load(std::memory_order_acquire); 217 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { 218 base_internal::CallOnceImpl( 219 once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL, 220 std::forward<Callable>(fn), std::forward<Args>(args)...); 221 } 222 } 223 224 ABSL_NAMESPACE_END 225 } // namespace absl 226 227 #endif // ABSL_BASE_CALL_ONCE_H_