futex.h (5690B)
1 // Copyright 2020 The Abseil Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 #ifndef ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 15 #define ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 16 17 #include "absl/base/config.h" 18 19 #ifndef _WIN32 20 #include <sys/time.h> 21 #include <unistd.h> 22 #endif 23 24 #ifdef __linux__ 25 #include <linux/futex.h> 26 #include <sys/syscall.h> 27 #endif 28 29 #include <errno.h> 30 #include <stdio.h> 31 #include <time.h> 32 33 #include <atomic> 34 #include <cstdint> 35 #include <limits> 36 37 #include "absl/base/optimization.h" 38 #include "absl/synchronization/internal/kernel_timeout.h" 39 40 #ifdef ABSL_INTERNAL_HAVE_FUTEX 41 #error ABSL_INTERNAL_HAVE_FUTEX may not be set on the command line 42 #elif defined(__BIONIC__) 43 // Bionic supports all the futex operations we need even when some of the futex 44 // definitions are missing. 45 #define ABSL_INTERNAL_HAVE_FUTEX 46 #elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME) 47 // FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28. 48 #define ABSL_INTERNAL_HAVE_FUTEX 49 #endif 50 51 #ifdef ABSL_INTERNAL_HAVE_FUTEX 52 53 namespace absl { 54 ABSL_NAMESPACE_BEGIN 55 namespace synchronization_internal { 56 57 // Some Android headers are missing these definitions even though they 58 // support these futex operations. 59 #ifdef __BIONIC__ 60 #ifndef SYS_futex 61 #define SYS_futex __NR_futex 62 #endif 63 #ifndef FUTEX_WAIT_BITSET 64 #define FUTEX_WAIT_BITSET 9 65 #endif 66 #ifndef FUTEX_PRIVATE_FLAG 67 #define FUTEX_PRIVATE_FLAG 128 68 #endif 69 #ifndef FUTEX_CLOCK_REALTIME 70 #define FUTEX_CLOCK_REALTIME 256 71 #endif 72 #ifndef FUTEX_BITSET_MATCH_ANY 73 #define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF 74 #endif 75 #endif 76 77 #if defined(__NR_futex_time64) && !defined(SYS_futex_time64) 78 #define SYS_futex_time64 __NR_futex_time64 79 #endif 80 81 #if defined(SYS_futex_time64) && !defined(SYS_futex) 82 #define SYS_futex SYS_futex_time64 83 using FutexTimespec = struct timespec; 84 #else 85 // Some libc implementations have switched to an unconditional 64-bit `time_t` 86 // definition. This means that `struct timespec` may not match the layout 87 // expected by the kernel ABI on 32-bit platforms. So we define the 88 // FutexTimespec that matches the kernel timespec definition. It should be safe 89 // to use this struct for 64-bit userspace builds too, since it will use another 90 // SYS_futex kernel call with 64-bit tv_sec inside timespec. 91 struct FutexTimespec { 92 long tv_sec; // NOLINT 93 long tv_nsec; // NOLINT 94 }; 95 #endif 96 97 class FutexImpl { 98 public: 99 // Atomically check that `*v == val`, and if it is, then sleep until the until 100 // woken by `Wake()`. 101 static int Wait(std::atomic<int32_t>* v, int32_t val) { 102 return WaitAbsoluteTimeout(v, val, nullptr); 103 } 104 105 // Atomically check that `*v == val`, and if it is, then sleep until 106 // CLOCK_REALTIME reaches `*abs_timeout`, or until woken by `Wake()`. 107 static int WaitAbsoluteTimeout(std::atomic<int32_t>* v, int32_t val, 108 const struct timespec* abs_timeout) { 109 FutexTimespec ts; 110 // https://locklessinc.com/articles/futex_cheat_sheet/ 111 // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time. 112 auto err = syscall( 113 SYS_futex, reinterpret_cast<int32_t*>(v), 114 FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val, 115 ToFutexTimespec(abs_timeout, &ts), nullptr, FUTEX_BITSET_MATCH_ANY); 116 if (err != 0) { 117 return -errno; 118 } 119 return 0; 120 } 121 122 // Atomically check that `*v == val`, and if it is, then sleep until 123 // `*rel_timeout` has elapsed, or until woken by `Wake()`. 124 static int WaitRelativeTimeout(std::atomic<int32_t>* v, int32_t val, 125 const struct timespec* rel_timeout) { 126 FutexTimespec ts; 127 // Atomically check that the futex value is still 0, and if it 128 // is, sleep until abs_timeout or until woken by FUTEX_WAKE. 129 auto err = 130 syscall(SYS_futex, reinterpret_cast<int32_t*>(v), FUTEX_PRIVATE_FLAG, 131 val, ToFutexTimespec(rel_timeout, &ts)); 132 if (err != 0) { 133 return -errno; 134 } 135 return 0; 136 } 137 138 // Wakes at most `count` waiters that have entered the sleep state on `v`. 139 static int Wake(std::atomic<int32_t>* v, int32_t count) { 140 auto err = syscall(SYS_futex, reinterpret_cast<int32_t*>(v), 141 FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count); 142 if (ABSL_PREDICT_FALSE(err < 0)) { 143 return -errno; 144 } 145 return 0; 146 } 147 148 private: 149 static FutexTimespec* ToFutexTimespec(const struct timespec* userspace_ts, 150 FutexTimespec* futex_ts) { 151 if (userspace_ts == nullptr) { 152 return nullptr; 153 } 154 155 using FutexSeconds = decltype(futex_ts->tv_sec); 156 using FutexNanoseconds = decltype(futex_ts->tv_nsec); 157 158 constexpr auto kMaxSeconds{(std::numeric_limits<FutexSeconds>::max)()}; 159 if (userspace_ts->tv_sec > kMaxSeconds) { 160 futex_ts->tv_sec = kMaxSeconds; 161 } else { 162 futex_ts->tv_sec = static_cast<FutexSeconds>(userspace_ts->tv_sec); 163 } 164 futex_ts->tv_nsec = static_cast<FutexNanoseconds>(userspace_ts->tv_nsec); 165 return futex_ts; 166 } 167 }; 168 169 class Futex : public FutexImpl {}; 170 171 } // namespace synchronization_internal 172 ABSL_NAMESPACE_END 173 } // namespace absl 174 175 #endif // ABSL_INTERNAL_HAVE_FUTEX 176 177 #endif // ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_