vdso_support.cc (7018B)
1 // Copyright 2017 The Abseil Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Allow dynamic symbol lookup in the kernel VDSO page. 16 // 17 // VDSOSupport -- a class representing kernel VDSO (if present). 18 19 #include "absl/debugging/internal/vdso_support.h" 20 21 #ifdef ABSL_HAVE_VDSO_SUPPORT // defined in vdso_support.h 22 23 #if !defined(__has_include) 24 #define __has_include(header) 0 25 #endif 26 27 #include <errno.h> 28 #include <fcntl.h> 29 #if __has_include(<syscall.h>) 30 #include <syscall.h> 31 #elif __has_include(<sys/syscall.h>) 32 #include <sys/syscall.h> 33 #endif 34 #include <unistd.h> 35 36 #if !defined(__UCLIBC__) && defined(__GLIBC__) && \ 37 (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 16)) 38 #define ABSL_HAVE_GETAUXVAL 39 #endif 40 41 #ifdef ABSL_HAVE_GETAUXVAL 42 #include <sys/auxv.h> 43 #endif 44 45 #include "absl/base/dynamic_annotations.h" 46 #include "absl/base/internal/raw_logging.h" 47 #include "absl/base/port.h" 48 49 #ifndef AT_SYSINFO_EHDR 50 #define AT_SYSINFO_EHDR 33 // for crosstoolv10 51 #endif 52 53 #if defined(__NetBSD__) 54 using Elf32_auxv_t = Aux32Info; 55 using Elf64_auxv_t = Aux64Info; 56 #endif 57 #if defined(__FreeBSD__) 58 #if defined(__ELF_WORD_SIZE) && __ELF_WORD_SIZE == 64 59 using Elf64_auxv_t = Elf64_Auxinfo; 60 #endif 61 using Elf32_auxv_t = Elf32_Auxinfo; 62 #endif 63 64 namespace absl { 65 ABSL_NAMESPACE_BEGIN 66 namespace debugging_internal { 67 68 ABSL_CONST_INIT 69 std::atomic<const void *> VDSOSupport::vdso_base_( 70 debugging_internal::ElfMemImage::kInvalidBase); 71 72 ABSL_CONST_INIT std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_( 73 &InitAndGetCPU); 74 75 VDSOSupport::VDSOSupport() 76 // If vdso_base_ is still set to kInvalidBase, we got here 77 // before VDSOSupport::Init has been called. Call it now. 78 : image_(vdso_base_.load(std::memory_order_relaxed) == 79 debugging_internal::ElfMemImage::kInvalidBase 80 ? Init() 81 : vdso_base_.load(std::memory_order_relaxed)) {} 82 83 // NOTE: we can't use GoogleOnceInit() below, because we can be 84 // called by tcmalloc, and none of the *once* stuff may be functional yet. 85 // 86 // In addition, we hope that the VDSOSupportHelper constructor 87 // causes this code to run before there are any threads, and before 88 // InitGoogle() has executed any chroot or setuid calls. 89 // 90 // Finally, even if there is a race here, it is harmless, because 91 // the operation should be idempotent. 92 const void *VDSOSupport::Init() { 93 const auto kInvalidBase = debugging_internal::ElfMemImage::kInvalidBase; 94 #ifdef ABSL_HAVE_GETAUXVAL 95 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { 96 errno = 0; 97 const void *const sysinfo_ehdr = 98 reinterpret_cast<const void *>(getauxval(AT_SYSINFO_EHDR)); 99 if (errno == 0) { 100 vdso_base_.store(sysinfo_ehdr, std::memory_order_relaxed); 101 } 102 } 103 #endif // ABSL_HAVE_GETAUXVAL 104 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { 105 int fd = open("/proc/self/auxv", O_RDONLY); 106 if (fd == -1) { 107 // Kernel too old to have a VDSO. 108 vdso_base_.store(nullptr, std::memory_order_relaxed); 109 getcpu_fn_.store(&GetCPUViaSyscall, std::memory_order_relaxed); 110 return nullptr; 111 } 112 ElfW(auxv_t) aux; 113 while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) { 114 if (aux.a_type == AT_SYSINFO_EHDR) { 115 #if defined(__NetBSD__) 116 vdso_base_.store(reinterpret_cast<void *>(aux.a_v), 117 std::memory_order_relaxed); 118 #else 119 vdso_base_.store(reinterpret_cast<void *>(aux.a_un.a_val), 120 std::memory_order_relaxed); 121 #endif 122 break; 123 } 124 } 125 close(fd); 126 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { 127 // Didn't find AT_SYSINFO_EHDR in auxv[]. 128 vdso_base_.store(nullptr, std::memory_order_relaxed); 129 } 130 } 131 GetCpuFn fn = &GetCPUViaSyscall; // default if VDSO not present. 132 if (vdso_base_.load(std::memory_order_relaxed)) { 133 VDSOSupport vdso; 134 SymbolInfo info; 135 if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) { 136 fn = reinterpret_cast<GetCpuFn>(const_cast<void *>(info.address)); 137 } 138 } 139 // Subtle: this code runs outside of any locks; prevent compiler 140 // from assigning to getcpu_fn_ more than once. 141 getcpu_fn_.store(fn, std::memory_order_relaxed); 142 return vdso_base_.load(std::memory_order_relaxed); 143 } 144 145 const void *VDSOSupport::SetBase(const void *base) { 146 ABSL_RAW_CHECK(base != debugging_internal::ElfMemImage::kInvalidBase, 147 "internal error"); 148 const void *old_base = vdso_base_.load(std::memory_order_relaxed); 149 vdso_base_.store(base, std::memory_order_relaxed); 150 image_.Init(base); 151 // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO. 152 getcpu_fn_.store(&InitAndGetCPU, std::memory_order_relaxed); 153 return old_base; 154 } 155 156 bool VDSOSupport::LookupSymbol(const char *name, 157 const char *version, 158 int type, 159 SymbolInfo *info) const { 160 return image_.LookupSymbol(name, version, type, info); 161 } 162 163 bool VDSOSupport::LookupSymbolByAddress(const void *address, 164 SymbolInfo *info_out) const { 165 return image_.LookupSymbolByAddress(address, info_out); 166 } 167 168 // NOLINT on 'long' because this routine mimics kernel api. 169 long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, // NOLINT(runtime/int) 170 void *, void *) { 171 #ifdef SYS_getcpu 172 return syscall(SYS_getcpu, cpu, nullptr, nullptr); 173 #else 174 // x86_64 never implemented sys_getcpu(), except as a VDSO call. 175 static_cast<void>(cpu); // Avoid an unused argument compiler warning. 176 errno = ENOSYS; 177 return -1; 178 #endif 179 } 180 181 // Use fast __vdso_getcpu if available. 182 long VDSOSupport::InitAndGetCPU(unsigned *cpu, // NOLINT(runtime/int) 183 void *x, void *y) { 184 Init(); 185 GetCpuFn fn = getcpu_fn_.load(std::memory_order_relaxed); 186 ABSL_RAW_CHECK(fn != &InitAndGetCPU, "Init() did not set getcpu_fn_"); 187 return (*fn)(cpu, x, y); 188 } 189 190 // This function must be very fast, and may be called from very 191 // low level (e.g. tcmalloc). Hence I avoid things like 192 // GoogleOnceInit() and ::operator new. 193 ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY 194 int GetCPU() { 195 unsigned cpu; 196 long ret_code = // NOLINT(runtime/int) 197 (*VDSOSupport::getcpu_fn_)(&cpu, nullptr, nullptr); 198 return ret_code == 0 ? static_cast<int>(cpu) : static_cast<int>(ret_code); 199 } 200 201 } // namespace debugging_internal 202 ABSL_NAMESPACE_END 203 } // namespace absl 204 205 #endif // ABSL_HAVE_VDSO_SUPPORT