FlushICache.cpp (5548B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "jit/FlushICache.h" 8 9 #ifdef JS_CODEGEN_ARM64 10 # include "jit/arm64/vixl/MozCachingDecoder.h" 11 # include "jit/arm64/vixl/Simulator-vixl.h" 12 #endif 13 14 #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) 15 16 # ifdef __linux__ 17 # include <linux/version.h> 18 # define LINUX_HAS_MEMBARRIER (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)) 19 # else 20 # define LINUX_HAS_MEMBARRIER 0 21 # endif 22 23 # if LINUX_HAS_MEMBARRIER || defined(__android__) 24 # include <string.h> 25 26 # if LINUX_HAS_MEMBARRIER 27 # include <linux/membarrier.h> 28 # include <sys/syscall.h> 29 # include <sys/utsname.h> 30 # include <unistd.h> 31 # elif defined(__android__) 32 # include <sys/syscall.h> 33 # include <unistd.h> 34 # else 35 # error "Missing platform-specific declarations for membarrier syscall!" 36 # endif // __linux__ / ANDROID 37 38 static int membarrier(int cmd, int flags) { 39 return syscall(__NR_membarrier, cmd, flags); 40 } 41 42 // These definitions come from the Linux kernel source, for kernels before 4.16 43 // which didn't have access to these membarrier commands. 44 # ifndef MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE 45 # define MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE (1 << 5) 46 # endif 47 48 # ifndef MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE 49 # define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE (1 << 6) 50 # endif 51 # endif // LINUX_HAS_MEMBARRIER || defined(__android__) 52 53 using namespace js; 54 using namespace js::jit; 55 56 namespace js { 57 namespace jit { 58 59 bool CanFlushExecutionContextForAllThreads() { 60 # if (LINUX_HAS_MEMBARRIER || defined(__android__)) 61 // We only need to decide this once, but we don't know which thread will ask 62 // first. Hence the atomic. 63 enum class MemBarrierAvailable : uint32_t { Unset, No, Yes }; 64 65 // The only allowable transitions are Unset->No and Unset->Yes. 66 static mozilla::Atomic<MemBarrierAvailable> state(MemBarrierAvailable::Unset); 67 68 MemBarrierAvailable localState = state; 69 if (MOZ_LIKELY(localState != MemBarrierAvailable::Unset)) { 70 return localState == MemBarrierAvailable::Yes; 71 } 72 73 // Otherwise we'll have to make enquiries. This needs to produce the same 74 // result if performed more than once. 75 76 // On linux, check the kernel supports membarrier(2). A prerequisite is that 77 // the kernel version is 4.16 or later. 78 // 79 // Note: this code has been extracted (August 2020) from 80 // https://android.googlesource.com/platform/art/+/58520dfba31d6eeef75f5babff15e09aa28e5db8/libartbase/base/membarrier.cc#50 81 static constexpr int kRequiredMajor = 4; 82 static constexpr int kRequiredMinor = 16; 83 84 struct utsname uts; 85 int major, minor; 86 bool memBarrierAvailable = 87 uname(&uts) == 0 && strcmp(uts.sysname, "Linux") == 0 && 88 sscanf(uts.release, "%d.%d", &major, &minor) == 2 && 89 major >= kRequiredMajor && 90 (major != kRequiredMajor || minor >= kRequiredMinor); 91 92 // Try to run the syscall with the command registering the intent to use the 93 // actual membarrier we'll want to carry out later. 94 // 95 // IMPORTANT: This is needs to succeed, since otherwise running the 96 // membarrier later won't actually interrupt the threads in this process. 97 // Per `man 2 membarrier`, "RETURN VALUE", we are assured that multiple calls 98 // to `membarrier` with the same commands always produce the same results. 99 if (memBarrierAvailable && 100 membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE, 0) != 0) { 101 memBarrierAvailable = false; 102 } 103 104 bool ok = state.compareExchange( 105 MemBarrierAvailable::Unset, 106 memBarrierAvailable ? MemBarrierAvailable::Yes : MemBarrierAvailable::No); 107 if (ok) { 108 return memBarrierAvailable; 109 } 110 111 // We got out-raced. Return the info that's already there. 112 MOZ_ASSERT(state != MemBarrierAvailable::Unset); 113 return state == MemBarrierAvailable::Yes; 114 115 # else 116 // On other platforms, we assume that the syscall for flushing the icache 117 // will flush the execution context for other cores. 118 return true; 119 # endif 120 } 121 122 void FlushExecutionContextForAllThreads() { 123 // Callers must check that this operation is available. 124 MOZ_RELEASE_ASSERT(CanFlushExecutionContextForAllThreads()); 125 126 # if defined(JS_SIMULATOR_ARM64) && defined(JS_CACHE_SIMULATOR_ARM64) 127 // Emulate what the real hardware would do by emitting a membarrier that'll 128 // interrupt and flush the execution context of all threads. 129 using js::jit::SimulatorProcess; 130 js::jit::AutoLockSimulatorCache alsc; 131 SimulatorProcess::membarrier(); 132 # elif (LINUX_HAS_MEMBARRIER || defined(__android__)) 133 // The caller has checked this can be performed, which will have registered 134 // this process to receive the membarrier. See above. 135 // 136 // membarrier will trigger an inter-processor-interrupt on any active threads 137 // of this process. This is an execution context synchronization event 138 // equivalent to running an `isb` instruction. 139 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE, 0) != 0) { 140 // Better safe than sorry. 141 MOZ_CRASH("membarrier can't be executed"); 142 } 143 # else 144 // On other platforms, we assume that the syscall for flushing the icache 145 // will flush the execution context for other cores. 146 # endif 147 } 148 149 } // namespace jit 150 } // namespace js 151 152 #endif