tor-browser

ProcessExecutableMemory.cpp (32356B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "jit/ProcessExecutableMemory.h"

#include "mozilla/Array.h"
#include "mozilla/Atomics.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/Maybe.h"
#include "mozilla/PodOperations.h"
#include "mozilla/TaggedAnonymousMemory.h"
#include "mozilla/XorShift128PlusRNG.h"

#include <errno.h>

#include "jsfriendapi.h"
#include "jsmath.h"

#include "gc/Memory.h"
#include "jit/FlushICache.h"  // js::jit::FlushICache
#include "jit/JitOptions.h"
#include "threading/LockGuard.h"
#include "threading/Mutex.h"
#include "util/Memory.h"
#include "util/Poison.h"
#include "util/WindowsWrapper.h"
#include "vm/MutexIDs.h"

#ifdef XP_WIN
#  include "mozilla/StackWalk_windows.h"
#elif defined(__wasi__)
#  if defined(JS_CODEGEN_WASM32)
#    include <cstdlib>
#  else
// Nothing.
#  endif
#else
#  include <sys/mman.h>
#  include <unistd.h>
#endif

#ifdef MOZ_VALGRIND
#  include <valgrind/valgrind.h>
#endif

using namespace js;
using namespace js::jit;

#ifdef XP_WIN
#  if defined(HAVE_64BIT_BUILD)
#    define NEED_JIT_UNWIND_HANDLING
#  endif

static void* ComputeRandomAllocationAddress() {
 /*
  * Inspiration is V8's OS::Allocate in platform-win32.cc.
  *
  * VirtualAlloc takes 64K chunks out of the virtual address space, so we
  * keep 16b alignment.
  *
  * x86: V8 comments say that keeping addresses in the [64MiB, 1GiB) range
  * tries to avoid system default DLL mapping space. In the end, we get 13
  * bits of randomness in our selection.
  * x64: [2GiB, 4TiB), with 25 bits of randomness.
  */
#  ifdef HAVE_64BIT_BUILD
 static const uintptr_t base = 0x0000000080000000;
 static const uintptr_t mask = 0x000003ffffff0000;
#  elif defined(_M_IX86) || defined(__i386__)
 static const uintptr_t base = 0x04000000;
 static const uintptr_t mask = 0x3fff0000;
#  else
#    error "Unsupported architecture"
#  endif

 uint64_t rand = js::GenerateRandomSeed();
 return (void*)(base | (rand & mask));
}

#  ifdef NEED_JIT_UNWIND_HANDLING
static js::JitExceptionHandler sJitExceptionHandler;
static bool sHasInstalledFunctionTable = false;
#  endif

JS_PUBLIC_API void js::SetJitExceptionHandler(JitExceptionHandler handler) {
#  ifdef NEED_JIT_UNWIND_HANDLING
 MOZ_ASSERT(!sJitExceptionHandler);
 sJitExceptionHandler = handler;
#  else
 // Just do nothing if unwind handling is disabled.
#  endif
}

#  ifdef NEED_JIT_UNWIND_HANDLING
#    if defined(_M_ARM64)
// See the ".xdata records" section of
// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
// These records can have various fields present or absent depending on the
// bits set in the header. Our struct will use one 32-bit slot for unwind codes,
// and no slots for epilog scopes.
struct UnwindData {
 uint32_t functionLength : 18;
 uint32_t version : 2;
 uint32_t hasExceptionHandler : 1;
 uint32_t packedEpilog : 1;
 uint32_t epilogCount : 5;
 uint32_t codeWords : 5;
 uint8_t unwindCodes[4];
 uint32_t exceptionHandler;
};

static const unsigned ThunkLength = 20;
#    else
// From documentation for UNWIND_INFO on
// https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64
struct UnwindInfo {
 uint8_t version : 3;
 uint8_t flags : 5;
 uint8_t sizeOfPrologue;
 uint8_t countOfUnwindCodes;
 uint8_t frameRegister : 4;
 uint8_t frameOffset : 4;
};
static const unsigned ThunkLength = 12;
union UnwindCode {
 struct {
   uint8_t codeOffset;
   uint8_t unwindOp : 4;
   uint8_t opInfo : 4;
 };
 uint16_t frameOffset;
};

static constexpr int kNumberOfUnwindCodes = 2;
static constexpr int kPushRbpInstructionLength = 1;
static constexpr int kMovRbpRspInstructionLength = 3;
static constexpr int kRbpPrefixCodes = 2;
static constexpr int kRbpPrefixLength =
   kPushRbpInstructionLength + kMovRbpRspInstructionLength;

struct UnwindData {
 UnwindInfo unwindInfo;
 UnwindCode unwindCodes[kNumberOfUnwindCodes];
 uint32_t exceptionHandler;

 UnwindData() {
   static constexpr int kOpPushNonvol = 0;
   static constexpr int kOpSetFPReg = 3;

   unwindInfo.version = 1;
   unwindInfo.flags = UNW_FLAG_EHANDLER;
   unwindInfo.sizeOfPrologue = kRbpPrefixLength;
   unwindInfo.countOfUnwindCodes = kRbpPrefixCodes;
   unwindInfo.frameRegister = 5;
   unwindInfo.frameOffset = 0;

   // Offset here are specified to beginning of the -next- instruction.
   unwindCodes[0].codeOffset = kRbpPrefixLength;  // movq rbp, rsp
   unwindCodes[0].unwindOp = kOpSetFPReg;
   unwindCodes[0].opInfo = 0;

   unwindCodes[1].codeOffset = kPushRbpInstructionLength;  // push rbp
   unwindCodes[1].unwindOp = kOpPushNonvol;
   unwindCodes[1].opInfo = 5;
 }
};
#    endif

struct ExceptionHandlerRecord {
 void* dynamicTable;
 UnwindData unwindData;
 uint8_t thunk[ThunkLength];
 RUNTIME_FUNCTION runtimeFunction;
};

// This function must match the function pointer type PEXCEPTION_HANDLER
// mentioned in:
//   http://msdn.microsoft.com/en-us/library/ssa62fwe.aspx.
// This type is rather elusive in documentation; Wine is the best I've found:
//   http://source.winehq.org/source/include/winnt.h
static DWORD ExceptionHandler(PEXCEPTION_RECORD exceptionRecord,
                             _EXCEPTION_REGISTRATION_RECORD*, PCONTEXT context,
                             _EXCEPTION_REGISTRATION_RECORD**) {
 if (sJitExceptionHandler) {
   return sJitExceptionHandler(exceptionRecord, context);
 }

 return ExceptionContinueSearch;
}

// Required for enabling Stackwalking on windows using external tools.
extern "C" NTSYSAPI DWORD NTAPI RtlAddGrowableFunctionTable(
   PVOID* DynamicTable, PRUNTIME_FUNCTION FunctionTable, DWORD EntryCount,
   DWORD MaximumEntryCount, ULONG_PTR RangeBase, ULONG_PTR RangeEnd);

// For an explanation of the problem being solved here, see
// SetJitExceptionFilter in jsfriendapi.h.
static bool RegisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
 if (!VirtualAlloc(p, pageSize, MEM_COMMIT, PAGE_READWRITE)) {
   MOZ_CRASH();
 }

 // A page was reserved inside this structure for the record. This is because
 // all entries in the record are describes as an offset from the start of the
 // memory region. We construct the record there.
 ExceptionHandlerRecord* r = new (p) ExceptionHandlerRecord();
 void* handler = JS_FUNC_TO_DATA_PTR(void*, ExceptionHandler);

 // Because the .xdata format on ARM64 can only encode sizes up to 1M (much
 // too small for our JIT code regions), we register a function table callback
 // to provide RUNTIME_FUNCTIONs at runtime. Windows doesn't seem to care about
 // the size fields on RUNTIME_FUNCTIONs that are created in this way, so the
 // same RUNTIME_FUNCTION can work for any address in the region. We'll set up
 // a generic one now and the callback can just return a pointer to it.

 // All these fields are specified to be offsets from the base of the
 // executable code (which is 'p'), even if they have 'Address' in their
 // names. In particular, exceptionHandler is a ULONG offset which is a
 // 32-bit integer. Since 'p' can be farther than INT32_MAX away from
 // sJitExceptionHandler, we must generate a little thunk inside the
 // record. The record is put on its own page so that we can take away write
 // access to protect against accidental clobbering.

#    if defined(_M_ARM64)
 if (!sJitExceptionHandler) {
   return false;
 }

 r->runtimeFunction.BeginAddress = pageSize;
 r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindData);
 static_assert(offsetof(ExceptionHandlerRecord, unwindData) % 4 == 0,
               "The ARM64 .pdata format requires that exception information "
               "RVAs be 4-byte aligned.");

 memset(&r->unwindData, 0, sizeof(r->unwindData));
 r->unwindData.hasExceptionHandler = true;
 r->unwindData.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);

 // Use a fake unwind code to make the Windows unwinder do _something_. If the
 // PC and SP both stay unchanged, we'll fail the unwinder's sanity checks and
 // it won't call our exception handler.
 r->unwindData.codeWords = 1;  // one 32-bit word gives us up to 4 codes
 r->unwindData.unwindCodes[0] =
     0b00000001;  // alloc_s small stack of size 1*16
 r->unwindData.unwindCodes[1] = 0b11100100;  // end

 uint32_t* thunk = (uint32_t*)r->thunk;
 uint16_t* addr = (uint16_t*)&handler;

 // xip0/r16 should be safe to clobber: Windows just used it to call our thunk.
 const uint8_t reg = 16;

 // Say `handler` is 0x4444333322221111, then:
 thunk[0] = 0xd2800000 | addr[0] << 5 | reg;  // mov  xip0, 1111
 thunk[1] = 0xf2a00000 | addr[1] << 5 | reg;  // movk xip0, 2222 lsl #0x10
 thunk[2] = 0xf2c00000 | addr[2] << 5 | reg;  // movk xip0, 3333 lsl #0x20
 thunk[3] = 0xf2e00000 | addr[3] << 5 | reg;  // movk xip0, 4444 lsl #0x30
 thunk[4] = 0xd61f0000 | reg << 5;            // br xip0
#    else
 r->runtimeFunction.BeginAddress = pageSize;
 r->runtimeFunction.EndAddress = (DWORD)bytes;
 r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindData);
 r->unwindData.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);

 // mov imm64, rax
 r->thunk[0] = 0x48;
 r->thunk[1] = 0xb8;
 memcpy(&r->thunk[2], &handler, 8);

 // jmp rax
 r->thunk[10] = 0xff;
 r->thunk[11] = 0xe0;
#    endif

 // RtlAddGrowableFunctionTable will write into the region. We must therefore
 // only write-protect is after this has been called.

 // XXX NB: The profiler believes this function is only called from the main
 // thread. If that ever becomes untrue, the profiler must be updated
 // immediately.
 {
   AutoSuppressStackWalking suppress;
   DWORD result = RtlAddGrowableFunctionTable(
       &r->dynamicTable, &r->runtimeFunction, 1, 1, (ULONG_PTR)p,
       (ULONG_PTR)p + bytes - pageSize);
   if (result != S_OK) {
     return false;
   }
 }

 DWORD oldProtect;
 if (!VirtualProtect(p, pageSize, PAGE_EXECUTE_READ, &oldProtect)) {
   MOZ_CRASH();
 }

 return true;
}

static void UnregisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
 // There's no such thing as RtlUninstallFunctionTableCallback, so there's
 // nothing to do here.
}
#  endif

static void* ReserveProcessExecutableMemory(size_t bytes) {
#  ifdef NEED_JIT_UNWIND_HANDLING
 size_t pageSize = gc::SystemPageSize();
 // Always reserve space for the unwind information.
 bytes += pageSize;
#  endif

 void* p = nullptr;
 for (size_t i = 0; i < 10; i++) {
   void* randomAddr = ComputeRandomAllocationAddress();
   p = VirtualAlloc(randomAddr, bytes, MEM_RESERVE, PAGE_NOACCESS);
   if (p) {
     break;
   }
 }

 if (!p) {
   // Try again without randomization.
   p = VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_NOACCESS);
   if (!p) {
     return nullptr;
   }
 }

#  ifdef NEED_JIT_UNWIND_HANDLING
 if (RegisterExecutableMemory(p, bytes, pageSize)) {
   sHasInstalledFunctionTable = true;
 } else {
   if (sJitExceptionHandler) {
     // This should have succeeded if we have an exception handler. Bail.
     VirtualFree(p, 0, MEM_RELEASE);
     return nullptr;
   }
 }

 // Skip the first page where we might have allocated an exception handler
 // record.
 p = (uint8_t*)p + pageSize;
 bytes -= pageSize;

 RegisterJitCodeRegion((uint8_t*)p, bytes);
#  endif
 return p;
}

static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
#  ifdef NEED_JIT_UNWIND_HANDLING
 UnregisterJitCodeRegion((uint8_t*)addr, bytes);

 size_t pageSize = gc::SystemPageSize();
 addr = (uint8_t*)addr - pageSize;

 if (sHasInstalledFunctionTable) {
   UnregisterExecutableMemory(addr, bytes, pageSize);
 }
#  endif

 VirtualFree(addr, 0, MEM_RELEASE);
}

static DWORD ProtectionSettingToFlags(ProtectionSetting protection) {
 if (!JitOptions.writeProtectCode) {
   return PAGE_EXECUTE_READWRITE;
 }
 switch (protection) {
   case ProtectionSetting::Writable:
     return PAGE_READWRITE;
   case ProtectionSetting::Executable:
     return PAGE_EXECUTE_READ;
 }
 MOZ_CRASH();
}

[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
                                     ProtectionSetting protection) {
 void* p = VirtualAlloc(addr, bytes, MEM_COMMIT,
                        ProtectionSettingToFlags(protection));
 if (!p) {
   return false;
 }
 MOZ_RELEASE_ASSERT(p == addr);
 return true;
}

static void DecommitPages(void* addr, size_t bytes) {
 if (!VirtualFree(addr, bytes, MEM_DECOMMIT)) {
   MOZ_CRASH("DecommitPages failed");
 }
}
#elif defined(__wasi__)
#  if defined(JS_CODEGEN_WASM32)
static void* ReserveProcessExecutableMemory(size_t bytes) {
 return malloc(bytes);
}

static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 free(addr);
}

[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
                                     ProtectionSetting protection) {
 return true;
}

static void DecommitPages(void* addr, size_t bytes) {}

#  else
static void* ReserveProcessExecutableMemory(size_t bytes) {
 MOZ_CRASH("NYI for WASI.");
 return nullptr;
}
static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 MOZ_CRASH("NYI for WASI.");
}
[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
                                     ProtectionSetting protection) {
 MOZ_CRASH("NYI for WASI.");
 return false;
}
static void DecommitPages(void* addr, size_t bytes) {
 MOZ_CRASH("NYI for WASI.");
}
#  endif
#else  // !XP_WIN && !__wasi__
#  ifndef MAP_NORESERVE
#    define MAP_NORESERVE 0
#  endif

static void* ComputeRandomAllocationAddress() {
#  ifdef __OpenBSD__
 // OpenBSD already has random mmap and the idea that all x64 cpus
 // have 48-bit address space is not correct. Returning nullptr
 // allows OpenBSD do to the right thing.
 return nullptr;
#  else
 uint64_t rand = js::GenerateRandomSeed();

#    ifdef HAVE_64BIT_BUILD
 // x64 CPUs have a 48-bit address space and on some platforms the OS will
 // give us access to 47 bits, so to be safe we right shift by 18 to leave
 // 46 bits.
 rand >>= 18;
#    else
 // On 32-bit, right shift by 34 to leave 30 bits, range [0, 1GiB). Then add
 // 512MiB to get range [512MiB, 1.5GiB), or [0x20000000, 0x60000000). This
 // is based on V8 comments in platform-posix.cc saying this range is
 // relatively unpopulated across a variety of kernels.
 rand >>= 34;
 rand += 512 * 1024 * 1024;
#    endif

 // Ensure page alignment.
 uintptr_t mask = ~uintptr_t(gc::SystemPageSize() - 1);
 return (void*)uintptr_t(rand & mask);
#  endif
}

static void DecommitPages(void* addr, size_t bytes);

static void* ReserveProcessExecutableMemory(size_t bytes) {
 // On most Unix platforms our strategy is as follows:
 //
 // * Reserve:  mmap with PROT_NONE
 // * Commit:   mmap with MAP_FIXED, PROT_READ | ...
 // * Decommit: mmap with MAP_FIXED, PROT_NONE
 //
 // On Apple Silicon this only works if we use mprotect to implement W^X. To
 // use RWX pages with the faster pthread_jit_write_protect_np API for
 // thread-local writable/executable switching, the kernel enforces the
 // following rules:
 //
 // * The initial mmap must be called with MAP_JIT.
 // * MAP_FIXED can't be used with MAP_JIT.
 // * Since macOS 11.2, mprotect can't be used to change permissions of RWX JIT
 //   pages (even PROT_NONE fails).
 //   See https://developer.apple.com/forums/thread/672804.
 //
 // This means we have to use the following strategy on Apple Silicon:
 //
 // * Reserve:  1) mmap with PROT_READ | PROT_WRITE | PROT_EXEC and MAP_JIT
 //             2) decommit
 // * Commit:   madvise with MADV_FREE_REUSE
 // * Decommit: madvise with MADV_FREE_REUSABLE
 //
 // On Intel Macs we also need to use MAP_JIT, to be compatible with the
 // Hardened Runtime (with com.apple.security.cs.allow-jit = true). The
 // pthread_jit_write_protect_np API is not available on Intel and MAP_JIT
 // can't be used with MAP_FIXED, so we have to use a hybrid of the above two
 // strategies:
 //
 // * Reserve:  1) mmap with PROT_NONE and MAP_JIT
 //             2) decommit
 // * Commit:   1) madvise with MADV_FREE_REUSE
 //             2) mprotect with PROT_READ | ...
 // * Decommit: 1) mprotect with PROT_NONE
 //             2) madvise with MADV_FREE_REUSABLE
 //
 // This is inspired by V8's code in OS::SetPermissions.

 // Note that randomAddr is just a hint: if the address is not available
 // mmap will pick a different address.
 void* randomAddr = ComputeRandomAllocationAddress();
 unsigned protection = PROT_NONE;
 unsigned flags = MAP_NORESERVE | MAP_PRIVATE | MAP_ANON;
#  if defined(XP_DARWIN)
 flags |= MAP_JIT;
#    if defined(JS_USE_APPLE_FAST_WX)
 protection = PROT_READ | PROT_WRITE | PROT_EXEC;
#    endif
#  endif
 void* p = MozTaggedAnonymousMmap(randomAddr, bytes, protection, flags, -1, 0,
                                  "js-executable-memory");
 if (p == MAP_FAILED) {
   return nullptr;
 }
#  if defined(XP_DARWIN)
 DecommitPages(p, bytes);
#  endif
 return p;
}

static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 mozilla::DebugOnly<int> result = munmap(addr, bytes);
 MOZ_ASSERT(!result || errno == ENOMEM);
}

static unsigned ProtectionSettingToFlags(ProtectionSetting protection) {
 if (!JitOptions.writeProtectCode) {
   return PROT_READ | PROT_WRITE | PROT_EXEC;
 }
#  ifdef MOZ_VALGRIND
 // If we're configured for Valgrind and running on it, use a slacker
 // scheme that doesn't change execute permissions, since doing so causes
 // Valgrind a lot of extra overhead re-JITting code that loses and later
 // regains execute permission.  See bug 1338179.
 if (RUNNING_ON_VALGRIND) {
   switch (protection) {
     case ProtectionSetting::Writable:
       return PROT_READ | PROT_WRITE | PROT_EXEC;
     case ProtectionSetting::Executable:
       return PROT_READ | PROT_EXEC;
   }
   MOZ_CRASH();
 }
 // If we get here, we're configured for Valgrind but not running on
 // it, so use the standard scheme.
#  endif
 switch (protection) {
   case ProtectionSetting::Writable:
     return PROT_READ | PROT_WRITE;
   case ProtectionSetting::Executable:
     return PROT_READ | PROT_EXEC;
 }
 MOZ_CRASH();
}

[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
                                     ProtectionSetting protection) {
 // See the comment in ReserveProcessExecutableMemory.
#  if defined(XP_DARWIN)
 int ret;
 do {
   ret = madvise(addr, bytes, MADV_FREE_REUSE);
 } while (ret != 0 && errno == EAGAIN);
 if (ret != 0) {
   return false;
 }
#    if !defined(JS_USE_APPLE_FAST_WX)
 unsigned flags = ProtectionSettingToFlags(protection);
 if (mprotect(addr, bytes, flags)) {
   return false;
 }
#    endif
 return true;
#  else
 unsigned flags = ProtectionSettingToFlags(protection);
 void* p = MozTaggedAnonymousMmap(addr, bytes, flags,
                                  MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
                                  "js-executable-memory");
 if (p == MAP_FAILED) {
   return false;
 }
 MOZ_RELEASE_ASSERT(p == addr);
 return true;
#  endif
}

static void DecommitPages(void* addr, size_t bytes) {
 // See the comment in ReserveProcessExecutableMemory.
#  if defined(XP_DARWIN)
 int ret;
#    if !defined(JS_USE_APPLE_FAST_WX)
 ret = mprotect(addr, bytes, PROT_NONE);
 MOZ_RELEASE_ASSERT(ret == 0);
#    endif
 do {
   ret = madvise(addr, bytes, MADV_FREE_REUSABLE);
 } while (ret != 0 && errno == EAGAIN);
 MOZ_RELEASE_ASSERT(ret == 0);
#  else
 // Use mmap with MAP_FIXED and PROT_NONE. Inspired by jemalloc's
 // pages_decommit.
 void* p = MozTaggedAnonymousMmap(addr, bytes, PROT_NONE,
                                  MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
                                  "js-executable-memory");
 MOZ_RELEASE_ASSERT(addr == p);
#  endif
}
#endif

template <size_t NumBits>
class PageBitSet {
 using WordType = uint32_t;
 static const size_t BitsPerWord = sizeof(WordType) * 8;

 static_assert((NumBits % BitsPerWord) == 0,
               "NumBits must be a multiple of BitsPerWord");
 static const size_t NumWords = NumBits / BitsPerWord;

 mozilla::Array<WordType, NumWords> words_;

 uint32_t indexToWord(uint32_t index) const {
   MOZ_ASSERT(index < NumBits);
   return index / BitsPerWord;
 }
 WordType indexToBit(uint32_t index) const {
   MOZ_ASSERT(index < NumBits);
   return WordType(1) << (index % BitsPerWord);
 }

public:
 void init() { mozilla::PodArrayZero(words_); }
 bool contains(size_t index) const {
   uint32_t word = indexToWord(index);
   return words_[word] & indexToBit(index);
 }
 void insert(size_t index) {
   MOZ_ASSERT(!contains(index));
   uint32_t word = indexToWord(index);
   words_[word] |= indexToBit(index);
 }
 void remove(size_t index) {
   MOZ_ASSERT(contains(index));
   uint32_t word = indexToWord(index);
   words_[word] &= ~indexToBit(index);
 }

#ifdef DEBUG
 bool empty() const {
   for (size_t i = 0; i < NumWords; i++) {
     if (words_[i] != 0) {
       return false;
     }
   }
   return true;
 }
#endif
};

// Per-process executable memory allocator. It reserves a block of memory of
// MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
//
// This has a number of benefits compared to raw mmap/VirtualAlloc:
//
// * More resillient against certain attacks.
//
// * Behaves more consistently across platforms: it avoids the 64K granularity
//   issues on Windows, for instance.
//
// * On x64, near jumps can be used for jumps to other JIT pages.
//
// * On Win64, we have to register the exception handler only once (at process
//   startup). This saves some memory and avoids RtlAddFunctionTable profiler
//   deadlocks.
class ProcessExecutableMemory {
 static_assert(
     (MaxCodeBytesPerProcess % ExecutableCodePageSize) == 0,
     "MaxCodeBytesPerProcess must be a multiple of ExecutableCodePageSize");
 static const size_t MaxCodePages =
     MaxCodeBytesPerProcess / ExecutableCodePageSize;

 // Start of the MaxCodeBytesPerProcess memory block or nullptr if
 // uninitialized. Note that this is NOT guaranteed to be aligned to
 // ExecutableCodePageSize.
 uint8_t* base_;

 // The fields below should only be accessed while we hold the lock.
 Mutex lock_ MOZ_UNANNOTATED;

 // pagesAllocated_ is an Atomic so that bytesAllocated does not have to
 // take the lock.
 mozilla::Atomic<size_t, mozilla::ReleaseAcquire> pagesAllocated_;

 // Page where we should try to allocate next.
 size_t cursor_;

 mozilla::Maybe<mozilla::non_crypto::XorShift128PlusRNG> rng_;
 PageBitSet<MaxCodePages> pages_;

public:
 ProcessExecutableMemory()
     : base_(nullptr),
       lock_(mutexid::ProcessExecutableRegion),
       pagesAllocated_(0),
       cursor_(0),
       pages_() {}

 [[nodiscard]] bool init() {
   pages_.init();

   MOZ_RELEASE_ASSERT(!initialized());
   MOZ_RELEASE_ASSERT(HasJitBackend());
   MOZ_RELEASE_ASSERT(gc::SystemPageSize() <= ExecutableCodePageSize);

   void* p = ReserveProcessExecutableMemory(MaxCodeBytesPerProcess);
   if (!p) {
     return false;
   }

   base_ = static_cast<uint8_t*>(p);

   mozilla::Array<uint64_t, 2> seed;
   GenerateXorShift128PlusSeed(seed);
   rng_.emplace(seed[0], seed[1]);
   return true;
 }

 uint8_t* base() const { return base_; }

 bool initialized() const { return base_ != nullptr; }

 size_t bytesAllocated() const {
   MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
   return pagesAllocated_ * ExecutableCodePageSize;
 }

 void release() {
   MOZ_ASSERT(initialized());
   MOZ_ASSERT(pages_.empty());
   MOZ_ASSERT(pagesAllocated_ == 0);
   DeallocateProcessExecutableMemory(base_, MaxCodeBytesPerProcess);
   base_ = nullptr;
   rng_.reset();
   MOZ_ASSERT(!initialized());
 }

 void assertValidAddress(void* p, size_t bytes) const {
   MOZ_RELEASE_ASSERT(p >= base_ &&
                      uintptr_t(p) + bytes <=
                          uintptr_t(base_) + MaxCodeBytesPerProcess);
 }

 bool containsAddress(const void* p) const {
   return p >= base_ &&
          uintptr_t(p) < uintptr_t(base_) + MaxCodeBytesPerProcess;
 }

 void* allocate(size_t bytes, ProtectionSetting protection,
                MemCheckKind checkKind);
 void deallocate(void* addr, size_t bytes, bool decommit);
};

void* ProcessExecutableMemory::allocate(size_t bytes,
                                       ProtectionSetting protection,
                                       MemCheckKind checkKind) {
 MOZ_ASSERT(initialized());
 MOZ_ASSERT(HasJitBackend());
 MOZ_ASSERT(bytes > 0);
 MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);

 size_t numPages = bytes / ExecutableCodePageSize;

 // Take the lock and try to allocate.
 void* p = nullptr;
 {
   LockGuard<Mutex> guard(lock_);
   MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);

   // Check if we have enough pages available.
   if (pagesAllocated_ + numPages >= MaxCodePages) {
     return nullptr;
   }

   MOZ_ASSERT(bytes <= MaxCodeBytesPerProcess);

   // Maybe skip a page to make allocations less predictable.
   size_t page = cursor_ + (rng_.ref().next() % 2);

   for (size_t i = 0; i < MaxCodePages; i++) {
     // Make sure page + numPages - 1 is a valid index.
     if (page + numPages > MaxCodePages) {
       page = 0;
     }

     bool available = true;
     for (size_t j = 0; j < numPages; j++) {
       if (pages_.contains(page + j)) {
         available = false;
         break;
       }
     }
     if (!available) {
       page++;
       continue;
     }

     // Mark the pages as unavailable.
     for (size_t j = 0; j < numPages; j++) {
       pages_.insert(page + j);
     }

     pagesAllocated_ += numPages;
     MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);

     // If we allocated a small number of pages, move cursor_ to the
     // next page. We don't do this for larger allocations to avoid
     // skipping a large number of small holes.
     if (numPages <= 2) {
       cursor_ = page + numPages;
     }

     p = base_ + page * ExecutableCodePageSize;
     break;
   }
   if (!p) {
     return nullptr;
   }
 }

 // Commit the pages after releasing the lock.
 if (!CommitPages(p, bytes, protection)) {
   deallocate(p, bytes, /* decommit = */ false);
   return nullptr;
 }

#if !defined(__wasi__)
 gc::RecordMemoryAlloc(bytes);
#endif

 SetMemCheckKind(p, bytes, checkKind);

 return p;
}

void ProcessExecutableMemory::deallocate(void* addr, size_t bytes,
                                        bool decommit) {
 MOZ_ASSERT(initialized());
 MOZ_ASSERT(addr);
 MOZ_ASSERT((uintptr_t(addr) % gc::SystemPageSize()) == 0);
 MOZ_ASSERT(bytes > 0);
 MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);

 assertValidAddress(addr, bytes);

 size_t firstPage =
     (static_cast<uint8_t*>(addr) - base_) / ExecutableCodePageSize;
 size_t numPages = bytes / ExecutableCodePageSize;

 // Decommit before taking the lock.
 MOZ_MAKE_MEM_NOACCESS(addr, bytes);
 if (decommit) {
   DecommitPages(addr, bytes);
#if !defined(__wasi__)
   gc::RecordMemoryFree(bytes);
#endif
 }

 LockGuard<Mutex> guard(lock_);
 MOZ_ASSERT(numPages <= pagesAllocated_);
 pagesAllocated_ -= numPages;

 for (size_t i = 0; i < numPages; i++) {
   pages_.remove(firstPage + i);
 }

 // Move the cursor back so we can reuse pages instead of fragmenting the
 // whole region.
 if (firstPage < cursor_) {
   cursor_ = firstPage;
 }
}

MOZ_RUNINIT static ProcessExecutableMemory execMemory;

void* js::jit::AllocateExecutableMemory(size_t bytes,
                                       ProtectionSetting protection,
                                       MemCheckKind checkKind) {
 return execMemory.allocate(bytes, protection, checkKind);
}

void js::jit::DeallocateExecutableMemory(void* addr, size_t bytes) {
 execMemory.deallocate(addr, bytes, /* decommit = */ true);
}

bool js::jit::InitProcessExecutableMemory() { return execMemory.init(); }

void js::jit::ReleaseProcessExecutableMemory() { execMemory.release(); }

size_t js::jit::LikelyAvailableExecutableMemory() {
 // Round down available memory to the closest MB.
 return MaxCodeBytesPerProcess -
        AlignBytes(execMemory.bytesAllocated(), 0x100000U);
}

bool js::jit::CanLikelyAllocateMoreExecutableMemory() {
 // Use a 8 MB buffer.
 static const size_t BufferSize = 8 * 1024 * 1024;

 MOZ_ASSERT(execMemory.bytesAllocated() <= MaxCodeBytesPerProcess);

 return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
}

bool js::jit::AddressIsInExecutableMemory(const void* p) {
 return execMemory.containsAddress(p);
}

bool js::jit::ReprotectRegion(void* start, size_t size,
                             ProtectionSetting protection,
                             MustFlushICache flushICache) {
#if defined(JS_CODEGEN_WASM32)
 return true;
#endif

 // Flush ICache when making code executable, before we modify |size|.
 if (flushICache == MustFlushICache::Yes) {
   MOZ_ASSERT(protection == ProtectionSetting::Executable);
   jit::FlushICache(start, size);
 }

 // Calculate the start of the page containing this region,
 // and account for this extra memory within size.
 size_t pageSize = gc::SystemPageSize();
 intptr_t startPtr = reinterpret_cast<intptr_t>(start);
 intptr_t pageStartPtr = startPtr & ~(pageSize - 1);
 void* pageStart = reinterpret_cast<void*>(pageStartPtr);
 size += (startPtr - pageStartPtr);

 // Round size up
 size += (pageSize - 1);
 size &= ~(pageSize - 1);

 MOZ_ASSERT((uintptr_t(pageStart) % pageSize) == 0);

 execMemory.assertValidAddress(pageStart, size);

 // On weak memory systems, make sure new code is visible on all cores before
 // addresses of the code are made public.  Now is the latest moment in time
 // when we can do that, and we're assuming that every other thread that has
 // written into the memory that is being reprotected here has synchronized
 // with this thread in such a way that the memory writes have become visible
 // and we therefore only need to execute the fence once here.  See bug 1529933
 // for a longer discussion of why this is both necessary and sufficient.
 //
 // We use the C++ fence here -- and not AtomicOperations::fenceSeqCst() --
 // primarily because ReprotectRegion will be called while we construct our own
 // jitted atomics.  But the C++ fence is sufficient and correct, too.
#ifdef __wasi__
 MOZ_CRASH("NYI FOR WASI.");
#else
 std::atomic_thread_fence(std::memory_order_seq_cst);

 if (!JitOptions.writeProtectCode) {
   return true;
 }

#  ifdef JS_USE_APPLE_FAST_WX
 MOZ_CRASH("writeProtectCode should always be false on Apple Silicon");
#  endif

#  ifdef XP_WIN
 DWORD flags = ProtectionSettingToFlags(protection);
 // This is a essentially a VirtualProtect, but with lighter impact on
 // antivirus analysis. See bug 1823634.
 if (!VirtualAlloc(pageStart, size, MEM_COMMIT, flags)) {
   return false;
 }
#  else
 unsigned flags = ProtectionSettingToFlags(protection);
 if (mprotect(pageStart, size, flags)) {
   return false;
 }
#  endif
#endif  // __wasi__

 execMemory.assertValidAddress(pageStart, size);
 return true;
}

#if defined(JS_USE_APPLE_FAST_WX) && !defined(XP_IOS)
void js::jit::AutoMarkJitCodeWritableForThread::markExecutable(
   bool executable) {
 if (__builtin_available(macOS 11.0, *)) {
   pthread_jit_write_protect_np(executable);
 } else {
   MOZ_CRASH("pthread_jit_write_protect_np must be available");
 }
}
#endif

#ifdef DEBUG
static MOZ_THREAD_LOCAL(bool) sMarkingWritable;

void js::jit::AutoMarkJitCodeWritableForThread::checkConstructor() {
 if (!sMarkingWritable.initialized()) {
   sMarkingWritable.infallibleInit();
 }
 MOZ_ASSERT(!sMarkingWritable.get(),
            "AutoMarkJitCodeWritableForThread shouldn't be nested");
 sMarkingWritable.set(true);
}

void js::jit::AutoMarkJitCodeWritableForThread::checkDestructor() {
 MOZ_ASSERT(sMarkingWritable.get());
 sMarkingWritable.set(false);
}
#endif