tor-browser

BufferAllocator.h (27820B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// GC-internal header file for the buffer allocator.

#ifndef gc_BufferAllocator_h
#define gc_BufferAllocator_h

#include "mozilla/Array.h"
#include "mozilla/Atomics.h"
#include "mozilla/BitSet.h"
#include "mozilla/HashTable.h"
#include "mozilla/Maybe.h"
#include "mozilla/TimeStamp.h"

#include <cstdint>
#include <stddef.h>

#include "jstypes.h"  // JS_PUBLIC_API

#include "ds/SlimLinkedList.h"
#include "js/HeapAPI.h"
#include "threading/LockGuard.h"
#include "threading/Mutex.h"
#include "threading/ProtectedData.h"

class JS_PUBLIC_API JSTracer;

namespace JS {
class JS_PUBLIC_API Zone;
}  // namespace JS

namespace js {

class GCMarker;
class Nursery;

namespace gc {

struct BufferChunk;
struct Cell;
class GCRuntime;
struct LargeBuffer;
struct SmallBufferRegion;

// BufferAllocator allocates dynamically sized blocks of memory which can be
// reclaimed by the garbage collector and are associated with GC things.
//
// Although these blocks can be reclaimed by GC, explicit free and resize are
// also supported. This is important for buffers that can grow or shrink.
//
// The allocator uses a different strategy depending on the size of the
// allocation requested. There are three size ranges, divided as follows:
//
//   Size:            Kind:   Allocator implementation:
//    16 B  -   4 KB  Small   Uses a free list allocator from 16KB regions
//     4 KB - 512 KB  Medium  Uses a free list allocator from 1 MB chunks
//     1 MB -         Large   Uses the OS page allocator (e.g. mmap)
//
// The smallest supported allocation size is 16 bytes. This is used for a
// dynamic slots allocation with zero slots to set a unique ID on a native
// object. This is the size of two JS::Values.
//
// These size ranges are chosen to be the same as those used by jemalloc
// (although that uses a different kind of allocator for its equivalent of small
// allocations).
//
// Supported operations
// --------------------
//
//  - Allocate a buffer. Buffers are always owned by a GC cell, and the
//    allocator tracks whether the owner is in the nursery or the tenured heap.
//
//  - Trace an edge to buffer. When the owning cell is traced it must trace the
//    edge to the buffer. This will mark the buffer in a GC and prevent it from
//    being swept. This does not trace the buffer contents.
//
//  - Free a buffer. This allows uniquely owned buffers to be freed and reused
//    without waiting for the next GC. It is a hint only and is not supported
//    for all buffer kinds.
//
//  - Reallocate a buffer. This allows uniquely owned buffers to be resized,
//    possibly in-place. This is important for performance on some benchmarks.
//
// Integration with the rest of the GC
// -----------------------------------
//
// The GC calls the allocator at several points during minor and major GC (at
// the start, when sweeping starts and at the end). Allocations are swept on a
// background thread and the memory used by unmarked allocations is reclaimed.
//
// The allocator tries hard to avoid locking, and where it is necessary tries to
// minimize the time spent holding locks. A lock is required for the following
// operations:
//
//  - allocating a new chunk
//  - main thread operations on large buffers while sweeping off-thread
//  - merging back swept data on the main thread
//
// No locks are required to allocate on the main thread, even when off-thread
// sweeping is taking place. This is achieved by moving data to be swept to
// separate containers from those used for allocation on the main thread.
//
// Small and medium allocations
// ----------------------------
//
// These are allocated in their own zone-specific chunks using a segregated free
// list strategy. This is the main part of the allocator.
//
// The requested allocation size is used to pick a size class, which is used to
// find a free regions of suitable size to satisfy that request. In general size
// classes for allocations are calculated by rounding up to the next power of
// two; size classes for free regions are calculated by rounding down to the
// previous power of two. This means that free regions of a particular size
// class are always large enough to satisfy an allocation of that size class.
//
// The allocation size class is used to index into an array giving a list of
// free regions of that size class. The first region in the list is used and
// its start address updated; it may also be moved to a different list if it is
// now empty or too small to satisfy further allocations for this size class.
//
// Medium allocations are allocated out of chunks directly and small allocations
// out of 16KB sub-regions (which are essentially medium allocations
// themselves).
//
// Data about allocations is stored in a header in the chunk or region, using
// bitmaps for boolean flags.
//
// Sweeping works by processing a list of chunks, scanning each one for
// allocated but unmarked buffers and rebuilding the free region data for that
// chunk. Sweeping happens separately for minor and major GC and only
// nursery-owned or tenured-owned buffers are swept at one time. This means that
// chunks containing nursery-owned allocations are swept twice during a major
// GC, once to sweep nursery-owned allocations and once for tenured-owned
// allocations. This is required because the sweeping happens at different
// times.
//
// Chunks containing nursery-owned buffers are stored in a separate list to
// chunks that only contain tenured-owned buffers to reduce the number of chunks
// that need to be swept for minor GC. They are stored in |mixedChunks| and are
// moved to |mixedChunksToSweep| at the start of minor GC. They are then swept
// on a background thread and are placed in |sweptMixedChunks| if they are not
// freed. From there they can merged back into one of the main thread lists
// (since they may no longer contain nursery-owned buffers).
//
// Chunks containing tenured-owned buffers are stored in |tenuredChunks| and are
// moved to |tenuredChunksToSweep| at the start of major GC. They are
// unavailable for allocation after this point and will be swept on a background
// thread and placed in |sweptTenuredChunks| if they are not freed. From there
// they will be merged back into |tenuredChunks|. This means that allocation
// during an incremental GC will allocate a new chunk.
//
// Merging swept data requires taking a lock and so only happens when
// necessary. This happens when a new chunk is needed or at various points
// during GC. During sweeping no additional synchronization is required for
// allocation.
//
// Since major GC requires doing a minor GC at the start and we don't want to
// have to wait for minor GC sweeping to finish there is an optimization where
// chunks containing nursery-owned buffers swept as part of minor GC at the
// start of a major GC are moved directly from |sweptMixedChunks| to
// |tenuredChunksToSweep| at the end of minor GC sweeping. This is controlled by
// the |majorStartedWhileMinorSweeping| flag.
//
// Similarly, if a major GC finishes while minor GC is sweeping then rather than
// waiting for it to finish we set the |majorFinishedWhileMinorSweeping| flag so
// that we clear the |allocatedDuringCollection| for these chunks the end of the
// minor sweeping.
//
// Free works by extending neighboring free regions to cover the freed
// allocation or adding a new one if necessary. Free regions are found by
// checking the allocated bitmap. Free is not supported if the containing chunk
// is currently being swept off-thread. Free is only supported for medium sized
// allocations.
//
// Reallocation works by resizing in place if possible. It is always possible to
// shrink a medium allocation in place if the target size is still medium.
// In-place growth is possible if there is enough free space following the
// allocation. This is not supported if the containing chunk is currently being
// swept off-thread.  If not possible, a new allocation is made and the contents
// of the buffer copied. Resizing in place is only supported for medium sized
// allocations.
//
// Large allocations
// -----------------
//
// These are implemented using the OS page allocator. Allocations of this size
// are relatively rare and not much attempt is made to optimize them. They are
// chunk aligned which allows them to be distinguished from the other allocation
// kinds by checking the low bits the pointer.
//
// Shrinking large allocations in place is only supported on Posix-like systems.
//
// Naming conventions
// ------------------
//
// The following conventions are used in the code:
//  - alloc:  client pointer to allocated memory
//  - buffer: pointer to per-allocation metadata
//

class BufferAllocator : public SlimLinkedListElement<BufferAllocator> {
public:
 static constexpr size_t MinSmallAllocShift = 4;    // 16 B
 static constexpr size_t MinMediumAllocShift = 12;  //  4 KB
 static constexpr size_t MinLargeAllocShift = 20;   //  1 MB

 // TODO: Ideally this would equal MinSmallAllocShift but we're constrained by
 // the size of FreeRegion which won't fit into 16 bytes.
 static constexpr size_t MinSizeClassShift = 5;  // 32 B
 static_assert(MinSizeClassShift >= MinSmallAllocShift);

 static constexpr size_t SmallSizeClasses =
     MinMediumAllocShift - MinSizeClassShift + 1;
 static constexpr size_t MediumSizeClasses =
     MinLargeAllocShift - MinMediumAllocShift + 1;
 static constexpr size_t AllocSizeClasses =
     SmallSizeClasses + MediumSizeClasses;

 static constexpr size_t FullChunkSizeClass = AllocSizeClasses;

 // An RAII guard to lock and unlock the buffer allocator lock.
 class AutoLock : public LockGuard<Mutex> {
  public:
   explicit AutoLock(GCRuntime* gc);
   explicit AutoLock(BufferAllocator* allocator);
   friend class UnlockGuard<AutoLock>;
 };

 // A lock guard that is locked only when needed.
 using MaybeLock = mozilla::Maybe<AutoLock>;

private:
 template <typename Derived, size_t SizeBytes, size_t GranularityBytes>
 friend struct AllocSpace;

 using BufferChunkList = SlimLinkedList<BufferChunk>;

 struct FreeRegion;
 using FreeList = SlimLinkedList<FreeRegion>;

 using SizeClassBitSet = mozilla::BitSet<AllocSizeClasses, uint32_t>;

 // Segregated free list: an array of free lists, one per size class.
 class FreeLists {
   using FreeListArray = mozilla::Array<FreeList, AllocSizeClasses>;

   FreeListArray lists;
   SizeClassBitSet available;

  public:
   class FreeListIter;
   class FreeRegionIter;

   FreeLists() = default;

   FreeLists(FreeLists&& other);
   FreeLists& operator=(FreeLists&& other);

   FreeListIter freeListIter();
   FreeRegionIter freeRegionIter();

   bool isEmpty() const { return available.IsEmpty(); }

   bool hasSizeClass(size_t sizeClass) const;
   const auto& availableSizeClasses() const { return available; }

   // Returns SIZE_MAX if none available.
   size_t getFirstAvailableSizeClass(size_t minSizeClass,
                                     size_t maxSizeClass) const;
   size_t getLastAvailableSizeClass(size_t minSizeClass,
                                    size_t maxSizeClass) const;

   FreeRegion* getFirstRegion(size_t sizeClass);

   void pushFront(size_t sizeClass, FreeRegion* region);
   void pushBack(size_t sizeClass, FreeRegion* region);

   void append(FreeLists&& other);
   void prepend(FreeLists&& other);

   void remove(size_t sizeClass, FreeRegion* region);

   void clear();

   template <typename Func>
   void forEachRegion(Func&& func);

   void assertEmpty() const;
   void assertContains(size_t sizeClass, FreeRegion* region) const;
   void checkAvailable() const;
 };

 class ChunkLists {
   using ChunkListArray =
       mozilla::Array<BufferChunkList, AllocSizeClasses + 1>;
   using AvailableBitSet = mozilla::BitSet<AllocSizeClasses + 1, uint32_t>;

   ChunkListArray lists;
   AvailableBitSet available;

  public:
   class ChunkListIter;
   class ChunkIter;

   ChunkLists() = default;

   ChunkLists(const ChunkLists& other) = delete;
   ChunkLists& operator=(const ChunkLists& other) = delete;

   ChunkListIter chunkListIter();
   ChunkIter chunkIter();
   const auto& availableSizeClasses() const { return available; }

   void pushFront(size_t sizeClass, BufferChunk* chunk);
   void pushBack(BufferChunk* chunk);
   void pushBack(size_t sizeClass, BufferChunk* chunk);

   // Returns SIZE_MAX if none available.
   size_t getFirstAvailableSizeClass(size_t minSizeClass,
                                     size_t maxSizeClass) const;

   BufferChunk* popFirstChunk(size_t sizeClass);

   void remove(size_t sizeClass, BufferChunk* chunk);

   BufferChunkList extractAllChunks();

   bool isEmpty() const;
   void checkAvailable() const;
 };

 using LargeAllocList = SlimLinkedList<LargeBuffer>;

 using LargeAllocMap =
     mozilla::HashMap<void*, LargeBuffer*, PointerHasher<void*>>;

 enum class State : uint8_t { NotCollecting, Marking, Sweeping };

 enum class SizeKind : uint8_t { Small, Medium };

 enum class SweepKind : uint8_t { Tenured = 0, Nursery };

 // The zone this allocator is associated with.
 MainThreadOrGCTaskData<JS::Zone*> zone;

 // Chunks containing medium and small buffers. They may contain both
 // nursery-owned and tenured-owned buffers.
 MainThreadData<BufferChunkList> mixedChunks;

 // Chunks containing only tenured-owned small and medium buffers.
 MainThreadData<BufferChunkList> tenuredChunks;

 // Free lists for the small and medium buffers in |mixedChunks| and
 // |tenuredChunks|. Used for allocation.
 MainThreadData<FreeLists> freeLists;

 // Chunks that may contain nursery-owned buffers waiting to be swept during a
 // minor GC. Populated from |mixedChunks|.
 MainThreadOrGCTaskData<BufferChunkList> mixedChunksToSweep;

 // Chunks that contain only tenured-owned buffers waiting to be swept during a
 // major GC. Populated from |tenuredChunks|.
 MainThreadOrGCTaskData<BufferChunkList> tenuredChunksToSweep;

 // Chunks that have been swept. Populated by a background thread.
 MutexData<BufferChunkList> sweptMixedChunks;
 MutexData<BufferChunkList> sweptTenuredChunks;

 // Chunks that have been swept and are available for allocation but have not
 // had their free regions merged into |freeLists|. Owned by the main thread.
 MainThreadData<ChunkLists> availableMixedChunks;
 MainThreadData<ChunkLists> availableTenuredChunks;

 // List of large nursery-owned buffers.
 MainThreadData<LargeAllocList> largeNurseryAllocs;

 // List of large tenured-owned buffers.
 MainThreadData<LargeAllocList> largeTenuredAllocs;

 // Map from allocation pointer to buffer metadata for large buffers.
 // Access requires holding the mutex during sweeping.
 MainThreadOrGCTaskData<LargeAllocMap> largeAllocMap;

 // Large buffers waiting to be swept.
 MainThreadOrGCTaskData<LargeAllocList> largeNurseryAllocsToSweep;
 MainThreadOrGCTaskData<LargeAllocList> largeTenuredAllocsToSweep;

 // Large buffers that have been swept.
 MutexData<LargeAllocList> sweptLargeTenuredAllocs;

 // Flag to indicate that data from minor sweeping is available to be
 // merged. This includes chunks in the |sweptMixedChunks| or
 // |sweptTenuredChunks| lists and the minorSweepingFinished flag.
 mozilla::Atomic<bool, mozilla::Relaxed> hasMinorSweepDataToMerge;

 // GC state for minor and major GC.
 MainThreadData<State> minorState;
 MainThreadData<State> majorState;

 // Flags to tell the main thread that sweeping has finished and the state
 // should be updated.
 MutexData<bool> minorSweepingFinished;
 MutexData<bool> majorSweepingFinished;

 // A major GC was started while a minor GC was still sweeping. Chunks by the
 // minor GC will be moved directly to the list of chunks to sweep for the
 // major GC. This happens for the minor GC at the start of every major GC.
 MainThreadData<bool> majorStartedWhileMinorSweeping;

 // A major GC finished while a minor GC was still sweeping. Some post major GC
 // cleanup will be deferred to the end of the minor sweeping.
 MainThreadData<bool> majorFinishedWhileMinorSweeping;

public:
 explicit BufferAllocator(JS::Zone* zone);
 ~BufferAllocator();

 static inline size_t GetGoodAllocSize(size_t requiredBytes);
 static inline size_t GetGoodElementCount(size_t requiredElements,
                                          size_t elementSize);
 static inline size_t GetGoodPower2AllocSize(size_t requiredBytes);
 static inline size_t GetGoodPower2ElementCount(size_t requiredElements,
                                                size_t elementSize);
 static bool IsBufferAlloc(void* alloc);

 void* alloc(size_t bytes, bool nurseryOwned);
 void* allocInGC(size_t bytes, bool nurseryOwned);
 void* realloc(void* alloc, size_t bytes, bool nurseryOwned);
 void free(void* alloc);
 size_t getAllocSize(void* alloc);
 bool isNurseryOwned(void* alloc);

 void startMinorCollection(MaybeLock& lock);
 bool startMinorSweeping();
 void sweepForMinorCollection();

 void startMajorCollection(MaybeLock& lock);
 void startMajorSweeping(MaybeLock& lock);
 void sweepForMajorCollection(bool shouldDecommit);
 void finishMajorCollection(const AutoLock& lock);
 void clearMarkStateAfterBarrierVerification();
 void clearChunkMarkBits(BufferChunk* chunk);

 bool isEmpty() const;

 void traceEdge(JSTracer* trc, Cell* owner, void** bufferp, const char* name);
 bool markTenuredAlloc(void* alloc);
 bool isMarkedBlack(void* alloc);

 // For debugging, used to implement GetMarkInfo. Returns false for allocations
 // being swept on another thread.
 bool isPointerWithinBuffer(void* ptr);

 Mutex& lock() const;

 size_t getSizeOfNurseryBuffers();

 void addSizeOfExcludingThis(size_t* usedBytesOut, size_t* freeBytesOut,
                             size_t* adminBytesOut);

 static void printStatsHeader(FILE* file);
 static void printStats(GCRuntime* gc, mozilla::TimeStamp creationTime,
                        bool isMajorGC, FILE* file);

 struct Stats {
   size_t usedBytes = 0;
   size_t freeBytes = 0;
   size_t adminBytes = 0;
   size_t mixedSmallRegions = 0;
   size_t tenuredSmallRegions = 0;
   size_t mixedChunks = 0;
   size_t tenuredChunks = 0;
   size_t availableMixedChunks = 0;
   size_t availableTenuredChunks = 0;
   size_t freeRegions = 0;
   size_t largeNurseryAllocs = 0;
   size_t largeTenuredAllocs = 0;
 };
 void getStats(Stats& stats);

#ifdef DEBUG
 bool hasAlloc(void* alloc);
 void checkGCStateNotInUse();
 void checkGCStateNotInUse(MaybeLock& lock);
 void checkGCStateNotInUse(const AutoLock& lock);
#endif

private:
 void markNurseryOwnedAlloc(void* alloc, bool nurseryOwned);
 friend class js::Nursery;

 void maybeMergeSweptData();
 void maybeMergeSweptData(MaybeLock& lock);
 void mergeSweptData();
 void mergeSweptData(const AutoLock& lock);
 void abortMajorSweeping(const AutoLock& lock);
 void clearAllocatedDuringCollectionState(const AutoLock& lock);

 // Small allocation methods:

 static inline bool IsSmallAllocSize(size_t bytes);
 static bool IsSmallAlloc(void* alloc);

 void* allocSmall(size_t bytes, bool nurseryOwned, bool inGC);
 void* retrySmallAlloc(size_t requestedBytes, size_t sizeClass, bool inGC);
 bool allocNewSmallRegion(bool inGC);
 void traceSmallAlloc(JSTracer* trc, Cell* owner, void** allocp,
                      const char* name);
 void markSmallNurseryOwnedBuffer(void* alloc, bool nurseryOwned);
 bool markSmallTenuredAlloc(void* alloc);

 // Medium allocation methods:

 static bool IsMediumAlloc(void* alloc);
 static bool CanSweepAlloc(bool nurseryOwned,
                           BufferAllocator::SweepKind sweepKind);

 void* allocMedium(size_t bytes, bool nurseryOwned, bool inGC);
 void* retryMediumAlloc(size_t requestedBytes, size_t sizeClass, bool inGC);
 template <typename Alloc, typename GrowHeap>
 void* refillFreeListsAndRetryAlloc(size_t sizeClass, size_t maxSizeClass,
                                    Alloc&& alloc, GrowHeap&& growHeap);
 enum class RefillResult { Fail = 0, Success, Retry };
 template <typename GrowHeap>
 RefillResult refillFreeLists(size_t sizeClass, size_t maxSizeClass,
                              GrowHeap&& growHeap);
 bool useAvailableChunk(size_t sizeClass, size_t maxSizeClass);
 bool useAvailableChunk(size_t sizeClass, size_t maxSizeClass, ChunkLists& src,
                        BufferChunkList& dst);
 SizeClassBitSet getChunkSizeClassesToMove(size_t maxSizeClass,
                                           ChunkLists& src) const;
 void* bumpAlloc(size_t bytes, size_t sizeClass, size_t maxSizeClass);
 void* allocFromRegion(FreeRegion* region, size_t bytes, size_t sizeClass);
 void* allocMediumAligned(size_t bytes, bool inGC);
 void* retryAlignedAlloc(size_t sizeClass, bool inGC);
 void* alignedAlloc(size_t sizeClass);
 void* alignedAllocFromRegion(FreeRegion* region, size_t sizeClass);
 void updateFreeListsAfterAlloc(FreeLists* freeLists, FreeRegion* region,
                                size_t sizeClass);
 void setAllocated(void* alloc, size_t bytes, bool nurseryOwned, bool inGC);
 void setChunkHasNurseryAllocs(BufferChunk* chunk);
 void recommitRegion(FreeRegion* region);
 bool allocNewChunk(bool inGC);
 bool sweepChunk(BufferChunk* chunk, SweepKind sweepKind, bool shouldDecommit);
 void addSweptRegion(BufferChunk* chunk, uintptr_t freeStart,
                     uintptr_t freeEnd, bool shouldDecommit,
                     bool expectUnchanged, FreeLists& freeLists);
 bool sweepSmallBufferRegion(BufferChunk* chunk, SmallBufferRegion* region,
                             SweepKind sweepKind);
 void addSweptRegion(SmallBufferRegion* region, uintptr_t freeStart,
                     uintptr_t freeEnd, bool expectUnchanged,
                     FreeLists& freeLists);
 void freeMedium(void* alloc);
 bool growMedium(void* alloc, size_t newBytes);
 bool shrinkMedium(void* alloc, size_t newBytes);
 enum class ListPosition { Front, Back };
 FreeRegion* addFreeRegion(FreeLists* freeLists, uintptr_t start,
                           uintptr_t bytes, SizeKind kind, bool anyDecommitted,
                           ListPosition position,
                           bool expectUnchanged = false);
 void updateFreeRegionStart(FreeLists* freeLists, FreeRegion* region,
                            uintptr_t newStart, SizeKind kind);
 FreeLists* getChunkFreeLists(BufferChunk* chunk);
 ChunkLists* getChunkAvailableLists(BufferChunk* chunk);
 void maybeUpdateAvailableLists(ChunkLists* availableChunks,
                                BufferChunk* chunk, size_t oldChunkSizeClass);
 bool isSweepingChunk(BufferChunk* chunk);
 void traceMediumAlloc(JSTracer* trc, Cell* owner, void** allocp,
                       const char* name);
 bool isMediumBufferNurseryOwned(void* alloc) const;
 void markMediumNurseryOwnedBuffer(void* alloc, bool nurseryOwned);
 bool markMediumTenuredAlloc(void* alloc);

 // Determine whether a size class is for a small or medium allocation.
 static SizeKind SizeClassKind(size_t sizeClass);

 // Get the size class for an allocation. This rounds up to a class that is
 // large enough to hold the required size.
 static size_t SizeClassForSmallAlloc(size_t bytes);
 static size_t SizeClassForMediumAlloc(size_t bytes);

 // Get the maximum size class of allocations that can use a free region. This
 // rounds down to the largest class that can fit in this region.
 static size_t SizeClassForFreeRegion(size_t bytes, SizeKind kind);

 static size_t SizeClassBytes(size_t sizeClass);
 friend struct BufferChunk;

 static void ClearAllocatedDuringCollection(ChunkLists& chunks);
 static void ClearAllocatedDuringCollection(BufferChunkList& list);
 static void ClearAllocatedDuringCollection(LargeAllocList& list);

 // Large allocation methods:

 static inline bool IsLargeAllocSize(size_t bytes);
 static bool IsLargeAlloc(void* alloc);

 void* allocLarge(size_t bytes, bool nurseryOwned, bool inGC);
 bool isLargeTenuredMarked(LargeBuffer* buffer);
 void freeLarge(void* alloc);
 bool shrinkLarge(LargeBuffer* buffer, size_t newBytes);
 void unmapLarge(LargeBuffer* buffer, bool isSweeping, MaybeLock& lock);
 void unregisterLarge(LargeBuffer* buffer, bool isSweeping, MaybeLock& lock);
 void traceLargeAlloc(JSTracer* trc, Cell* owner, void** allocp,
                      const char* name);
 void markLargeNurseryOwnedBuffer(LargeBuffer* buffer, bool nurseryOwned);
 bool markLargeTenuredBuffer(LargeBuffer* buffer);

 // Lookup a large buffer by pointer in the map.
 LargeBuffer* lookupLargeBuffer(void* alloc);
 LargeBuffer* lookupLargeBuffer(void* alloc, MaybeLock& lock);
 bool needLockToAccessBufferMap() const;

 void increaseHeapSize(size_t bytes, bool nurseryOwned, bool checkThresholds,
                       bool updateRetainedSize);
 void decreaseHeapSize(size_t bytes, bool nurseryOwned,
                       bool updateRetainedSize);

 // Testing functions we allow access.
 friend void* TestAllocAligned(JS::Zone* zone, size_t bytes);
 friend size_t TestGetAllocSizeKind(void* alloc);

#ifdef DEBUG
 void checkChunkListsGCStateNotInUse(ChunkLists& chunkLists,
                                     bool hasNurseryOwnedAllocs,
                                     bool allowAllocatedDuringCollection);
 void checkChunkListGCStateNotInUse(BufferChunkList& chunks,
                                    bool hasNurseryOwnedAllocs,
                                    bool allowAllocatedDuringCollection,
                                    bool allowFreeLists);
 void checkChunkGCStateNotInUse(BufferChunk* chunk,
                                bool allowAllocatedDuringCollection,
                                bool allowFreeLists);
 void checkAllocListGCStateNotInUse(LargeAllocList& list, bool isNurseryOwned);
 void verifyChunk(BufferChunk* chunk, bool hasNurseryOwnedAllocs);
 void verifyFreeRegion(BufferChunk* chunk, uintptr_t endOffset,
                       size_t expectedSize, size_t& freeRegionCount);
 void verifySmallBufferRegion(SmallBufferRegion* region,
                              size_t& freeRegionCount);
 void verifyFreeRegion(SmallBufferRegion* chunk, uintptr_t endOffset,
                       size_t expectedSize, size_t& freeRegionCount);
#endif
};

static constexpr size_t SmallAllocGranularityShift =
   BufferAllocator::MinSmallAllocShift;
static constexpr size_t MediumAllocGranularityShift =
   BufferAllocator::MinMediumAllocShift;

static constexpr size_t SmallAllocGranularity = 1 << SmallAllocGranularityShift;
static constexpr size_t MediumAllocGranularity = 1
                                                << MediumAllocGranularityShift;

static constexpr size_t MinSmallAllocSize =
   1 << BufferAllocator::MinSmallAllocShift;
static constexpr size_t MinMediumAllocSize =
   1 << BufferAllocator::MinMediumAllocShift;
static constexpr size_t MinLargeAllocSize =
   1 << BufferAllocator::MinLargeAllocShift;

static constexpr size_t MinAllocSize = MinSmallAllocSize;

static constexpr size_t MaxSmallAllocSize =
   MinMediumAllocSize - SmallAllocGranularity;
static constexpr size_t MaxMediumAllocSize =
   MinLargeAllocSize - MediumAllocGranularity;
static constexpr size_t MaxAlignedAllocSize = MinLargeAllocSize / 4;

}  // namespace gc
}  // namespace js

#endif  // gc_BufferAllocator_h