tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

Heap-inl.h (7787B)

      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #ifndef gc_Heap_inl_h
      8 #define gc_Heap_inl_h
      9 
     10 #include "gc/Heap.h"
     11 
     12 #include "gc/StoreBuffer.h"
     13 #include "gc/Zone.h"
     14 #include "util/Poison.h"
     15 #include "vm/Runtime.h"
     16 
     17 inline void js::gc::Arena::init(GCRuntime* gc, JS::Zone* zone, AllocKind kind) {
     18  MOZ_ASSERT(zone);
     19  MOZ_ASSERT(IsValidAllocKind(kind));
     20 
     21  MOZ_MAKE_MEM_UNDEFINED(this, ArenaSize);
     22 
     23  allocKind = kind;
     24  zone_ = zone;
     25  next = nullptr;
     26  isNewlyCreated_ = 1;
     27  onDelayedMarkingList_ = 0;
     28  hasDelayedBlackMarking_ = 0;
     29  hasDelayedGrayMarking_ = 0;
     30  nextDelayedMarkingArena_ = 0;
     31  if (zone_->isAtomsZone()) {
     32    atomBitmapStart() = gc->atomMarking.allocateIndex(gc);
     33  } else {
     34    bufferedCells() = &ArenaCellSet::Empty;
     35  }
     36 
     37  setAsFullyUnused();  // Initializes firstFreeSpan.
     38 
     39 #ifdef DEBUG
     40  checkNoMarkedCells();
     41 #endif
     42 }
     43 
     44 inline void js::gc::Arena::freeAtomMarkingBitmapIndex(GCRuntime* gc,
     45                                                      const AutoLockGC& lock) {
     46  MOZ_ASSERT(zone_->isAtomsZone());
     47  gc->atomMarking.freeIndex(atomBitmapStart(), lock);
     48 #ifdef DEBUG
     49  atomBitmapStart() = 0;  // Also zeroed by write to bufferedCells_ in release.
     50 #endif
     51 }
     52 
     53 inline void js::gc::Arena::release() {
     54  MOZ_ASSERT(allocated());
     55 
     56  // Clients should call freeAtomMarkingBitmapIndex() if necessary.
     57  MOZ_ASSERT_IF(zone_->isAtomsZone(), atomBitmapStart_ == 0);
     58 
     59  // Poison zone pointer to highlight UAF on released arenas in crash data.
     60  AlwaysPoison(&zone_, JS_FREED_ARENA_PATTERN, sizeof(zone_),
     61               MemCheckKind::MakeNoAccess);
     62 
     63  firstFreeSpan.initAsEmpty();
     64  allocKind = AllocKind::LIMIT;
     65  onDelayedMarkingList_ = 0;
     66  hasDelayedBlackMarking_ = 0;
     67  hasDelayedGrayMarking_ = 0;
     68  nextDelayedMarkingArena_ = 0;
     69  bufferedCells_ = nullptr;
     70 
     71  MOZ_ASSERT(!allocated());
     72 }
     73 
     74 inline js::gc::ArenaCellSet*& js::gc::Arena::bufferedCells() {
     75  MOZ_ASSERT(zone_ && !zone_->isAtomsZone());
     76  return bufferedCells_;
     77 }
     78 
     79 inline size_t& js::gc::Arena::atomBitmapStart() {
     80  MOZ_ASSERT(zone_ && zone_->isAtomsZone());
     81  return atomBitmapStart_;
     82 }
     83 
     84 // Mark bitmap API:
     85 
     86 // Unless noted otherwise, the following methods that update the mark bits are
     87 // not thread safe and must not be called in parallel with each other.
     88 //
     89 // They use separate read and write operations to avoid an unnecessarily strict
     90 // atomic update on the marking bitmap.
     91 //
     92 // They may be called in parallel with read operations on the mark bitmap where
     93 // there is no required ordering between the operations. This happens when gray
     94 // unmarking occurs in parallel with background sweeping.
     95 
     96 // The return value indicates if the cell went from unmarked to marked.
     97 MOZ_ALWAYS_INLINE bool js::gc::ChunkMarkBitmap::markIfUnmarked(
     98    const void* cell, MarkColor color) {
     99  Word* word;
    100  uintptr_t mask;
    101  getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
    102  if (*word & mask) {
    103    return false;
    104  }
    105  if (color == MarkColor::Black) {
    106    uintptr_t bits = *word;
    107    *word = bits | mask;
    108  } else {
    109    // We use getMarkWordAndMask to recalculate both mask and word as doing just
    110    // mask << color may overflow the mask.
    111    getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask);
    112    if (*word & mask) {
    113      return false;
    114    }
    115    uintptr_t bits = *word;
    116    *word = bits | mask;
    117  }
    118  return true;
    119 }
    120 
    121 // This version of the method is safe in the face of concurrent writes to the
    122 // mark bitmap but if two threads attempt to mark the same cell at the same time
    123 // then both calls can succeed and return true.
    124 //
    125 // This method is used for parallel marking where the extra synchronization
    126 // required to avoid this results in worse performance overall.
    127 MOZ_ALWAYS_INLINE bool js::gc::ChunkMarkBitmap::markIfUnmarkedThreadSafe(
    128    const void* cell, MarkColor color) {
    129  Word* word;
    130  uintptr_t mask;
    131  getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
    132  if (*word & mask) {
    133    return false;
    134  }
    135  if (color == MarkColor::Black) {
    136    *word |= mask;
    137  } else {
    138    // We use getMarkWordAndMask to recalculate both mask and word as doing just
    139    // mask << color may overflow the mask.
    140    getMarkWordAndMask(cell, ColorBit::GrayOrBlackBit, &word, &mask);
    141    if (*word & mask) {
    142      return false;
    143    }
    144    *word |= mask;
    145  }
    146  return true;
    147 }
    148 
    149 MOZ_ALWAYS_INLINE void js::gc::ChunkMarkBitmap::markBlack(const void* cell) {
    150  Word* word;
    151  uintptr_t mask;
    152  getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
    153  uintptr_t bits = *word;
    154  *word = bits | mask;
    155 }
    156 
    157 MOZ_ALWAYS_INLINE void js::gc::ChunkMarkBitmap::markBlackAtomic(
    158    const void* cell) {
    159  Word* word;
    160  uintptr_t mask;
    161  getMarkWordAndMask(cell, ColorBit::BlackBit, &word, &mask);
    162  *word |= mask;
    163 }
    164 
    165 MOZ_ALWAYS_INLINE void js::gc::ChunkMarkBitmap::copyMarkBit(
    166    TenuredCell* dst, const TenuredCell* src, ColorBit colorBit) {
    167  ArenaChunkBase* srcChunk = detail::GetCellChunkBase(src);
    168  Word* srcWord;
    169  uintptr_t srcMask;
    170  srcChunk->markBits.getMarkWordAndMask(src, colorBit, &srcWord, &srcMask);
    171 
    172  Word* dstWord;
    173  uintptr_t dstMask;
    174  getMarkWordAndMask(dst, colorBit, &dstWord, &dstMask);
    175 
    176  uintptr_t bits = *dstWord;
    177  bits &= ~dstMask;
    178  if (*srcWord & srcMask) {
    179    bits |= dstMask;
    180  }
    181  *dstWord = bits;
    182 }
    183 
    184 MOZ_ALWAYS_INLINE void js::gc::ChunkMarkBitmap::unmark(const void* cell) {
    185  unmarkOneBit(cell, ColorBit::BlackBit);
    186  unmarkOneBit(cell, ColorBit::GrayOrBlackBit);
    187 }
    188 
    189 MOZ_ALWAYS_INLINE void js::gc::ChunkMarkBitmap::unmarkOneBit(
    190    const void* cell, ColorBit colorBit) {
    191  Word* word;
    192  uintptr_t mask;
    193  uintptr_t bits;
    194  getMarkWordAndMask(cell, colorBit, &word, &mask);
    195  bits = *word;
    196  *word = bits & ~mask;
    197 }
    198 
    199 inline js::gc::AtomicBitmapWord* js::gc::ChunkMarkBitmap::arenaBits(
    200    Arena* arena) {
    201  static_assert(
    202      ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD,
    203      "We assume that the part of the bitmap corresponding to the arena "
    204      "has the exact number of words so we do not need to deal with a word "
    205      "that covers bits from two arenas.");
    206 
    207  Word* word;
    208  uintptr_t unused;
    209  getMarkWordAndMask(arena, ColorBit::BlackBit, &word, &unused);
    210  return word;
    211 }
    212 
    213 inline void js::gc::ChunkMarkBitmap::copyFrom(const ChunkMarkBitmap& other) {
    214  Bitmap::copyFrom(other);
    215 }
    216 
    217 template <size_t N>
    218 void js::gc::AtomicBitmap<N>::copyFrom(const AtomicBitmap& other) {
    219  for (size_t i = 0; i < WordCount; i++) {
    220    bitmap[i] = uintptr_t(other.bitmap[i]);
    221  }
    222 }
    223 
    224 template <size_t N>
    225 void js::gc::AtomicBitmap<N>::clear() {
    226  for (size_t i = 0; i < WordCount; i++) {
    227    bitmap[i] = 0;
    228  }
    229 }
    230 
    231 template <size_t N>
    232 bool js::gc::AtomicBitmap<N>::isEmpty() const {
    233  for (size_t i = 0; i < WordCount; i++) {
    234    if (bitmap[i]) {
    235      return false;
    236    }
    237  }
    238 
    239  return true;
    240 }
    241 
    242 bool js::gc::TenuredCell::markIfUnmarked(MarkColor color /* = Black */) const {
    243  return chunk()->markBits.markIfUnmarked(this, color);
    244 }
    245 
    246 bool js::gc::TenuredCell::markIfUnmarkedThreadSafe(MarkColor color) const {
    247  return chunk()->markBits.markIfUnmarkedThreadSafe(this, color);
    248 }
    249 
    250 void js::gc::TenuredCell::markBlack() const {
    251  chunk()->markBits.markBlack(this);
    252 }
    253 
    254 void js::gc::TenuredCell::markBlackAtomic() const {
    255  chunk()->markBits.markBlackAtomic(this);
    256 }
    257 
    258 void js::gc::TenuredCell::copyMarkBitsFrom(const TenuredCell* src) {
    259  ChunkMarkBitmap& markBits = chunk()->markBits;
    260  markBits.copyMarkBit(this, src, ColorBit::BlackBit);
    261  markBits.copyMarkBit(this, src, ColorBit::GrayOrBlackBit);
    262 }
    263 
    264 void js::gc::TenuredCell::unmark() { chunk()->markBits.unmark(this); }
    265 
    266 #endif