Nursery-inl.h (13574B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=4 sw=2 et tw=80 ft=cpp: 3 * 4 * This Source Code Form is subject to the terms of the Mozilla Public 5 * License, v. 2.0. If a copy of the MPL was not distributed with this 6 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 7 8 #ifndef gc_Nursery_inl_h 9 #define gc_Nursery_inl_h 10 11 #include "gc/Nursery.h" 12 13 #include "gc/GCRuntime.h" 14 #include "gc/RelocationOverlay.h" 15 #include "js/TracingAPI.h" 16 #include "vm/JSContext.h" 17 #include "vm/NativeObject.h" 18 #include "vm/StringType.h" 19 20 namespace js { 21 namespace gc { 22 struct Cell; 23 } // namespace gc 24 } // namespace js 25 26 inline JSRuntime* js::Nursery::runtime() const { return gc->rt; } 27 28 template <typename T> 29 bool js::Nursery::isInside(const SharedMem<T>& p) const { 30 return isInside(p.unwrap(/*safe - used for value in comparison above*/)); 31 } 32 33 inline void js::Nursery::addMallocedBufferBytes(size_t nbytes) { 34 MOZ_ASSERT(nbytes > 0); 35 toSpace.mallocedBufferBytes += nbytes; 36 if (MOZ_UNLIKELY(toSpace.mallocedBufferBytes > capacity() * 8)) { 37 requestMinorGC(JS::GCReason::NURSERY_MALLOC_BUFFERS); 38 } 39 } 40 41 inline void js::Nursery::removeMallocedBufferBytes(size_t nbytes) { 42 MOZ_ASSERT(nbytes > 0); 43 MOZ_ASSERT(toSpace.mallocedBufferBytes >= nbytes); 44 toSpace.mallocedBufferBytes -= nbytes; 45 } 46 47 inline bool js::Nursery::addStringBuffer(JSLinearString* s) { 48 MOZ_ASSERT(IsInsideNursery(s)); 49 MOZ_ASSERT(isEnabled()); 50 MOZ_ASSERT(s->hasStringBuffer()); 51 52 auto* buffer = s->stringBuffer(); 53 if (!stringBuffers_.emplaceBack(s, buffer)) { 54 return false; 55 } 56 57 // Note: update mallocedBufferBytes only if the buffer has a refcount of 1, to 58 // avoid double counting when the same buffer is used by multiple nursery 59 // strings. 60 if (!buffer->HasMultipleReferences()) { 61 addMallocedBufferBytes(buffer->AllocationSize()); 62 } 63 return true; 64 } 65 66 inline bool js::Nursery::addExtensibleStringBuffer( 67 JSLinearString* s, mozilla::StringBuffer* buffer, bool updateMallocBytes) { 68 MOZ_ASSERT(IsInsideNursery(s)); 69 MOZ_ASSERT(isEnabled()); 70 if (!extensibleStringBuffers_.putNew(s, buffer)) { 71 return false; 72 } 73 MOZ_ASSERT(!buffer->HasMultipleReferences()); 74 if (updateMallocBytes) { 75 addMallocedBufferBytes(buffer->AllocationSize()); 76 } 77 return true; 78 } 79 80 inline void js::Nursery::removeMallocedBuffer(void* buffer, size_t nbytes) { 81 MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting()); 82 MOZ_ASSERT(toSpace.mallocedBuffers.has(buffer)); 83 MOZ_ASSERT(nbytes > 0); 84 removeMallocedBufferBytes(nbytes); 85 toSpace.mallocedBuffers.remove(buffer); 86 } 87 88 void js::Nursery::removeMallocedBufferDuringMinorGC(void* buffer) { 89 MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting()); 90 MOZ_ASSERT(fromSpace.mallocedBuffers.has(buffer)); 91 fromSpace.mallocedBuffers.remove(buffer); 92 } 93 94 inline void js::Nursery::removeExtensibleStringBuffer(JSLinearString* s, 95 bool updateMallocBytes) { 96 MOZ_ASSERT(gc::IsInsideNursery(s)); 97 extensibleStringBuffers_.remove(s); 98 99 if (updateMallocBytes) { 100 size_t nbytes = s->stringBuffer()->AllocationSize(); 101 removeMallocedBufferBytes(nbytes); 102 } 103 } 104 105 inline bool js::Nursery::shouldTenure(gc::Cell* cell) { 106 MOZ_ASSERT(semispaceEnabled()); 107 MOZ_ASSERT(inCollectedRegion(cell)); 108 109 size_t offset = fromSpace.offsetFromAddress(uintptr_t(cell)); 110 MOZ_ASSERT(offset >= 111 fromSpace.offsetFromExclusiveAddress(fromSpace.startPosition_)); 112 return offset <= tenureThreshold_; 113 } 114 115 inline bool js::Nursery::inCollectedRegion(const gc::Cell* cell) const { 116 return gc::InCollectedNurseryRegion(cell); 117 } 118 119 inline bool js::Nursery::inCollectedRegion(void* ptr) const { 120 if (!semispaceEnabled()) { 121 return toSpace.isInside(ptr); 122 } 123 124 return fromSpace.isInside(ptr); 125 } 126 127 inline size_t js::Nursery::Space::offsetFromExclusiveAddress( 128 uintptr_t addr) const { 129 if ((addr & gc::ChunkMask) == 0) { 130 // |addr| points one past the end of the previous chunk. 131 return offsetFromAddress(addr - 1) + 1; 132 } 133 134 return offsetFromAddress(addr); 135 } 136 137 inline size_t js::Nursery::Space::offsetFromAddress(uintptr_t addr) const { 138 gc::ChunkBase* chunk = 139 gc::detail::GetCellChunkBase(reinterpret_cast<gc::Cell*>(addr)); 140 MOZ_ASSERT(chunk->getKind() == kind); 141 MOZ_ASSERT(findChunkIndex(addr & ~gc::ChunkMask) == chunk->nurseryChunkIndex); 142 143 uint32_t offset = addr & gc::ChunkMask; 144 MOZ_ASSERT(offset >= sizeof(gc::ChunkBase)); 145 return (chunk->nurseryChunkIndex << gc::ChunkShift) | offset; 146 } 147 148 MOZ_ALWAYS_INLINE /* static */ bool js::Nursery::getForwardedPointer( 149 js::gc::Cell** ref) { 150 js::gc::Cell* cell = (*ref); 151 MOZ_ASSERT(IsInsideNursery(cell)); 152 if (!cell->isForwarded()) { 153 return false; 154 } 155 const gc::RelocationOverlay* overlay = gc::RelocationOverlay::fromCell(cell); 156 *ref = overlay->forwardingAddress(); 157 return true; 158 } 159 160 inline void js::Nursery::maybeSetForwardingPointer(JSTracer* trc, void* oldData, 161 void* newData, bool direct) { 162 if (trc->isTenuringTracer()) { 163 setForwardingPointerWhileTenuring(oldData, newData, direct); 164 } 165 } 166 167 inline void js::Nursery::setForwardingPointerWhileTenuring(void* oldData, 168 void* newData, 169 bool direct) { 170 if (isInside(oldData)) { 171 setForwardingPointer(oldData, newData, direct); 172 } 173 } 174 175 inline void js::Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, 176 HeapSlot* newSlots, 177 uint32_t nslots) { 178 // Slot arrays always have enough space for a forwarding pointer, since the 179 // number of slots is never zero. 180 MOZ_ASSERT(nslots > 0); 181 setDirectForwardingPointer(oldSlots, newSlots); 182 } 183 184 inline void js::Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, 185 ObjectElements* newHeader, 186 uint32_t capacity) { 187 // Only use a direct forwarding pointer if there is enough space for one. 188 setForwardingPointer(oldHeader->elements(), newHeader->elements(), 189 capacity > 0); 190 } 191 192 inline void js::Nursery::setForwardingPointer(void* oldData, void* newData, 193 bool direct) { 194 if (direct) { 195 setDirectForwardingPointer(oldData, newData); 196 return; 197 } 198 199 setIndirectForwardingPointer(oldData, newData); 200 } 201 202 inline void js::Nursery::setDirectForwardingPointer(void* oldData, 203 void* newData) { 204 MOZ_ASSERT(isInside(oldData)); 205 MOZ_ASSERT_IF(isInside(newData), !inCollectedRegion(newData)); 206 207 new (oldData) BufferRelocationOverlay{newData}; 208 } 209 210 inline void* js::Nursery::tryAllocateCell(gc::AllocSite* site, size_t size, 211 JS::TraceKind kind) { 212 // Ensure there's enough space to replace the contents with a 213 // RelocationOverlay. 214 // MOZ_ASSERT(size >= sizeof(RelocationOverlay)); 215 MOZ_ASSERT(size % gc::CellAlignBytes == 0); 216 MOZ_ASSERT(size_t(kind) < gc::NurseryTraceKinds); 217 MOZ_ASSERT_IF(kind == JS::TraceKind::String, canAllocateStrings()); 218 MOZ_ASSERT_IF(kind == JS::TraceKind::BigInt, canAllocateBigInts()); 219 220 void* ptr = tryAllocate(sizeof(gc::NurseryCellHeader) + size); 221 if (MOZ_UNLIKELY(!ptr)) { 222 return nullptr; 223 } 224 225 new (ptr) gc::NurseryCellHeader(site, kind); 226 227 void* cell = 228 reinterpret_cast<void*>(uintptr_t(ptr) + sizeof(gc::NurseryCellHeader)); 229 if (!cell) { 230 MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE( 231 "Successful allocation cannot result in nullptr"); 232 } 233 234 // Update the allocation site. This code is also inlined in 235 // MacroAssembler::updateAllocSite. 236 uint32_t allocCount = site->incAllocCount(); 237 if (allocCount == gc::NormalSiteAttentionThreshold) { 238 pretenuringNursery.insertIntoAllocatedList(site); 239 } 240 MOZ_ASSERT_IF( 241 site->isNormal() && allocCount >= gc::NormalSiteAttentionThreshold, 242 site->isInAllocatedList()); 243 244 gc::gcprobes::NurseryAlloc(cell, kind); 245 return cell; 246 } 247 248 inline void* js::Nursery::tryAllocate(size_t size) { 249 MOZ_ASSERT(isEnabled()); 250 MOZ_ASSERT_IF(JS::RuntimeHeapIsBusy(), JS::RuntimeHeapIsMinorCollecting()); 251 MOZ_ASSERT_IF(currentChunk() == startChunk(), position() >= startPosition()); 252 MOZ_ASSERT(size % gc::CellAlignBytes == 0); 253 MOZ_ASSERT(position() % gc::CellAlignBytes == 0); 254 255 if (MOZ_UNLIKELY(currentEnd() < position() + size)) { 256 return nullptr; 257 } 258 259 void* ptr = reinterpret_cast<void*>(position()); 260 if (!ptr) { 261 MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE( 262 "Successful allocation cannot result in nullptr"); 263 } 264 265 toSpace.position_ = position() + size; 266 267 DebugOnlyPoison(ptr, JS_ALLOCATED_NURSERY_PATTERN, size, 268 MemCheckKind::MakeUndefined); 269 270 return ptr; 271 } 272 273 inline void* js::Nursery::allocateBuffer(Zone* zone, js::gc::Cell* owner, 274 size_t nbytes, size_t maxNurserySize) { 275 MOZ_ASSERT(owner); 276 MOZ_ASSERT(zone == owner->zone()); 277 MOZ_ASSERT(nbytes > 0); 278 MOZ_ASSERT(nbytes <= SIZE_MAX - gc::CellAlignBytes); 279 nbytes = RoundUp(nbytes, gc::CellAlignBytes); 280 281 if (!IsInsideNursery(owner)) { 282 return gc::AllocBuffer(zone, nbytes, false); 283 } 284 285 if (nbytes <= maxNurserySize) { 286 void* buffer = allocateInternalBuffer(zone, nbytes); 287 if (buffer) { 288 return buffer; 289 } 290 } 291 292 return gc::AllocBuffer(zone, nbytes, true); 293 } 294 295 namespace js { 296 297 // The allocation methods below will not run the garbage collector. If the 298 // nursery cannot accomodate the allocation, the malloc heap will be used 299 // instead. 300 301 template <typename T> 302 static inline T* AllocNurseryOrMallocBuffer(Nursery& nursery, gc::Cell* cell, 303 uint32_t count) { 304 size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value)); 305 return static_cast<T*>(nursery.allocNurseryOrMallocBuffer( 306 cell->zone(), cell, nbytes, js::MallocArena)); 307 } 308 309 template <typename T> 310 static inline T* AllocNurseryOrMallocBuffer(JSContext* cx, gc::Cell* cell, 311 uint32_t count) { 312 T* buffer = AllocNurseryOrMallocBuffer<T>(cx->nursery(), cell, count); 313 if (!buffer) { 314 ReportOutOfMemory(cx); 315 return nullptr; 316 } 317 318 return buffer; 319 } 320 321 template <typename T> 322 static inline T* AllocateCellBuffer( 323 Nursery& nursery, JS::Zone* zone, gc::Cell* cell, uint32_t count, 324 size_t maxNurserySize = Nursery::MaxNurseryBufferSize) { 325 MOZ_ASSERT(zone == cell->zone()); 326 327 size_t nbytes = RoundUp(count * sizeof(T), sizeof(Value)); 328 return static_cast<T*>( 329 nursery.allocateBuffer(zone, cell, nbytes, maxNurserySize)); 330 } 331 332 template <typename T> 333 static inline T* AllocateCellBuffer( 334 JSContext* cx, gc::Cell* cell, uint32_t count, 335 size_t maxNurserySize = Nursery::MaxNurseryBufferSize) { 336 T* buffer = AllocateCellBuffer<T>(cx->nursery(), cx->zone(), cell, count, 337 maxNurserySize); 338 if (!buffer) { 339 ReportOutOfMemory(cx); 340 return nullptr; 341 } 342 343 return buffer; 344 } 345 346 // If this returns null then the old buffer will be left alone. 347 template <typename T> 348 static inline T* ReallocNurseryOrMallocBuffer(JSContext* cx, gc::Cell* cell, 349 T* oldBuffer, uint32_t oldCount, 350 uint32_t newCount, 351 arena_id_t arenaId) { 352 size_t oldBytes = RoundUp(oldCount * sizeof(T), sizeof(Value)); 353 size_t newBytes = RoundUp(newCount * sizeof(T), sizeof(Value)); 354 355 T* buffer = static_cast<T*>(cx->nursery().reallocNurseryOrMallocBuffer( 356 cell->zone(), cell, oldBuffer, oldBytes, newBytes, arenaId)); 357 if (!buffer) { 358 ReportOutOfMemory(cx); 359 } 360 361 return buffer; 362 } 363 364 // If this returns null then the old buffer will be left alone. 365 template <typename T> 366 static inline T* ReallocateCellBuffer( 367 Nursery& nursery, JS::Zone* zone, gc::Cell* cell, T* oldBuffer, 368 uint32_t oldCount, uint32_t newCount, 369 size_t maxNurserySize = Nursery::MaxNurseryBufferSize) { 370 MOZ_ASSERT(zone == cell->zone()); 371 372 size_t oldBytes = RoundUp(oldCount * sizeof(T), sizeof(Value)); 373 size_t newBytes = RoundUp(newCount * sizeof(T), sizeof(Value)); 374 375 return static_cast<T*>(nursery.reallocateBuffer( 376 zone, cell, oldBuffer, oldBytes, newBytes, maxNurserySize)); 377 } 378 379 // If this returns null then the old buffer will be left alone. 380 template <typename T> 381 static inline T* ReallocateCellBuffer( 382 JSContext* cx, gc::Cell* cell, T* oldBuffer, uint32_t oldCount, 383 uint32_t newCount, size_t maxNurserySize = Nursery::MaxNurseryBufferSize) { 384 T* buffer = 385 ReallocateCellBuffer<T>(cx->nursery(), cx->zone(), cell, oldBuffer, 386 oldCount, newCount, maxNurserySize); 387 if (!buffer) { 388 ReportOutOfMemory(cx); 389 } 390 391 return buffer; 392 } 393 394 template <typename T> 395 static inline void FreeCellBuffer(Nursery& nursery, JS::Zone* zone, 396 gc::Cell* cell, T* buffer, uint32_t count) { 397 MOZ_ASSERT(zone == cell->zone()); 398 399 size_t bytes = RoundUp(count * sizeof(T), sizeof(Value)); 400 401 nursery.freeBuffer(zone, cell, buffer, bytes); 402 } 403 404 template <typename T> 405 static inline void FreeCellBuffer(JSContext* cx, gc::Cell* cell, T* buffer, 406 uint32_t count) { 407 FreeCellBuffer<T>(cx->nursery(), cx->zone(), cell, buffer, count); 408 } 409 410 } // namespace js 411 412 #endif /* gc_Nursery_inl_h */