WasmGcObject-inl.h (13327B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef wasm_WasmGcObject_inl_h 8 #define wasm_WasmGcObject_inl_h 9 10 #include "wasm/WasmGcObject.h" 11 12 #include "mozilla/Attributes.h" 13 #include "mozilla/DebugOnly.h" 14 #include "util/Memory.h" 15 16 #include "gc/Nursery-inl.h" 17 #include "gc/ObjectKind-inl.h" 18 #include "vm/JSContext-inl.h" 19 20 //========================================================================= 21 // WasmStructObject inlineable allocation methods 22 23 // Maximum size of trailer block to allocate directly in the nursery. 24 // 25 // For objects that die in the nursery, direct nursery allocation is faster and 26 // is better for cache locality. For objects that survive, direct nursery 27 // allocation incurs the overhead of copying the data. This parameter should be 28 // chosen to balance these based on the expected allocation sizes and tenuring 29 // rates in workloads we care about. 30 // 31 // This is set to a lower value than the default (Nursery::MaxNurseryBufferSize) 32 // because we tend to get higher tenuring rates in Wasm GC benchmarks. 33 static constexpr size_t MaxNurseryTrailerSize = 256; 34 static_assert(MaxNurseryTrailerSize < js::gc::ChunkSize); 35 36 namespace js { 37 38 /* static */ 39 template <bool ZeroFields> 40 MOZ_ALWAYS_INLINE WasmStructObject* WasmStructObject::createStructIL( 41 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 42 gc::AllocSite* allocSite, js::gc::Heap initialHeap) { 43 // It is up to our caller to ensure that `typeDefData` refers to a type that 44 // doesn't need OOL storage. 45 MOZ_ASSERT(typeDefData->cached.strukt.totalSizeOOL == 0); 46 47 MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp)); 48 MOZ_ASSERT(!typeDefData->clasp->isNativeObject()); 49 MOZ_ASSERT(!IsFinalizedKind(typeDefData->cached.strukt.allocKind)); 50 51 AutoSetNewObjectMetadata metadata(cx); 52 debugCheckNewObject(typeDefData->shape, typeDefData->cached.strukt.allocKind, 53 initialHeap); 54 55 mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef; 56 MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct); 57 58 // This doesn't need to be rooted, since all we do with it prior to 59 // return is to zero out the fields (and then only if ZeroFields is true). 60 WasmStructObject* structObj = (WasmStructObject*)cx->newCell<WasmGcObject>( 61 typeDefData->cached.strukt.allocKind, initialHeap, typeDefData->clasp, 62 allocSite); 63 if (MOZ_UNLIKELY(!structObj)) { 64 ReportOutOfMemory(cx); 65 return nullptr; 66 } 67 68 structObj->initShape(typeDefData->shape); 69 structObj->superTypeVector_ = typeDefData->superTypeVector; 70 if constexpr (ZeroFields) { 71 size_t headerSize = typeDefData->cached.strukt.payloadOffsetIL; 72 memset((uint8_t*)structObj + headerSize, 0, 73 typeDefData->cached.strukt.totalSizeIL - headerSize); 74 } 75 76 MOZ_ASSERT(typeDefData->clasp->shouldDelayMetadataBuilder()); 77 cx->realm()->setObjectPendingMetadata(structObj); 78 79 js::gc::gcprobes::CreateObject(structObj); 80 probes::CreateObject(cx, structObj); 81 82 return structObj; 83 } 84 85 /* static */ 86 template <bool ZeroFields> 87 MOZ_ALWAYS_INLINE WasmStructObject* WasmStructObject::createStructOOL( 88 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 89 gc::AllocSite* allocSite, js::gc::Heap initialHeap) { 90 // It is up to our caller to ensure that `typeDefData` refers to a type that 91 // needs OOL storage. 92 MOZ_ASSERT(typeDefData->cached.strukt.totalSizeOOL > 0); 93 94 MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp)); 95 MOZ_ASSERT(!typeDefData->clasp->isNativeObject()); 96 MOZ_ASSERT(!IsFinalizedKind(typeDefData->cached.strukt.allocKind)); 97 98 AutoSetNewObjectMetadata metadata(cx); 99 debugCheckNewObject(typeDefData->shape, typeDefData->cached.strukt.allocKind, 100 initialHeap); 101 102 mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef; 103 MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct); 104 105 uint32_t outlineBytes = typeDefData->cached.strukt.totalSizeOOL; 106 107 // This doesn't need to be Rooted because the AllocateCellBuffer call that 108 // follows can't trigger GC. 109 auto* structObj = (WasmStructObject*)cx->newCell<WasmGcObject>( 110 typeDefData->cached.strukt.allocKind, initialHeap, typeDefData->clasp, 111 allocSite); 112 if (MOZ_UNLIKELY(!structObj)) { 113 ReportOutOfMemory(cx); 114 return nullptr; 115 } 116 117 structObj->initShape(typeDefData->shape); 118 structObj->superTypeVector_ = typeDefData->superTypeVector; 119 120 uint8_t* outlineData = AllocateCellBuffer<uint8_t>( 121 cx, structObj, outlineBytes, MaxNurseryTrailerSize); 122 if (MOZ_UNLIKELY(!outlineData)) { 123 // AllocateCellBuffer will have called ReportOutOfMemory(cx) itself, 124 // so no need to do that here. 125 structObj->setOOLPointer(typeDefData, nullptr); 126 return nullptr; 127 } 128 129 // Initialize the inline and outline data fields 130 if constexpr (ZeroFields) { 131 size_t headerSize = typeDefData->cached.strukt.payloadOffsetIL; 132 memset((uint8_t*)structObj + headerSize, 0, 133 typeDefData->cached.strukt.totalSizeIL - headerSize); 134 memset(outlineData, 0, outlineBytes); 135 } 136 137 structObj->setOOLPointer(typeDefData, outlineData); 138 139 MOZ_ASSERT(typeDefData->clasp->shouldDelayMetadataBuilder()); 140 cx->realm()->setObjectPendingMetadata(structObj); 141 142 js::gc::gcprobes::CreateObject(structObj); 143 probes::CreateObject(cx, structObj); 144 145 return structObj; 146 } 147 148 //========================================================================= 149 // WasmArrayObject inlineable allocation methods 150 151 /* static */ 152 inline gc::AllocKind WasmArrayObject::allocKindForOOL() { 153 gc::AllocKind allocKind = 154 gc::GetGCObjectKindForBytes(sizeof(WasmArrayObject)); 155 return gc::GetFinalizedAllocKindForClass(allocKind, &WasmArrayObject::class_); 156 } 157 158 /* static */ 159 inline gc::AllocKind WasmArrayObject::allocKindForIL(uint32_t storageBytes) { 160 gc::AllocKind allocKind = 161 gc::GetGCObjectKindForBytes(sizeof(WasmArrayObject) + storageBytes); 162 return gc::GetFinalizedAllocKindForClass(allocKind, &WasmArrayObject::class_); 163 } 164 165 inline gc::AllocKind WasmArrayObject::allocKind() const { 166 if (isDataInline()) { 167 // numElements_ was validated to not overflow when constructing this object 168 uint32_t storageBytes = calcStorageBytesUnchecked( 169 typeDef().arrayType().elementType().size(), numElements_); 170 return allocKindForIL(storageBytes); 171 } 172 173 return allocKindForOOL(); 174 } 175 176 /* static */ 177 template <bool ZeroFields> 178 MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArrayOOL( 179 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 180 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 181 uint32_t numElements, uint32_t storageBytes) { 182 STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; 183 184 MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp)); 185 MOZ_ASSERT(!typeDefData->clasp->isNativeObject()); 186 gc::AllocKind allocKind = allocKindForOOL(); 187 AutoSetNewObjectMetadata metadata(cx); 188 debugCheckNewObject(typeDefData->shape, allocKind, initialHeap); 189 190 mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef; 191 MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Array); 192 193 // This routine is for large arrays with out-of-line data only. For small 194 // arrays use createArrayIL. 195 MOZ_ASSERT(storageBytes > WasmArrayObject_MaxInlineBytes); 196 197 auto* arrayObj = (WasmArrayObject*)cx->newCell<WasmGcObject>( 198 allocKind, initialHeap, typeDefData->clasp, allocSite); 199 if (MOZ_UNLIKELY(!arrayObj)) { 200 ReportOutOfMemory(cx); 201 return nullptr; 202 } 203 204 uint8_t* outlineAlloc = AllocateCellBuffer<uint8_t>( 205 cx, arrayObj, storageBytes, MaxNurseryTrailerSize); 206 if (MOZ_UNLIKELY(!outlineAlloc)) { 207 arrayObj->numElements_ = 0; 208 arrayObj->data_ = nullptr; 209 ReportOutOfMemory(cx); 210 return nullptr; 211 } 212 213 DataHeader* outlineHeader = (DataHeader*)outlineAlloc; 214 *outlineHeader = DataIsOOL; 215 uint8_t* outlineData = dataHeaderToDataPointer(outlineHeader); 216 217 arrayObj->initShape(typeDefData->shape); 218 arrayObj->superTypeVector_ = typeDefData->superTypeVector; 219 arrayObj->numElements_ = numElements; 220 arrayObj->data_ = outlineData; 221 if constexpr (ZeroFields) { 222 uint32_t dataBytes = storageBytes - sizeof(DataHeader); 223 MOZ_ASSERT(dataBytes >= numElements * typeDefData->cached.array.elemSize); 224 memset(arrayObj->data_, 0, dataBytes); 225 } 226 227 MOZ_ASSERT(!arrayObj->isDataInline()); 228 229 MOZ_ASSERT(typeDefData->clasp->shouldDelayMetadataBuilder()); 230 cx->realm()->setObjectPendingMetadata(arrayObj); 231 232 js::gc::gcprobes::CreateObject(arrayObj); 233 probes::CreateObject(cx, arrayObj); 234 235 return arrayObj; 236 } 237 238 template WasmArrayObject* WasmArrayObject::createArrayOOL<true>( 239 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 240 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 241 uint32_t numElements, uint32_t storageBytes); 242 template WasmArrayObject* WasmArrayObject::createArrayOOL<false>( 243 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 244 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 245 uint32_t numElements, uint32_t storageBytes); 246 247 /* static */ 248 template <bool ZeroFields> 249 MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArrayIL( 250 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 251 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 252 uint32_t numElements, uint32_t storageBytes) { 253 STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; 254 255 MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp)); 256 MOZ_ASSERT(!typeDefData->clasp->isNativeObject()); 257 AutoSetNewObjectMetadata metadata(cx); 258 gc::AllocKind allocKind = allocKindForIL(storageBytes); 259 debugCheckNewObject(typeDefData->shape, allocKind, initialHeap); 260 261 mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef; 262 MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Array); 263 264 MOZ_ASSERT(storageBytes <= WasmArrayObject_MaxInlineBytes); 265 266 // There's no need for `arrayObj` to be rooted, since the only thing we're 267 // going to do is fill in some bits of it, then return it. 268 WasmArrayObject* arrayObj = (WasmArrayObject*)cx->newCell<WasmGcObject>( 269 allocKind, initialHeap, typeDefData->clasp, allocSite); 270 if (MOZ_UNLIKELY(!arrayObj)) { 271 ReportOutOfMemory(cx); 272 return nullptr; 273 } 274 275 arrayObj->initShape(typeDefData->shape); 276 arrayObj->superTypeVector_ = typeDefData->superTypeVector; 277 arrayObj->numElements_ = numElements; 278 279 DataHeader* inlineHeader = 280 WasmArrayObject::addressOfInlineDataHeader(arrayObj); 281 uint8_t* inlineData = WasmArrayObject::addressOfInlineData(arrayObj); 282 *inlineHeader = DataIsIL; 283 arrayObj->data_ = inlineData; 284 285 if constexpr (ZeroFields) { 286 uint32_t dataBytes = storageBytes - sizeof(DataHeader); 287 MOZ_ASSERT(dataBytes >= numElements * typeDefData->cached.array.elemSize); 288 289 if (numElements > 0) { 290 memset(arrayObj->data_, 0, dataBytes); 291 } 292 } 293 294 MOZ_ASSERT(arrayObj->isDataInline()); 295 296 MOZ_ASSERT(typeDefData->clasp->shouldDelayMetadataBuilder()); 297 cx->realm()->setObjectPendingMetadata(arrayObj); 298 299 js::gc::gcprobes::CreateObject(arrayObj); 300 probes::CreateObject(cx, arrayObj); 301 302 return arrayObj; 303 } 304 305 template WasmArrayObject* WasmArrayObject::createArrayIL<true>( 306 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 307 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 308 uint32_t numElements, uint32_t storageBytes); 309 template WasmArrayObject* WasmArrayObject::createArrayIL<false>( 310 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 311 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 312 uint32_t numElements, uint32_t storageBytes); 313 314 /* static */ 315 template <bool ZeroFields> 316 MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArray( 317 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 318 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 319 uint32_t numElements) { 320 MOZ_ASSERT(typeDefData->cached.array.elemSize == 321 typeDefData->typeDef->arrayType().elementType().size()); 322 mozilla::CheckedUint32 storageBytes = 323 calcStorageBytesChecked(typeDefData->cached.array.elemSize, numElements); 324 if (!storageBytes.isValid() || 325 storageBytes.value() > uint32_t(wasm::MaxArrayPayloadBytes)) { 326 js::ReportOversizedAllocation(cx, JSMSG_WASM_ARRAY_IMP_LIMIT); 327 wasm::MarkPendingExceptionAsTrap(cx); 328 return nullptr; 329 } 330 331 if (storageBytes.value() <= WasmArrayObject_MaxInlineBytes) { 332 return createArrayIL<ZeroFields>(cx, typeDefData, allocSite, initialHeap, 333 numElements, storageBytes.value()); 334 } 335 336 return createArrayOOL<ZeroFields>(cx, typeDefData, allocSite, initialHeap, 337 numElements, storageBytes.value()); 338 } 339 340 template WasmArrayObject* WasmArrayObject::createArray<true>( 341 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 342 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 343 uint32_t numElements); 344 template WasmArrayObject* WasmArrayObject::createArray<false>( 345 JSContext* cx, wasm::TypeDefInstanceData* typeDefData, 346 js::gc::AllocSite* allocSite, js::gc::Heap initialHeap, 347 uint32_t numElements); 348 349 } // namespace js 350 351 #endif /* wasm_WasmGcObject_inl_h */