memory_manager_internal.cc (5258B)
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved. 2 // 3 // Use of this source code is governed by a BSD-style 4 // license that can be found in the LICENSE file. 5 6 #include "lib/jxl/memory_manager_internal.h" 7 8 #include <jxl/memory_manager.h> 9 #include <jxl/types.h> 10 11 #include <atomic> 12 #include <cstddef> 13 #include <cstdio> 14 #include <cstdlib> 15 #include <cstring> // memcpy 16 #include <hwy/base.h> // kMaxVectorSize 17 18 #include "lib/jxl/base/common.h" 19 #include "lib/jxl/base/status.h" 20 #include "lib/jxl/simd_util.h" 21 22 namespace jxl { 23 24 namespace { 25 26 void* MemoryManagerDefaultAlloc(void* opaque, size_t size) { 27 return malloc(size); 28 } 29 30 void MemoryManagerDefaultFree(void* opaque, void* address) { free(address); } 31 32 } // namespace 33 34 void* MemoryManagerAlloc(const JxlMemoryManager* memory_manager, size_t size) { 35 return memory_manager->alloc(memory_manager->opaque, size); 36 } 37 38 void MemoryManagerFree(const JxlMemoryManager* memory_manager, void* address) { 39 memory_manager->free(memory_manager->opaque, address); 40 } 41 42 Status MemoryManagerInit(JxlMemoryManager* self, 43 const JxlMemoryManager* memory_manager) { 44 if (memory_manager) { 45 *self = *memory_manager; 46 } else { 47 memset(self, 0, sizeof(*self)); 48 } 49 bool is_default_alloc = (self->alloc == nullptr); 50 bool is_default_free = (self->free == nullptr); 51 if (is_default_alloc != is_default_free) { 52 return false; 53 } 54 if (is_default_alloc) self->alloc = jxl::MemoryManagerDefaultAlloc; 55 if (is_default_free) self->free = jxl::MemoryManagerDefaultFree; 56 57 return true; 58 } 59 60 size_t BytesPerRow(const size_t xsize, const size_t sizeof_t) { 61 // Special case: we don't allow any ops -> don't need extra padding/ 62 if (xsize == 0) { 63 return 0; 64 } 65 66 const size_t vec_size = MaxVectorSize(); 67 size_t valid_bytes = xsize * sizeof_t; 68 69 // Allow unaligned accesses starting at the last valid value. 70 // Skip for the scalar case because no extra lanes will be loaded. 71 if (vec_size != 0) { 72 valid_bytes += vec_size - sizeof_t; 73 } 74 75 // Round up to vector and cache line size. 76 const size_t align = std::max(vec_size, memory_manager_internal::kAlignment); 77 size_t bytes_per_row = RoundUpTo(valid_bytes, align); 78 79 // During the lengthy window before writes are committed to memory, CPUs 80 // guard against read after write hazards by checking the address, but 81 // only the lower 11 bits. We avoid a false dependency between writes to 82 // consecutive rows by ensuring their sizes are not multiples of 2 KiB. 83 // Avoid2K prevents the same problem for the planes of an Image3. 84 if (bytes_per_row % memory_manager_internal::kAlias == 0) { 85 bytes_per_row += align; 86 } 87 88 JXL_DASSERT(bytes_per_row % align == 0); 89 return bytes_per_row; 90 } 91 92 StatusOr<AlignedMemory> AlignedMemory::Create(JxlMemoryManager* memory_manager, 93 size_t size, size_t pre_padding) { 94 JXL_ENSURE(pre_padding <= memory_manager_internal::kAlias); 95 size_t allocation_size = size + pre_padding + memory_manager_internal::kAlias; 96 if (size > allocation_size) { 97 return JXL_FAILURE("Requested allocation is too large"); 98 } 99 JXL_ENSURE(memory_manager); 100 void* allocated = 101 memory_manager->alloc(memory_manager->opaque, allocation_size); 102 if (allocated == nullptr) { 103 return JXL_FAILURE("Allocation failed"); 104 } 105 return AlignedMemory(memory_manager, allocated, pre_padding); 106 } 107 108 AlignedMemory::AlignedMemory(JxlMemoryManager* memory_manager, void* allocation, 109 size_t pre_padding) 110 : allocation_(allocation), memory_manager_(memory_manager) { 111 // Congruence to `offset` (mod kAlias) reduces cache conflicts and load/store 112 // stalls, especially with large allocations that would otherwise have similar 113 // alignments. 114 static std::atomic<uint32_t> next_group{0}; 115 size_t group = 116 static_cast<size_t>(next_group.fetch_add(1, std::memory_order_relaxed)); 117 group &= (memory_manager_internal::kNumAlignmentGroups - 1); 118 size_t offset = memory_manager_internal::kAlignment * group; 119 120 // Actual allocation. 121 uintptr_t address = reinterpret_cast<uintptr_t>(allocation) + pre_padding; 122 123 // Aligned address, but might land before allocation (50%/50%) or not have 124 // enough pre-padding. 125 uintptr_t aligned_address = 126 (address & ~(memory_manager_internal::kAlias - 1)) + offset; 127 if (aligned_address < address) 128 aligned_address += memory_manager_internal::kAlias; 129 130 address_ = reinterpret_cast<void*>(aligned_address); // NOLINT 131 } 132 133 AlignedMemory::AlignedMemory(AlignedMemory&& other) noexcept { 134 allocation_ = other.allocation_; 135 memory_manager_ = other.memory_manager_; 136 address_ = other.address_; 137 other.memory_manager_ = nullptr; 138 } 139 140 AlignedMemory& AlignedMemory::operator=(AlignedMemory&& other) noexcept { 141 if (this == &other) return *this; 142 if (memory_manager_ && allocation_) { 143 memory_manager_->free(memory_manager_->opaque, allocation_); 144 } 145 allocation_ = other.allocation_; 146 memory_manager_ = other.memory_manager_; 147 address_ = other.address_; 148 other.memory_manager_ = nullptr; 149 return *this; 150 } 151 152 AlignedMemory::~AlignedMemory() { 153 if (memory_manager_ == nullptr) return; 154 memory_manager_->free(memory_manager_->opaque, allocation_); 155 } 156 157 } // namespace jxl