mem.c (10388B)
1 /* 2 * Copyright © 2020, VideoLAN and dav1d authors 3 * Copyright © 2020, Two Orioles, LLC 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, this 10 * list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "config.h" 29 30 #include <stdint.h> 31 32 #include "src/internal.h" 33 34 #if TRACK_HEAP_ALLOCATIONS 35 #include <stdio.h> 36 37 #include "src/log.h" 38 39 #define DEFAULT_ALIGN 16 40 41 typedef struct { 42 size_t sz; 43 unsigned align; 44 enum AllocationType type; 45 } Dav1dAllocationData; 46 47 typedef struct { 48 size_t curr_sz; 49 size_t peak_sz; 50 unsigned num_allocs; 51 unsigned num_reuses; 52 } AllocStats; 53 54 static AllocStats tracked_allocs[N_ALLOC_TYPES]; 55 static size_t curr_total_sz; 56 static size_t peak_total_sz; 57 static pthread_mutex_t track_alloc_mutex = PTHREAD_MUTEX_INITIALIZER; 58 59 static void *track_alloc(const enum AllocationType type, char *ptr, 60 const size_t sz, const size_t align) 61 { 62 assert(align >= sizeof(Dav1dAllocationData)); 63 if (ptr) { 64 ptr += align; 65 Dav1dAllocationData *const d = &((Dav1dAllocationData*)ptr)[-1]; 66 AllocStats *const s = &tracked_allocs[type]; 67 68 d->sz = sz; 69 d->align = (unsigned)align; 70 d->type = type; 71 72 pthread_mutex_lock(&track_alloc_mutex); 73 s->num_allocs++; 74 s->curr_sz += sz; 75 if (s->curr_sz > s->peak_sz) 76 s->peak_sz = s->curr_sz; 77 78 curr_total_sz += sz; 79 if (curr_total_sz > peak_total_sz) 80 peak_total_sz = curr_total_sz; 81 pthread_mutex_unlock(&track_alloc_mutex); 82 } 83 return ptr; 84 } 85 86 static void *track_free(char *const ptr) { 87 const Dav1dAllocationData *const d = &((Dav1dAllocationData*)ptr)[-1]; 88 const size_t sz = d->sz; 89 90 pthread_mutex_lock(&track_alloc_mutex); 91 tracked_allocs[d->type].curr_sz -= sz; 92 curr_total_sz -= sz; 93 pthread_mutex_unlock(&track_alloc_mutex); 94 95 return ptr - d->align; 96 } 97 98 static void dav1d_track_reuse(const enum AllocationType type) { 99 pthread_mutex_lock(&track_alloc_mutex); 100 tracked_allocs[type].num_reuses++; 101 pthread_mutex_unlock(&track_alloc_mutex); 102 } 103 104 void *dav1d_malloc(const enum AllocationType type, const size_t sz) { 105 void *const ptr = malloc(sz + DEFAULT_ALIGN); 106 return track_alloc(type, ptr, sz, DEFAULT_ALIGN); 107 } 108 109 void *dav1d_alloc_aligned(const enum AllocationType type, 110 const size_t sz, const size_t align) 111 { 112 void *const ptr = dav1d_alloc_aligned_internal(align, sz + align); 113 return track_alloc(type, ptr, sz, align); 114 } 115 116 void *dav1d_realloc(const enum AllocationType type, 117 void *ptr, const size_t sz) 118 { 119 if (!ptr) 120 return dav1d_malloc(type, sz); 121 ptr = realloc((char*)ptr - DEFAULT_ALIGN, sz + DEFAULT_ALIGN); 122 if (ptr) 123 ptr = track_free((char*)ptr + DEFAULT_ALIGN); 124 return track_alloc(type, ptr, sz, DEFAULT_ALIGN); 125 } 126 127 void dav1d_free(void *ptr) { 128 if (ptr) 129 free(track_free(ptr)); 130 } 131 132 void dav1d_free_aligned(void *ptr) { 133 if (ptr) { 134 dav1d_free_aligned_internal(track_free(ptr)); 135 } 136 } 137 138 static COLD int cmp_stats(const void *const a, const void *const b) { 139 const size_t a_sz = ((const AllocStats*)a)->peak_sz; 140 const size_t b_sz = ((const AllocStats*)b)->peak_sz; 141 return a_sz < b_sz ? -1 : a_sz > b_sz; 142 } 143 144 /* Insert spaces as thousands separators for better readability */ 145 static COLD int format_tsep(char *const s, const size_t n, const size_t value) { 146 if (value < 1000) 147 return snprintf(s, n, "%u", (unsigned)value); 148 149 const int len = format_tsep(s, n, value / 1000); 150 assert((size_t)len < n); 151 return len + snprintf(s + len, n - len, " %03u", (unsigned)(value % 1000)); 152 } 153 154 COLD void dav1d_log_alloc_stats(Dav1dContext *const c) { 155 static const char *const type_names[N_ALLOC_TYPES] = { 156 [ALLOC_BLOCK ] = "Block data", 157 [ALLOC_CDEF ] = "CDEF line buffers", 158 [ALLOC_CDF ] = "CDF contexts", 159 [ALLOC_COEF ] = "Coefficient data", 160 [ALLOC_COMMON_CTX] = "Common context data", 161 [ALLOC_DAV1DDATA ] = "Dav1dData", 162 [ALLOC_IPRED ] = "Intra pred edges", 163 [ALLOC_LF ] = "Loopfilter data", 164 [ALLOC_LR ] = "Looprestoration data", 165 [ALLOC_OBU_HDR ] = "OBU headers", 166 [ALLOC_OBU_META ] = "OBU metadata", 167 [ALLOC_PAL ] = "Palette data", 168 [ALLOC_PIC ] = "Picture buffers", 169 [ALLOC_PIC_CTX ] = "Picture context data", 170 [ALLOC_REFMVS ] = "Reference mv data", 171 [ALLOC_SEGMAP ] = "Segmentation maps", 172 [ALLOC_THREAD_CTX] = "Thread context data", 173 [ALLOC_TILE ] = "Tile data", 174 }; 175 176 struct { 177 AllocStats stats; 178 enum AllocationType type; 179 } data[N_ALLOC_TYPES]; 180 unsigned total_allocs = 0; 181 unsigned total_reuses = 0; 182 183 pthread_mutex_lock(&track_alloc_mutex); 184 for (int i = 0; i < N_ALLOC_TYPES; i++) { 185 AllocStats *const s = &data[i].stats; 186 *s = tracked_allocs[i]; 187 data[i].type = i; 188 total_allocs += s->num_allocs; 189 total_reuses += s->num_reuses; 190 } 191 size_t total_sz = peak_total_sz; 192 pthread_mutex_unlock(&track_alloc_mutex); 193 194 /* Sort types by memory usage */ 195 qsort(&data, N_ALLOC_TYPES, sizeof(*data), cmp_stats); 196 197 const double inv_total_share = 100.0 / total_sz; 198 char total_sz_buf[32]; 199 const int sz_len = 4 + format_tsep(total_sz_buf, sizeof(total_sz_buf), total_sz); 200 201 dav1d_log(c, "\n Type Allocs Reuses Share Peak size\n" 202 "---------------------------------------------------------------------\n"); 203 for (int i = N_ALLOC_TYPES - 1; i >= 0; i--) { 204 const AllocStats *const s = &data[i].stats; 205 if (s->num_allocs) { 206 const double share = s->peak_sz * inv_total_share; 207 char sz_buf[32]; 208 format_tsep(sz_buf, sizeof(sz_buf), s->peak_sz); 209 dav1d_log(c, " %-20s%10u%10u%8.1f%%%*s\n", type_names[data[i].type], 210 s->num_allocs, s->num_reuses, share, sz_len, sz_buf); 211 } 212 } 213 dav1d_log(c, "---------------------------------------------------------------------\n" 214 "%31u%10u %s\n", 215 total_allocs, total_reuses, total_sz_buf); 216 } 217 #endif /* TRACK_HEAP_ALLOCATIONS */ 218 219 static COLD void mem_pool_destroy(Dav1dMemPool *const pool) { 220 pthread_mutex_destroy(&pool->lock); 221 dav1d_free(pool); 222 } 223 224 void dav1d_mem_pool_push(Dav1dMemPool *const pool, Dav1dMemPoolBuffer *const buf) { 225 pthread_mutex_lock(&pool->lock); 226 const int ref_cnt = --pool->ref_cnt; 227 if (!pool->end) { 228 buf->next = pool->buf; 229 pool->buf = buf; 230 pthread_mutex_unlock(&pool->lock); 231 assert(ref_cnt > 0); 232 } else { 233 pthread_mutex_unlock(&pool->lock); 234 dav1d_free_aligned(buf->data); 235 if (!ref_cnt) mem_pool_destroy(pool); 236 } 237 } 238 239 Dav1dMemPoolBuffer *dav1d_mem_pool_pop(Dav1dMemPool *const pool, const size_t size) { 240 assert(!(size & (sizeof(void*) - 1))); 241 pthread_mutex_lock(&pool->lock); 242 Dav1dMemPoolBuffer *buf = pool->buf; 243 pool->ref_cnt++; 244 uint8_t *data; 245 if (buf) { 246 pool->buf = buf->next; 247 pthread_mutex_unlock(&pool->lock); 248 data = buf->data; 249 if ((uintptr_t)buf - (uintptr_t)data != size) { 250 /* Reallocate if the size has changed */ 251 dav1d_free_aligned(data); 252 goto alloc; 253 } 254 #if TRACK_HEAP_ALLOCATIONS 255 dav1d_track_reuse(pool->type); 256 #endif 257 } else { 258 pthread_mutex_unlock(&pool->lock); 259 alloc: 260 data = dav1d_alloc_aligned(pool->type, 261 size + sizeof(Dav1dMemPoolBuffer), 64); 262 if (!data) { 263 pthread_mutex_lock(&pool->lock); 264 const int ref_cnt = --pool->ref_cnt; 265 pthread_mutex_unlock(&pool->lock); 266 if (!ref_cnt) mem_pool_destroy(pool); 267 return NULL; 268 } 269 buf = (Dav1dMemPoolBuffer*)(data + size); 270 buf->data = data; 271 } 272 273 return buf; 274 } 275 276 COLD int dav1d_mem_pool_init(const enum AllocationType type, 277 Dav1dMemPool **const ppool) 278 { 279 Dav1dMemPool *const pool = dav1d_malloc(ALLOC_COMMON_CTX, 280 sizeof(Dav1dMemPool)); 281 if (pool) { 282 if (!pthread_mutex_init(&pool->lock, NULL)) { 283 pool->buf = NULL; 284 pool->ref_cnt = 1; 285 pool->end = 0; 286 #if TRACK_HEAP_ALLOCATIONS 287 pool->type = type; 288 #endif 289 *ppool = pool; 290 return 0; 291 } 292 dav1d_free(pool); 293 } 294 *ppool = NULL; 295 return DAV1D_ERR(ENOMEM); 296 } 297 298 COLD void dav1d_mem_pool_end(Dav1dMemPool *const pool) { 299 if (pool) { 300 pthread_mutex_lock(&pool->lock); 301 Dav1dMemPoolBuffer *buf = pool->buf; 302 const int ref_cnt = --pool->ref_cnt; 303 pool->buf = NULL; 304 pool->end = 1; 305 pthread_mutex_unlock(&pool->lock); 306 307 while (buf) { 308 void *const data = buf->data; 309 buf = buf->next; 310 dav1d_free_aligned(data); 311 } 312 if (!ref_cnt) mem_pool_destroy(pool); 313 } 314 }