av1_noise_estimate.c (11362B)
1 /* 2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <assert.h> 13 #include <limits.h> 14 #include <math.h> 15 16 #include "config/aom_dsp_rtcd.h" 17 #include "aom_dsp/aom_dsp_common.h" 18 #include "aom_scale/yv12config.h" 19 #include "aom/aom_integer.h" 20 #include "av1/encoder/context_tree.h" 21 #include "av1/encoder/av1_noise_estimate.h" 22 #include "av1/encoder/encoder.h" 23 #if CONFIG_AV1_TEMPORAL_DENOISING 24 #include "av1/encoder/av1_temporal_denoiser.h" 25 #endif 26 27 #if CONFIG_AV1_TEMPORAL_DENOISING 28 // For SVC: only do noise estimation on top spatial layer. 29 static inline int noise_est_svc(const struct AV1_COMP *const cpi) { 30 return (!cpi->ppi->use_svc || 31 (cpi->ppi->use_svc && 32 cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1)); 33 } 34 #endif 35 36 void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) { 37 const int64_t area = (int64_t)width * height; 38 ne->enabled = 0; 39 ne->level = (area < 1280 * 720) ? kLowLow : kLow; 40 ne->value = 0; 41 ne->count = 0; 42 ne->thresh = 90; 43 ne->last_w = 0; 44 ne->last_h = 0; 45 if (area >= 1920 * 1080) { 46 ne->thresh = 200; 47 } else if (area >= 1280 * 720) { 48 ne->thresh = 140; 49 } else if (area >= 640 * 360) { 50 ne->thresh = 115; 51 } 52 ne->num_frames_estimate = 15; 53 ne->adapt_thresh = (3 * ne->thresh) >> 1; 54 } 55 56 static int enable_noise_estimation(AV1_COMP *const cpi) { 57 const int resize_pending = is_frame_resize_pending(cpi); 58 59 #if CONFIG_AV1_HIGHBITDEPTH 60 if (cpi->common.seq_params->use_highbitdepth) return 0; 61 #endif 62 // Enable noise estimation if denoising is on. 63 #if CONFIG_AV1_TEMPORAL_DENOISING 64 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) && 65 cpi->common.width >= 320 && cpi->common.height >= 180) 66 return 1; 67 #endif 68 // Only allow noise estimate under certain encoding mode. 69 // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original. 70 // Not enabled for SVC mode and screen_content_mode. 71 // Not enabled for low resolutions. 72 if (cpi->oxcf.pass == AOM_RC_ONE_PASS && cpi->oxcf.rc_cfg.mode == AOM_CBR && 73 cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 && 74 resize_pending == 0 && !cpi->ppi->use_svc && 75 cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN && 76 cpi->common.width * cpi->common.height >= 640 * 360) 77 return 1; 78 else 79 return 0; 80 } 81 82 #if CONFIG_AV1_TEMPORAL_DENOISING 83 static void copy_frame(YV12_BUFFER_CONFIG *const dest, 84 const YV12_BUFFER_CONFIG *const src) { 85 const uint8_t *srcbuf = src->y_buffer; 86 uint8_t *destbuf = dest->y_buffer; 87 88 assert(dest->y_width == src->y_width); 89 assert(dest->y_height == src->y_height); 90 91 for (int r = 0; r < dest->y_height; ++r) { 92 memcpy(destbuf, srcbuf, dest->y_width); 93 destbuf += dest->y_stride; 94 srcbuf += src->y_stride; 95 } 96 } 97 #endif // CONFIG_AV1_TEMPORAL_DENOISING 98 99 NOISE_LEVEL av1_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) { 100 int noise_level = kLowLow; 101 if (ne->value > (ne->thresh << 1)) { 102 noise_level = kHigh; 103 } else { 104 if (ne->value > ne->thresh) 105 noise_level = kMedium; 106 else if (ne->value > (ne->thresh >> 1)) 107 noise_level = kLow; 108 else 109 noise_level = kLowLow; 110 } 111 return noise_level; 112 } 113 114 void av1_update_noise_estimate(AV1_COMP *const cpi) { 115 const AV1_COMMON *const cm = &cpi->common; 116 const CommonModeInfoParams *const mi_params = &cm->mi_params; 117 118 NOISE_ESTIMATE *const ne = &cpi->noise_estimate; 119 const int low_res = (cm->width <= 352 && cm->height <= 288); 120 // Estimate of noise level every frame_period frames. 121 int frame_period = 8; 122 int thresh_consec_zeromv = 2; 123 int frame_counter = cm->current_frame.frame_number; 124 // Estimate is between current source and last source. 125 YV12_BUFFER_CONFIG *last_source = cpi->last_source; 126 #if CONFIG_AV1_TEMPORAL_DENOISING 127 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) { 128 last_source = &cpi->denoiser.last_source; 129 // Tune these thresholds for different resolutions when denoising is 130 // enabled. 131 if (cm->width > 640 && cm->width <= 1920) { 132 thresh_consec_zeromv = 2; 133 } 134 } 135 #endif 136 ne->enabled = enable_noise_estimation(cpi); 137 if (cpi->svc.number_spatial_layers > 1) 138 frame_counter = cpi->svc.current_superframe; 139 if (!ne->enabled || frame_counter % frame_period != 0 || 140 last_source == NULL || 141 (cpi->svc.number_spatial_layers == 1 && 142 (ne->last_w != cm->width || ne->last_h != cm->height))) { 143 #if CONFIG_AV1_TEMPORAL_DENOISING 144 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) 145 copy_frame(&cpi->denoiser.last_source, cpi->source); 146 #endif 147 if (last_source != NULL) { 148 ne->last_w = cm->width; 149 ne->last_h = cm->height; 150 } 151 return; 152 } else if (frame_counter > 60 && cpi->svc.num_encoded_top_layer > 1 && 153 cpi->rc.frames_since_key > cpi->svc.number_spatial_layers && 154 cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 && 155 cpi->rc.avg_frame_low_motion < (low_res ? 60 : 40)) { 156 // Force noise estimation to 0 and denoiser off if content has high motion. 157 ne->level = kLowLow; 158 ne->count = 0; 159 ne->num_frames_estimate = 10; 160 #if CONFIG_AV1_TEMPORAL_DENOISING 161 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) && 162 cpi->svc.current_superframe > 1) { 163 av1_denoiser_set_noise_level(cpi, ne->level); 164 copy_frame(&cpi->denoiser.last_source, cpi->source); 165 } 166 #endif 167 return; 168 } else { 169 unsigned int bin_size = 100; 170 unsigned int hist[MAX_VAR_HIST_BINS] = { 0 }; 171 unsigned int hist_avg[MAX_VAR_HIST_BINS]; 172 unsigned int max_bin = 0; 173 unsigned int max_bin_count = 0; 174 unsigned int bin_cnt; 175 BLOCK_SIZE bsize = BLOCK_16X16; 176 // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have 177 // been encoded as zero/small mv at least x consecutive frames, compute 178 // the variance to update estimate of noise in the source. 179 const uint8_t *src_y = cpi->source->y_buffer; 180 const int src_ystride = cpi->source->y_stride; 181 const uint8_t *last_src_y = last_source->y_buffer; 182 const int last_src_ystride = last_source->y_stride; 183 int mi_row, mi_col; 184 int num_low_motion = 0; 185 int frame_low_motion = 1; 186 for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row += 2) { 187 for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col += 2) { 188 int bl_index = 189 (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1); 190 if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv) 191 num_low_motion++; 192 } 193 } 194 if (num_low_motion < 195 (((3 * (mi_params->mi_rows * mi_params->mi_cols) >> 2)) >> 3)) 196 frame_low_motion = 0; 197 for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row++) { 198 for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col++) { 199 // 16x16 blocks, 1/4 sample of frame. 200 if (mi_row % 8 == 0 && mi_col % 8 == 0 && 201 mi_row < mi_params->mi_rows - 3 && 202 mi_col < mi_params->mi_cols - 3) { 203 int bl_index = 204 (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1); 205 int bl_index1 = bl_index + 1; 206 int bl_index2 = bl_index + (mi_params->mi_cols >> 1); 207 int bl_index3 = bl_index2 + 1; 208 int consec_zeromv = 209 AOMMIN(cpi->consec_zero_mv[bl_index], 210 AOMMIN(cpi->consec_zero_mv[bl_index1], 211 AOMMIN(cpi->consec_zero_mv[bl_index2], 212 cpi->consec_zero_mv[bl_index3]))); 213 // Only consider blocks that are likely steady background. i.e, have 214 // been encoded as zero/low motion x (= thresh_consec_zeromv) frames 215 // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all 216 // 4 sub-blocks for 16x16 block. And exclude this frame if 217 // high_source_sad is true (i.e., scene/content change). 218 if (frame_low_motion && consec_zeromv > thresh_consec_zeromv && 219 !cpi->rc.high_source_sad) { 220 unsigned int sse; 221 // Compute variance between co-located blocks from current and 222 // last input frames. 223 unsigned int variance = cpi->ppi->fn_ptr[bsize].vf( 224 src_y, src_ystride, last_src_y, last_src_ystride, &sse); 225 unsigned int hist_index = variance / bin_size; 226 if (hist_index < MAX_VAR_HIST_BINS) 227 hist[hist_index]++; 228 else if (hist_index < 3 * (MAX_VAR_HIST_BINS >> 1)) 229 hist[MAX_VAR_HIST_BINS - 1]++; // Account for the tail 230 } 231 } 232 src_y += 4; 233 last_src_y += 4; 234 } 235 src_y += (src_ystride << 2) - (mi_params->mi_cols << 2); 236 last_src_y += (last_src_ystride << 2) - (mi_params->mi_cols << 2); 237 } 238 ne->last_w = cm->width; 239 ne->last_h = cm->height; 240 // Adjust histogram to account for effect that histogram flattens 241 // and shifts to zero as scene darkens. 242 if (hist[0] > 10 && (hist[MAX_VAR_HIST_BINS - 1] > hist[0] >> 2)) { 243 hist[0] = 0; 244 hist[1] >>= 2; 245 hist[2] >>= 2; 246 hist[3] >>= 2; 247 hist[4] >>= 1; 248 hist[5] >>= 1; 249 hist[6] = 3 * hist[6] >> 1; 250 hist[MAX_VAR_HIST_BINS - 1] >>= 1; 251 } 252 253 // Average hist[] and find largest bin 254 for (bin_cnt = 0; bin_cnt < MAX_VAR_HIST_BINS; bin_cnt++) { 255 if (bin_cnt == 0) 256 hist_avg[bin_cnt] = (hist[0] + hist[1] + hist[2]) / 3; 257 else if (bin_cnt == MAX_VAR_HIST_BINS - 1) 258 hist_avg[bin_cnt] = hist[MAX_VAR_HIST_BINS - 1] >> 2; 259 else if (bin_cnt == MAX_VAR_HIST_BINS - 2) 260 hist_avg[bin_cnt] = (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + 261 (hist[bin_cnt + 1] >> 1) + 2) >> 262 2; 263 else 264 hist_avg[bin_cnt] = 265 (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + hist[bin_cnt + 1] + 2) >> 266 2; 267 268 if (hist_avg[bin_cnt] > max_bin_count) { 269 max_bin_count = hist_avg[bin_cnt]; 270 max_bin = bin_cnt; 271 } 272 } 273 // Scale by 40 to work with existing thresholds 274 ne->value = (int)((3 * ne->value + max_bin * 40) >> 2); 275 // Quickly increase VNR strength when the noise level increases suddenly. 276 if (ne->level < kMedium && ne->value > ne->adapt_thresh) { 277 ne->count = ne->num_frames_estimate; 278 } else { 279 ne->count++; 280 } 281 if (ne->count == ne->num_frames_estimate) { 282 // Reset counter and check noise level condition. 283 ne->num_frames_estimate = 30; 284 ne->count = 0; 285 ne->level = av1_noise_estimate_extract_level(ne); 286 #if CONFIG_AV1_TEMPORAL_DENOISING 287 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) 288 av1_denoiser_set_noise_level(cpi, ne->level); 289 #endif 290 } 291 } 292 #if CONFIG_AV1_TEMPORAL_DENOISING 293 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) 294 copy_frame(&cpi->denoiser.last_source, cpi->source); 295 #endif 296 }