superres_scale.c (17233B)
1 /* 2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include "av1/encoder/encoder_alloc.h" 13 #include "av1/encoder/superres_scale.h" 14 #include "av1/encoder/random.h" 15 16 // Compute the horizontal frequency components' energy in a frame 17 // by calculuating the 16x4 Horizontal DCT. This is to be used to 18 // decide the superresolution parameters. 19 static void analyze_hor_freq(const AV1_COMP *cpi, double *energy) { 20 uint64_t freq_energy[16] = { 0 }; 21 const YV12_BUFFER_CONFIG *buf = cpi->source; 22 const int bd = cpi->td.mb.e_mbd.bd; 23 const int width = buf->y_crop_width; 24 const int height = buf->y_crop_height; 25 DECLARE_ALIGNED(16, int32_t, coeff[16 * 4]); 26 int n = 0; 27 memset(freq_energy, 0, sizeof(freq_energy)); 28 if (buf->flags & YV12_FLAG_HIGHBITDEPTH) { 29 const int16_t *src16 = (const int16_t *)CONVERT_TO_SHORTPTR(buf->y_buffer); 30 for (int i = 0; i < height - 4; i += 4) { 31 for (int j = 0; j < width - 16; j += 16) { 32 av1_fwd_txfm2d_16x4(src16 + i * buf->y_stride + j, coeff, buf->y_stride, 33 H_DCT, bd); 34 for (int k = 1; k < 16; ++k) { 35 const uint64_t this_energy = 36 ((int64_t)coeff[k] * coeff[k]) + 37 ((int64_t)coeff[k + 16] * coeff[k + 16]) + 38 ((int64_t)coeff[k + 32] * coeff[k + 32]) + 39 ((int64_t)coeff[k + 48] * coeff[k + 48]); 40 freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2 + 2 * (bd - 8)); 41 } 42 n++; 43 } 44 } 45 } else { 46 assert(bd == 8); 47 DECLARE_ALIGNED(16, int16_t, src16[16 * 4]); 48 for (int i = 0; i < height - 4; i += 4) { 49 for (int j = 0; j < width - 16; j += 16) { 50 for (int ii = 0; ii < 4; ++ii) 51 for (int jj = 0; jj < 16; ++jj) 52 src16[ii * 16 + jj] = 53 buf->y_buffer[(i + ii) * buf->y_stride + (j + jj)]; 54 av1_fwd_txfm2d_16x4(src16, coeff, 16, H_DCT, bd); 55 for (int k = 1; k < 16; ++k) { 56 const uint64_t this_energy = 57 ((int64_t)coeff[k] * coeff[k]) + 58 ((int64_t)coeff[k + 16] * coeff[k + 16]) + 59 ((int64_t)coeff[k + 32] * coeff[k + 32]) + 60 ((int64_t)coeff[k + 48] * coeff[k + 48]); 61 freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2); 62 } 63 n++; 64 } 65 } 66 } 67 if (n) { 68 for (int k = 1; k < 16; ++k) energy[k] = (double)freq_energy[k] / n; 69 // Convert to cumulative energy 70 for (int k = 14; k > 0; --k) energy[k] += energy[k + 1]; 71 } else { 72 for (int k = 1; k < 16; ++k) energy[k] = 1e+20; 73 } 74 } 75 76 static uint8_t calculate_next_resize_scale(const AV1_COMP *cpi) { 77 // Choose an arbitrary random number 78 static unsigned int seed = 56789; 79 const ResizeCfg *resize_cfg = &cpi->oxcf.resize_cfg; 80 if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR; 81 uint8_t new_denom = SCALE_NUMERATOR; 82 83 if (cpi->common.seq_params->reduced_still_picture_hdr) return SCALE_NUMERATOR; 84 switch (resize_cfg->resize_mode) { 85 case RESIZE_NONE: new_denom = SCALE_NUMERATOR; break; 86 case RESIZE_FIXED: 87 if (cpi->common.current_frame.frame_type == KEY_FRAME) 88 new_denom = resize_cfg->resize_kf_scale_denominator; 89 else 90 new_denom = resize_cfg->resize_scale_denominator; 91 break; 92 case RESIZE_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break; 93 default: assert(0); 94 } 95 return new_denom; 96 } 97 98 int av1_superres_in_recode_allowed(const AV1_COMP *const cpi) { 99 const AV1EncoderConfig *const oxcf = &cpi->oxcf; 100 // Empirically found to not be beneficial for image coding. 101 return oxcf->superres_cfg.superres_mode == AOM_SUPERRES_AUTO && 102 cpi->sf.hl_sf.superres_auto_search_type != SUPERRES_AUTO_SOLO && 103 cpi->rc.frames_to_key > 1; 104 } 105 106 #define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO 0.012 107 #define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME 0.008 108 #define SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME 0.008 109 #define SUPERRES_ENERGY_BY_AC_THRESH 0.2 110 111 static double get_energy_by_q2_thresh(const GF_GROUP *gf_group, 112 const RATE_CONTROL *rc, 113 int gf_frame_index) { 114 // TODO(now): Return keyframe thresh * factor based on frame type / pyramid 115 // level. 116 if (gf_group->update_type[gf_frame_index] == ARF_UPDATE) { 117 return SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME; 118 } else if (gf_group->update_type[gf_frame_index] == KF_UPDATE) { 119 if (rc->frames_to_key <= 1) 120 return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO; 121 else 122 return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME; 123 } else { 124 assert(0); 125 } 126 return 0; 127 } 128 129 static uint8_t get_superres_denom_from_qindex_energy(int qindex, double *energy, 130 double threshq, 131 double threshp) { 132 const double q = av1_convert_qindex_to_q(qindex, AOM_BITS_8); 133 const double tq = threshq * q * q; 134 const double tp = threshp * energy[1]; 135 const double thresh = AOMMIN(tq, tp); 136 int k; 137 for (k = SCALE_NUMERATOR * 2; k > SCALE_NUMERATOR; --k) { 138 if (energy[k - 1] > thresh) break; 139 } 140 return 3 * SCALE_NUMERATOR - k; 141 } 142 143 static uint8_t get_superres_denom_for_qindex(const AV1_COMP *cpi, int qindex, 144 int sr_kf, int sr_arf) { 145 // Use superres for Key-frames and Alt-ref frames only. 146 const GF_GROUP *gf_group = &cpi->ppi->gf_group; 147 if (gf_group->update_type[cpi->gf_frame_index] != KF_UPDATE && 148 gf_group->update_type[cpi->gf_frame_index] != ARF_UPDATE) { 149 return SCALE_NUMERATOR; 150 } 151 if (gf_group->update_type[cpi->gf_frame_index] == KF_UPDATE && !sr_kf) { 152 return SCALE_NUMERATOR; 153 } 154 if (gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE && !sr_arf) { 155 return SCALE_NUMERATOR; 156 } 157 158 double energy[16]; 159 analyze_hor_freq(cpi, energy); 160 161 const double energy_by_q2_thresh = 162 get_energy_by_q2_thresh(gf_group, &cpi->rc, cpi->gf_frame_index); 163 int denom = get_superres_denom_from_qindex_energy( 164 qindex, energy, energy_by_q2_thresh, SUPERRES_ENERGY_BY_AC_THRESH); 165 /* 166 printf("\nenergy = ["); 167 for (int k = 1; k < 16; ++k) printf("%f, ", energy[k]); 168 printf("]\n"); 169 printf("boost = %d\n", 170 (gf_group->update_type[cpi->gf_frame_index] == KF_UPDATE) 171 ? cpi->ppi->p_rc.kf_boost 172 : cpi->rc.gfu_boost); 173 printf("denom = %d\n", denom); 174 */ 175 if (av1_superres_in_recode_allowed(cpi)) { 176 assert(cpi->superres_mode != AOM_SUPERRES_NONE); 177 // Force superres to be tried in the recode loop, as full-res is also going 178 // to be tried anyway. 179 denom = AOMMAX(denom, SCALE_NUMERATOR + 1); 180 } 181 return denom; 182 } 183 184 static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) { 185 // Choose an arbitrary random number 186 static unsigned int seed = 34567; 187 const AV1EncoderConfig *oxcf = &cpi->oxcf; 188 const SuperResCfg *const superres_cfg = &oxcf->superres_cfg; 189 const FrameDimensionCfg *const frm_dim_cfg = &oxcf->frm_dim_cfg; 190 const RateControlCfg *const rc_cfg = &oxcf->rc_cfg; 191 192 if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR; 193 uint8_t new_denom = SCALE_NUMERATOR; 194 195 // Make sure that superres mode of the frame is consistent with the 196 // sequence-level flag. 197 assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_NONE, 198 cpi->common.seq_params->enable_superres)); 199 assert(IMPLIES(!cpi->common.seq_params->enable_superres, 200 superres_cfg->superres_mode == AOM_SUPERRES_NONE)); 201 // Make sure that superres mode for current encoding is consistent with user 202 // provided superres mode. 203 assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_AUTO, 204 cpi->superres_mode == superres_cfg->superres_mode)); 205 206 // Note: we must look at the current superres_mode to be tried in 'cpi' here, 207 // not the user given mode in 'oxcf'. 208 switch (cpi->superres_mode) { 209 case AOM_SUPERRES_NONE: new_denom = SCALE_NUMERATOR; break; 210 case AOM_SUPERRES_FIXED: 211 if (cpi->common.current_frame.frame_type == KEY_FRAME) 212 new_denom = superres_cfg->superres_kf_scale_denominator; 213 else 214 new_denom = superres_cfg->superres_scale_denominator; 215 break; 216 case AOM_SUPERRES_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break; 217 case AOM_SUPERRES_QTHRESH: { 218 // Do not use superres when screen content tools are used. 219 if (cpi->common.features.allow_screen_content_tools) break; 220 if (rc_cfg->mode == AOM_VBR || rc_cfg->mode == AOM_CQ) 221 av1_set_target_rate(cpi, frm_dim_cfg->width, frm_dim_cfg->height); 222 223 // Now decide the use of superres based on 'q'. 224 int bottom_index, top_index; 225 const int q = av1_rc_pick_q_and_bounds( 226 cpi, frm_dim_cfg->width, frm_dim_cfg->height, cpi->gf_frame_index, 227 &bottom_index, &top_index); 228 229 const int qthresh = (frame_is_intra_only(&cpi->common)) 230 ? superres_cfg->superres_kf_qthresh 231 : superres_cfg->superres_qthresh; 232 if (q <= qthresh) { 233 new_denom = SCALE_NUMERATOR; 234 } else { 235 new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1); 236 } 237 break; 238 } 239 case AOM_SUPERRES_AUTO: { 240 if (cpi->common.features.allow_screen_content_tools) break; 241 if (rc_cfg->mode == AOM_VBR || rc_cfg->mode == AOM_CQ) 242 av1_set_target_rate(cpi, frm_dim_cfg->width, frm_dim_cfg->height); 243 244 // Now decide the use of superres based on 'q'. 245 int bottom_index, top_index; 246 const int q = av1_rc_pick_q_and_bounds( 247 cpi, frm_dim_cfg->width, frm_dim_cfg->height, cpi->gf_frame_index, 248 &bottom_index, &top_index); 249 250 const SUPERRES_AUTO_SEARCH_TYPE sr_search_type = 251 cpi->sf.hl_sf.superres_auto_search_type; 252 const int qthresh = (sr_search_type == SUPERRES_AUTO_SOLO) ? 128 : 0; 253 if (q <= qthresh) { 254 new_denom = SCALE_NUMERATOR; // Don't use superres. 255 } else { 256 if (sr_search_type == SUPERRES_AUTO_ALL) { 257 if (cpi->common.current_frame.frame_type == KEY_FRAME) 258 new_denom = superres_cfg->superres_kf_scale_denominator; 259 else 260 new_denom = superres_cfg->superres_scale_denominator; 261 } else { 262 new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1); 263 } 264 } 265 break; 266 } 267 default: assert(0); 268 } 269 return new_denom; 270 } 271 272 static int dimension_is_ok(int orig_dim, int resized_dim, int denom) { 273 return (resized_dim * SCALE_NUMERATOR >= orig_dim * denom / 2); 274 } 275 276 static int dimensions_are_ok(int owidth, int oheight, size_params_type *rsz) { 277 // Only need to check the width, as scaling is horizontal only. 278 (void)oheight; 279 return dimension_is_ok(owidth, rsz->resize_width, rsz->superres_denom); 280 } 281 282 static int validate_size_scales(RESIZE_MODE resize_mode, 283 aom_superres_mode superres_mode, int owidth, 284 int oheight, size_params_type *rsz) { 285 if (dimensions_are_ok(owidth, oheight, rsz)) { // Nothing to do. 286 return 1; 287 } 288 289 // Calculate current resize scale. 290 int resize_denom = 291 AOMMAX(DIVIDE_AND_ROUND(owidth * SCALE_NUMERATOR, rsz->resize_width), 292 DIVIDE_AND_ROUND(oheight * SCALE_NUMERATOR, rsz->resize_height)); 293 294 if (resize_mode != RESIZE_RANDOM && superres_mode == AOM_SUPERRES_RANDOM) { 295 // Alter superres scale as needed to enforce conformity. 296 rsz->superres_denom = 297 (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / resize_denom; 298 if (!dimensions_are_ok(owidth, oheight, rsz)) { 299 if (rsz->superres_denom > SCALE_NUMERATOR) --rsz->superres_denom; 300 } 301 } else if (resize_mode == RESIZE_RANDOM && 302 superres_mode != AOM_SUPERRES_RANDOM) { 303 // Alter resize scale as needed to enforce conformity. 304 resize_denom = 305 (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / rsz->superres_denom; 306 rsz->resize_width = owidth; 307 rsz->resize_height = oheight; 308 av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height, 309 resize_denom); 310 if (!dimensions_are_ok(owidth, oheight, rsz)) { 311 if (resize_denom > SCALE_NUMERATOR) { 312 --resize_denom; 313 rsz->resize_width = owidth; 314 rsz->resize_height = oheight; 315 av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height, 316 resize_denom); 317 } 318 } 319 } else if (resize_mode == RESIZE_RANDOM && 320 superres_mode == AOM_SUPERRES_RANDOM) { 321 // Alter both resize and superres scales as needed to enforce conformity. 322 do { 323 if (resize_denom > rsz->superres_denom) 324 --resize_denom; 325 else 326 --rsz->superres_denom; 327 rsz->resize_width = owidth; 328 rsz->resize_height = oheight; 329 av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height, 330 resize_denom); 331 } while (!dimensions_are_ok(owidth, oheight, rsz) && 332 (resize_denom > SCALE_NUMERATOR || 333 rsz->superres_denom > SCALE_NUMERATOR)); 334 } else { // We are allowed to alter neither resize scale nor superres 335 // scale. 336 return 0; 337 } 338 return dimensions_are_ok(owidth, oheight, rsz); 339 } 340 341 // Calculates resize and superres params for next frame 342 static size_params_type calculate_next_size_params(AV1_COMP *cpi) { 343 const AV1EncoderConfig *oxcf = &cpi->oxcf; 344 ResizePendingParams *resize_pending_params = &cpi->resize_pending_params; 345 const FrameDimensionCfg *const frm_dim_cfg = &oxcf->frm_dim_cfg; 346 size_params_type rsz = { frm_dim_cfg->width, frm_dim_cfg->height, 347 SCALE_NUMERATOR }; 348 int resize_denom = SCALE_NUMERATOR; 349 if (has_no_stats_stage(cpi) && cpi->ppi->use_svc && 350 (cpi->common.width != cpi->oxcf.frm_dim_cfg.width || 351 cpi->common.height != cpi->oxcf.frm_dim_cfg.height)) { 352 rsz.resize_width = cpi->common.width; 353 rsz.resize_height = cpi->common.height; 354 return rsz; 355 } 356 if (is_stat_generation_stage(cpi)) return rsz; 357 if (resize_pending_params->width && resize_pending_params->height) { 358 rsz.resize_width = resize_pending_params->width; 359 rsz.resize_height = resize_pending_params->height; 360 resize_pending_params->width = resize_pending_params->height = 0; 361 if (oxcf->superres_cfg.superres_mode == AOM_SUPERRES_NONE) return rsz; 362 } else { 363 resize_denom = calculate_next_resize_scale(cpi); 364 rsz.resize_width = frm_dim_cfg->width; 365 rsz.resize_height = frm_dim_cfg->height; 366 av1_calculate_scaled_size(&rsz.resize_width, &rsz.resize_height, 367 resize_denom); 368 } 369 rsz.superres_denom = calculate_next_superres_scale(cpi); 370 if (!validate_size_scales(oxcf->resize_cfg.resize_mode, cpi->superres_mode, 371 frm_dim_cfg->width, frm_dim_cfg->height, &rsz)) 372 assert(0 && "Invalid scale parameters"); 373 return rsz; 374 } 375 376 static void setup_frame_size_from_params(AV1_COMP *cpi, 377 const size_params_type *rsz) { 378 int encode_width = rsz->resize_width; 379 int encode_height = rsz->resize_height; 380 381 AV1_COMMON *cm = &cpi->common; 382 cm->superres_upscaled_width = encode_width; 383 cm->superres_upscaled_height = encode_height; 384 cm->superres_scale_denominator = rsz->superres_denom; 385 av1_calculate_scaled_superres_size(&encode_width, &encode_height, 386 rsz->superres_denom); 387 av1_set_frame_size(cpi, encode_width, encode_height); 388 } 389 390 void av1_setup_frame_size(AV1_COMP *cpi) { 391 AV1_COMMON *cm = &cpi->common; 392 // Reset superres params from previous frame. 393 cm->superres_scale_denominator = SCALE_NUMERATOR; 394 const size_params_type rsz = calculate_next_size_params(cpi); 395 setup_frame_size_from_params(cpi, &rsz); 396 397 assert(av1_is_min_tile_width_satisfied(cm)); 398 } 399 400 void av1_superres_post_encode(AV1_COMP *cpi) { 401 AV1_COMMON *cm = &cpi->common; 402 403 assert(cpi->oxcf.superres_cfg.enable_superres); 404 assert(!is_lossless_requested(&cpi->oxcf.rc_cfg)); 405 assert(!cm->features.all_lossless); 406 407 av1_superres_upscale(cm, NULL, cpi->alloc_pyramid); 408 409 // If regular resizing is occurring the source will need to be downscaled to 410 // match the upscaled superres resolution. Otherwise the original source is 411 // used. 412 if (!av1_resize_scaled(cm)) { 413 cpi->source = cpi->unscaled_source; 414 if (cpi->last_source != NULL) cpi->last_source = cpi->unscaled_last_source; 415 } else { 416 assert(cpi->unscaled_source->y_crop_width != cm->superres_upscaled_width); 417 assert(cpi->unscaled_source->y_crop_height != cm->superres_upscaled_height); 418 // Do downscale. cm->(width|height) has been updated by 419 // av1_superres_upscale 420 cpi->source = realloc_and_scale_source(cpi, cm->superres_upscaled_width, 421 cm->superres_upscaled_height); 422 } 423 }