rdopt_utils.h (33867B)
1 /* 2 * Copyright (c) 2019, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #ifndef AOM_AV1_ENCODER_RDOPT_UTILS_H_ 13 #define AOM_AV1_ENCODER_RDOPT_UTILS_H_ 14 15 #include "aom/aom_integer.h" 16 #include "av1/encoder/block.h" 17 #include "av1/common/cfl.h" 18 #include "av1/common/pred_common.h" 19 #include "av1/encoder/rdopt_data_defs.h" 20 21 #ifdef __cplusplus 22 extern "C" { 23 #endif 24 25 #define MAX_REF_MV_SEARCH 3 26 #define MAX_TX_RD_GATE_LEVEL 5 27 #define INTER_INTRA_RD_THRESH_SCALE 9 28 #define INTER_INTRA_RD_THRESH_SHIFT 4 29 30 typedef struct { 31 PREDICTION_MODE mode; 32 MV_REFERENCE_FRAME ref_frame[2]; 33 } MODE_DEFINITION; 34 35 // This array defines the mapping from the enums in THR_MODES to the actual 36 // prediction modes and refrence frames 37 static const MODE_DEFINITION av1_mode_defs[MAX_MODES] = { 38 { NEARESTMV, { LAST_FRAME, NONE_FRAME } }, 39 { NEARESTMV, { LAST2_FRAME, NONE_FRAME } }, 40 { NEARESTMV, { LAST3_FRAME, NONE_FRAME } }, 41 { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } }, 42 { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } }, 43 { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } }, 44 { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } }, 45 46 { NEWMV, { LAST_FRAME, NONE_FRAME } }, 47 { NEWMV, { LAST2_FRAME, NONE_FRAME } }, 48 { NEWMV, { LAST3_FRAME, NONE_FRAME } }, 49 { NEWMV, { BWDREF_FRAME, NONE_FRAME } }, 50 { NEWMV, { ALTREF2_FRAME, NONE_FRAME } }, 51 { NEWMV, { ALTREF_FRAME, NONE_FRAME } }, 52 { NEWMV, { GOLDEN_FRAME, NONE_FRAME } }, 53 54 { NEARMV, { LAST_FRAME, NONE_FRAME } }, 55 { NEARMV, { LAST2_FRAME, NONE_FRAME } }, 56 { NEARMV, { LAST3_FRAME, NONE_FRAME } }, 57 { NEARMV, { BWDREF_FRAME, NONE_FRAME } }, 58 { NEARMV, { ALTREF2_FRAME, NONE_FRAME } }, 59 { NEARMV, { ALTREF_FRAME, NONE_FRAME } }, 60 { NEARMV, { GOLDEN_FRAME, NONE_FRAME } }, 61 62 { GLOBALMV, { LAST_FRAME, NONE_FRAME } }, 63 { GLOBALMV, { LAST2_FRAME, NONE_FRAME } }, 64 { GLOBALMV, { LAST3_FRAME, NONE_FRAME } }, 65 { GLOBALMV, { BWDREF_FRAME, NONE_FRAME } }, 66 { GLOBALMV, { ALTREF2_FRAME, NONE_FRAME } }, 67 { GLOBALMV, { ALTREF_FRAME, NONE_FRAME } }, 68 { GLOBALMV, { GOLDEN_FRAME, NONE_FRAME } }, 69 70 // TODO(zoeliu): May need to reconsider the order on the modes to check 71 72 { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, 73 { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, 74 { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, 75 { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 76 { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, 77 { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, 78 { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, 79 { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 80 { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } }, 81 { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } }, 82 { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } }, 83 { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 84 85 { NEAREST_NEARESTMV, { LAST_FRAME, LAST2_FRAME } }, 86 { NEAREST_NEARESTMV, { LAST_FRAME, LAST3_FRAME } }, 87 { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } }, 88 { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } }, 89 90 { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, 91 { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, 92 { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } }, 93 { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, 94 { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } }, 95 { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } }, 96 { GLOBAL_GLOBALMV, { LAST_FRAME, BWDREF_FRAME } }, 97 98 { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, 99 { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, 100 { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } }, 101 { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, 102 { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } }, 103 { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } }, 104 { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF_FRAME } }, 105 106 { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, 107 { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, 108 { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } }, 109 { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, 110 { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } }, 111 { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } }, 112 { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF_FRAME } }, 113 114 { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, 115 { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, 116 { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } }, 117 { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, 118 { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } }, 119 { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } }, 120 { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF_FRAME } }, 121 122 { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 123 { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 124 { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 125 { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 126 { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 127 { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 128 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF_FRAME } }, 129 130 { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, 131 { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, 132 { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } }, 133 { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, 134 { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } }, 135 { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } }, 136 { GLOBAL_GLOBALMV, { LAST2_FRAME, BWDREF_FRAME } }, 137 138 { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, 139 { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, 140 { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } }, 141 { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, 142 { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } }, 143 { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } }, 144 { GLOBAL_GLOBALMV, { LAST3_FRAME, BWDREF_FRAME } }, 145 146 { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 147 { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 148 { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 149 { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 150 { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 151 { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 152 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, BWDREF_FRAME } }, 153 154 { NEAR_NEARMV, { LAST_FRAME, ALTREF2_FRAME } }, 155 { NEW_NEWMV, { LAST_FRAME, ALTREF2_FRAME } }, 156 { NEW_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } }, 157 { NEAREST_NEWMV, { LAST_FRAME, ALTREF2_FRAME } }, 158 { NEW_NEARMV, { LAST_FRAME, ALTREF2_FRAME } }, 159 { NEAR_NEWMV, { LAST_FRAME, ALTREF2_FRAME } }, 160 { GLOBAL_GLOBALMV, { LAST_FRAME, ALTREF2_FRAME } }, 161 162 { NEAR_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } }, 163 { NEW_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } }, 164 { NEW_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } }, 165 { NEAREST_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } }, 166 { NEW_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } }, 167 { NEAR_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } }, 168 { GLOBAL_GLOBALMV, { LAST2_FRAME, ALTREF2_FRAME } }, 169 170 { NEAR_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } }, 171 { NEW_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } }, 172 { NEW_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } }, 173 { NEAREST_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } }, 174 { NEW_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } }, 175 { NEAR_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } }, 176 { GLOBAL_GLOBALMV, { LAST3_FRAME, ALTREF2_FRAME } }, 177 178 { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 179 { NEW_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 180 { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 181 { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 182 { NEW_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 183 { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 184 { GLOBAL_GLOBALMV, { GOLDEN_FRAME, ALTREF2_FRAME } }, 185 186 { NEAR_NEARMV, { LAST_FRAME, LAST2_FRAME } }, 187 { NEW_NEWMV, { LAST_FRAME, LAST2_FRAME } }, 188 { NEW_NEARESTMV, { LAST_FRAME, LAST2_FRAME } }, 189 { NEAREST_NEWMV, { LAST_FRAME, LAST2_FRAME } }, 190 { NEW_NEARMV, { LAST_FRAME, LAST2_FRAME } }, 191 { NEAR_NEWMV, { LAST_FRAME, LAST2_FRAME } }, 192 { GLOBAL_GLOBALMV, { LAST_FRAME, LAST2_FRAME } }, 193 194 { NEAR_NEARMV, { LAST_FRAME, LAST3_FRAME } }, 195 { NEW_NEWMV, { LAST_FRAME, LAST3_FRAME } }, 196 { NEW_NEARESTMV, { LAST_FRAME, LAST3_FRAME } }, 197 { NEAREST_NEWMV, { LAST_FRAME, LAST3_FRAME } }, 198 { NEW_NEARMV, { LAST_FRAME, LAST3_FRAME } }, 199 { NEAR_NEWMV, { LAST_FRAME, LAST3_FRAME } }, 200 { GLOBAL_GLOBALMV, { LAST_FRAME, LAST3_FRAME } }, 201 202 { NEAR_NEARMV, { LAST_FRAME, GOLDEN_FRAME } }, 203 { NEW_NEWMV, { LAST_FRAME, GOLDEN_FRAME } }, 204 { NEW_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } }, 205 { NEAREST_NEWMV, { LAST_FRAME, GOLDEN_FRAME } }, 206 { NEW_NEARMV, { LAST_FRAME, GOLDEN_FRAME } }, 207 { NEAR_NEWMV, { LAST_FRAME, GOLDEN_FRAME } }, 208 { GLOBAL_GLOBALMV, { LAST_FRAME, GOLDEN_FRAME } }, 209 210 { NEAR_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } }, 211 { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } }, 212 { NEW_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } }, 213 { NEAREST_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } }, 214 { NEW_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } }, 215 { NEAR_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } }, 216 { GLOBAL_GLOBALMV, { BWDREF_FRAME, ALTREF_FRAME } }, 217 218 // intra modes 219 { DC_PRED, { INTRA_FRAME, NONE_FRAME } }, 220 { PAETH_PRED, { INTRA_FRAME, NONE_FRAME } }, 221 { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } }, 222 { SMOOTH_V_PRED, { INTRA_FRAME, NONE_FRAME } }, 223 { SMOOTH_H_PRED, { INTRA_FRAME, NONE_FRAME } }, 224 { H_PRED, { INTRA_FRAME, NONE_FRAME } }, 225 { V_PRED, { INTRA_FRAME, NONE_FRAME } }, 226 { D135_PRED, { INTRA_FRAME, NONE_FRAME } }, 227 { D203_PRED, { INTRA_FRAME, NONE_FRAME } }, 228 { D157_PRED, { INTRA_FRAME, NONE_FRAME } }, 229 { D67_PRED, { INTRA_FRAME, NONE_FRAME } }, 230 { D113_PRED, { INTRA_FRAME, NONE_FRAME } }, 231 { D45_PRED, { INTRA_FRAME, NONE_FRAME } }, 232 }; 233 234 // Number of winner modes allowed for different values of the speed feature 235 // multi_winner_mode_type. 236 static const int winner_mode_count_allowed[MULTI_WINNER_MODE_LEVELS] = { 237 1, // MULTI_WINNER_MODE_OFF 238 2, // MULTI_WINNER_MODE_FAST 239 3 // MULTI_WINNER_MODE_DEFAULT 240 }; 241 242 static inline void restore_dst_buf(MACROBLOCKD *xd, const BUFFER_SET dst, 243 const int num_planes) { 244 for (int i = 0; i < num_planes; i++) { 245 xd->plane[i].dst.buf = dst.plane[i]; 246 xd->plane[i].dst.stride = dst.stride[i]; 247 } 248 } 249 250 static inline void swap_dst_buf(MACROBLOCKD *xd, const BUFFER_SET *dst_bufs[2], 251 int num_planes) { 252 const BUFFER_SET *buf0 = dst_bufs[0]; 253 dst_bufs[0] = dst_bufs[1]; 254 dst_bufs[1] = buf0; 255 restore_dst_buf(xd, *dst_bufs[0], num_planes); 256 } 257 258 /* clang-format on */ 259 // Calculate rd threshold based on ref best rd and relevant scaling factors 260 static inline int64_t get_rd_thresh_from_best_rd(int64_t ref_best_rd, 261 int mul_factor, 262 int div_factor) { 263 int64_t rd_thresh = ref_best_rd; 264 if (div_factor != 0) { 265 rd_thresh = ref_best_rd < (div_factor * (INT64_MAX / mul_factor)) 266 ? ((ref_best_rd / div_factor) * mul_factor) 267 : INT64_MAX; 268 } 269 return rd_thresh; 270 } 271 272 static inline THR_MODES get_prediction_mode_idx( 273 PREDICTION_MODE this_mode, MV_REFERENCE_FRAME ref_frame, 274 MV_REFERENCE_FRAME second_ref_frame) { 275 if (this_mode < INTRA_MODE_END) { 276 assert(ref_frame == INTRA_FRAME); 277 assert(second_ref_frame == NONE_FRAME); 278 return intra_to_mode_idx[this_mode - INTRA_MODE_START]; 279 } 280 if (this_mode >= SINGLE_INTER_MODE_START && 281 this_mode < SINGLE_INTER_MODE_END) { 282 assert((ref_frame > INTRA_FRAME) && (ref_frame <= ALTREF_FRAME)); 283 return single_inter_to_mode_idx[this_mode - SINGLE_INTER_MODE_START] 284 [ref_frame]; 285 } 286 if (this_mode >= COMP_INTER_MODE_START && this_mode < COMP_INTER_MODE_END && 287 second_ref_frame != NONE_FRAME) { 288 assert((ref_frame > INTRA_FRAME) && (ref_frame <= ALTREF_FRAME)); 289 assert((second_ref_frame > INTRA_FRAME) && 290 (second_ref_frame <= ALTREF_FRAME)); 291 return comp_inter_to_mode_idx[this_mode - COMP_INTER_MODE_START][ref_frame] 292 [second_ref_frame]; 293 } 294 assert(0); 295 return THR_INVALID; 296 } 297 298 static inline int inter_mode_data_block_idx(BLOCK_SIZE bsize) { 299 if (bsize == BLOCK_4X4 || bsize == BLOCK_4X8 || bsize == BLOCK_8X4 || 300 bsize == BLOCK_4X16 || bsize == BLOCK_16X4) { 301 return -1; 302 } 303 return 1; 304 } 305 306 // Get transform block visible dimensions cropped to the MI units. 307 static inline void get_txb_dimensions(const MACROBLOCKD *xd, int plane, 308 BLOCK_SIZE plane_bsize, int blk_row, 309 int blk_col, BLOCK_SIZE tx_bsize, 310 int *width, int *height, 311 int *visible_width, int *visible_height) { 312 assert(tx_bsize <= plane_bsize); 313 const int txb_height = block_size_high[tx_bsize]; 314 const int txb_width = block_size_wide[tx_bsize]; 315 const struct macroblockd_plane *const pd = &xd->plane[plane]; 316 317 // TODO(aconverse@google.com): Investigate using crop_width/height here rather 318 // than the MI size 319 if (xd->mb_to_bottom_edge >= 0) { 320 *visible_height = txb_height; 321 } else { 322 const int block_height = block_size_high[plane_bsize]; 323 const int block_rows = 324 (xd->mb_to_bottom_edge >> (3 + pd->subsampling_y)) + block_height; 325 *visible_height = 326 clamp(block_rows - (blk_row << MI_SIZE_LOG2), 0, txb_height); 327 } 328 if (height) *height = txb_height; 329 330 if (xd->mb_to_right_edge >= 0) { 331 *visible_width = txb_width; 332 } else { 333 const int block_width = block_size_wide[plane_bsize]; 334 const int block_cols = 335 (xd->mb_to_right_edge >> (3 + pd->subsampling_x)) + block_width; 336 *visible_width = 337 clamp(block_cols - (blk_col << MI_SIZE_LOG2), 0, txb_width); 338 } 339 if (width) *width = txb_width; 340 } 341 342 static inline int bsize_to_num_blk(BLOCK_SIZE bsize) { 343 int num_blk = 1 << (num_pels_log2_lookup[bsize] - 2 * MI_SIZE_LOG2); 344 return num_blk; 345 } 346 347 static inline int check_txfm_eval(MACROBLOCK *const x, BLOCK_SIZE bsize, 348 int64_t best_skip_rd, int64_t skip_rd, 349 int level, int is_luma_only) { 350 int eval_txfm = 1; 351 // Derive aggressiveness factor for gating the transform search 352 // Lower value indicates more aggressiveness. Be more conservative (high 353 // value) for (i) low quantizers (ii) regions where prediction is poor 354 const int scale[MAX_TX_RD_GATE_LEVEL + 1] = { INT_MAX, 4, 3, 2, 2, 1 }; 355 const int qslope = 2 * (!is_luma_only); 356 const int level_to_qindex_map[MAX_TX_RD_GATE_LEVEL + 1] = { 0, 0, 0, 357 80, 100, 140 }; 358 int aggr_factor = 4; 359 assert(level <= MAX_TX_RD_GATE_LEVEL); 360 const int pred_qindex_thresh = level_to_qindex_map[level]; 361 if (!is_luma_only && level <= 2) { 362 aggr_factor = 4 * AOMMAX(1, ROUND_POWER_OF_TWO((MAXQ - x->qindex) * qslope, 363 QINDEX_BITS)); 364 } 365 if ((best_skip_rd > 366 (x->source_variance << (num_pels_log2_lookup[bsize] + RDDIV_BITS))) && 367 (x->qindex >= pred_qindex_thresh)) 368 aggr_factor *= scale[level]; 369 // For level setting 1, be more conservative for non-luma-only case even when 370 // prediction is good. 371 else if ((level <= 1) && !is_luma_only) 372 aggr_factor = (aggr_factor >> 2) * 6; 373 374 // Be more conservative for luma only cases (called from compound type rd) 375 // since best_skip_rd is computed after and skip_rd is computed (with 8-bit 376 // prediction signals blended for WEDGE/DIFFWTD rather than 16-bit) before 377 // interpolation filter search 378 const int luma_mul[MAX_TX_RD_GATE_LEVEL + 1] = { 379 INT_MAX, 32, 29, 17, 17, 17 380 }; 381 int mul_factor = is_luma_only ? luma_mul[level] : 16; 382 int64_t rd_thresh = 383 (best_skip_rd == INT64_MAX) 384 ? best_skip_rd 385 : (int64_t)(best_skip_rd * aggr_factor * mul_factor >> 6); 386 if (skip_rd > rd_thresh) eval_txfm = 0; 387 return eval_txfm; 388 } 389 390 static TX_MODE select_tx_mode( 391 const AV1_COMMON *cm, const TX_SIZE_SEARCH_METHOD tx_size_search_method) { 392 if (cm->features.coded_lossless) return ONLY_4X4; 393 if (tx_size_search_method == USE_LARGESTALL) { 394 return TX_MODE_LARGEST; 395 } else { 396 assert(tx_size_search_method == USE_FULL_RD || 397 tx_size_search_method == USE_FAST_RD); 398 return TX_MODE_SELECT; 399 } 400 } 401 402 // Checks the conditions to disable winner mode processing 403 static inline int bypass_winner_mode_processing(const MACROBLOCK *const x, 404 const SPEED_FEATURES *sf, 405 int use_txfm_skip, 406 int actual_txfm_skip, 407 PREDICTION_MODE best_mode) { 408 const int prune_winner_mode_eval_level = 409 sf->winner_mode_sf.prune_winner_mode_eval_level; 410 411 // Disable winner mode processing for blocks with low source variance. 412 // The aggressiveness of this pruning logic reduces as qindex increases. 413 // The threshold decreases linearly from 64 as qindex varies from 0 to 255. 414 if (prune_winner_mode_eval_level == 1) { 415 const unsigned int src_var_thresh = 64 - 48 * x->qindex / (MAXQ + 1); 416 if (x->source_variance < src_var_thresh) return 1; 417 } else if (prune_winner_mode_eval_level == 2) { 418 // Skip winner mode processing of blocks for which transform turns out to be 419 // skip due to nature of eob alone except NEWMV mode. 420 if (!have_newmv_in_inter_mode(best_mode) && actual_txfm_skip) return 1; 421 } else if (prune_winner_mode_eval_level == 3) { 422 // Skip winner mode processing of blocks for which transform turns out to be 423 // skip except NEWMV mode and considered based on the quantizer. 424 // At high quantizers: Take conservative approach by considering transform 425 // skip based on eob alone. 426 // At low quantizers: Consider transform skip based on eob nature or RD cost 427 // evaluation. 428 const int is_txfm_skip = 429 x->qindex > 127 ? actual_txfm_skip : actual_txfm_skip || use_txfm_skip; 430 431 if (!have_newmv_in_inter_mode(best_mode) && is_txfm_skip) return 1; 432 } else if (prune_winner_mode_eval_level >= 4) { 433 // Do not skip winner mode evaluation at low quantizers if normal mode's 434 // transform search was too aggressive. 435 if (sf->rd_sf.perform_coeff_opt >= 5 && x->qindex <= 70) return 0; 436 437 if (use_txfm_skip || actual_txfm_skip) return 1; 438 } 439 440 return 0; 441 } 442 443 // Checks the conditions to enable winner mode processing 444 static inline int is_winner_mode_processing_enabled(const struct AV1_COMP *cpi, 445 const MACROBLOCK *const x, 446 MB_MODE_INFO *const mbmi, 447 int actual_txfm_skip) { 448 const SPEED_FEATURES *sf = &cpi->sf; 449 const PREDICTION_MODE best_mode = mbmi->mode; 450 451 if (bypass_winner_mode_processing(x, sf, mbmi->skip_txfm, actual_txfm_skip, 452 best_mode)) 453 return 0; 454 455 // TODO(any): Move block independent condition checks to frame level 456 if (is_inter_block(mbmi)) { 457 if (is_inter_mode(best_mode) && 458 (sf->tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh != INT_MAX) && 459 !cpi->oxcf.txfm_cfg.use_inter_dct_only) 460 return 1; 461 } else { 462 if (sf->tx_sf.tx_type_search.fast_intra_tx_type_search && 463 !cpi->oxcf.txfm_cfg.use_intra_default_tx_only && 464 !cpi->oxcf.txfm_cfg.use_intra_dct_only) 465 return 1; 466 } 467 468 // Check speed feature related to winner mode processing 469 if (sf->winner_mode_sf.enable_winner_mode_for_coeff_opt && 470 cpi->optimize_seg_arr[mbmi->segment_id] != NO_TRELLIS_OPT && 471 cpi->optimize_seg_arr[mbmi->segment_id] != FINAL_PASS_TRELLIS_OPT) 472 return 1; 473 if (sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch) return 1; 474 475 return 0; 476 } 477 478 static inline void set_tx_size_search_method( 479 const AV1_COMMON *cm, const WinnerModeParams *winner_mode_params, 480 TxfmSearchParams *txfm_params, int enable_winner_mode_for_tx_size_srch, 481 int is_winner_mode) { 482 // Populate transform size search method/transform mode appropriately 483 txfm_params->tx_size_search_method = 484 winner_mode_params->tx_size_search_methods[DEFAULT_EVAL]; 485 if (enable_winner_mode_for_tx_size_srch) { 486 if (is_winner_mode) 487 txfm_params->tx_size_search_method = 488 winner_mode_params->tx_size_search_methods[WINNER_MODE_EVAL]; 489 else 490 txfm_params->tx_size_search_method = 491 winner_mode_params->tx_size_search_methods[MODE_EVAL]; 492 } 493 txfm_params->tx_mode_search_type = 494 select_tx_mode(cm, txfm_params->tx_size_search_method); 495 } 496 497 static inline void set_tx_type_prune(const SPEED_FEATURES *sf, 498 TxfmSearchParams *txfm_params, 499 int winner_mode_tx_type_pruning, 500 int is_winner_mode) { 501 // Populate prune transform mode appropriately 502 txfm_params->prune_2d_txfm_mode = sf->tx_sf.tx_type_search.prune_2d_txfm_mode; 503 if (!winner_mode_tx_type_pruning) return; 504 505 const int prune_mode[4][2] = { { TX_TYPE_PRUNE_3, TX_TYPE_PRUNE_0 }, 506 { TX_TYPE_PRUNE_4, TX_TYPE_PRUNE_0 }, 507 { TX_TYPE_PRUNE_5, TX_TYPE_PRUNE_2 }, 508 { TX_TYPE_PRUNE_5, TX_TYPE_PRUNE_3 } }; 509 txfm_params->prune_2d_txfm_mode = 510 prune_mode[winner_mode_tx_type_pruning - 1][is_winner_mode]; 511 } 512 513 static inline void set_tx_domain_dist_params( 514 const WinnerModeParams *winner_mode_params, TxfmSearchParams *txfm_params, 515 int enable_winner_mode_for_tx_domain_dist, int is_winner_mode) { 516 if (txfm_params->use_qm_dist_metric) { 517 // QM-weighted PSNR is computed in transform space, so we need to forcibly 518 // enable the use of tx domain distortion. 519 txfm_params->use_transform_domain_distortion = 1; 520 txfm_params->tx_domain_dist_threshold = 0; 521 return; 522 } 523 524 if (!enable_winner_mode_for_tx_domain_dist) { 525 txfm_params->use_transform_domain_distortion = 526 winner_mode_params->use_transform_domain_distortion[DEFAULT_EVAL]; 527 txfm_params->tx_domain_dist_threshold = 528 winner_mode_params->tx_domain_dist_threshold[DEFAULT_EVAL]; 529 return; 530 } 531 532 if (is_winner_mode) { 533 txfm_params->use_transform_domain_distortion = 534 winner_mode_params->use_transform_domain_distortion[WINNER_MODE_EVAL]; 535 txfm_params->tx_domain_dist_threshold = 536 winner_mode_params->tx_domain_dist_threshold[WINNER_MODE_EVAL]; 537 } else { 538 txfm_params->use_transform_domain_distortion = 539 winner_mode_params->use_transform_domain_distortion[MODE_EVAL]; 540 txfm_params->tx_domain_dist_threshold = 541 winner_mode_params->tx_domain_dist_threshold[MODE_EVAL]; 542 } 543 } 544 545 // This function sets mode parameters for different mode evaluation stages 546 static inline void set_mode_eval_params(const struct AV1_COMP *cpi, 547 MACROBLOCK *x, 548 MODE_EVAL_TYPE mode_eval_type) { 549 const AV1_COMMON *cm = &cpi->common; 550 const SPEED_FEATURES *sf = &cpi->sf; 551 const WinnerModeParams *winner_mode_params = &cpi->winner_mode_params; 552 TxfmSearchParams *txfm_params = &x->txfm_search_params; 553 554 txfm_params->use_qm_dist_metric = 555 cpi->oxcf.tune_cfg.dist_metric == AOM_DIST_METRIC_QM_PSNR; 556 557 switch (mode_eval_type) { 558 case DEFAULT_EVAL: 559 txfm_params->default_inter_tx_type_prob_thresh = INT_MAX; 560 txfm_params->use_default_intra_tx_type = 0; 561 txfm_params->skip_txfm_level = 562 winner_mode_params->skip_txfm_level[DEFAULT_EVAL]; 563 txfm_params->predict_dc_level = 564 winner_mode_params->predict_dc_level[DEFAULT_EVAL]; 565 // Set default transform domain distortion type 566 set_tx_domain_dist_params(winner_mode_params, txfm_params, 0, 0); 567 568 // Get default threshold for R-D optimization of coefficients 569 get_rd_opt_coeff_thresh(winner_mode_params->coeff_opt_thresholds, 570 txfm_params, 0, 0); 571 572 // Set default transform size search method 573 set_tx_size_search_method(cm, winner_mode_params, txfm_params, 0, 0); 574 // Set default transform type prune 575 set_tx_type_prune(sf, txfm_params, 0, 0); 576 break; 577 case MODE_EVAL: 578 txfm_params->use_default_intra_tx_type = 579 (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search || 580 cpi->oxcf.txfm_cfg.use_intra_default_tx_only); 581 txfm_params->default_inter_tx_type_prob_thresh = 582 cpi->sf.tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh; 583 txfm_params->skip_txfm_level = 584 winner_mode_params->skip_txfm_level[MODE_EVAL]; 585 txfm_params->predict_dc_level = 586 winner_mode_params->predict_dc_level[MODE_EVAL]; 587 // Set transform domain distortion type for mode evaluation 588 set_tx_domain_dist_params( 589 winner_mode_params, txfm_params, 590 sf->winner_mode_sf.enable_winner_mode_for_use_tx_domain_dist, 0); 591 592 // Get threshold for R-D optimization of coefficients during mode 593 // evaluation 594 get_rd_opt_coeff_thresh( 595 winner_mode_params->coeff_opt_thresholds, txfm_params, 596 sf->winner_mode_sf.enable_winner_mode_for_coeff_opt, 0); 597 598 // Set the transform size search method for mode evaluation 599 set_tx_size_search_method( 600 cm, winner_mode_params, txfm_params, 601 sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch, 0); 602 // Set transform type prune for mode evaluation 603 set_tx_type_prune(sf, txfm_params, 604 sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning, 605 0); 606 break; 607 case WINNER_MODE_EVAL: 608 txfm_params->default_inter_tx_type_prob_thresh = INT_MAX; 609 txfm_params->use_default_intra_tx_type = 0; 610 txfm_params->skip_txfm_level = 611 winner_mode_params->skip_txfm_level[WINNER_MODE_EVAL]; 612 txfm_params->predict_dc_level = 613 winner_mode_params->predict_dc_level[WINNER_MODE_EVAL]; 614 615 // Set transform domain distortion type for winner mode evaluation 616 set_tx_domain_dist_params( 617 winner_mode_params, txfm_params, 618 sf->winner_mode_sf.enable_winner_mode_for_use_tx_domain_dist, 1); 619 620 // Get threshold for R-D optimization of coefficients for winner mode 621 // evaluation 622 get_rd_opt_coeff_thresh( 623 winner_mode_params->coeff_opt_thresholds, txfm_params, 624 sf->winner_mode_sf.enable_winner_mode_for_coeff_opt, 1); 625 626 // Set the transform size search method for winner mode evaluation 627 set_tx_size_search_method( 628 cm, winner_mode_params, txfm_params, 629 sf->winner_mode_sf.enable_winner_mode_for_tx_size_srch, 1); 630 // Set default transform type prune mode for winner mode evaluation 631 set_tx_type_prune(sf, txfm_params, 632 sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning, 633 1); 634 break; 635 default: assert(0); 636 } 637 638 // Rd record collected at a specific mode evaluation stage can not be used 639 // across other evaluation stages as the transform parameters are different. 640 // Hence, reset mb rd record whenever mode evaluation stage type changes. 641 if (txfm_params->mode_eval_type != mode_eval_type) 642 reset_mb_rd_record(x->txfm_search_info.mb_rd_record); 643 644 txfm_params->mode_eval_type = mode_eval_type; 645 } 646 647 // Similar to store_cfl_required(), but for use during the RDO process, 648 // where we haven't yet determined whether this block uses CfL. 649 static inline CFL_ALLOWED_TYPE store_cfl_required_rdo(const AV1_COMMON *cm, 650 const MACROBLOCK *x) { 651 const MACROBLOCKD *xd = &x->e_mbd; 652 653 if (cm->seq_params->monochrome || !xd->is_chroma_ref) return CFL_DISALLOWED; 654 655 if (!xd->is_chroma_ref) { 656 // For non-chroma-reference blocks, we should always store the luma pixels, 657 // in case the corresponding chroma-reference block uses CfL. 658 // Note that this can only happen for block sizes which are <8 on 659 // their shortest side, as otherwise they would be chroma reference 660 // blocks. 661 return CFL_ALLOWED; 662 } 663 664 // For chroma reference blocks, we should store data in the encoder iff we're 665 // allowed to try out CfL. 666 return is_cfl_allowed(xd); 667 } 668 669 static inline void init_sbuv_mode(MB_MODE_INFO *const mbmi) { 670 mbmi->uv_mode = UV_DC_PRED; 671 mbmi->palette_mode_info.palette_size[1] = 0; 672 } 673 674 // Store best mode stats for winner mode processing 675 static inline void store_winner_mode_stats( 676 const AV1_COMMON *const cm, MACROBLOCK *x, const MB_MODE_INFO *mbmi, 677 RD_STATS *rd_cost, RD_STATS *rd_cost_y, RD_STATS *rd_cost_uv, 678 THR_MODES mode_index, uint8_t *color_map, BLOCK_SIZE bsize, int64_t this_rd, 679 int multi_winner_mode_type, int txfm_search_done) { 680 WinnerModeStats *winner_mode_stats = x->winner_mode_stats; 681 int mode_idx = 0; 682 int is_palette_mode = mbmi->palette_mode_info.palette_size[PLANE_TYPE_Y] > 0; 683 // Mode stat is not required when multiwinner mode processing is disabled 684 if (multi_winner_mode_type == MULTI_WINNER_MODE_OFF) return; 685 // Ignore mode with maximum rd 686 if (this_rd == INT64_MAX) return; 687 // TODO(any): Winner mode processing is currently not applicable for palette 688 // mode in Inter frames. Clean-up the following code, once support is added 689 if (!frame_is_intra_only(cm) && is_palette_mode) return; 690 691 int max_winner_mode_count = winner_mode_count_allowed[multi_winner_mode_type]; 692 assert(x->winner_mode_count >= 0 && 693 x->winner_mode_count <= max_winner_mode_count); 694 695 if (x->winner_mode_count) { 696 // Find the mode which has higher rd cost than this_rd 697 for (mode_idx = 0; mode_idx < x->winner_mode_count; mode_idx++) 698 if (winner_mode_stats[mode_idx].rd > this_rd) break; 699 700 if (mode_idx == max_winner_mode_count) { 701 // No mode has higher rd cost than this_rd 702 return; 703 } else if (mode_idx < max_winner_mode_count - 1) { 704 // Create a slot for current mode and move others to the next slot 705 memmove( 706 &winner_mode_stats[mode_idx + 1], &winner_mode_stats[mode_idx], 707 (max_winner_mode_count - mode_idx - 1) * sizeof(*winner_mode_stats)); 708 } 709 } 710 // Add a mode stat for winner mode processing 711 winner_mode_stats[mode_idx].mbmi = *mbmi; 712 winner_mode_stats[mode_idx].rd = this_rd; 713 winner_mode_stats[mode_idx].mode_index = mode_index; 714 715 // Update rd stats required for inter frame 716 if (!frame_is_intra_only(cm) && rd_cost && rd_cost_y && rd_cost_uv) { 717 const MACROBLOCKD *xd = &x->e_mbd; 718 const int skip_ctx = av1_get_skip_txfm_context(xd); 719 const int is_intra_mode = av1_mode_defs[mode_index].mode < INTRA_MODE_END; 720 const int skip_txfm = mbmi->skip_txfm && !is_intra_mode; 721 722 winner_mode_stats[mode_idx].rd_cost = *rd_cost; 723 if (txfm_search_done) { 724 winner_mode_stats[mode_idx].rate_y = 725 rd_cost_y->rate + 726 x->mode_costs 727 .skip_txfm_cost[skip_ctx][rd_cost->skip_txfm || skip_txfm]; 728 winner_mode_stats[mode_idx].rate_uv = rd_cost_uv->rate; 729 } 730 } 731 732 if (color_map) { 733 // Store color_index_map for palette mode 734 const MACROBLOCKD *const xd = &x->e_mbd; 735 int block_width, block_height; 736 av1_get_block_dimensions(bsize, AOM_PLANE_Y, xd, &block_width, 737 &block_height, NULL, NULL); 738 memcpy(winner_mode_stats[mode_idx].color_index_map, color_map, 739 block_width * block_height * sizeof(color_map[0])); 740 } 741 742 x->winner_mode_count = 743 AOMMIN(x->winner_mode_count + 1, max_winner_mode_count); 744 } 745 746 unsigned int av1_get_perpixel_variance(const AV1_COMP *cpi, 747 const MACROBLOCKD *xd, 748 const struct buf_2d *ref, 749 BLOCK_SIZE bsize, int plane, 750 int use_hbd); 751 752 unsigned int av1_get_perpixel_variance_facade(const struct AV1_COMP *cpi, 753 const MACROBLOCKD *xd, 754 const struct buf_2d *ref, 755 BLOCK_SIZE bsize, int plane); 756 757 static inline int is_mode_intra(PREDICTION_MODE mode) { 758 return mode < INTRA_MODE_END; 759 } 760 761 // This function will copy usable ref_mv_stack[ref_frame][4] and 762 // weight[ref_frame][4] information from ref_mv_stack[ref_frame][8] and 763 // weight[ref_frame][8]. 764 static inline void av1_copy_usable_ref_mv_stack_and_weight( 765 const MACROBLOCKD *xd, MB_MODE_INFO_EXT *const mbmi_ext, 766 MV_REFERENCE_FRAME ref_frame) { 767 memcpy(mbmi_ext->weight[ref_frame], xd->weight[ref_frame], 768 USABLE_REF_MV_STACK_SIZE * sizeof(xd->weight[0][0])); 769 memcpy(mbmi_ext->ref_mv_stack[ref_frame], xd->ref_mv_stack[ref_frame], 770 USABLE_REF_MV_STACK_SIZE * sizeof(xd->ref_mv_stack[0][0])); 771 } 772 773 // Get transform rd gate level for the given transform search case. 774 static inline int get_txfm_rd_gate_level( 775 const int is_masked_compound_enabled, 776 const int txfm_rd_gate_level[TX_SEARCH_CASES], BLOCK_SIZE bsize, 777 TX_SEARCH_CASE tx_search_case, int eval_motion_mode) { 778 assert(tx_search_case < TX_SEARCH_CASES); 779 if (tx_search_case == TX_SEARCH_MOTION_MODE && !eval_motion_mode && 780 num_pels_log2_lookup[bsize] > 8) 781 return txfm_rd_gate_level[TX_SEARCH_MOTION_MODE]; 782 // Enable aggressive gating of transform search only when masked compound type 783 // is enabled. 784 else if (tx_search_case == TX_SEARCH_COMP_TYPE_MODE && 785 is_masked_compound_enabled) 786 return txfm_rd_gate_level[TX_SEARCH_COMP_TYPE_MODE]; 787 788 return txfm_rd_gate_level[TX_SEARCH_DEFAULT]; 789 } 790 791 #ifdef __cplusplus 792 } // extern "C" 793 #endif 794 795 #endif // AOM_AV1_ENCODER_RDOPT_UTILS_H_