global_motion_facade.c (19228B)
1 /* 2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include "aom_dsp/binary_codes_writer.h" 13 14 #include "aom_dsp/flow_estimation/corner_detect.h" 15 #include "aom_dsp/flow_estimation/flow_estimation.h" 16 #include "aom_dsp/pyramid.h" 17 #include "av1/common/warped_motion.h" 18 #include "av1/encoder/encoder.h" 19 #include "av1/encoder/ethread.h" 20 #include "av1/encoder/rdopt.h" 21 #include "av1/encoder/global_motion_facade.h" 22 23 // Range of model types to search 24 #define FIRST_GLOBAL_TRANS_TYPE ROTZOOM 25 #define LAST_GLOBAL_TRANS_TYPE ROTZOOM 26 27 // Computes the cost for the warp parameters. 28 static int gm_get_params_cost(const WarpedMotionParams *gm, 29 const WarpedMotionParams *ref_gm, int allow_hp) { 30 int params_cost = 0; 31 int trans_bits, trans_prec_diff; 32 switch (gm->wmtype) { 33 case AFFINE: 34 case ROTZOOM: 35 params_cost += aom_count_signed_primitive_refsubexpfin( 36 GM_ALPHA_MAX + 1, SUBEXPFIN_K, 37 (ref_gm->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS), 38 (gm->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS)); 39 params_cost += aom_count_signed_primitive_refsubexpfin( 40 GM_ALPHA_MAX + 1, SUBEXPFIN_K, 41 (ref_gm->wmmat[3] >> GM_ALPHA_PREC_DIFF), 42 (gm->wmmat[3] >> GM_ALPHA_PREC_DIFF)); 43 if (gm->wmtype >= AFFINE) { 44 params_cost += aom_count_signed_primitive_refsubexpfin( 45 GM_ALPHA_MAX + 1, SUBEXPFIN_K, 46 (ref_gm->wmmat[4] >> GM_ALPHA_PREC_DIFF), 47 (gm->wmmat[4] >> GM_ALPHA_PREC_DIFF)); 48 params_cost += aom_count_signed_primitive_refsubexpfin( 49 GM_ALPHA_MAX + 1, SUBEXPFIN_K, 50 (ref_gm->wmmat[5] >> GM_ALPHA_PREC_DIFF) - 51 (1 << GM_ALPHA_PREC_BITS), 52 (gm->wmmat[5] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS)); 53 } 54 AOM_FALLTHROUGH_INTENDED; 55 case TRANSLATION: 56 trans_bits = (gm->wmtype == TRANSLATION) 57 ? GM_ABS_TRANS_ONLY_BITS - !allow_hp 58 : GM_ABS_TRANS_BITS; 59 trans_prec_diff = (gm->wmtype == TRANSLATION) 60 ? GM_TRANS_ONLY_PREC_DIFF + !allow_hp 61 : GM_TRANS_PREC_DIFF; 62 params_cost += aom_count_signed_primitive_refsubexpfin( 63 (1 << trans_bits) + 1, SUBEXPFIN_K, 64 (ref_gm->wmmat[0] >> trans_prec_diff), 65 (gm->wmmat[0] >> trans_prec_diff)); 66 params_cost += aom_count_signed_primitive_refsubexpfin( 67 (1 << trans_bits) + 1, SUBEXPFIN_K, 68 (ref_gm->wmmat[1] >> trans_prec_diff), 69 (gm->wmmat[1] >> trans_prec_diff)); 70 AOM_FALLTHROUGH_INTENDED; 71 case IDENTITY: break; 72 default: assert(0); 73 } 74 return (params_cost << AV1_PROB_COST_SHIFT); 75 } 76 77 // For the given reference frame, computes the global motion parameters for 78 // different motion models and finds the best. 79 static inline void compute_global_motion_for_ref_frame( 80 AV1_COMP *cpi, struct aom_internal_error_info *error_info, 81 YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame, 82 MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w, 83 const int segment_map_h, const WarpedMotionParams *ref_params) { 84 AV1_COMMON *const cm = &cpi->common; 85 MACROBLOCKD *const xd = &cpi->td.mb.e_mbd; 86 int src_width = cpi->source->y_crop_width; 87 int src_height = cpi->source->y_crop_height; 88 int src_stride = cpi->source->y_stride; 89 assert(ref_buf[frame] != NULL); 90 int bit_depth = cpi->common.seq_params->bit_depth; 91 GlobalMotionMethod global_motion_method = default_global_motion_method; 92 int downsample_level = cpi->sf.gm_sf.downsample_level; 93 int num_refinements = cpi->sf.gm_sf.num_refinement_steps; 94 int gm_erroradv_tr_level = cpi->sf.gm_sf.gm_erroradv_tr_level; 95 bool mem_alloc_failed = false; 96 97 assert(gm_erroradv_tr_level < 2); 98 // Select the best model based on fractional error reduction. 99 // By initializing this to erroradv_tr, the same logic which is used to 100 // select the best model will automatically filter out any model which 101 // doesn't meet the required quality threshold 102 double best_erroradv = erroradv_tr[gm_erroradv_tr_level]; 103 for (TransformationType model = FIRST_GLOBAL_TRANS_TYPE; 104 model <= LAST_GLOBAL_TRANS_TYPE; ++model) { 105 if (!aom_compute_global_motion(model, cpi->source, ref_buf[frame], 106 bit_depth, global_motion_method, 107 downsample_level, motion_models, 108 RANSAC_NUM_MOTIONS, &mem_alloc_failed)) { 109 if (mem_alloc_failed) { 110 aom_internal_error(error_info, AOM_CODEC_MEM_ERROR, 111 "Failed to allocate global motion buffers"); 112 } 113 continue; 114 } 115 116 for (int i = 0; i < RANSAC_NUM_MOTIONS; ++i) { 117 if (motion_models[i].num_inliers == 0) continue; 118 119 WarpedMotionParams tmp_wm_params; 120 av1_convert_model_to_params(motion_models[i].params, &tmp_wm_params); 121 122 // Check that the generated model is warp-able 123 if (!av1_get_shear_params(&tmp_wm_params)) continue; 124 125 // Skip models that we won't use (IDENTITY or TRANSLATION) 126 // 127 // For IDENTITY type models, we don't need to evaluate anything because 128 // all the following logic is effectively comparing the estimated model 129 // to an identity model. 130 // 131 // For TRANSLATION type global motion models, gm_get_motion_vector() gives 132 // the wrong motion vector (see comments in that function for details). 133 // As translation-type models do not give much gain, we can avoid this bug 134 // by never choosing a TRANSLATION type model 135 if (tmp_wm_params.wmtype <= TRANSLATION) continue; 136 137 av1_compute_feature_segmentation_map( 138 segment_map, segment_map_w, segment_map_h, motion_models[i].inliers, 139 motion_models[i].num_inliers); 140 141 int64_t ref_frame_error = av1_segmented_frame_error( 142 is_cur_buf_hbd(xd), xd->bd, ref_buf[frame]->y_buffer, 143 ref_buf[frame]->y_stride, cpi->source->y_buffer, src_stride, 144 src_width, src_height, segment_map, segment_map_w); 145 146 if (ref_frame_error == 0) continue; 147 148 const int64_t warp_error = av1_refine_integerized_param( 149 &tmp_wm_params, tmp_wm_params.wmtype, is_cur_buf_hbd(xd), xd->bd, 150 ref_buf[frame]->y_buffer, ref_buf[frame]->y_crop_width, 151 ref_buf[frame]->y_crop_height, ref_buf[frame]->y_stride, 152 cpi->source->y_buffer, src_width, src_height, src_stride, 153 num_refinements, ref_frame_error, segment_map, segment_map_w, 154 erroradv_tr[gm_erroradv_tr_level]); 155 156 // av1_refine_integerized_param() can return a simpler model type than 157 // its input, so re-check model type here 158 if (tmp_wm_params.wmtype <= TRANSLATION) continue; 159 160 double erroradvantage = (double)warp_error / ref_frame_error; 161 162 // Check that the model signaling cost is not too high 163 if (!av1_is_enough_erroradvantage( 164 erroradvantage, 165 gm_get_params_cost(&tmp_wm_params, ref_params, 166 cm->features.allow_high_precision_mv), 167 erroradv_tr[gm_erroradv_tr_level])) { 168 continue; 169 } 170 171 if (erroradvantage < best_erroradv) { 172 best_erroradv = erroradvantage; 173 // Save the wm_params modified by 174 // av1_refine_integerized_param() rather than motion index to 175 // avoid rerunning refine() below. 176 memcpy(&(cm->global_motion[frame]), &tmp_wm_params, 177 sizeof(WarpedMotionParams)); 178 } 179 } 180 } 181 } 182 183 // Computes global motion for the given reference frame. 184 void av1_compute_gm_for_valid_ref_frames( 185 AV1_COMP *cpi, struct aom_internal_error_info *error_info, 186 YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame, 187 MotionModel *motion_models, uint8_t *segment_map, int segment_map_w, 188 int segment_map_h) { 189 AV1_COMMON *const cm = &cpi->common; 190 const WarpedMotionParams *ref_params = 191 cm->prev_frame ? &cm->prev_frame->global_motion[frame] 192 : &default_warp_params; 193 194 compute_global_motion_for_ref_frame(cpi, error_info, ref_buf, frame, 195 motion_models, segment_map, segment_map_w, 196 segment_map_h, ref_params); 197 } 198 199 // Loops over valid reference frames and computes global motion estimation. 200 static inline void compute_global_motion_for_references( 201 AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], 202 FrameDistPair reference_frame[REF_FRAMES - 1], int num_ref_frames, 203 MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w, 204 const int segment_map_h) { 205 AV1_COMMON *const cm = &cpi->common; 206 struct aom_internal_error_info *const error_info = 207 cpi->td.mb.e_mbd.error_info; 208 // Compute global motion w.r.t. reference frames starting from the nearest ref 209 // frame in a given direction. 210 for (int frame = 0; frame < num_ref_frames; frame++) { 211 int ref_frame = reference_frame[frame].frame; 212 av1_compute_gm_for_valid_ref_frames(cpi, error_info, ref_buf, ref_frame, 213 motion_models, segment_map, 214 segment_map_w, segment_map_h); 215 // If global motion w.r.t. current ref frame is 216 // INVALID/TRANSLATION/IDENTITY, skip the evaluation of global motion w.r.t 217 // the remaining ref frames in that direction. 218 if (cpi->sf.gm_sf.prune_ref_frame_for_gm_search && 219 cm->global_motion[ref_frame].wmtype <= TRANSLATION) 220 break; 221 } 222 } 223 224 // Compares the distance in 'a' and 'b'. Returns 1 if the frame corresponding to 225 // 'a' is farther, -1 if the frame corresponding to 'b' is farther, 0 otherwise. 226 static int compare_distance(const void *a, const void *b) { 227 const int diff = 228 ((FrameDistPair *)a)->distance - ((FrameDistPair *)b)->distance; 229 if (diff > 0) 230 return 1; 231 else if (diff < 0) 232 return -1; 233 return 0; 234 } 235 236 static int disable_gm_search_based_on_stats(const AV1_COMP *const cpi) { 237 int is_gm_present = 1; 238 239 // Check number of GM models only in GF groups with ARF frames. GM param 240 // estimation is always done in the case of GF groups with no ARF frames (flat 241 // gops) 242 if (cpi->ppi->gf_group.arf_index > -1) { 243 // valid_gm_model_found is initialized to INT32_MAX in the beginning of 244 // every GF group. 245 // Therefore, GM param estimation is always done for all frames until 246 // at least 1 frame each of ARF_UPDATE, INTNL_ARF_UPDATE and LF_UPDATE are 247 // encoded in a GF group For subsequent frames, GM param estimation is 248 // disabled, if no valid models have been found in all the three update 249 // types. 250 is_gm_present = (cpi->ppi->valid_gm_model_found[ARF_UPDATE] != 0) || 251 (cpi->ppi->valid_gm_model_found[INTNL_ARF_UPDATE] != 0) || 252 (cpi->ppi->valid_gm_model_found[LF_UPDATE] != 0); 253 } 254 return !is_gm_present; 255 } 256 257 // Prunes reference frames for global motion estimation based on the speed 258 // feature 'gm_search_type'. 259 static int do_gm_search_logic(SPEED_FEATURES *const sf, int frame) { 260 (void)frame; 261 switch (sf->gm_sf.gm_search_type) { 262 case GM_FULL_SEARCH: return 1; 263 case GM_REDUCED_REF_SEARCH_SKIP_L2_L3: 264 return !(frame == LAST2_FRAME || frame == LAST3_FRAME); 265 case GM_REDUCED_REF_SEARCH_SKIP_L2_L3_ARF2: 266 return !(frame == LAST2_FRAME || frame == LAST3_FRAME || 267 (frame == ALTREF2_FRAME)); 268 case GM_SEARCH_CLOSEST_REFS_ONLY: return 1; 269 case GM_DISABLE_SEARCH: return 0; 270 default: assert(0); 271 } 272 return 1; 273 } 274 275 // Populates valid reference frames in past/future directions in 276 // 'reference_frames' and their count in 'num_ref_frames'. 277 static inline void update_valid_ref_frames_for_gm( 278 AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], 279 FrameDistPair reference_frames[MAX_DIRECTIONS][REF_FRAMES - 1], 280 int *num_ref_frames) { 281 AV1_COMMON *const cm = &cpi->common; 282 int *num_past_ref_frames = &num_ref_frames[0]; 283 int *num_future_ref_frames = &num_ref_frames[1]; 284 const GF_GROUP *gf_group = &cpi->ppi->gf_group; 285 int ref_pruning_enabled = is_frame_eligible_for_ref_pruning( 286 gf_group, cpi->sf.inter_sf.selective_ref_frame, 1, cpi->gf_frame_index); 287 int cur_frame_gm_disabled = 0; 288 int pyr_lvl = cm->cur_frame->pyramid_level; 289 290 if (cpi->sf.gm_sf.disable_gm_search_based_on_stats) { 291 cur_frame_gm_disabled = disable_gm_search_based_on_stats(cpi); 292 } 293 294 for (int frame = ALTREF_FRAME; frame >= LAST_FRAME; --frame) { 295 const MV_REFERENCE_FRAME ref_frame[2] = { frame, NONE_FRAME }; 296 RefCntBuffer *buf = get_ref_frame_buf(cm, frame); 297 const int ref_disabled = 298 !(cpi->ref_frame_flags & av1_ref_frame_flag_list[frame]); 299 ref_buf[frame] = NULL; 300 cm->global_motion[frame] = default_warp_params; 301 // Skip global motion estimation for invalid ref frames 302 if (buf == NULL || 303 (ref_disabled && cpi->sf.hl_sf.recode_loop != DISALLOW_RECODE)) { 304 continue; 305 } else { 306 ref_buf[frame] = &buf->buf; 307 } 308 309 int prune_ref_frames = 310 ref_pruning_enabled && 311 prune_ref_by_selective_ref_frame(cpi, NULL, ref_frame, 312 cm->cur_frame->ref_display_order_hint); 313 int ref_pyr_lvl = buf->pyramid_level; 314 315 if (ref_buf[frame]->y_crop_width == cpi->source->y_crop_width && 316 ref_buf[frame]->y_crop_height == cpi->source->y_crop_height && 317 do_gm_search_logic(&cpi->sf, frame) && !prune_ref_frames && 318 ref_pyr_lvl <= pyr_lvl && !cur_frame_gm_disabled) { 319 assert(ref_buf[frame] != NULL); 320 const int relative_frame_dist = av1_encoder_get_relative_dist( 321 buf->display_order_hint, cm->cur_frame->display_order_hint); 322 // Populate past and future ref frames. 323 // reference_frames[0][] indicates past direction and 324 // reference_frames[1][] indicates future direction. 325 if (relative_frame_dist == 0) { 326 // Skip global motion estimation for frames at the same nominal instant. 327 // This will generally be either a "real" frame coded against a 328 // temporal filtered version, or a higher spatial layer coded against 329 // a lower spatial layer. In either case, the optimal motion model will 330 // be IDENTITY, so we don't need to search explicitly. 331 } else if (relative_frame_dist < 0) { 332 reference_frames[0][*num_past_ref_frames].distance = 333 abs(relative_frame_dist); 334 reference_frames[0][*num_past_ref_frames].frame = frame; 335 (*num_past_ref_frames)++; 336 } else { 337 reference_frames[1][*num_future_ref_frames].distance = 338 abs(relative_frame_dist); 339 reference_frames[1][*num_future_ref_frames].frame = frame; 340 (*num_future_ref_frames)++; 341 } 342 } 343 } 344 } 345 346 // Initializes parameters used for computing global motion. 347 static inline void setup_global_motion_info_params(AV1_COMP *cpi) { 348 GlobalMotionInfo *const gm_info = &cpi->gm_info; 349 YV12_BUFFER_CONFIG *source = cpi->source; 350 351 gm_info->segment_map_w = 352 (source->y_crop_width + WARP_ERROR_BLOCK - 1) >> WARP_ERROR_BLOCK_LOG; 353 gm_info->segment_map_h = 354 (source->y_crop_height + WARP_ERROR_BLOCK - 1) >> WARP_ERROR_BLOCK_LOG; 355 356 memset(gm_info->reference_frames, -1, 357 sizeof(gm_info->reference_frames[0][0]) * MAX_DIRECTIONS * 358 (REF_FRAMES - 1)); 359 av1_zero(gm_info->num_ref_frames); 360 361 // Populate ref_buf for valid ref frames in global motion 362 update_valid_ref_frames_for_gm(cpi, gm_info->ref_buf, 363 gm_info->reference_frames, 364 gm_info->num_ref_frames); 365 366 // Sort the past and future ref frames in the ascending order of their 367 // distance from the current frame. reference_frames[0] => past direction 368 // and reference_frames[1] => future direction. 369 qsort(gm_info->reference_frames[0], gm_info->num_ref_frames[0], 370 sizeof(gm_info->reference_frames[0][0]), compare_distance); 371 qsort(gm_info->reference_frames[1], gm_info->num_ref_frames[1], 372 sizeof(gm_info->reference_frames[1][0]), compare_distance); 373 374 if (cpi->sf.gm_sf.gm_search_type == GM_SEARCH_CLOSEST_REFS_ONLY) { 375 // Filter down to the nearest two ref frames. 376 // Prefer one past and one future ref over two past refs, even if 377 // the second past ref is closer 378 if (gm_info->num_ref_frames[1] > 0) { 379 gm_info->num_ref_frames[0] = AOMMIN(gm_info->num_ref_frames[0], 1); 380 gm_info->num_ref_frames[1] = AOMMIN(gm_info->num_ref_frames[1], 1); 381 } else { 382 gm_info->num_ref_frames[0] = AOMMIN(gm_info->num_ref_frames[0], 2); 383 } 384 } 385 } 386 387 // Computes global motion w.r.t. valid reference frames. 388 static inline void global_motion_estimation(AV1_COMP *cpi) { 389 GlobalMotionInfo *const gm_info = &cpi->gm_info; 390 GlobalMotionData *gm_data = &cpi->td.gm_data; 391 392 // Compute global motion w.r.t. past reference frames and future reference 393 // frames 394 for (int dir = 0; dir < MAX_DIRECTIONS; dir++) { 395 if (gm_info->num_ref_frames[dir] > 0) 396 compute_global_motion_for_references( 397 cpi, gm_info->ref_buf, gm_info->reference_frames[dir], 398 gm_info->num_ref_frames[dir], gm_data->motion_models, 399 gm_data->segment_map, gm_info->segment_map_w, gm_info->segment_map_h); 400 } 401 } 402 403 // Global motion estimation for the current frame is computed.This computation 404 // happens once per frame and the winner motion model parameters are stored in 405 // cm->cur_frame->global_motion. 406 void av1_compute_global_motion_facade(AV1_COMP *cpi) { 407 AV1_COMMON *const cm = &cpi->common; 408 GlobalMotionInfo *const gm_info = &cpi->gm_info; 409 410 if (cpi->oxcf.tool_cfg.enable_global_motion) { 411 if (cpi->gf_frame_index == 0) { 412 for (int i = 0; i < FRAME_UPDATE_TYPES; i++) { 413 cpi->ppi->valid_gm_model_found[i] = INT32_MAX; 414 #if CONFIG_FPMT_TEST 415 if (cpi->ppi->fpmt_unit_test_cfg == PARALLEL_SIMULATION_ENCODE) 416 cpi->ppi->temp_valid_gm_model_found[i] = INT32_MAX; 417 #endif 418 } 419 } 420 } 421 422 if (cpi->common.current_frame.frame_type == INTER_FRAME && cpi->source && 423 cpi->oxcf.tool_cfg.enable_global_motion && !gm_info->search_done && 424 cpi->sf.gm_sf.gm_search_type != GM_DISABLE_SEARCH) { 425 setup_global_motion_info_params(cpi); 426 // Terminate early if the total number of reference frames is zero. 427 if (cpi->gm_info.num_ref_frames[0] || cpi->gm_info.num_ref_frames[1]) { 428 gm_alloc_data(cpi, &cpi->td.gm_data); 429 if (cpi->mt_info.num_workers > 1) 430 av1_global_motion_estimation_mt(cpi); 431 else 432 global_motion_estimation(cpi); 433 gm_dealloc_data(&cpi->td.gm_data); 434 gm_info->search_done = 1; 435 } 436 } 437 memcpy(cm->cur_frame->global_motion, cm->global_motion, 438 sizeof(cm->cur_frame->global_motion)); 439 }