tx_search.h (9599B)
1 /* 2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #ifndef AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_ 13 #define AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_ 14 15 #include "av1/common/pred_common.h" 16 #include "av1/encoder/encoder.h" 17 18 #ifdef __cplusplus 19 extern "C" { 20 #endif 21 22 // Set this macro as 1 to collect data about tx size selection. 23 #define COLLECT_TX_SIZE_DATA 0 24 25 #if COLLECT_TX_SIZE_DATA 26 static const char av1_tx_size_data_output_file[] = "tx_size_data.txt"; 27 #endif 28 29 enum { 30 FTXS_NONE = 0, 31 FTXS_DCT_AND_1D_DCT_ONLY = 1 << 0, 32 FTXS_DISABLE_TRELLIS_OPT = 1 << 1, 33 FTXS_USE_TRANSFORM_DOMAIN = 1 << 2 34 } UENUM1BYTE(FAST_TX_SEARCH_MODE); 35 36 static inline int tx_size_cost(const MACROBLOCK *const x, BLOCK_SIZE bsize, 37 TX_SIZE tx_size) { 38 assert(bsize == x->e_mbd.mi[0]->bsize); 39 if (x->txfm_search_params.tx_mode_search_type != TX_MODE_SELECT || 40 !block_signals_txsize(bsize)) 41 return 0; 42 43 const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize); 44 const int depth = tx_size_to_depth(tx_size, bsize); 45 const MACROBLOCKD *const xd = &x->e_mbd; 46 const int tx_size_ctx = get_tx_size_context(xd); 47 return x->mode_costs.tx_size_cost[tx_size_cat][tx_size_ctx][depth]; 48 } 49 50 /*!\brief Compute the pixel domain distortion. 51 * 52 * \ingroup transform_search 53 * Compute the pixel domain distortion from diff on all visible 4x4s in the 54 * transform block. 55 * 56 * \param[in] x Pointer to structure holding the data for the 57 current encoding macroblock 58 * \param[in] plane Plane index 59 * \param[in] blk_row Block row index 60 * \param[in] blk_col Block col index 61 * \param[in] plane_bsize Current plane block size 62 * \param[in] tx_bsize Transform size 63 * \param[in] block_mse_q8 Block mse 64 * \return An int64_t value that is the block sse. 65 */ 66 int64_t av1_pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row, 67 int blk_col, const BLOCK_SIZE plane_bsize, 68 const BLOCK_SIZE tx_bsize, 69 unsigned int *block_mse_q8); 70 71 int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, 72 RD_STATS *rd_stats, int64_t ref_best_rd, 73 BLOCK_SIZE bs, TX_SIZE tx_size); 74 75 /*!\brief Recursive transform size and type search. 76 * 77 * \ingroup transform_search 78 * Search for best transform size and type for luma inter blocks. The transform 79 * block partitioning can be recursive resulting in non-uniform transform sizes. 80 * The best transform size and type, if found, will be saved in the MB_MODE_INFO 81 * structure, and the corresponding RD stats will be saved in rd_stats. 82 * 83 * \param[in] cpi Top-level encoder structure 84 * \param[in] x Pointer to structure holding the data for the 85 current encoding macroblock 86 * \param[in] rd_stats Pointer to struct to keep track of the RD stats 87 * \param[in] bsize Current macroblock size 88 * \param[in] ref_best_rd Best RD cost seen for this block so far 89 * \remark Nothing is returned. The selected transform size and type will 90 be saved in the MB_MODE_INFO structure 91 */ 92 void av1_pick_recursive_tx_size_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, 93 RD_STATS *rd_stats, BLOCK_SIZE bsize, 94 int64_t ref_best_rd); 95 96 /*!\brief Uniform transform size and type search. 97 * 98 * \ingroup transform_search 99 * Search for the best transform size and type for current macroblock block, 100 * with the assumption that all the transform blocks have a uniform size 101 * (VP9 style). The selected transform size and type will be saved in the 102 * MB_MODE_INFO structure; the corresponding RD stats will be saved in rd_stats. 103 * This function may be used for both intra and inter predicted blocks. 104 * 105 * \param[in] cpi Top-level encoder structure 106 * \param[in] x Pointer to structure holding the data for the 107 current encoding macroblock 108 * \param[in] rd_stats Pointer to struct to keep track of the RD stats 109 * \param[in] bs Current macroblock size 110 * \param[in] ref_best_rd Best RD cost seen for this block so far 111 * \remark Nothing is returned. The selected transform size and type will 112 be saved in the MB_MODE_INFO structure 113 */ 114 void av1_pick_uniform_tx_size_type_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, 115 RD_STATS *rd_stats, BLOCK_SIZE bs, 116 int64_t ref_best_rd); 117 118 /*!\brief Chroma block transform search. 119 * 120 * \ingroup transform_search 121 * Calculate the transform coefficient RD cost for the given chroma macroblock 122 * If the current mode is intra, then this function will compute the predictor. 123 * 124 * \param[in] cpi Top-level encoder structure 125 * \param[in] x Pointer to structure holding the data for the 126 current encoding macroblock 127 * \param[in] rd_stats Pointer to struct to keep track of the RD stats 128 * \param[in] bsize Current macroblock size 129 * \param[in] ref_best_rd Best RD cost seen for this block so far 130 * \return An integer value is returned. 0: early termination triggered, 131 no valid rd cost available; 1: rd cost values are valid. 132 */ 133 int av1_txfm_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, 134 BLOCK_SIZE bsize, int64_t ref_best_rd); 135 136 /*!\brief Transform type search with fixed transform size. 137 * 138 * \ingroup transform_search 139 * Search for the best transform type and calculate the transform coefficients 140 * RD cost of the current transform block with the specified (uniform) transform 141 * size and plane. The RD results will be saved in rd_stats. 142 * 143 * \param[in] x Pointer to structure holding the data for the 144 current encoding macroblock 145 * \param[in] cpi Top-level encoder structure 146 * \param[in] rd_stats Pointer to struct to keep track of the RD stats 147 * \param[in] ref_best_rd Best RD cost seen for this block so far 148 * \param[in] current_rd Current RD cost for this block so far 149 * \param[in] plane Plane index 150 * \param[in] plane_bsize Size of the current macroblock considering 151 sup-sampling 152 * \param[in] tx_size The given transform size 153 * \param[in] ftxs_mode Transform search mode specifying desired speed 154 and quality tradeoff 155 * \param[in] skip_trellis Binary flag indicating if trellis optimization 156 should be skipped 157 * 158 * \remark Nothing is returned. The RD results will be saved in rd_stats. 159 */ 160 void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi, 161 RD_STATS *rd_stats, int64_t ref_best_rd, 162 int64_t current_rd, int plane, BLOCK_SIZE plane_bsize, 163 TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode, 164 int skip_trellis); 165 166 /*!\brief Recursive transform size and type search. 167 * 168 * \ingroup transform_search 169 * This function combines y and uv planes' transform search processes together 170 * for inter-predicted blocks (including IntraBC), when the prediction is 171 * already generated. It first does subtraction to obtain the prediction error. 172 * Then it calls 173 * av1_pick_recursive_tx_size_type_yrd/av1_pick_uniform_tx_size_type_yrd and 174 * av1_txfm_uvrd sequentially and handles possible early terminations. 175 * The RD metrics are calculated and stored in rd_stats/_y/_uv. 176 * 177 * \param[in] cpi Top-level encoder structure 178 * \param[in] x Pointer to structure holding the data for the 179 current encoding macroblock 180 * \param[in] bsize Current macroblock size 181 * \param[in] rd_stats Pointer to struct to keep track of the overal RD 182 stats 183 * \param[in] rd_stats_y Pointer to struct to keep track of the RD 184 stats for the luma plane 185 * \param[in] rd_stats_uv Pointer to struct to keep track of the RD 186 stats for the chroma planes 187 * \param[in] mode_rate Rate cost to encode the prediction mode info. of 188 the current macroblock 189 * \param[in] ref_best_rd Best RD cost seen for this block so far 190 * 191 * \return An integer value is returned indicating if a valid transform 192 candidate is found (1) or not (0). 193 */ 194 int av1_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, 195 RD_STATS *rd_stats, RD_STATS *rd_stats_y, 196 RD_STATS *rd_stats_uv, int mode_rate, int64_t ref_best_rd); 197 198 #ifdef __cplusplus 199 } // extern "C" 200 #endif 201 202 #endif // AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_