intra_mode_search.h (16102B)
1 /* 2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 /*!\file 13 * \brief Declares high level functions to search through intra modes. 14 */ 15 #ifndef AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_ 16 #define AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_ 17 18 #include <stdbool.h> 19 20 #include "av1/encoder/encoder.h" 21 22 #ifdef __cplusplus 23 extern "C" { 24 #endif 25 26 /*! \brief Variables related to intra-mode search during inter frame coding. 27 * 28 * \ingroup intra_mode_search 29 * This is a set of variables used during intra-mode search for inter frames. 30 * This includes an histogram of gradient speed features and a cache of uv 31 * prediction to avoid repeated search of chroma prediction. 32 */ 33 typedef struct IntraModeSearchState { 34 /*! 35 * \brief The best luma intra-mode found so far 36 */ 37 PREDICTION_MODE best_intra_mode; 38 39 /** \name Speed feature variables 40 * Variables to help with pruning some luma intra-modes during inter frame 41 * coding process. 42 */ 43 /**@{*/ 44 /*! 45 * \brief Whether to terminate all intra mode search. 46 */ 47 int skip_intra_modes; 48 /*! 49 * \brief Whether a directional mode is pruned. 50 */ 51 uint8_t directional_mode_skip_mask[INTRA_MODES]; 52 /*! 53 * \brief Whether \ref directional_mode_skip_mask is valid for pruning. 54 */ 55 int dir_mode_skip_mask_ready; 56 /**@}*/ 57 58 /** \name Chroma mode search cache 59 * A cache of the best chroma prediction mode to avoid having to search for 60 * chroma predictions repeatedly in \ref 61 * av1_search_intra_uv_modes_in_interframe() 62 */ 63 /**@{*/ 64 int rate_uv_intra; /*!< \brief Total rate to transmit uv_mode */ 65 int rate_uv_tokenonly; /*!< \brief Rate transmit txfm tokens */ 66 int64_t dist_uvs; /*!< \brief Distortion of the uv_mode's recon */ 67 uint8_t skip_uvs; /*!< \brief Whether the uv txfm is skippable */ 68 UV_PREDICTION_MODE mode_uv; /*!< \brief The best uv mode */ 69 PALETTE_MODE_INFO pmi_uv; /*!< \brief Color map if mode_uv is palette */ 70 int8_t uv_angle_delta; /*!< \brief Angle delta if mode_uv directional */ 71 /**@}*/ 72 } IntraModeSearchState; 73 74 /*!\brief Evaluate a given luma intra-mode for inter frames. 75 * 76 * \ingroup intra_mode_search 77 * \callgraph 78 * \callergraph 79 * This function handles an intra-mode luma prediction when the current frame 80 * is an inter frame. This is the intra-mode counterpart of handle_inter_mode. 81 * This function performs an intra luma prediction using the mode specified by 82 * x->e_mbd.mi[0]->mode. This function does *not* support palette mode 83 * prediction in the luma channel. 84 * 85 * \param[in,out] intra_search_state Structure to intra search state. 86 * \param[in] cpi Top-level encoder structure. 87 * \param[in,out] x Pointer to structure holding all the 88 * data for the current macroblock. 89 * \param[in] bsize Current partition block size. 90 * \param[in] ref_frame_cost The entropy cost for signaling that the 91 * current ref frame is an intra frame. 92 * \param[in] ctx Structure to hold the number of 4x4 blks 93 * to copy tx_type and txfm_skip arrays. 94 * \param[out] rd_stats_y Struct to keep track of the current 95 * intra-mode's rd_stats (luma only). 96 * \param[in] best_rd Best RD seen for this block so far. 97 * \param[out] mode_cost_y The cost needed to signal the current 98 * intra mode. 99 * \param[out] rd_y The rdcost of the chosen mode. 100 * \param[in] best_model_rd Best model RD seen for this block so far 101 * \param[in] top_intra_model_rd Top intra model RD seen for this 102 * block so far. 103 * 104 * \return Returns 1 if a valid intra mode is found, 0 otherwise. 105 * The corresponding values in x->e_mbd.mi[0], rd_stats_y, mode_cost_y, and 106 * rd_y are also updated. Moreover, in the first evaluation with directional 107 * mode, a prune_mask computed with histogram of gradient is also stored in 108 * intra_search_state. 109 */ 110 int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state, 111 const AV1_COMP *cpi, MACROBLOCK *x, 112 BLOCK_SIZE bsize, unsigned int ref_frame_cost, 113 const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y, 114 int64_t best_rd, int *mode_cost_y, int64_t *rd_y, 115 int64_t *best_model_rd, 116 int64_t top_intra_model_rd[]); 117 118 /*!\brief Search through all chroma intra-modes for inter frames. 119 * 120 * \ingroup intra_mode_search 121 * \callgraph 122 * \callergraph 123 * This function handles intra-mode chroma prediction when the current frame 124 * is an inter frame. This is done by calling \ref av1_rd_pick_intra_sbuv_mode 125 * with some additional book-keeping. 126 * 127 * \param[in,out] intra_search_state Structure to intra search state. 128 * \param[in] cpi Top-level encoder structure. 129 * \param[in,out] x Pointer to structure holding all the 130 * data for the current macroblock. 131 * \param[in] bsize Current partition block size. 132 * \param[out] rd_stats Struct to keep track of the current 133 * intra-mode's rd_stats (all planes). 134 * \param[out] rd_stats_y Struct to keep track of the current 135 * intra-mode's rd_stats (luma only). 136 * \param[out] rd_stats_uv Struct to keep track of the current 137 * intra-mode's rd_stats (chroma only). 138 * \param[in] best_rd Best RD seen for this block so far. 139 * 140 * \return Returns 1 if a valid intra mode is found, 0 otherwise. 141 * The corresponding values in x->e_mbd.mi[0], rd_stats(_y|_uv) are also 142 * updated. Moreover, in the first evocation of the function, the chroma intra 143 * mode result is cached in intra_search_state to be used in subsequent calls. 144 */ 145 int av1_search_intra_uv_modes_in_interframe( 146 IntraModeSearchState *intra_search_state, const AV1_COMP *cpi, 147 MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats, 148 const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd); 149 150 /*!\brief Evaluate luma palette mode for inter frames. 151 * 152 * \ingroup intra_mode_search 153 * \callergraph 154 * \callgraph 155 * This function handles luma palette mode when the current frame is an 156 * inter frame. 157 * 158 * \param[in] intra_search_state Structure to hold the best luma intra mode 159 * and cache chroma prediction for speed up. 160 * \param[in] cpi Top-level encoder structure. 161 * \param[in] x Pointer to structure holding all the data 162 * for the current macroblock. 163 * \param[in] bsize Current partition block size. 164 * \param[in] ref_frame_cost The entropy cost for signaling that the 165 * current ref frame is an intra frame. 166 * \param[in] ctx Structure to hold the number of 4x4 blks to 167 * copy the tx_type and txfm_skip arrays. 168 * \param[in] this_rd_cost Struct to keep track of palette mode's 169 * rd_stats. 170 * \param[in] best_rd Best RD seen for this block so far. 171 * 172 * \return Returns whether luma palette mode can skip the txfm. The 173 * corresponding mbmi, this_rd_costs, intra_search_state, and tx_type arrays in 174 * ctx are also updated. 175 */ 176 int av1_search_palette_mode(IntraModeSearchState *intra_search_state, 177 const AV1_COMP *cpi, MACROBLOCK *x, 178 BLOCK_SIZE bsize, unsigned int ref_frame_cost, 179 PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost, 180 int64_t best_rd); 181 182 /*!\brief Evaluate luma palette mode for inter frames. 183 * 184 * \ingroup intra_mode_search 185 * \callergraph 186 * \callgraph 187 * This function handles luma palette mode when the current frame is an 188 * inter frame. 189 * 190 * \param[in] cpi Top-level encoder structure. 191 * \param[in] x Pointer to structure holding all the data 192 * for the current macroblock. 193 * \param[in] bsize Current partition block size. 194 * \param[in] ref_frame_cost The entropy cost for signaling that the 195 * current ref frame is an intra frame. 196 * \param[in] ctx Structure to hold the number of 4x4 blks to 197 * copy the tx_type and txfm_skip arrays. 198 * \param[in] this_rd_cost Struct to keep track of palette mode's 199 * rd_stats. 200 * \param[in] best_rd Best RD seen for this block so far. 201 */ 202 void av1_search_palette_mode_luma(const AV1_COMP *cpi, MACROBLOCK *x, 203 BLOCK_SIZE bsize, unsigned int ref_frame_cost, 204 PICK_MODE_CONTEXT *ctx, 205 RD_STATS *this_rd_cost, int64_t best_rd); 206 207 /*!\brief Perform intra-mode search on luma channels for intra frames. 208 * 209 * \ingroup intra_mode_search 210 * \callgraph 211 * \callergraph 212 * This function performs intra-mode search on the luma channel when the 213 * current frame is intra-only. This function does not search intrabc mode, 214 * but it does search palette and filter_intra. 215 * 216 * \param[in] cpi Top-level encoder structure. 217 * \param[in] x Pointer to structure holding all the data 218 * for the current macroblock. 219 * \param[in] rate The total rate needed to predict the current 220 * chroma block. 221 * \param[in] rate_tokenonly The rate without the cost of sending the 222 * prediction modes. 223 * chroma block. 224 * after the reconstruction. 225 * \param[in] distortion The chroma distortion of the best prediction 226 * after the reconstruction. 227 * \param[in] skippable Whether we can skip txfm process. 228 * \param[in] bsize Current partition block size. 229 * \param[in] best_rd Best RD seen for this block so far. 230 * \param[in] ctx Structure to hold the number of 4x4 blks to 231 * copy the tx_type and txfm_skip arrays. 232 * 233 * \return Returns the rd_cost if this function finds a mode better than 234 * best_rd, otherwise returns INT64_MAX. This also updates the mbmi, the rate 235 * and distortion, and the tx_type arrays in ctx. 236 */ 237 int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, 238 int *rate, int *rate_tokenonly, 239 int64_t *distortion, uint8_t *skippable, 240 BLOCK_SIZE bsize, int64_t best_rd, 241 PICK_MODE_CONTEXT *ctx); 242 243 /*!\brief Perform intra-mode search on chroma channels. 244 * 245 * \ingroup intra_mode_search 246 * \callergraph 247 * \callgraph 248 * This function performs intra-mode search on the chroma channels. Just like 249 * \ref av1_rd_pick_intra_sby_mode(), this function searches over palette mode 250 * (filter_intra is not available on chroma planes). Unlike \ref 251 * av1_rd_pick_intra_sby_mode() this function is used by both inter and intra 252 * frames. 253 * 254 * \param[in] cpi Top-level encoder structure. 255 * \param[in] x Pointer to structure holding all the data 256 * for the current macroblock. 257 * \param[in] rate The total rate needed to predict the current 258 * chroma block. 259 * \param[in] rate_tokenonly The rate without the cost of sending the 260 * prediction modes. 261 * chroma block. 262 * after the reconstruction. 263 * \param[in] distortion The chroma distortion of the best prediction 264 * after the reconstruction. 265 * \param[in] skippable Whether we can skip txfm process. 266 * \param[in] bsize Current partition block size. 267 * \param[in] max_tx_size The maximum tx_size available 268 * 269 * \return Returns the rd_cost of the best uv mode found. This also updates the 270 * mbmi, the rate and distortion, distortion. 271 */ 272 int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x, 273 int *rate, int *rate_tokenonly, 274 int64_t *distortion, uint8_t *skippable, 275 BLOCK_SIZE bsize, TX_SIZE max_tx_size); 276 277 /*! \brief Return the number of colors in src. Used by palette mode. 278 */ 279 void av1_count_colors(const uint8_t *src, int stride, int rows, int cols, 280 int *val_count, int *num_colors); 281 282 /*! \brief See \ref av1_count_colors(), but for highbd. 283 */ 284 void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, 285 int cols, int bit_depth, int *val_count, 286 int *val_count_8bit, int *num_color_bins, 287 int *num_colors); 288 289 /*! \brief Set \a *num_colors to the number of colors in src. 290 291 Exits early if the number of colors exceeds a specified threshold. 292 Used by screen content detection mode 2. 293 \return Returns true if the number of colors does not exceed the threshold. 294 \note If the function returns true, \a *num_colors is accurate (and is <= 295 \a num_colors_threshold). If the function returns false, the function exited 296 early, and \a *num_colors may be smaller than the actual number of colors 297 (and is > \a num_colors_threshold). 298 */ 299 bool av1_count_colors_with_threshold(const uint8_t *src, int stride, int rows, 300 int cols, int num_colors_threshold, 301 int *num_colors); 302 303 /*! \brief Initializes the \ref IntraModeSearchState struct. 304 */ 305 static inline void init_intra_mode_search_state( 306 IntraModeSearchState *intra_search_state) { 307 memset(intra_search_state, 0, sizeof(*intra_search_state)); 308 intra_search_state->rate_uv_intra = INT_MAX; 309 } 310 311 /*! \brief set the luma intra mode and delta angles for a given mode index. 312 * 313 * The total number of luma intra mode is LUMA_MODE_COUNT = 61. 314 * The first 13 modes are from DC_PRED to PAETH_PRED, followed by directional 315 * modes. Each of the main 8 directional modes have 6 = MAX_ANGLE_DELTA * 2 316 * delta angles. 317 * \param[in] mode_idx mode index in intra mode decision 318 * process. 319 * \param[in] mbmi Pointer to structure holding the mode 320 * info for the current macroblock. 321 * \param[in] reorder_delta_angle_eval Indicates whether to reorder the 322 * evaluation of delta angle modes. 323 */ 324 void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi, 325 int reorder_delta_angle_eval); 326 #ifdef __cplusplus 327 } // extern "C" 328 #endif 329 330 #endif // AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_