txb_rdopt_utils.h (9031B)
1 /* 2 * Copyright (c) 2021, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #ifndef AOM_AV1_ENCODER_TXB_RDOPT_UTILS_H_ 13 #define AOM_AV1_ENCODER_TXB_RDOPT_UTILS_H_ 14 15 #include "av1/encoder/encodetxb.h" 16 17 static const int golomb_bits_cost[32] = { 18 0, 512, 512 * 3, 512 * 3, 512 * 5, 512 * 5, 512 * 5, 512 * 5, 19 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 20 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 21 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9 22 }; 23 24 static const int golomb_cost_diff[32] = { 25 0, 512, 512 * 2, 0, 512 * 2, 0, 0, 0, 512 * 2, 0, 0, 0, 0, 0, 0, 0, 26 512 * 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 27 }; 28 29 // Look up table of individual cost of coefficient by its quantization level. 30 // determined based on Laplacian distribution conditioned on estimated context 31 static const int costLUT[15] = { -1143, 53, 545, 825, 1031, 32 1209, 1393, 1577, 1763, 1947, 33 2132, 2317, 2501, 2686, 2871 }; 34 35 static const int const_term = (1 << AV1_PROB_COST_SHIFT); 36 37 static const int loge_par = ((14427 << AV1_PROB_COST_SHIFT) + 5000) / 10000; 38 39 static inline int get_dqv(const int16_t *dequant, int coeff_idx, 40 const qm_val_t *iqmatrix) { 41 int dqv = dequant[!!coeff_idx]; 42 if (iqmatrix != NULL) 43 dqv = 44 ((iqmatrix[coeff_idx] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; 45 return dqv; 46 } 47 48 static inline int64_t get_coeff_dist(tran_low_t tcoeff, tran_low_t dqcoeff, 49 int shift, const qm_val_t *qmatrix, 50 int coeff_idx) { 51 int64_t diff = (tcoeff - dqcoeff) * (1 << shift); 52 if (qmatrix == NULL) { 53 return diff * diff; 54 } 55 // When AOM_DIST_METRIC_QM_PSNR is enabled, this mirrors the rate-distortion 56 // computation done in av1_block_error_qm, improving visual quality. 57 // The maximum value of `shift` is 2, `tcoeff` and `dqcoeff` are at most 22 58 // bits, and AOM_QM_BITS is 5, so `diff` should fit in 29-bits. The 59 // multiplication `diff * diff` then does not risk overflowing. 60 diff *= qmatrix[coeff_idx]; 61 const int64_t error = 62 (diff * diff + (1 << (2 * AOM_QM_BITS - 1))) >> (2 * AOM_QM_BITS); 63 return error; 64 } 65 66 static int get_eob_cost(int eob, const LV_MAP_EOB_COST *txb_eob_costs, 67 const LV_MAP_COEFF_COST *txb_costs, TX_CLASS tx_class) { 68 int eob_extra; 69 const int eob_pt = av1_get_eob_pos_token(eob, &eob_extra); 70 int eob_cost = 0; 71 const int eob_multi_ctx = (tx_class == TX_CLASS_2D) ? 0 : 1; 72 eob_cost = txb_eob_costs->eob_cost[eob_multi_ctx][eob_pt - 1]; 73 74 if (av1_eob_offset_bits[eob_pt] > 0) { 75 const int eob_ctx = eob_pt - 3; 76 const int eob_shift = av1_eob_offset_bits[eob_pt] - 1; 77 const int bit = (eob_extra & (1 << eob_shift)) ? 1 : 0; 78 eob_cost += txb_costs->eob_extra_cost[eob_ctx][bit]; 79 const int offset_bits = av1_eob_offset_bits[eob_pt]; 80 if (offset_bits > 1) eob_cost += av1_cost_literal(offset_bits - 1); 81 } 82 return eob_cost; 83 } 84 85 static inline int get_golomb_cost(int abs_qc) { 86 if (abs_qc >= 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) { 87 const int r = abs_qc - COEFF_BASE_RANGE - NUM_BASE_LEVELS; 88 const int length = get_msb(r) + 1; 89 return av1_cost_literal(2 * length - 1); 90 } 91 return 0; 92 } 93 94 static inline int get_br_cost(tran_low_t level, const int *coeff_lps) { 95 const int base_range = AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); 96 return coeff_lps[base_range] + get_golomb_cost(level); 97 } 98 99 static inline int get_br_cost_with_diff(tran_low_t level, const int *coeff_lps, 100 int *diff) { 101 const int base_range = AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE); 102 int golomb_bits = 0; 103 if (level <= COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS) 104 *diff += coeff_lps[base_range + COEFF_BASE_RANGE + 1]; 105 106 if (level >= COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS) { 107 int r = level - COEFF_BASE_RANGE - NUM_BASE_LEVELS; 108 if (r < 32) { 109 golomb_bits = golomb_bits_cost[r]; 110 *diff += golomb_cost_diff[r]; 111 } else { 112 golomb_bits = get_golomb_cost(level); 113 *diff += (r & (r - 1)) == 0 ? 1024 : 0; 114 } 115 } 116 117 return coeff_lps[base_range] + golomb_bits; 118 } 119 120 static AOM_FORCE_INLINE int get_two_coeff_cost_simple( 121 int ci, tran_low_t abs_qc, int coeff_ctx, 122 const LV_MAP_COEFF_COST *txb_costs, int bhl, TX_CLASS tx_class, 123 const uint8_t *levels, int *cost_low) { 124 // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0) 125 // and not the last (scan_idx != eob - 1) 126 assert(ci > 0); 127 int cost = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)]; 128 int diff = 0; 129 if (abs_qc <= 3) diff = txb_costs->base_cost[coeff_ctx][abs_qc + 4]; 130 if (abs_qc) { 131 cost += av1_cost_literal(1); 132 if (abs_qc > NUM_BASE_LEVELS) { 133 const int br_ctx = get_br_ctx(levels, ci, bhl, tx_class); 134 int brcost_diff = 0; 135 cost += get_br_cost_with_diff(abs_qc, txb_costs->lps_cost[br_ctx], 136 &brcost_diff); 137 diff += brcost_diff; 138 } 139 } 140 *cost_low = cost - diff; 141 142 return cost; 143 } 144 145 static inline int get_coeff_cost_eob(int ci, tran_low_t abs_qc, int sign, 146 int coeff_ctx, int dc_sign_ctx, 147 const LV_MAP_COEFF_COST *txb_costs, 148 int bhl, TX_CLASS tx_class) { 149 int cost = 0; 150 cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1]; 151 if (abs_qc != 0) { 152 if (ci == 0) { 153 cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign]; 154 } else { 155 cost += av1_cost_literal(1); 156 } 157 if (abs_qc > NUM_BASE_LEVELS) { 158 int br_ctx; 159 br_ctx = get_br_ctx_eob(ci, bhl, tx_class); 160 cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]); 161 } 162 } 163 return cost; 164 } 165 166 static inline int get_coeff_cost_general(int is_last, int ci, tran_low_t abs_qc, 167 int sign, int coeff_ctx, 168 int dc_sign_ctx, 169 const LV_MAP_COEFF_COST *txb_costs, 170 int bhl, TX_CLASS tx_class, 171 const uint8_t *levels) { 172 int cost = 0; 173 if (is_last) { 174 cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1]; 175 } else { 176 cost += txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)]; 177 } 178 if (abs_qc != 0) { 179 if (ci == 0) { 180 cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign]; 181 } else { 182 cost += av1_cost_literal(1); 183 } 184 if (abs_qc > NUM_BASE_LEVELS) { 185 int br_ctx; 186 if (is_last) 187 br_ctx = get_br_ctx_eob(ci, bhl, tx_class); 188 else 189 br_ctx = get_br_ctx(levels, ci, bhl, tx_class); 190 cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]); 191 } 192 } 193 return cost; 194 } 195 196 static inline void get_qc_dqc_low(tran_low_t abs_qc, int sign, int dqv, 197 int shift, tran_low_t *qc_low, 198 tran_low_t *dqc_low) { 199 tran_low_t abs_qc_low = abs_qc - 1; 200 *qc_low = (-sign ^ abs_qc_low) + sign; 201 assert((sign ? -abs_qc_low : abs_qc_low) == *qc_low); 202 tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift; 203 *dqc_low = (-sign ^ abs_dqc_low) + sign; 204 assert((sign ? -abs_dqc_low : abs_dqc_low) == *dqc_low); 205 } 206 207 static inline void update_coeff_eob_fast(int *eob, int shift, 208 const int16_t *dequant_ptr, 209 const int16_t *scan, 210 const tran_low_t *coeff_ptr, 211 tran_low_t *qcoeff_ptr, 212 tran_low_t *dqcoeff_ptr) { 213 // TODO(sarahparker) make this work for aomqm 214 int eob_out = *eob; 215 int zbin[2] = { dequant_ptr[0] + ROUND_POWER_OF_TWO(dequant_ptr[0] * 70, 7), 216 dequant_ptr[1] + ROUND_POWER_OF_TWO(dequant_ptr[1] * 70, 7) }; 217 218 for (int i = *eob - 1; i >= 0; i--) { 219 const int rc = scan[i]; 220 const int qcoeff = qcoeff_ptr[rc]; 221 const int coeff = coeff_ptr[rc]; 222 const int coeff_sign = AOMSIGN(coeff); 223 int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign; 224 225 if (((abs_coeff << (1 + shift)) < zbin[rc != 0]) || (qcoeff == 0)) { 226 eob_out--; 227 qcoeff_ptr[rc] = 0; 228 dqcoeff_ptr[rc] = 0; 229 } else { 230 break; 231 } 232 } 233 234 *eob = eob_out; 235 } 236 #endif // AOM_AV1_ENCODER_TXB_RDOPT_UTILS_H_