highbd_quantize_neon.c (19145B)
1 /* 2 * Copyright (c) 2022, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <arm_neon.h> 13 #include <assert.h> 14 #include <string.h> 15 16 #include "config/aom_config.h" 17 #include "config/aom_dsp_rtcd.h" 18 19 #include "aom_dsp/quantize.h" 20 21 static inline uint32_t sum_abs_coeff(const uint32x4_t a) { 22 #if AOM_ARCH_AARCH64 23 return vaddvq_u32(a); 24 #else 25 const uint64x2_t b = vpaddlq_u32(a); 26 const uint64x1_t c = vadd_u64(vget_low_u64(b), vget_high_u64(b)); 27 return (uint32_t)vget_lane_u64(c, 0); 28 #endif 29 } 30 31 static inline uint16x4_t quantize_4( 32 const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr, 33 tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32, 34 int32x4_t v_round_s32, int32x4_t v_zbin_s32, int32x4_t v_quant_shift_s32, 35 int log_scale) { 36 const int32x4_t v_coeff = vld1q_s32(coeff_ptr); 37 const int32x4_t v_coeff_sign = 38 vreinterpretq_s32_u32(vcltq_s32(v_coeff, vdupq_n_s32(0))); 39 const int32x4_t v_abs_coeff = vabsq_s32(v_coeff); 40 // if (abs_coeff < zbins[rc != 0]), 41 const uint32x4_t v_zbin_mask = vcgeq_s32(v_abs_coeff, v_zbin_s32); 42 const int32x4_t v_log_scale = vdupq_n_s32(log_scale); 43 // const int64_t tmp = (int64_t)abs_coeff + log_scaled_round; 44 const int32x4_t v_tmp = vaddq_s32(v_abs_coeff, v_round_s32); 45 // const int32_t tmpw32 = tmp * wt; 46 const int32x4_t v_tmpw32 = vmulq_s32(v_tmp, vdupq_n_s32((1 << AOM_QM_BITS))); 47 // const int32_t tmp2 = (int32_t)((tmpw32 * quant64) >> 16); 48 const int32x4_t v_tmp2 = vqdmulhq_s32(v_tmpw32, v_quant_s32); 49 // const int32_t tmp3 = 50 // ((((tmp2 + tmpw32)<< log_scale) * (int64_t)(quant_shift << 15)) >> 32); 51 const int32x4_t v_tmp3 = vqdmulhq_s32( 52 vshlq_s32(vaddq_s32(v_tmp2, v_tmpw32), v_log_scale), v_quant_shift_s32); 53 // const int abs_qcoeff = vmask ? (int)tmp3 >> AOM_QM_BITS : 0; 54 const int32x4_t v_abs_qcoeff = vandq_s32(vreinterpretq_s32_u32(v_zbin_mask), 55 vshrq_n_s32(v_tmp3, AOM_QM_BITS)); 56 // const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant_iwt) >> log_scale; 57 // vshlq_s32 will shift right if shift value is negative. 58 const int32x4_t v_abs_dqcoeff = 59 vshlq_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), vnegq_s32(v_log_scale)); 60 // qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); 61 const int32x4_t v_qcoeff = 62 vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign); 63 // dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign); 64 const int32x4_t v_dqcoeff = 65 vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign); 66 67 vst1q_s32(qcoeff_ptr, v_qcoeff); 68 vst1q_s32(dqcoeff_ptr, v_dqcoeff); 69 70 // Used to find eob. 71 const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, vdupq_n_s32(0)); 72 return vmovn_u32(nz_qcoeff_mask); 73 } 74 75 static inline int16x8_t get_max_lane_eob(const int16_t *iscan, 76 int16x8_t v_eobmax, 77 uint16x8_t v_mask) { 78 const int16x8_t v_iscan = vld1q_s16(&iscan[0]); 79 const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, vdupq_n_s16(1)); 80 const int16x8_t v_nz_iscan = vbslq_s16(v_mask, v_iscan_plus1, vdupq_n_s16(0)); 81 return vmaxq_s16(v_eobmax, v_nz_iscan); 82 } 83 84 #if !CONFIG_REALTIME_ONLY 85 static inline void get_min_max_lane_eob(const int16_t *iscan, 86 int16x8_t *v_eobmin, 87 int16x8_t *v_eobmax, uint16x8_t v_mask, 88 intptr_t n_coeffs) { 89 const int16x8_t v_iscan = vld1q_s16(&iscan[0]); 90 const int16x8_t v_nz_iscan_max = vbslq_s16(v_mask, v_iscan, vdupq_n_s16(-1)); 91 #if SKIP_EOB_FACTOR_ADJUST 92 const int16x8_t v_nz_iscan_min = 93 vbslq_s16(v_mask, v_iscan, vdupq_n_s16((int16_t)n_coeffs)); 94 *v_eobmin = vminq_s16(*v_eobmin, v_nz_iscan_min); 95 #else 96 (void)v_eobmin; 97 #endif 98 *v_eobmax = vmaxq_s16(*v_eobmax, v_nz_iscan_max); 99 } 100 #endif // !CONFIG_REALTIME_ONLY 101 102 static inline uint16_t get_max_eob(int16x8_t v_eobmax) { 103 #if AOM_ARCH_AARCH64 104 return (uint16_t)vmaxvq_s16(v_eobmax); 105 #else 106 const int16x4_t v_eobmax_3210 = 107 vmax_s16(vget_low_s16(v_eobmax), vget_high_s16(v_eobmax)); 108 const int64x1_t v_eobmax_xx32 = 109 vshr_n_s64(vreinterpret_s64_s16(v_eobmax_3210), 32); 110 const int16x4_t v_eobmax_tmp = 111 vmax_s16(v_eobmax_3210, vreinterpret_s16_s64(v_eobmax_xx32)); 112 const int64x1_t v_eobmax_xxx3 = 113 vshr_n_s64(vreinterpret_s64_s16(v_eobmax_tmp), 16); 114 const int16x4_t v_eobmax_final = 115 vmax_s16(v_eobmax_tmp, vreinterpret_s16_s64(v_eobmax_xxx3)); 116 return (uint16_t)vget_lane_s16(v_eobmax_final, 0); 117 #endif 118 } 119 120 #if SKIP_EOB_FACTOR_ADJUST && !CONFIG_REALTIME_ONLY 121 static inline uint16_t get_min_eob(int16x8_t v_eobmin) { 122 #if AOM_ARCH_AARCH64 123 return (uint16_t)vminvq_s16(v_eobmin); 124 #else 125 const int16x4_t v_eobmin_3210 = 126 vmin_s16(vget_low_s16(v_eobmin), vget_high_s16(v_eobmin)); 127 const int64x1_t v_eobmin_xx32 = 128 vshr_n_s64(vreinterpret_s64_s16(v_eobmin_3210), 32); 129 const int16x4_t v_eobmin_tmp = 130 vmin_s16(v_eobmin_3210, vreinterpret_s16_s64(v_eobmin_xx32)); 131 const int64x1_t v_eobmin_xxx3 = 132 vshr_n_s64(vreinterpret_s64_s16(v_eobmin_tmp), 16); 133 const int16x4_t v_eobmin_final = 134 vmin_s16(v_eobmin_tmp, vreinterpret_s16_s64(v_eobmin_xxx3)); 135 return (uint16_t)vget_lane_s16(v_eobmin_final, 0); 136 #endif 137 } 138 #endif // SKIP_EOB_FACTOR_ADJUST && !CONFIG_REALTIME_ONLY 139 140 static void highbd_quantize_b_neon( 141 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 142 const int16_t *round_ptr, const int16_t *quant_ptr, 143 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 144 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 145 const int16_t *scan, const int16_t *iscan, const int log_scale) { 146 (void)scan; 147 const int16x4_t v_quant = vld1_s16(quant_ptr); 148 const int16x4_t v_dequant = vld1_s16(dequant_ptr); 149 const int16x4_t v_zero = vdup_n_s16(0); 150 const uint16x4_t v_round_select = vcgt_s16(vdup_n_s16(log_scale), v_zero); 151 const int16x4_t v_round_no_scale = vld1_s16(round_ptr); 152 const int16x4_t v_round_log_scale = 153 vqrdmulh_n_s16(v_round_no_scale, (int16_t)(1 << (15 - log_scale))); 154 const int16x4_t v_round = 155 vbsl_s16(v_round_select, v_round_log_scale, v_round_no_scale); 156 const int16x4_t v_quant_shift = vld1_s16(quant_shift_ptr); 157 const int16x4_t v_zbin_no_scale = vld1_s16(zbin_ptr); 158 const int16x4_t v_zbin_log_scale = 159 vqrdmulh_n_s16(v_zbin_no_scale, (int16_t)(1 << (15 - log_scale))); 160 const int16x4_t v_zbin = 161 vbsl_s16(v_round_select, v_zbin_log_scale, v_zbin_no_scale); 162 int32x4_t v_round_s32 = vmovl_s16(v_round); 163 int32x4_t v_quant_s32 = vshlq_n_s32(vmovl_s16(v_quant), 15); 164 int32x4_t v_dequant_s32 = vmovl_s16(v_dequant); 165 int32x4_t v_quant_shift_s32 = vshlq_n_s32(vmovl_s16(v_quant_shift), 15); 166 int32x4_t v_zbin_s32 = vmovl_s16(v_zbin); 167 uint16x4_t v_mask_lo, v_mask_hi; 168 int16x8_t v_eobmax = vdupq_n_s16(-1); 169 170 intptr_t non_zero_count = n_coeffs; 171 172 assert(n_coeffs > 8); 173 // Pre-scan pass 174 const int32x4_t v_zbin_s32x = vdupq_lane_s32(vget_low_s32(v_zbin_s32), 1); 175 intptr_t i = n_coeffs; 176 do { 177 const int32x4_t v_coeff_a = vld1q_s32(coeff_ptr + i - 4); 178 const int32x4_t v_coeff_b = vld1q_s32(coeff_ptr + i - 8); 179 const int32x4_t v_abs_coeff_a = vabsq_s32(v_coeff_a); 180 const int32x4_t v_abs_coeff_b = vabsq_s32(v_coeff_b); 181 const uint32x4_t v_mask_a = vcgeq_s32(v_abs_coeff_a, v_zbin_s32x); 182 const uint32x4_t v_mask_b = vcgeq_s32(v_abs_coeff_b, v_zbin_s32x); 183 // If the coefficient is in the base ZBIN range, then discard. 184 if (sum_abs_coeff(v_mask_a) + sum_abs_coeff(v_mask_b) == 0) { 185 non_zero_count -= 8; 186 } else { 187 break; 188 } 189 i -= 8; 190 } while (i > 0); 191 192 const intptr_t remaining_zcoeffs = n_coeffs - non_zero_count; 193 memset(qcoeff_ptr + non_zero_count, 0, 194 remaining_zcoeffs * sizeof(*qcoeff_ptr)); 195 memset(dqcoeff_ptr + non_zero_count, 0, 196 remaining_zcoeffs * sizeof(*dqcoeff_ptr)); 197 198 // DC and first 3 AC 199 v_mask_lo = 200 quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32, v_dequant_s32, 201 v_round_s32, v_zbin_s32, v_quant_shift_s32, log_scale); 202 203 // overwrite the DC constants with AC constants 204 v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1); 205 v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1); 206 v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1); 207 v_quant_shift_s32 = vdupq_lane_s32(vget_low_s32(v_quant_shift_s32), 1); 208 v_zbin_s32 = vdupq_lane_s32(vget_low_s32(v_zbin_s32), 1); 209 210 // 4 more AC 211 v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4, 212 v_quant_s32, v_dequant_s32, v_round_s32, v_zbin_s32, 213 v_quant_shift_s32, log_scale); 214 215 v_eobmax = 216 get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi)); 217 218 intptr_t count = non_zero_count - 8; 219 for (; count > 0; count -= 8) { 220 coeff_ptr += 8; 221 qcoeff_ptr += 8; 222 dqcoeff_ptr += 8; 223 iscan += 8; 224 v_mask_lo = quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32, 225 v_dequant_s32, v_round_s32, v_zbin_s32, 226 v_quant_shift_s32, log_scale); 227 v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4, 228 v_quant_s32, v_dequant_s32, v_round_s32, v_zbin_s32, 229 v_quant_shift_s32, log_scale); 230 // Find the max lane eob for 8 coeffs. 231 v_eobmax = 232 get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi)); 233 } 234 235 *eob_ptr = get_max_eob(v_eobmax); 236 } 237 238 void aom_highbd_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, 239 const int16_t *zbin_ptr, 240 const int16_t *round_ptr, 241 const int16_t *quant_ptr, 242 const int16_t *quant_shift_ptr, 243 tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, 244 const int16_t *dequant_ptr, uint16_t *eob_ptr, 245 const int16_t *scan, const int16_t *iscan) { 246 highbd_quantize_b_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, 247 quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, 248 eob_ptr, scan, iscan, 0); 249 } 250 251 void aom_highbd_quantize_b_32x32_neon( 252 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 253 const int16_t *round_ptr, const int16_t *quant_ptr, 254 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 255 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 256 const int16_t *scan, const int16_t *iscan) { 257 highbd_quantize_b_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, 258 quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, 259 eob_ptr, scan, iscan, 1); 260 } 261 262 void aom_highbd_quantize_b_64x64_neon( 263 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 264 const int16_t *round_ptr, const int16_t *quant_ptr, 265 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 266 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 267 const int16_t *scan, const int16_t *iscan) { 268 highbd_quantize_b_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, 269 quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, 270 eob_ptr, scan, iscan, 2); 271 } 272 273 #if !CONFIG_REALTIME_ONLY 274 static void highbd_quantize_b_adaptive_neon( 275 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 276 const int16_t *round_ptr, const int16_t *quant_ptr, 277 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 278 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 279 const int16_t *scan, const int16_t *iscan, const int log_scale) { 280 (void)scan; 281 const int16x4_t v_quant = vld1_s16(quant_ptr); 282 const int16x4_t v_dequant = vld1_s16(dequant_ptr); 283 const int16x4_t v_zero = vdup_n_s16(0); 284 const uint16x4_t v_round_select = vcgt_s16(vdup_n_s16(log_scale), v_zero); 285 const int16x4_t v_round_no_scale = vld1_s16(round_ptr); 286 const int16x4_t v_round_log_scale = 287 vqrdmulh_n_s16(v_round_no_scale, (int16_t)(1 << (15 - log_scale))); 288 const int16x4_t v_round = 289 vbsl_s16(v_round_select, v_round_log_scale, v_round_no_scale); 290 const int16x4_t v_quant_shift = vld1_s16(quant_shift_ptr); 291 const int16x4_t v_zbin_no_scale = vld1_s16(zbin_ptr); 292 const int16x4_t v_zbin_log_scale = 293 vqrdmulh_n_s16(v_zbin_no_scale, (int16_t)(1 << (15 - log_scale))); 294 const int16x4_t v_zbin = 295 vbsl_s16(v_round_select, v_zbin_log_scale, v_zbin_no_scale); 296 int32x4_t v_round_s32 = vmovl_s16(v_round); 297 int32x4_t v_quant_s32 = vshlq_n_s32(vmovl_s16(v_quant), 15); 298 int32x4_t v_dequant_s32 = vmovl_s16(v_dequant); 299 int32x4_t v_quant_shift_s32 = vshlq_n_s32(vmovl_s16(v_quant_shift), 15); 300 int32x4_t v_zbin_s32 = vmovl_s16(v_zbin); 301 uint16x4_t v_mask_lo, v_mask_hi; 302 int16x8_t v_eobmax = vdupq_n_s16(-1); 303 int16x8_t v_eobmin = vdupq_n_s16((int16_t)n_coeffs); 304 305 assert(n_coeffs > 8); 306 // Pre-scan pass 307 const int32x4_t v_zbin_s32x = vdupq_lane_s32(vget_low_s32(v_zbin_s32), 1); 308 const int prescan_add_1 = 309 ROUND_POWER_OF_TWO(dequant_ptr[1] * EOB_FACTOR, 7 + AOM_QM_BITS); 310 const int32x4_t v_zbin_prescan = 311 vaddq_s32(v_zbin_s32x, vdupq_n_s32(prescan_add_1)); 312 intptr_t non_zero_count = n_coeffs; 313 intptr_t i = n_coeffs; 314 do { 315 const int32x4_t v_coeff_a = vld1q_s32(coeff_ptr + i - 4); 316 const int32x4_t v_coeff_b = vld1q_s32(coeff_ptr + i - 8); 317 const int32x4_t v_abs_coeff_a = vabsq_s32(v_coeff_a); 318 const int32x4_t v_abs_coeff_b = vabsq_s32(v_coeff_b); 319 const uint32x4_t v_mask_a = vcgeq_s32(v_abs_coeff_a, v_zbin_prescan); 320 const uint32x4_t v_mask_b = vcgeq_s32(v_abs_coeff_b, v_zbin_prescan); 321 // If the coefficient is in the base ZBIN range, then discard. 322 if (sum_abs_coeff(v_mask_a) + sum_abs_coeff(v_mask_b) == 0) { 323 non_zero_count -= 8; 324 } else { 325 break; 326 } 327 i -= 8; 328 } while (i > 0); 329 330 const intptr_t remaining_zcoeffs = n_coeffs - non_zero_count; 331 memset(qcoeff_ptr + non_zero_count, 0, 332 remaining_zcoeffs * sizeof(*qcoeff_ptr)); 333 memset(dqcoeff_ptr + non_zero_count, 0, 334 remaining_zcoeffs * sizeof(*dqcoeff_ptr)); 335 336 // DC and first 3 AC 337 v_mask_lo = 338 quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32, v_dequant_s32, 339 v_round_s32, v_zbin_s32, v_quant_shift_s32, log_scale); 340 341 // overwrite the DC constants with AC constants 342 v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1); 343 v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1); 344 v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1); 345 v_quant_shift_s32 = vdupq_lane_s32(vget_low_s32(v_quant_shift_s32), 1); 346 v_zbin_s32 = vdupq_lane_s32(vget_low_s32(v_zbin_s32), 1); 347 348 // 4 more AC 349 v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4, 350 v_quant_s32, v_dequant_s32, v_round_s32, v_zbin_s32, 351 v_quant_shift_s32, log_scale); 352 353 get_min_max_lane_eob(iscan, &v_eobmin, &v_eobmax, 354 vcombine_u16(v_mask_lo, v_mask_hi), n_coeffs); 355 356 intptr_t count = non_zero_count - 8; 357 for (; count > 0; count -= 8) { 358 coeff_ptr += 8; 359 qcoeff_ptr += 8; 360 dqcoeff_ptr += 8; 361 iscan += 8; 362 v_mask_lo = quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32, 363 v_dequant_s32, v_round_s32, v_zbin_s32, 364 v_quant_shift_s32, log_scale); 365 v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4, 366 v_quant_s32, v_dequant_s32, v_round_s32, v_zbin_s32, 367 v_quant_shift_s32, log_scale); 368 369 get_min_max_lane_eob(iscan, &v_eobmin, &v_eobmax, 370 vcombine_u16(v_mask_lo, v_mask_hi), n_coeffs); 371 } 372 373 int eob = get_max_eob(v_eobmax); 374 375 #if SKIP_EOB_FACTOR_ADJUST 376 const int first = get_min_eob(v_eobmin); 377 if (eob >= 0 && first == eob) { 378 const int rc = scan[eob]; 379 if (qcoeff_ptr[rc] == 1 || qcoeff_ptr[rc] == -1) { 380 const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), 381 ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) }; 382 const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; 383 const qm_val_t wt = (1 << AOM_QM_BITS); 384 const int coeff = coeff_ptr[rc] * wt; 385 const int factor = EOB_FACTOR + SKIP_EOB_FACTOR_ADJUST; 386 const int prescan_add_val = 387 ROUND_POWER_OF_TWO(dequant_ptr[rc != 0] * factor, 7); 388 if (coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS) + prescan_add_val) && 389 coeff > (nzbins[rc != 0] * (1 << AOM_QM_BITS) - prescan_add_val)) { 390 qcoeff_ptr[rc] = 0; 391 dqcoeff_ptr[rc] = 0; 392 eob = -1; 393 } 394 } 395 } 396 #endif // SKIP_EOB_FACTOR_ADJUST 397 *eob_ptr = eob + 1; 398 } 399 400 void aom_highbd_quantize_b_adaptive_neon( 401 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 402 const int16_t *round_ptr, const int16_t *quant_ptr, 403 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 404 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 405 const int16_t *scan, const int16_t *iscan) { 406 highbd_quantize_b_adaptive_neon( 407 coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, quant_shift_ptr, 408 qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 0); 409 } 410 411 void aom_highbd_quantize_b_32x32_adaptive_neon( 412 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 413 const int16_t *round_ptr, const int16_t *quant_ptr, 414 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 415 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 416 const int16_t *scan, const int16_t *iscan) { 417 highbd_quantize_b_adaptive_neon( 418 coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, quant_shift_ptr, 419 qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 1); 420 } 421 422 void aom_highbd_quantize_b_64x64_adaptive_neon( 423 const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, 424 const int16_t *round_ptr, const int16_t *quant_ptr, 425 const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, 426 tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, 427 const int16_t *scan, const int16_t *iscan) { 428 highbd_quantize_b_adaptive_neon( 429 coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, quant_shift_ptr, 430 qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 2); 431 } 432 #endif // !CONFIG_REALTIME_ONLY