av1_convolve_scale_neon_dotprod.c (17159B)
1 /* 2 * Copyright (c) 2024, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <assert.h> 13 #include <arm_neon.h> 14 #include <stddef.h> 15 #include <stdint.h> 16 17 #include "config/aom_config.h" 18 #include "config/av1_rtcd.h" 19 20 #include "aom_dsp/aom_dsp_common.h" 21 #include "aom_dsp/aom_filter.h" 22 #include "aom_dsp/arm/mem_neon.h" 23 #include "aom_dsp/arm/transpose_neon.h" 24 #include "aom_ports/mem.h" 25 #include "av1/common/arm/convolve_scale_neon.h" 26 #include "av1/common/convolve.h" 27 #include "av1/common/enums.h" 28 #include "av1/common/filter.h" 29 30 // clang-format off 31 DECLARE_ALIGNED(16, static const uint8_t, kScale2DotProdPermuteTbl[32]) = { 32 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 33 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 34 }; 35 // clang-format on 36 37 static inline int16x4_t convolve8_4_h(const uint8x8_t s0, const uint8x8_t s1, 38 const uint8x8_t s2, const uint8x8_t s3, 39 const int8x8_t filter, 40 const int32x4_t horiz_const) { 41 const int8x16_t filters = vcombine_s8(filter, filter); 42 43 uint8x16_t s01 = vcombine_u8(s0, s1); 44 uint8x16_t s23 = vcombine_u8(s2, s3); 45 46 // Transform sample range to [-128, 127] for 8-bit signed dot product. 47 int8x16_t s01_128 = vreinterpretq_s8_u8(vsubq_u8(s01, vdupq_n_u8(128))); 48 int8x16_t s23_128 = vreinterpretq_s8_u8(vsubq_u8(s23, vdupq_n_u8(128))); 49 50 int32x4_t sum01 = vdotq_s32(horiz_const, s01_128, filters); 51 int32x4_t sum23 = vdotq_s32(horiz_const, s23_128, filters); 52 53 int32x4_t sum = vpaddq_s32(sum01, sum23); 54 55 // We halved the filter values so -1 from right shift. 56 return vshrn_n_s32(sum, ROUND0_BITS - 1); 57 } 58 59 static inline int16x8_t convolve8_8_h(const uint8x8_t s0, const uint8x8_t s1, 60 const uint8x8_t s2, const uint8x8_t s3, 61 const uint8x8_t s4, const uint8x8_t s5, 62 const uint8x8_t s6, const uint8x8_t s7, 63 const int8x8_t filter, 64 const int32x4_t horiz_const) { 65 const int8x16_t filters = vcombine_s8(filter, filter); 66 67 uint8x16_t s01 = vcombine_u8(s0, s1); 68 uint8x16_t s23 = vcombine_u8(s2, s3); 69 uint8x16_t s45 = vcombine_u8(s4, s5); 70 uint8x16_t s67 = vcombine_u8(s6, s7); 71 72 // Transform sample range to [-128, 127] for 8-bit signed dot product. 73 int8x16_t s01_128 = vreinterpretq_s8_u8(vsubq_u8(s01, vdupq_n_u8(128))); 74 int8x16_t s23_128 = vreinterpretq_s8_u8(vsubq_u8(s23, vdupq_n_u8(128))); 75 int8x16_t s45_128 = vreinterpretq_s8_u8(vsubq_u8(s45, vdupq_n_u8(128))); 76 int8x16_t s67_128 = vreinterpretq_s8_u8(vsubq_u8(s67, vdupq_n_u8(128))); 77 78 int32x4_t sum01 = vdotq_s32(horiz_const, s01_128, filters); 79 int32x4_t sum23 = vdotq_s32(horiz_const, s23_128, filters); 80 int32x4_t sum45 = vdotq_s32(horiz_const, s45_128, filters); 81 int32x4_t sum67 = vdotq_s32(horiz_const, s67_128, filters); 82 83 int32x4_t sum0123 = vpaddq_s32(sum01, sum23); 84 int32x4_t sum4567 = vpaddq_s32(sum45, sum67); 85 86 // We halved the filter values so -1 from right shift. 87 return vcombine_s16(vshrn_n_s32(sum0123, ROUND0_BITS - 1), 88 vshrn_n_s32(sum4567, ROUND0_BITS - 1)); 89 } 90 91 static inline void convolve_horiz_scale_neon_dotprod( 92 const uint8_t *src, int src_stride, int16_t *dst, int dst_stride, int w, 93 int h, const int16_t *x_filter, const int subpel_x_qn, 94 const int x_step_qn) { 95 DECLARE_ALIGNED(16, int16_t, temp[8 * 8]); 96 const int bd = 8; 97 // A shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding 98 // shifts - which are generally faster than rounding shifts on modern CPUs. 99 const int32_t horiz_offset = 100 (1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)); 101 // The shim of 128 << FILTER_BITS is needed because we are subtracting 128 102 // from every source value. 103 const int32_t dotprod_offset = 128 << FILTER_BITS; 104 // Divide the total by 4: we halved the filter values and will use a pairwise 105 // add in the convolution kernel. 106 const int32x4_t horiz_offset_vec = 107 vdupq_n_s32((horiz_offset + dotprod_offset) >> 2); 108 109 if (w == 4) { 110 do { 111 int x_qn = subpel_x_qn; 112 113 // Process a 4x4 tile. 114 for (int r = 0; r < 4; r++) { 115 const uint8_t *const s = &src[x_qn >> SCALE_SUBPEL_BITS]; 116 117 const ptrdiff_t filter_offset = 118 SUBPEL_TAPS * ((x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS); 119 // Filter values are all even so halve them to fit in int8_t. 120 const int8x8_t filter = 121 vshrn_n_s16(vld1q_s16(x_filter + filter_offset), 1); 122 123 uint8x8_t t0, t1, t2, t3; 124 load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); 125 126 int16x4_t d0 = convolve8_4_h(t0, t1, t2, t3, filter, horiz_offset_vec); 127 128 vst1_s16(&temp[r * 4], d0); 129 130 x_qn += x_step_qn; 131 } 132 133 // Transpose the 4x4 result tile and store. 134 int16x4_t d0, d1, d2, d3; 135 load_s16_4x4(temp, 4, &d0, &d1, &d2, &d3); 136 137 transpose_elems_inplace_s16_4x4(&d0, &d1, &d2, &d3); 138 139 store_s16_4x4(dst, dst_stride, d0, d1, d2, d3); 140 141 dst += 4 * dst_stride; 142 src += 4 * src_stride; 143 h -= 4; 144 } while (h > 0); 145 } else { 146 do { 147 int x_qn = subpel_x_qn; 148 int16_t *d = dst; 149 int width = w; 150 151 do { 152 // Process an 8x8 tile. 153 for (int r = 0; r < 8; r++) { 154 const uint8_t *const s = &src[(x_qn >> SCALE_SUBPEL_BITS)]; 155 156 const ptrdiff_t filter_offset = 157 SUBPEL_TAPS * ((x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS); 158 // Filter values are all even so halve them to fit in int8_t. 159 int8x8_t filter = vshrn_n_s16(vld1q_s16(x_filter + filter_offset), 1); 160 161 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7; 162 load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); 163 164 int16x8_t d0 = convolve8_8_h(t0, t1, t2, t3, t4, t5, t6, t7, filter, 165 horiz_offset_vec); 166 167 vst1q_s16(&temp[r * 8], d0); 168 169 x_qn += x_step_qn; 170 } 171 172 // Transpose the 8x8 result tile and store. 173 int16x8_t d0, d1, d2, d3, d4, d5, d6, d7; 174 load_s16_8x8(temp, 8, &d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7); 175 176 transpose_elems_inplace_s16_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7); 177 178 store_s16_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7); 179 180 d += 8; 181 width -= 8; 182 } while (width != 0); 183 184 dst += 8 * dst_stride; 185 src += 8 * src_stride; 186 h -= 8; 187 } while (h > 0); 188 } 189 } 190 191 static inline int16x4_t convolve8_4_h_scale_2(uint8x16_t samples, 192 const int8x8_t filters, 193 const int32x4_t horiz_const, 194 const uint8x16x2_t permute_tbl) { 195 // Transform sample range to [-128, 127] for 8-bit signed dot product. 196 int8x16_t samples_128 = 197 vreinterpretq_s8_u8(vsubq_u8(samples, vdupq_n_u8(128))); 198 199 // Permute samples ready for dot product. 200 // { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 } 201 // { 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 } 202 int8x16_t perm_samples[2] = { vqtbl1q_s8(samples_128, permute_tbl.val[0]), 203 vqtbl1q_s8(samples_128, permute_tbl.val[1]) }; 204 205 int32x4_t sum = vdotq_lane_s32(horiz_const, perm_samples[0], filters, 0); 206 sum = vdotq_lane_s32(sum, perm_samples[1], filters, 1); 207 208 // We halved the filter values so -1 from right shift. 209 return vshrn_n_s32(sum, ROUND0_BITS - 1); 210 } 211 212 static inline int16x8_t convolve8_8_h_scale_2(uint8x16_t samples[2], 213 const int8x8_t filters, 214 const int32x4_t horiz_const, 215 const uint8x16x2_t permute_tbl) { 216 // Transform sample range to [-128, 127] for 8-bit signed dot product. 217 int8x16_t samples0_128 = 218 vreinterpretq_s8_u8(vsubq_u8(samples[0], vdupq_n_u8(128))); 219 int8x16_t samples1_128 = 220 vreinterpretq_s8_u8(vsubq_u8(samples[1], vdupq_n_u8(128))); 221 222 // Permute samples ready for dot product. 223 // { 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 } 224 // { 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 } 225 int8x16_t perm_samples[4] = { vqtbl1q_s8(samples0_128, permute_tbl.val[0]), 226 vqtbl1q_s8(samples0_128, permute_tbl.val[1]), 227 vqtbl1q_s8(samples1_128, permute_tbl.val[0]), 228 vqtbl1q_s8(samples1_128, permute_tbl.val[1]) }; 229 230 // First 4 output values. 231 int32x4_t sum0123 = vdotq_lane_s32(horiz_const, perm_samples[0], filters, 0); 232 sum0123 = vdotq_lane_s32(sum0123, perm_samples[1], filters, 1); 233 // Second 4 output values. 234 int32x4_t sum4567 = vdotq_lane_s32(horiz_const, perm_samples[2], filters, 0); 235 sum4567 = vdotq_lane_s32(sum4567, perm_samples[3], filters, 1); 236 237 // We halved the filter values so -1 from right shift. 238 return vcombine_s16(vshrn_n_s32(sum0123, ROUND0_BITS - 1), 239 vshrn_n_s32(sum4567, ROUND0_BITS - 1)); 240 } 241 242 static inline void convolve_horiz_scale_2_neon_dotprod( 243 const uint8_t *src, int src_stride, int16_t *dst, int dst_stride, int w, 244 int h, const int16_t *x_filter) { 245 const int bd = 8; 246 // A shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding 247 // shifts - which are generally faster than rounding shifts on modern CPUs. 248 const int32_t horiz_offset = 249 (1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1)); 250 // The shim of 128 << FILTER_BITS is needed because we are subtracting 128 251 // from every source value. 252 const int32_t dotprod_offset = 128 << FILTER_BITS; 253 // Divide the total by 2 because we halved the filter values. 254 const int32x4_t horiz_offset_vec = 255 vdupq_n_s32((horiz_offset + dotprod_offset) >> 1); 256 257 const uint8x16x2_t permute_tbl = vld1q_u8_x2(kScale2DotProdPermuteTbl); 258 // Filter values are all even so halve them to fit in int8_t. 259 const int8x8_t filter = vshrn_n_s16(vld1q_s16(x_filter), 1); 260 261 if (w == 4) { 262 do { 263 const uint8_t *s = src; 264 int16_t *d = dst; 265 int width = w; 266 267 do { 268 uint8x16_t s0, s1, s2, s3; 269 load_u8_16x4(s, src_stride, &s0, &s1, &s2, &s3); 270 271 int16x4_t d0 = 272 convolve8_4_h_scale_2(s0, filter, horiz_offset_vec, permute_tbl); 273 int16x4_t d1 = 274 convolve8_4_h_scale_2(s1, filter, horiz_offset_vec, permute_tbl); 275 int16x4_t d2 = 276 convolve8_4_h_scale_2(s2, filter, horiz_offset_vec, permute_tbl); 277 int16x4_t d3 = 278 convolve8_4_h_scale_2(s3, filter, horiz_offset_vec, permute_tbl); 279 280 store_s16_4x4(d, dst_stride, d0, d1, d2, d3); 281 282 s += 8; 283 d += 4; 284 width -= 4; 285 } while (width != 0); 286 287 dst += 4 * dst_stride; 288 src += 4 * src_stride; 289 h -= 4; 290 } while (h > 0); 291 } else { 292 do { 293 const uint8_t *s = src; 294 int16_t *d = dst; 295 int width = w; 296 297 do { 298 uint8x16_t s0[2], s1[2], s2[2], s3[2]; 299 load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]); 300 load_u8_16x4(s + 8, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]); 301 302 int16x8_t d0 = 303 convolve8_8_h_scale_2(s0, filter, horiz_offset_vec, permute_tbl); 304 int16x8_t d1 = 305 convolve8_8_h_scale_2(s1, filter, horiz_offset_vec, permute_tbl); 306 int16x8_t d2 = 307 convolve8_8_h_scale_2(s2, filter, horiz_offset_vec, permute_tbl); 308 int16x8_t d3 = 309 convolve8_8_h_scale_2(s3, filter, horiz_offset_vec, permute_tbl); 310 311 store_s16_8x4(d, dst_stride, d0, d1, d2, d3); 312 313 s += 16; 314 d += 8; 315 width -= 8; 316 } while (width != 0); 317 318 dst += 4 * dst_stride; 319 src += 4 * src_stride; 320 h -= 4; 321 } while (h > 0); 322 } 323 } 324 325 void av1_convolve_2d_scale_neon_dotprod( 326 const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, 327 int h, const InterpFilterParams *filter_params_x, 328 const InterpFilterParams *filter_params_y, const int subpel_x_qn, 329 const int x_step_qn, const int subpel_y_qn, const int y_step_qn, 330 ConvolveParams *conv_params) { 331 if (w < 4 || h < 4) { 332 av1_convolve_2d_scale_c(src, src_stride, dst, dst_stride, w, h, 333 filter_params_x, filter_params_y, subpel_x_qn, 334 x_step_qn, subpel_y_qn, y_step_qn, conv_params); 335 return; 336 } 337 338 // For the interpolation 8-tap filters are used. 339 assert(filter_params_y->taps <= 8 && filter_params_x->taps <= 8); 340 341 DECLARE_ALIGNED(32, int16_t, 342 im_block[(2 * MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); 343 int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) + 344 filter_params_y->taps; 345 int im_stride = MAX_SB_SIZE; 346 CONV_BUF_TYPE *dst16 = conv_params->dst; 347 const int dst16_stride = conv_params->dst_stride; 348 349 // Account for needing filter_taps / 2 - 1 lines prior and filter_taps / 2 350 // lines post both horizontally and vertically. 351 const ptrdiff_t horiz_offset = filter_params_x->taps / 2 - 1; 352 const ptrdiff_t vert_offset = (filter_params_y->taps / 2 - 1) * src_stride; 353 354 // Horizontal filter 355 if (x_step_qn != 2 * (1 << SCALE_SUBPEL_BITS)) { 356 convolve_horiz_scale_neon_dotprod( 357 src - horiz_offset - vert_offset, src_stride, im_block, im_stride, w, 358 im_h, filter_params_x->filter_ptr, subpel_x_qn, x_step_qn); 359 } else { 360 assert(subpel_x_qn < (1 << SCALE_SUBPEL_BITS)); 361 // The filter index is calculated using the 362 // ((subpel_x_qn + x * x_step_qn) & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS 363 // equation, where the values of x are from 0 to w. If x_step_qn is a 364 // multiple of SCALE_SUBPEL_MASK we can leave it out of the equation. 365 const ptrdiff_t filter_offset = 366 SUBPEL_TAPS * ((subpel_x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS); 367 const int16_t *x_filter = filter_params_x->filter_ptr + filter_offset; 368 369 // The source index is calculated using the (subpel_x_qn + x * x_step_qn) >> 370 // SCALE_SUBPEL_BITS, where the values of x are from 0 to w. If subpel_x_qn 371 // < (1 << SCALE_SUBPEL_BITS) and x_step_qn % (1 << SCALE_SUBPEL_BITS) == 0, 372 // the source index can be determined using the value x * (x_step_qn / 373 // (1 << SCALE_SUBPEL_BITS)). 374 convolve_horiz_scale_2_neon_dotprod(src - horiz_offset - vert_offset, 375 src_stride, im_block, im_stride, w, 376 im_h, x_filter); 377 } 378 379 // Vertical filter 380 if (filter_params_y->interp_filter == MULTITAP_SHARP) { 381 if (UNLIKELY(conv_params->is_compound)) { 382 if (conv_params->do_average) { 383 if (conv_params->use_dist_wtd_comp_avg) { 384 compound_dist_wtd_convolve_vert_scale_8tap_neon( 385 im_block, im_stride, dst, dst_stride, dst16, dst16_stride, w, h, 386 filter_params_y->filter_ptr, conv_params, subpel_y_qn, y_step_qn); 387 } else { 388 compound_avg_convolve_vert_scale_8tap_neon( 389 im_block, im_stride, dst, dst_stride, dst16, dst16_stride, w, h, 390 filter_params_y->filter_ptr, subpel_y_qn, y_step_qn); 391 } 392 } else { 393 compound_convolve_vert_scale_8tap_neon( 394 im_block, im_stride, dst16, dst16_stride, w, h, 395 filter_params_y->filter_ptr, subpel_y_qn, y_step_qn); 396 } 397 } else { 398 convolve_vert_scale_8tap_neon(im_block, im_stride, dst, dst_stride, w, h, 399 filter_params_y->filter_ptr, subpel_y_qn, 400 y_step_qn); 401 } 402 } else { 403 if (UNLIKELY(conv_params->is_compound)) { 404 if (conv_params->do_average) { 405 if (conv_params->use_dist_wtd_comp_avg) { 406 compound_dist_wtd_convolve_vert_scale_6tap_neon( 407 im_block + im_stride, im_stride, dst, dst_stride, dst16, 408 dst16_stride, w, h, filter_params_y->filter_ptr, conv_params, 409 subpel_y_qn, y_step_qn); 410 } else { 411 compound_avg_convolve_vert_scale_6tap_neon( 412 im_block + im_stride, im_stride, dst, dst_stride, dst16, 413 dst16_stride, w, h, filter_params_y->filter_ptr, subpel_y_qn, 414 y_step_qn); 415 } 416 } else { 417 compound_convolve_vert_scale_6tap_neon( 418 im_block + im_stride, im_stride, dst16, dst16_stride, w, h, 419 filter_params_y->filter_ptr, subpel_y_qn, y_step_qn); 420 } 421 } else { 422 convolve_vert_scale_6tap_neon( 423 im_block + im_stride, im_stride, dst, dst_stride, w, h, 424 filter_params_y->filter_ptr, subpel_y_qn, y_step_qn); 425 } 426 } 427 }