highbd_convolve8_neon.c (14437B)
1 /* 2 * Copyright (c) 2014 The WebM project authors. All rights reserved. 3 * Copyright (c) 2023, Alliance for Open Media. All rights reserved. 4 * 5 * This source code is subject to the terms of the BSD 2 Clause License and 6 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 7 * was not distributed with this source code in the LICENSE file, you can 8 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 9 * Media Patent License 1.0 was not distributed with this source code in the 10 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 11 */ 12 13 #include <arm_neon.h> 14 #include <assert.h> 15 16 #include "config/aom_config.h" 17 #include "config/aom_dsp_rtcd.h" 18 19 #include "aom/aom_integer.h" 20 #include "aom_dsp/aom_dsp_common.h" 21 #include "aom_dsp/aom_filter.h" 22 #include "aom_dsp/arm/aom_filter.h" 23 #include "aom_dsp/arm/highbd_convolve8_neon.h" 24 #include "aom_dsp/arm/mem_neon.h" 25 #include "aom_dsp/arm/transpose_neon.h" 26 #include "aom_ports/mem.h" 27 28 static inline uint16x4_t highbd_convolve8_4( 29 const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, 30 const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, 31 const int16x4_t s6, const int16x4_t s7, const int16x8_t filter, 32 const uint16x4_t max) { 33 const int16x4_t filter_lo = vget_low_s16(filter); 34 const int16x4_t filter_hi = vget_high_s16(filter); 35 36 int32x4_t sum = vmull_lane_s16(s0, filter_lo, 0); 37 sum = vmlal_lane_s16(sum, s1, filter_lo, 1); 38 sum = vmlal_lane_s16(sum, s2, filter_lo, 2); 39 sum = vmlal_lane_s16(sum, s3, filter_lo, 3); 40 sum = vmlal_lane_s16(sum, s4, filter_hi, 0); 41 sum = vmlal_lane_s16(sum, s5, filter_hi, 1); 42 sum = vmlal_lane_s16(sum, s6, filter_hi, 2); 43 sum = vmlal_lane_s16(sum, s7, filter_hi, 3); 44 45 uint16x4_t res = vqrshrun_n_s32(sum, FILTER_BITS); 46 47 return vmin_u16(res, max); 48 } 49 50 static inline uint16x8_t highbd_convolve8_8( 51 const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, 52 const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, 53 const int16x8_t s6, const int16x8_t s7, const int16x8_t filter, 54 const uint16x8_t max) { 55 const int16x4_t filter_lo = vget_low_s16(filter); 56 const int16x4_t filter_hi = vget_high_s16(filter); 57 58 int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), filter_lo, 0); 59 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filter_lo, 1); 60 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filter_lo, 2); 61 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter_lo, 3); 62 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filter_hi, 0); 63 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filter_hi, 1); 64 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), filter_hi, 2); 65 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), filter_hi, 3); 66 67 int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), filter_lo, 0); 68 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filter_lo, 1); 69 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filter_lo, 2); 70 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter_lo, 3); 71 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filter_hi, 0); 72 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filter_hi, 1); 73 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), filter_hi, 2); 74 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), filter_hi, 3); 75 76 uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS), 77 vqrshrun_n_s32(sum1, FILTER_BITS)); 78 79 return vminq_u16(res, max); 80 } 81 82 static void highbd_convolve_horiz_8tap_neon( 83 const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, 84 ptrdiff_t dst_stride, const int16_t *x_filter_ptr, int w, int h, int bd) { 85 assert(w >= 4 && h >= 4); 86 const int16x8_t x_filter = vld1q_s16(x_filter_ptr); 87 88 if (w == 4) { 89 const uint16x4_t max = vdup_n_u16((1 << bd) - 1); 90 const int16_t *s = (const int16_t *)src_ptr; 91 uint16_t *d = dst_ptr; 92 93 do { 94 int16x4_t s0[8], s1[8], s2[8], s3[8]; 95 load_s16_4x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3], 96 &s0[4], &s0[5], &s0[6], &s0[7]); 97 load_s16_4x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3], 98 &s1[4], &s1[5], &s1[6], &s1[7]); 99 load_s16_4x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3], 100 &s2[4], &s2[5], &s2[6], &s2[7]); 101 load_s16_4x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3], 102 &s3[4], &s3[5], &s3[6], &s3[7]); 103 104 uint16x4_t d0 = highbd_convolve8_4(s0[0], s0[1], s0[2], s0[3], s0[4], 105 s0[5], s0[6], s0[7], x_filter, max); 106 uint16x4_t d1 = highbd_convolve8_4(s1[0], s1[1], s1[2], s1[3], s1[4], 107 s1[5], s1[6], s1[7], x_filter, max); 108 uint16x4_t d2 = highbd_convolve8_4(s2[0], s2[1], s2[2], s2[3], s2[4], 109 s2[5], s2[6], s2[7], x_filter, max); 110 uint16x4_t d3 = highbd_convolve8_4(s3[0], s3[1], s3[2], s3[3], s3[4], 111 s3[5], s3[6], s3[7], x_filter, max); 112 113 store_u16_4x4(d, dst_stride, d0, d1, d2, d3); 114 115 s += 4 * src_stride; 116 d += 4 * dst_stride; 117 h -= 4; 118 } while (h > 0); 119 } else { 120 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1); 121 int height = h; 122 123 do { 124 int width = w; 125 const int16_t *s = (const int16_t *)src_ptr; 126 uint16_t *d = dst_ptr; 127 128 do { 129 int16x8_t s0[8], s1[8], s2[8], s3[8]; 130 load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3], 131 &s0[4], &s0[5], &s0[6], &s0[7]); 132 load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3], 133 &s1[4], &s1[5], &s1[6], &s1[7]); 134 load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3], 135 &s2[4], &s2[5], &s2[6], &s2[7]); 136 load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3], 137 &s3[4], &s3[5], &s3[6], &s3[7]); 138 139 uint16x8_t d0 = highbd_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4], 140 s0[5], s0[6], s0[7], x_filter, max); 141 uint16x8_t d1 = highbd_convolve8_8(s1[0], s1[1], s1[2], s1[3], s1[4], 142 s1[5], s1[6], s1[7], x_filter, max); 143 uint16x8_t d2 = highbd_convolve8_8(s2[0], s2[1], s2[2], s2[3], s2[4], 144 s2[5], s2[6], s2[7], x_filter, max); 145 uint16x8_t d3 = highbd_convolve8_8(s3[0], s3[1], s3[2], s3[3], s3[4], 146 s3[5], s3[6], s3[7], x_filter, max); 147 148 store_u16_8x4(d, dst_stride, d0, d1, d2, d3); 149 150 s += 8; 151 d += 8; 152 width -= 8; 153 } while (width > 0); 154 src_ptr += 4 * src_stride; 155 dst_ptr += 4 * dst_stride; 156 height -= 4; 157 } while (height > 0); 158 } 159 } 160 161 static void highbd_convolve_horiz_4tap_neon( 162 const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, 163 ptrdiff_t dst_stride, const int16_t *x_filter_ptr, int w, int h, int bd) { 164 assert(w >= 4 && h >= 4); 165 const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2); 166 167 if (w == 4) { 168 const uint16x4_t max = vdup_n_u16((1 << bd) - 1); 169 const int16_t *s = (const int16_t *)src_ptr; 170 uint16_t *d = dst_ptr; 171 172 do { 173 int16x4_t s0[4], s1[4], s2[4], s3[4]; 174 load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]); 175 load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]); 176 load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]); 177 load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]); 178 179 uint16x4_t d0 = 180 highbd_convolve4_4(s0[0], s0[1], s0[2], s0[3], x_filter, max); 181 uint16x4_t d1 = 182 highbd_convolve4_4(s1[0], s1[1], s1[2], s1[3], x_filter, max); 183 uint16x4_t d2 = 184 highbd_convolve4_4(s2[0], s2[1], s2[2], s2[3], x_filter, max); 185 uint16x4_t d3 = 186 highbd_convolve4_4(s3[0], s3[1], s3[2], s3[3], x_filter, max); 187 188 store_u16_4x4(d, dst_stride, d0, d1, d2, d3); 189 190 s += 4 * src_stride; 191 d += 4 * dst_stride; 192 h -= 4; 193 } while (h > 0); 194 } else { 195 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1); 196 int height = h; 197 198 do { 199 int width = w; 200 const int16_t *s = (const int16_t *)src_ptr; 201 uint16_t *d = dst_ptr; 202 203 do { 204 int16x8_t s0[4], s1[4], s2[4], s3[4]; 205 load_s16_8x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]); 206 load_s16_8x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]); 207 load_s16_8x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]); 208 load_s16_8x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]); 209 210 uint16x8_t d0 = 211 highbd_convolve4_8(s0[0], s0[1], s0[2], s0[3], x_filter, max); 212 uint16x8_t d1 = 213 highbd_convolve4_8(s1[0], s1[1], s1[2], s1[3], x_filter, max); 214 uint16x8_t d2 = 215 highbd_convolve4_8(s2[0], s2[1], s2[2], s2[3], x_filter, max); 216 uint16x8_t d3 = 217 highbd_convolve4_8(s3[0], s3[1], s3[2], s3[3], x_filter, max); 218 219 store_u16_8x4(d, dst_stride, d0, d1, d2, d3); 220 221 s += 8; 222 d += 8; 223 width -= 8; 224 } while (width > 0); 225 src_ptr += 4 * src_stride; 226 dst_ptr += 4 * dst_stride; 227 height -= 4; 228 } while (height > 0); 229 } 230 } 231 232 void aom_highbd_convolve8_horiz_neon(const uint8_t *src8, ptrdiff_t src_stride, 233 uint8_t *dst8, ptrdiff_t dst_stride, 234 const int16_t *filter_x, int x_step_q4, 235 const int16_t *filter_y, int y_step_q4, 236 int w, int h, int bd) { 237 if (x_step_q4 != 16) { 238 aom_highbd_convolve8_horiz_c(src8, src_stride, dst8, dst_stride, filter_x, 239 x_step_q4, filter_y, y_step_q4, w, h, bd); 240 } else { 241 (void)filter_y; 242 (void)y_step_q4; 243 244 uint16_t *src = CONVERT_TO_SHORTPTR(src8); 245 uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); 246 247 src -= SUBPEL_TAPS / 2 - 1; 248 249 const int filter_taps = get_filter_taps_convolve8(filter_x); 250 251 if (filter_taps == 2) { 252 highbd_convolve8_horiz_2tap_neon(src + 3, src_stride, dst, dst_stride, 253 filter_x, w, h, bd); 254 } else if (filter_taps == 4) { 255 highbd_convolve_horiz_4tap_neon(src + 2, src_stride, dst, dst_stride, 256 filter_x, w, h, bd); 257 } else { 258 highbd_convolve_horiz_8tap_neon(src, src_stride, dst, dst_stride, 259 filter_x, w, h, bd); 260 } 261 } 262 } 263 264 static void highbd_convolve_vert_8tap_neon( 265 const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr, 266 ptrdiff_t dst_stride, const int16_t *y_filter_ptr, int w, int h, int bd) { 267 assert(w >= 4 && h >= 4); 268 const int16x8_t y_filter = vld1q_s16(y_filter_ptr); 269 270 if (w == 4) { 271 const uint16x4_t max = vdup_n_u16((1 << bd) - 1); 272 const int16_t *s = (const int16_t *)src_ptr; 273 uint16_t *d = dst_ptr; 274 275 int16x4_t s0, s1, s2, s3, s4, s5, s6; 276 load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); 277 s += 7 * src_stride; 278 279 do { 280 int16x4_t s7, s8, s9, s10; 281 load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10); 282 283 uint16x4_t d0 = 284 highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, max); 285 uint16x4_t d1 = 286 highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, max); 287 uint16x4_t d2 = 288 highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, max); 289 uint16x4_t d3 = 290 highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, max); 291 292 store_u16_4x4(d, dst_stride, d0, d1, d2, d3); 293 294 s0 = s4; 295 s1 = s5; 296 s2 = s6; 297 s3 = s7; 298 s4 = s8; 299 s5 = s9; 300 s6 = s10; 301 302 s += 4 * src_stride; 303 d += 4 * dst_stride; 304 h -= 4; 305 } while (h > 0); 306 } else { 307 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1); 308 309 do { 310 int height = h; 311 const int16_t *s = (const int16_t *)src_ptr; 312 uint16_t *d = dst_ptr; 313 314 int16x8_t s0, s1, s2, s3, s4, s5, s6; 315 load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); 316 s += 7 * src_stride; 317 318 do { 319 int16x8_t s7, s8, s9, s10; 320 load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10); 321 322 uint16x8_t d0 = 323 highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, max); 324 uint16x8_t d1 = 325 highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, max); 326 uint16x8_t d2 = 327 highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, max); 328 uint16x8_t d3 = 329 highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, max); 330 331 store_u16_8x4(d, dst_stride, d0, d1, d2, d3); 332 333 s0 = s4; 334 s1 = s5; 335 s2 = s6; 336 s3 = s7; 337 s4 = s8; 338 s5 = s9; 339 s6 = s10; 340 341 s += 4 * src_stride; 342 d += 4 * dst_stride; 343 height -= 4; 344 } while (height > 0); 345 src_ptr += 8; 346 dst_ptr += 8; 347 w -= 8; 348 } while (w > 0); 349 } 350 } 351 352 void aom_highbd_convolve8_vert_neon(const uint8_t *src8, ptrdiff_t src_stride, 353 uint8_t *dst8, ptrdiff_t dst_stride, 354 const int16_t *filter_x, int x_step_q4, 355 const int16_t *filter_y, int y_step_q4, 356 int w, int h, int bd) { 357 if (y_step_q4 != 16) { 358 aom_highbd_convolve8_vert_c(src8, src_stride, dst8, dst_stride, filter_x, 359 x_step_q4, filter_y, y_step_q4, w, h, bd); 360 } else { 361 (void)filter_x; 362 (void)x_step_q4; 363 364 uint16_t *src = CONVERT_TO_SHORTPTR(src8); 365 uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); 366 367 src -= (SUBPEL_TAPS / 2 - 1) * src_stride; 368 369 const int filter_taps = get_filter_taps_convolve8(filter_y); 370 371 if (filter_taps == 2) { 372 highbd_convolve8_vert_2tap_neon(src + 3 * src_stride, src_stride, dst, 373 dst_stride, filter_y, w, h, bd); 374 } else if (filter_taps == 4) { 375 highbd_convolve8_vert_4tap_neon(src + 2 * src_stride, src_stride, dst, 376 dst_stride, filter_y, w, h, bd); 377 } else { 378 highbd_convolve_vert_8tap_neon(src, src_stride, dst, dst_stride, filter_y, 379 w, h, bd); 380 } 381 } 382 }