convolve_avx2.h (57612B)
1 /* 2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #ifndef AOM_AOM_DSP_X86_CONVOLVE_AVX2_H_ 13 #define AOM_AOM_DSP_X86_CONVOLVE_AVX2_H_ 14 15 #include <immintrin.h> 16 17 #include "aom_ports/mem.h" 18 19 #include "av1/common/convolve.h" 20 #include "av1/common/filter.h" 21 22 // filters for 16 23 DECLARE_ALIGNED(32, static const uint8_t, filt_global_avx2[]) = { 24 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1, 25 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5, 26 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6, 27 7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 28 10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 29 12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7, 30 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 31 }; 32 33 DECLARE_ALIGNED(32, static const uint8_t, filt_d4_global_avx2[]) = { 34 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 0, 1, 2, 3, 1, 2, 35 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 36 7, 8, 9, 10, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 37 }; 38 39 DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = { 40 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 41 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 42 }; 43 44 DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = { 45 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255, 46 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255 47 }; 48 49 DECLARE_ALIGNED(32, static const uint8_t, 50 filt1_global_avx2[32]) = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 51 6, 6, 7, 7, 8, 0, 1, 1, 2, 2, 3, 52 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; 53 54 DECLARE_ALIGNED(32, static const uint8_t, 55 filt2_global_avx2[32]) = { 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 56 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 57 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 }; 58 59 DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = { 60 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 61 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 62 }; 63 64 DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = { 65 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 66 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 67 }; 68 69 #define CONVOLVE_SR_HORIZONTAL_FILTER_4TAP \ 70 for (i = 0; i < (im_h - 2); i += 2) { \ 71 __m256i data = _mm256_castsi128_si256( \ 72 _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ 73 data = _mm256_inserti128_si256( \ 74 data, \ 75 _mm_loadu_si128( \ 76 (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ 77 1); \ 78 __m256i res = convolve_lowbd_x_4tap(data, coeffs_h + 1, filt); \ 79 res = \ 80 _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ 81 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ 82 } \ 83 __m256i data_1 = _mm256_castsi128_si256( \ 84 _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ 85 __m256i res = convolve_lowbd_x_4tap(data_1, coeffs_h + 1, filt); \ 86 res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ 87 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); 88 89 #define CONVOLVE_SR_VERTICAL_FILTER_4TAP \ 90 __m256i s[6]; \ 91 __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ 92 __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ 93 __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ 94 __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ 95 \ 96 s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ 97 s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ 98 s[3] = _mm256_unpackhi_epi16(src_0, src_1); \ 99 s[4] = _mm256_unpackhi_epi16(src_2, src_3); \ 100 \ 101 for (i = 0; i < h; i += 2) { \ 102 const int16_t *data = &im_block[i * im_stride]; \ 103 const __m256i s4 = _mm256_loadu_si256((__m256i *)(data + 4 * im_stride)); \ 104 const __m256i s5 = _mm256_loadu_si256((__m256i *)(data + 5 * im_stride)); \ 105 s[2] = _mm256_unpacklo_epi16(s4, s5); \ 106 s[5] = _mm256_unpackhi_epi16(s4, s5); \ 107 \ 108 __m256i res_a = convolve_4tap(s, coeffs_v + 1); \ 109 __m256i res_b = convolve_4tap(s + 3, coeffs_v + 1); \ 110 \ 111 res_a = \ 112 _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ 113 res_b = \ 114 _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ 115 const __m256i res_a_round = _mm256_sra_epi32( \ 116 _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ 117 const __m256i res_b_round = _mm256_sra_epi32( \ 118 _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ 119 const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ 120 const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ 121 const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ 122 const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ 123 \ 124 __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ 125 __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ 126 if (w - j > 4) { \ 127 _mm_storel_epi64(p_0, res_0); \ 128 _mm_storel_epi64(p_1, res_1); \ 129 } else if (w == 4) { \ 130 xx_storel_32(p_0, res_0); \ 131 xx_storel_32(p_1, res_1); \ 132 } else { \ 133 *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ 134 *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ 135 } \ 136 \ 137 s[0] = s[1]; \ 138 s[1] = s[2]; \ 139 s[3] = s[4]; \ 140 s[4] = s[5]; \ 141 } 142 143 #define CONVOLVE_SR_HORIZONTAL_FILTER_6TAP \ 144 for (i = 0; i < (im_h - 2); i += 2) { \ 145 __m256i data = _mm256_castsi128_si256( \ 146 _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ 147 data = _mm256_inserti128_si256( \ 148 data, \ 149 _mm_loadu_si128( \ 150 (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ 151 1); \ 152 \ 153 __m256i res = convolve_lowbd_x_6tap(data, coeffs_h, filt); \ 154 res = \ 155 _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ 156 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ 157 } \ 158 \ 159 __m256i data_1 = _mm256_castsi128_si256( \ 160 _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ 161 \ 162 __m256i res = convolve_lowbd_x_6tap(data_1, coeffs_h, filt); \ 163 \ 164 res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ 165 \ 166 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); 167 168 #define CONVOLVE_SR_VERTICAL_FILTER_6TAP \ 169 __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ 170 __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ 171 __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ 172 __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ 173 \ 174 __m256i s[8]; \ 175 s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ 176 s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ 177 \ 178 s[3] = _mm256_unpackhi_epi16(src_0, src_1); \ 179 s[4] = _mm256_unpackhi_epi16(src_2, src_3); \ 180 \ 181 for (i = 0; i < h; i += 2) { \ 182 const int16_t *data = &im_block[i * im_stride]; \ 183 \ 184 const __m256i s6 = _mm256_loadu_si256((__m256i *)(data + 4 * im_stride)); \ 185 const __m256i s7 = _mm256_loadu_si256((__m256i *)(data + 5 * im_stride)); \ 186 \ 187 s[2] = _mm256_unpacklo_epi16(s6, s7); \ 188 s[5] = _mm256_unpackhi_epi16(s6, s7); \ 189 \ 190 __m256i res_a = convolve_6tap(s, coeffs_v); \ 191 __m256i res_b = convolve_6tap(s + 3, coeffs_v); \ 192 \ 193 res_a = \ 194 _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ 195 res_b = \ 196 _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ 197 \ 198 const __m256i res_a_round = _mm256_sra_epi32( \ 199 _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ 200 const __m256i res_b_round = _mm256_sra_epi32( \ 201 _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ 202 \ 203 const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ 204 const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ 205 \ 206 const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ 207 const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ 208 \ 209 __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ 210 __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ 211 if (w - j > 4) { \ 212 _mm_storel_epi64(p_0, res_0); \ 213 _mm_storel_epi64(p_1, res_1); \ 214 } else if (w == 4) { \ 215 xx_storel_32(p_0, res_0); \ 216 xx_storel_32(p_1, res_1); \ 217 } else { \ 218 *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ 219 *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ 220 } \ 221 \ 222 s[0] = s[1]; \ 223 s[1] = s[2]; \ 224 \ 225 s[3] = s[4]; \ 226 s[4] = s[5]; \ 227 } 228 229 #define CONVOLVE_SR_HORIZONTAL_FILTER_8TAP \ 230 for (i = 0; i < (im_h - 2); i += 2) { \ 231 __m256i data = _mm256_castsi128_si256( \ 232 _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ 233 data = _mm256_inserti128_si256( \ 234 data, \ 235 _mm_loadu_si128( \ 236 (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ 237 1); \ 238 \ 239 __m256i res = convolve_lowbd_x(data, coeffs_h, filt); \ 240 res = \ 241 _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ 242 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ 243 } \ 244 \ 245 __m256i data_1 = _mm256_castsi128_si256( \ 246 _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ 247 \ 248 __m256i res = convolve_lowbd_x(data_1, coeffs_h, filt); \ 249 \ 250 res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ 251 \ 252 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); 253 254 #define CONVOLVE_SR_VERTICAL_FILTER_8TAP \ 255 __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ 256 __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ 257 __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ 258 __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ 259 __m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); \ 260 __m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); \ 261 \ 262 __m256i s[8]; \ 263 s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ 264 s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ 265 s[2] = _mm256_unpacklo_epi16(src_4, src_5); \ 266 \ 267 s[4] = _mm256_unpackhi_epi16(src_0, src_1); \ 268 s[5] = _mm256_unpackhi_epi16(src_2, src_3); \ 269 s[6] = _mm256_unpackhi_epi16(src_4, src_5); \ 270 \ 271 for (i = 0; i < h; i += 2) { \ 272 const int16_t *data = &im_block[i * im_stride]; \ 273 \ 274 const __m256i s6 = _mm256_loadu_si256((__m256i *)(data + 6 * im_stride)); \ 275 const __m256i s7 = _mm256_loadu_si256((__m256i *)(data + 7 * im_stride)); \ 276 \ 277 s[3] = _mm256_unpacklo_epi16(s6, s7); \ 278 s[7] = _mm256_unpackhi_epi16(s6, s7); \ 279 \ 280 __m256i res_a = convolve(s, coeffs_v); \ 281 __m256i res_b = convolve(s + 4, coeffs_v); \ 282 \ 283 res_a = \ 284 _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ 285 res_b = \ 286 _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ 287 \ 288 const __m256i res_a_round = _mm256_sra_epi32( \ 289 _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ 290 const __m256i res_b_round = _mm256_sra_epi32( \ 291 _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ 292 \ 293 const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ 294 const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ 295 \ 296 const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ 297 const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ 298 \ 299 __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ 300 __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ 301 if (w - j > 4) { \ 302 _mm_storel_epi64(p_0, res_0); \ 303 _mm_storel_epi64(p_1, res_1); \ 304 } else if (w == 4) { \ 305 xx_storel_32(p_0, res_0); \ 306 xx_storel_32(p_1, res_1); \ 307 } else { \ 308 *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ 309 *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ 310 } \ 311 \ 312 s[0] = s[1]; \ 313 s[1] = s[2]; \ 314 s[2] = s[3]; \ 315 \ 316 s[4] = s[5]; \ 317 s[5] = s[6]; \ 318 s[6] = s[7]; \ 319 } 320 321 #define CONVOLVE_SR_HORIZONTAL_FILTER_12TAP \ 322 const __m256i v_zero = _mm256_setzero_si256(); \ 323 __m256i s[12]; \ 324 if (w <= 4) { \ 325 for (i = 0; i < im_h; i += 2) { \ 326 const __m256i data = _mm256_permute2x128_si256( \ 327 _mm256_castsi128_si256( \ 328 _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride + j]))), \ 329 _mm256_castsi128_si256(_mm_loadu_si128( \ 330 (__m128i *)(&src_ptr[i * src_stride + src_stride + j]))), \ 331 0x20); \ 332 const __m256i s_16lo = _mm256_unpacklo_epi8(data, v_zero); \ 333 const __m256i s_16hi = _mm256_unpackhi_epi8(data, v_zero); \ 334 const __m256i s_lolo = _mm256_unpacklo_epi16(s_16lo, s_16lo); \ 335 const __m256i s_lohi = _mm256_unpackhi_epi16(s_16lo, s_16lo); \ 336 \ 337 const __m256i s_hilo = _mm256_unpacklo_epi16(s_16hi, s_16hi); \ 338 const __m256i s_hihi = _mm256_unpackhi_epi16(s_16hi, s_16hi); \ 339 \ 340 s[0] = _mm256_alignr_epi8(s_lohi, s_lolo, 2); \ 341 s[1] = _mm256_alignr_epi8(s_lohi, s_lolo, 10); \ 342 s[2] = _mm256_alignr_epi8(s_hilo, s_lohi, 2); \ 343 s[3] = _mm256_alignr_epi8(s_hilo, s_lohi, 10); \ 344 s[4] = _mm256_alignr_epi8(s_hihi, s_hilo, 2); \ 345 s[5] = _mm256_alignr_epi8(s_hihi, s_hilo, 10); \ 346 \ 347 const __m256i res_lo = convolve_12taps(s, coeffs_h); \ 348 \ 349 __m256i res_32b_lo = _mm256_sra_epi32( \ 350 _mm256_add_epi32(res_lo, round_const_h12), round_shift_h12); \ 351 __m256i res_16b_lo = _mm256_packs_epi32(res_32b_lo, res_32b_lo); \ 352 const __m128i res_0 = _mm256_extracti128_si256(res_16b_lo, 0); \ 353 const __m128i res_1 = _mm256_extracti128_si256(res_16b_lo, 1); \ 354 if (w > 2) { \ 355 _mm_storel_epi64((__m128i *)&im_block[i * im_stride], res_0); \ 356 _mm_storel_epi64((__m128i *)&im_block[i * im_stride + im_stride], \ 357 res_1); \ 358 } else { \ 359 uint32_t horiz_2; \ 360 horiz_2 = (uint32_t)_mm_cvtsi128_si32(res_0); \ 361 im_block[i * im_stride] = (uint16_t)horiz_2; \ 362 im_block[i * im_stride + 1] = (uint16_t)(horiz_2 >> 16); \ 363 horiz_2 = (uint32_t)_mm_cvtsi128_si32(res_1); \ 364 im_block[i * im_stride + im_stride] = (uint16_t)horiz_2; \ 365 im_block[i * im_stride + im_stride + 1] = (uint16_t)(horiz_2 >> 16); \ 366 } \ 367 } \ 368 } else { \ 369 for (i = 0; i < im_h; i++) { \ 370 const __m256i data = _mm256_permute2x128_si256( \ 371 _mm256_castsi128_si256( \ 372 _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride + j]))), \ 373 _mm256_castsi128_si256( \ 374 _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride + j + 4]))), \ 375 0x20); \ 376 const __m256i s_16lo = _mm256_unpacklo_epi8(data, v_zero); \ 377 const __m256i s_16hi = _mm256_unpackhi_epi8(data, v_zero); \ 378 \ 379 const __m256i s_lolo = _mm256_unpacklo_epi16(s_16lo, s_16lo); \ 380 const __m256i s_lohi = _mm256_unpackhi_epi16(s_16lo, s_16lo); \ 381 \ 382 const __m256i s_hilo = _mm256_unpacklo_epi16(s_16hi, s_16hi); \ 383 const __m256i s_hihi = _mm256_unpackhi_epi16(s_16hi, s_16hi); \ 384 \ 385 s[0] = _mm256_alignr_epi8(s_lohi, s_lolo, 2); \ 386 s[1] = _mm256_alignr_epi8(s_lohi, s_lolo, 10); \ 387 s[2] = _mm256_alignr_epi8(s_hilo, s_lohi, 2); \ 388 s[3] = _mm256_alignr_epi8(s_hilo, s_lohi, 10); \ 389 s[4] = _mm256_alignr_epi8(s_hihi, s_hilo, 2); \ 390 s[5] = _mm256_alignr_epi8(s_hihi, s_hilo, 10); \ 391 \ 392 const __m256i res_lo = convolve_12taps(s, coeffs_h); \ 393 \ 394 __m256i res_32b_lo = _mm256_sra_epi32( \ 395 _mm256_add_epi32(res_lo, round_const_h12), round_shift_h12); \ 396 \ 397 __m256i res_16b_lo = _mm256_packs_epi32(res_32b_lo, res_32b_lo); \ 398 _mm_store_si128((__m128i *)&im_block[i * im_stride], \ 399 _mm256_extracti128_si256( \ 400 _mm256_permute4x64_epi64(res_16b_lo, 0x88), 0)); \ 401 } \ 402 } 403 404 #define CONVOLVE_SR_VERTICAL_FILTER_12TAP \ 405 __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ 406 __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ 407 __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ 408 __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ 409 __m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); \ 410 __m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); \ 411 __m256i src_6 = _mm256_loadu_si256((__m256i *)(im_block + 6 * im_stride)); \ 412 __m256i src_7 = _mm256_loadu_si256((__m256i *)(im_block + 7 * im_stride)); \ 413 __m256i src_8 = _mm256_loadu_si256((__m256i *)(im_block + 8 * im_stride)); \ 414 __m256i src_9 = _mm256_loadu_si256((__m256i *)(im_block + 9 * im_stride)); \ 415 \ 416 s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ 417 s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ 418 s[2] = _mm256_unpacklo_epi16(src_4, src_5); \ 419 s[3] = _mm256_unpacklo_epi16(src_6, src_7); \ 420 s[4] = _mm256_unpacklo_epi16(src_8, src_9); \ 421 \ 422 s[6] = _mm256_unpackhi_epi16(src_0, src_1); \ 423 s[7] = _mm256_unpackhi_epi16(src_2, src_3); \ 424 s[8] = _mm256_unpackhi_epi16(src_4, src_5); \ 425 s[9] = _mm256_unpackhi_epi16(src_6, src_7); \ 426 s[10] = _mm256_unpackhi_epi16(src_8, src_9); \ 427 \ 428 for (i = 0; i < h; i += 2) { \ 429 const int16_t *data = &im_block[i * im_stride]; \ 430 \ 431 const __m256i s6 = _mm256_loadu_si256((__m256i *)(data + 10 * im_stride)); \ 432 const __m256i s7 = _mm256_loadu_si256((__m256i *)(data + 11 * im_stride)); \ 433 \ 434 s[5] = _mm256_unpacklo_epi16(s6, s7); \ 435 s[11] = _mm256_unpackhi_epi16(s6, s7); \ 436 \ 437 __m256i res_a = convolve_12taps(s, coeffs_v); \ 438 __m256i res_b = convolve_12taps(s + 6, coeffs_v); \ 439 \ 440 res_a = \ 441 _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ 442 res_b = \ 443 _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ 444 \ 445 const __m256i res_a_round = _mm256_sra_epi32( \ 446 _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ 447 const __m256i res_b_round = _mm256_sra_epi32( \ 448 _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ 449 \ 450 const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ 451 const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ 452 \ 453 const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ 454 const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ 455 \ 456 __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ 457 __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ 458 if (w - j > 4) { \ 459 _mm_storel_epi64(p_0, res_0); \ 460 _mm_storel_epi64(p_1, res_1); \ 461 } else if (w == 4) { \ 462 xx_storel_32(p_0, res_0); \ 463 xx_storel_32(p_1, res_1); \ 464 } else { \ 465 *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ 466 *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ 467 } \ 468 \ 469 s[0] = s[1]; \ 470 s[1] = s[2]; \ 471 s[2] = s[3]; \ 472 s[3] = s[4]; \ 473 s[4] = s[5]; \ 474 \ 475 s[6] = s[7]; \ 476 s[7] = s[8]; \ 477 s[8] = s[9]; \ 478 s[9] = s[10]; \ 479 s[10] = s[11]; \ 480 } 481 482 #define DIST_WTD_CONVOLVE_HORIZONTAL_FILTER_8TAP \ 483 do { \ 484 for (i = 0; i < im_h; i += 2) { \ 485 __m256i data = \ 486 _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)src_h)); \ 487 if (i + 1 < im_h) \ 488 data = _mm256_inserti128_si256( \ 489 data, _mm_loadu_si128((__m128i *)(src_h + src_stride)), 1); \ 490 src_h += (src_stride << 1); \ 491 __m256i res = convolve_lowbd_x(data, coeffs_x, filt); \ 492 \ 493 res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), \ 494 round_shift_h); \ 495 \ 496 _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ 497 } \ 498 } while (0) 499 500 #define DIST_WTD_CONVOLVE_VERTICAL_FILTER_8TAP \ 501 do { \ 502 __m256i s[8]; \ 503 __m256i s0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ 504 __m256i s1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ 505 __m256i s2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ 506 __m256i s3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ 507 __m256i s4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); \ 508 __m256i s5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); \ 509 \ 510 s[0] = _mm256_unpacklo_epi16(s0, s1); \ 511 s[1] = _mm256_unpacklo_epi16(s2, s3); \ 512 s[2] = _mm256_unpacklo_epi16(s4, s5); \ 513 \ 514 s[4] = _mm256_unpackhi_epi16(s0, s1); \ 515 s[5] = _mm256_unpackhi_epi16(s2, s3); \ 516 s[6] = _mm256_unpackhi_epi16(s4, s5); \ 517 \ 518 for (i = 0; i < h; i += 2) { \ 519 const int16_t *data = &im_block[i * im_stride]; \ 520 \ 521 const __m256i s6 = \ 522 _mm256_loadu_si256((__m256i *)(data + 6 * im_stride)); \ 523 const __m256i s7 = \ 524 _mm256_loadu_si256((__m256i *)(data + 7 * im_stride)); \ 525 \ 526 s[3] = _mm256_unpacklo_epi16(s6, s7); \ 527 s[7] = _mm256_unpackhi_epi16(s6, s7); \ 528 \ 529 const __m256i res_a = convolve(s, coeffs_y); \ 530 const __m256i res_a_round = _mm256_sra_epi32( \ 531 _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ 532 \ 533 if (w - j > 4) { \ 534 const __m256i res_b = convolve(s + 4, coeffs_y); \ 535 const __m256i res_b_round = _mm256_sra_epi32( \ 536 _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ 537 const __m256i res_16b = _mm256_packs_epi32(res_a_round, res_b_round); \ 538 const __m256i res_unsigned = _mm256_add_epi16(res_16b, offset_const); \ 539 \ 540 if (do_average) { \ 541 const __m256i data_ref_0 = \ 542 load_line2_avx2(&dst[i * dst_stride + j], \ 543 &dst[i * dst_stride + j + dst_stride]); \ 544 const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_unsigned, \ 545 &wt, use_dist_wtd_comp_avg); \ 546 \ 547 const __m256i round_result = convolve_rounding( \ 548 &comp_avg_res, &offset_const, &rounding_const, rounding_shift); \ 549 \ 550 const __m256i res_8 = \ 551 _mm256_packus_epi16(round_result, round_result); \ 552 const __m128i res_0 = _mm256_castsi256_si128(res_8); \ 553 const __m128i res_1 = _mm256_extracti128_si256(res_8, 1); \ 554 \ 555 _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0); \ 556 _mm_storel_epi64( \ 557 (__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1); \ 558 } else { \ 559 const __m128i res_0 = _mm256_castsi256_si128(res_unsigned); \ 560 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0); \ 561 \ 562 const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1); \ 563 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]), \ 564 res_1); \ 565 } \ 566 } else { \ 567 const __m256i res_16b = _mm256_packs_epi32(res_a_round, res_a_round); \ 568 const __m256i res_unsigned = _mm256_add_epi16(res_16b, offset_const); \ 569 \ 570 if (do_average) { \ 571 const __m256i data_ref_0 = \ 572 load_line2_avx2(&dst[i * dst_stride + j], \ 573 &dst[i * dst_stride + j + dst_stride]); \ 574 \ 575 const __m256i comp_avg_res = comp_avg(&data_ref_0, &res_unsigned, \ 576 &wt, use_dist_wtd_comp_avg); \ 577 \ 578 const __m256i round_result = convolve_rounding( \ 579 &comp_avg_res, &offset_const, &rounding_const, rounding_shift); \ 580 \ 581 const __m256i res_8 = \ 582 _mm256_packus_epi16(round_result, round_result); \ 583 const __m128i res_0 = _mm256_castsi256_si128(res_8); \ 584 const __m128i res_1 = _mm256_extracti128_si256(res_8, 1); \ 585 \ 586 *(int *)(&dst0[i * dst_stride0 + j]) = _mm_cvtsi128_si32(res_0); \ 587 *(int *)(&dst0[i * dst_stride0 + j + dst_stride0]) = \ 588 _mm_cvtsi128_si32(res_1); \ 589 \ 590 } else { \ 591 const __m128i res_0 = _mm256_castsi256_si128(res_unsigned); \ 592 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_0); \ 593 \ 594 const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1); \ 595 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]), \ 596 res_1); \ 597 } \ 598 } \ 599 \ 600 s[0] = s[1]; \ 601 s[1] = s[2]; \ 602 s[2] = s[3]; \ 603 \ 604 s[4] = s[5]; \ 605 s[5] = s[6]; \ 606 s[6] = s[7]; \ 607 } \ 608 } while (0) 609 610 static inline void prepare_coeffs_lowbd( 611 const InterpFilterParams *const filter_params, const int subpel_q4, 612 __m256i *const coeffs /* [4] */) { 613 const int16_t *const filter = av1_get_interp_filter_subpel_kernel( 614 filter_params, subpel_q4 & SUBPEL_MASK); 615 const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); 616 const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); 617 618 // right shift all filter co-efficients by 1 to reduce the bits required. 619 // This extra right shift will be taken care of at the end while rounding 620 // the result. 621 // Since all filter co-efficients are even, this change will not affect the 622 // end result 623 assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), 624 _mm_set1_epi16((short)0xffff))); 625 626 const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); 627 628 // coeffs 0 1 0 1 0 1 0 1 629 coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0200u)); 630 // coeffs 2 3 2 3 2 3 2 3 631 coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0604u)); 632 // coeffs 4 5 4 5 4 5 4 5 633 coeffs[2] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0a08u)); 634 // coeffs 6 7 6 7 6 7 6 7 635 coeffs[3] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0e0cu)); 636 } 637 638 static inline void prepare_coeffs_6t_lowbd( 639 const InterpFilterParams *const filter_params, const int subpel_q4, 640 __m256i *const coeffs /* [4] */) { 641 const int16_t *const filter = av1_get_interp_filter_subpel_kernel( 642 filter_params, subpel_q4 & SUBPEL_MASK); 643 const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); 644 const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); 645 646 // right shift all filter co-efficients by 1 to reduce the bits required. 647 // This extra right shift will be taken care of at the end while rounding 648 // the result. 649 // Since all filter co-efficients are even, this change will not affect the 650 // end result 651 assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), 652 _mm_set1_epi16((int16_t)0xffff))); 653 654 const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); 655 656 // coeffs 1 2 1 2 1 2 1 2 657 coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0402u)); 658 // coeffs 3 4 3 4 3 4 3 4 659 coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0806u)); 660 // coeffs 5 6 5 6 5 6 5 6 661 coeffs[2] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0c0au)); 662 } 663 664 static inline void prepare_coeffs_6t( 665 const InterpFilterParams *const filter_params, const int subpel_q4, 666 __m256i *const coeffs /* [4] */) { 667 const int16_t *filter = av1_get_interp_filter_subpel_kernel( 668 filter_params, subpel_q4 & SUBPEL_MASK); 669 670 const __m128i coeff_8 = _mm_loadu_si128((__m128i *)(filter + 1)); 671 const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); 672 673 // coeffs 1 2 1 2 1 2 1 2 674 coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); 675 // coeffs 3 4 3 4 3 4 3 4 676 coeffs[1] = _mm256_shuffle_epi32(coeff, 0x55); 677 // coeffs 5 6 5 6 5 6 5 6 678 coeffs[2] = _mm256_shuffle_epi32(coeff, 0xaa); 679 } 680 681 static inline void prepare_coeffs(const InterpFilterParams *const filter_params, 682 const int subpel_q4, 683 __m256i *const coeffs /* [4] */) { 684 const int16_t *filter = av1_get_interp_filter_subpel_kernel( 685 filter_params, subpel_q4 & SUBPEL_MASK); 686 687 const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); 688 const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); 689 690 // coeffs 0 1 0 1 0 1 0 1 691 coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); 692 // coeffs 2 3 2 3 2 3 2 3 693 coeffs[1] = _mm256_shuffle_epi32(coeff, 0x55); 694 // coeffs 4 5 4 5 4 5 4 5 695 coeffs[2] = _mm256_shuffle_epi32(coeff, 0xaa); 696 // coeffs 6 7 6 7 6 7 6 7 697 coeffs[3] = _mm256_shuffle_epi32(coeff, 0xff); 698 } 699 700 static inline void prepare_coeffs_12taps( 701 const InterpFilterParams *const filter_params, const int subpel_q4, 702 __m256i *const coeffs /* [4] */) { 703 const int16_t *filter = av1_get_interp_filter_subpel_kernel( 704 filter_params, subpel_q4 & SUBPEL_MASK); 705 706 __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); 707 __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); 708 709 // coeffs 0 1 0 1 0 1 0 1 710 coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); 711 // coeffs 2 3 2 3 2 3 2 3 712 coeffs[1] = _mm256_shuffle_epi32(coeff, 0x55); 713 // coeffs 4 5 4 5 4 5 4 5 714 coeffs[2] = _mm256_shuffle_epi32(coeff, 0xaa); 715 // coeffs 6 7 6 7 6 7 6 7 716 coeffs[3] = _mm256_shuffle_epi32(coeff, 0xff); 717 // coeffs 8 9 10 11 0 0 0 0 718 coeff_8 = _mm_loadl_epi64((__m128i *)(filter + 8)); 719 coeff = _mm256_broadcastq_epi64(coeff_8); 720 coeffs[4] = _mm256_shuffle_epi32(coeff, 0x00); // coeffs 8 9 8 9 8 9 8 9 721 coeffs[5] = _mm256_shuffle_epi32(coeff, 0x55); // coeffs 10 11 10 11.. 10 11 722 } 723 724 static inline __m256i convolve_lowbd(const __m256i *const s, 725 const __m256i *const coeffs) { 726 const __m256i res_01 = _mm256_maddubs_epi16(s[0], coeffs[0]); 727 const __m256i res_23 = _mm256_maddubs_epi16(s[1], coeffs[1]); 728 const __m256i res_45 = _mm256_maddubs_epi16(s[2], coeffs[2]); 729 const __m256i res_67 = _mm256_maddubs_epi16(s[3], coeffs[3]); 730 731 // order: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 732 const __m256i res = _mm256_add_epi16(_mm256_add_epi16(res_01, res_45), 733 _mm256_add_epi16(res_23, res_67)); 734 735 return res; 736 } 737 738 static inline __m256i convolve_lowbd_6tap(const __m256i *const s, 739 const __m256i *const coeffs) { 740 const __m256i res_01 = _mm256_maddubs_epi16(s[0], coeffs[0]); 741 const __m256i res_23 = _mm256_maddubs_epi16(s[1], coeffs[1]); 742 const __m256i res_45 = _mm256_maddubs_epi16(s[2], coeffs[2]); 743 744 // order: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 745 const __m256i res = 746 _mm256_add_epi16(_mm256_add_epi16(res_01, res_45), res_23); 747 748 return res; 749 } 750 751 static inline __m256i convolve_lowbd_4tap(const __m256i *const s, 752 const __m256i *const coeffs) { 753 const __m256i res_23 = _mm256_maddubs_epi16(s[0], coeffs[0]); 754 const __m256i res_45 = _mm256_maddubs_epi16(s[1], coeffs[1]); 755 756 // order: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 757 const __m256i res = _mm256_add_epi16(res_45, res_23); 758 759 return res; 760 } 761 762 static inline __m256i convolve_6tap(const __m256i *const s, 763 const __m256i *const coeffs) { 764 const __m256i res_0 = _mm256_madd_epi16(s[0], coeffs[0]); 765 const __m256i res_1 = _mm256_madd_epi16(s[1], coeffs[1]); 766 const __m256i res_2 = _mm256_madd_epi16(s[2], coeffs[2]); 767 768 const __m256i res = _mm256_add_epi32(_mm256_add_epi32(res_0, res_1), res_2); 769 770 return res; 771 } 772 773 static inline __m256i convolve_12taps(const __m256i *const s, 774 const __m256i *const coeffs) { 775 const __m256i res_0 = _mm256_madd_epi16(s[0], coeffs[0]); 776 const __m256i res_1 = _mm256_madd_epi16(s[1], coeffs[1]); 777 const __m256i res_2 = _mm256_madd_epi16(s[2], coeffs[2]); 778 const __m256i res_3 = _mm256_madd_epi16(s[3], coeffs[3]); 779 const __m256i res_4 = _mm256_madd_epi16(s[4], coeffs[4]); 780 const __m256i res_5 = _mm256_madd_epi16(s[5], coeffs[5]); 781 782 const __m256i res1 = _mm256_add_epi32(_mm256_add_epi32(res_0, res_1), 783 _mm256_add_epi32(res_2, res_3)); 784 const __m256i res = _mm256_add_epi32(_mm256_add_epi32(res_4, res_5), res1); 785 786 return res; 787 } 788 789 static inline __m256i convolve(const __m256i *const s, 790 const __m256i *const coeffs) { 791 const __m256i res_0 = _mm256_madd_epi16(s[0], coeffs[0]); 792 const __m256i res_1 = _mm256_madd_epi16(s[1], coeffs[1]); 793 const __m256i res_2 = _mm256_madd_epi16(s[2], coeffs[2]); 794 const __m256i res_3 = _mm256_madd_epi16(s[3], coeffs[3]); 795 796 const __m256i res = _mm256_add_epi32(_mm256_add_epi32(res_0, res_1), 797 _mm256_add_epi32(res_2, res_3)); 798 799 return res; 800 } 801 802 static inline __m256i convolve_4tap(const __m256i *const s, 803 const __m256i *const coeffs) { 804 const __m256i res_1 = _mm256_madd_epi16(s[0], coeffs[0]); 805 const __m256i res_2 = _mm256_madd_epi16(s[1], coeffs[1]); 806 807 const __m256i res = _mm256_add_epi32(res_1, res_2); 808 return res; 809 } 810 811 static inline __m256i convolve_lowbd_x(const __m256i data, 812 const __m256i *const coeffs, 813 const __m256i *const filt) { 814 __m256i s[4]; 815 816 s[0] = _mm256_shuffle_epi8(data, filt[0]); 817 s[1] = _mm256_shuffle_epi8(data, filt[1]); 818 s[2] = _mm256_shuffle_epi8(data, filt[2]); 819 s[3] = _mm256_shuffle_epi8(data, filt[3]); 820 821 return convolve_lowbd(s, coeffs); 822 } 823 824 static inline __m256i convolve_lowbd_x_6tap(const __m256i data, 825 const __m256i *const coeffs, 826 const __m256i *const filt) { 827 __m256i s[4]; 828 829 s[0] = _mm256_shuffle_epi8(data, filt[0]); 830 s[1] = _mm256_shuffle_epi8(data, filt[1]); 831 s[2] = _mm256_shuffle_epi8(data, filt[2]); 832 833 return convolve_lowbd_6tap(s, coeffs); 834 } 835 836 static inline __m256i convolve_lowbd_x_4tap(const __m256i data, 837 const __m256i *const coeffs, 838 const __m256i *const filt) { 839 __m256i s[2]; 840 841 s[0] = _mm256_shuffle_epi8(data, filt[0]); 842 s[1] = _mm256_shuffle_epi8(data, filt[1]); 843 844 return convolve_lowbd_4tap(s, coeffs); 845 } 846 847 static inline void add_store_aligned_256(CONV_BUF_TYPE *const dst, 848 const __m256i *const res, 849 const int do_average) { 850 __m256i d; 851 if (do_average) { 852 d = _mm256_load_si256((__m256i *)dst); 853 d = _mm256_add_epi32(d, *res); 854 d = _mm256_srai_epi32(d, 1); 855 } else { 856 d = *res; 857 } 858 _mm256_store_si256((__m256i *)dst, d); 859 } 860 861 static inline __m256i comp_avg(const __m256i *const data_ref_0, 862 const __m256i *const res_unsigned, 863 const __m256i *const wt, 864 const int use_dist_wtd_comp_avg) { 865 __m256i res; 866 if (use_dist_wtd_comp_avg) { 867 const __m256i data_lo = _mm256_unpacklo_epi16(*data_ref_0, *res_unsigned); 868 const __m256i data_hi = _mm256_unpackhi_epi16(*data_ref_0, *res_unsigned); 869 870 const __m256i wt_res_lo = _mm256_madd_epi16(data_lo, *wt); 871 const __m256i wt_res_hi = _mm256_madd_epi16(data_hi, *wt); 872 873 const __m256i res_lo = _mm256_srai_epi32(wt_res_lo, DIST_PRECISION_BITS); 874 const __m256i res_hi = _mm256_srai_epi32(wt_res_hi, DIST_PRECISION_BITS); 875 876 res = _mm256_packs_epi32(res_lo, res_hi); 877 } else { 878 const __m256i wt_res = _mm256_add_epi16(*data_ref_0, *res_unsigned); 879 res = _mm256_srai_epi16(wt_res, 1); 880 } 881 return res; 882 } 883 884 static inline __m256i convolve_rounding(const __m256i *const res_unsigned, 885 const __m256i *const offset_const, 886 const __m256i *const round_const, 887 const int round_shift) { 888 const __m256i res_signed = _mm256_sub_epi16(*res_unsigned, *offset_const); 889 const __m256i res_round = _mm256_srai_epi16( 890 _mm256_add_epi16(res_signed, *round_const), round_shift); 891 return res_round; 892 } 893 894 static inline __m256i highbd_comp_avg(const __m256i *const data_ref_0, 895 const __m256i *const res_unsigned, 896 const __m256i *const wt0, 897 const __m256i *const wt1, 898 const int use_dist_wtd_comp_avg) { 899 __m256i res; 900 if (use_dist_wtd_comp_avg) { 901 const __m256i wt0_res = _mm256_mullo_epi32(*data_ref_0, *wt0); 902 const __m256i wt1_res = _mm256_mullo_epi32(*res_unsigned, *wt1); 903 const __m256i wt_res = _mm256_add_epi32(wt0_res, wt1_res); 904 res = _mm256_srai_epi32(wt_res, DIST_PRECISION_BITS); 905 } else { 906 const __m256i wt_res = _mm256_add_epi32(*data_ref_0, *res_unsigned); 907 res = _mm256_srai_epi32(wt_res, 1); 908 } 909 return res; 910 } 911 912 static inline __m256i highbd_convolve_rounding( 913 const __m256i *const res_unsigned, const __m256i *const offset_const, 914 const __m256i *const round_const, const int round_shift) { 915 const __m256i res_signed = _mm256_sub_epi32(*res_unsigned, *offset_const); 916 const __m256i res_round = _mm256_srai_epi32( 917 _mm256_add_epi32(res_signed, *round_const), round_shift); 918 919 return res_round; 920 } 921 922 #endif // AOM_AOM_DSP_X86_CONVOLVE_AVX2_H_