fwd_txfm_impl_sse2.h (24059B)
1 /* 2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <emmintrin.h> // SSE2 13 14 #include "config/aom_dsp_rtcd.h" 15 16 #include "aom_dsp/txfm_common.h" 17 #include "aom_dsp/x86/fwd_txfm_sse2.h" 18 #include "aom_dsp/x86/txfm_common_sse2.h" 19 #include "aom_ports/mem.h" 20 21 // TODO(jingning) The high bit-depth functions need rework for performance. 22 // After we properly fix the high bit-depth function implementations, this 23 // file's dependency should be substantially simplified. 24 #if DCT_HIGH_BIT_DEPTH 25 #define ADD_EPI16 _mm_adds_epi16 26 #define SUB_EPI16 _mm_subs_epi16 27 28 #else 29 #define ADD_EPI16 _mm_add_epi16 30 #define SUB_EPI16 _mm_sub_epi16 31 #endif 32 33 #if defined(FDCT4x4_2D_HELPER) 34 static void FDCT4x4_2D_HELPER(const int16_t *input, int stride, __m128i *in0, 35 __m128i *in1) { 36 // Constants 37 // These are the coefficients used for the multiplies. 38 // In the comments, pN means cos(N pi /64) and mN is -cos(N pi /64), 39 // where cospi_N_64 = cos(N pi /64) 40 const __m128i k__cospi_A = 41 octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64, 42 cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64); 43 const __m128i k__cospi_B = 44 octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64, 45 cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64); 46 const __m128i k__cospi_C = 47 octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64, 48 cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64); 49 const __m128i k__cospi_D = 50 octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64, 51 cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64); 52 const __m128i k__cospi_E = 53 octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64, 54 cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64); 55 const __m128i k__cospi_F = 56 octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64, 57 cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64); 58 const __m128i k__cospi_G = 59 octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64, 60 -cospi_8_64, -cospi_24_64, -cospi_8_64, -cospi_24_64); 61 const __m128i k__cospi_H = 62 octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64, 63 -cospi_24_64, cospi_8_64, -cospi_24_64, cospi_8_64); 64 65 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 66 // This second rounding constant saves doing some extra adds at the end 67 const __m128i k__DCT_CONST_ROUNDING2 = 68 _mm_set1_epi32(DCT_CONST_ROUNDING + (DCT_CONST_ROUNDING << 1)); 69 const int DCT_CONST_BITS2 = DCT_CONST_BITS + 2; 70 const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1); 71 const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0); 72 73 // Load inputs. 74 *in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); 75 *in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); 76 *in1 = _mm_unpacklo_epi64( 77 *in1, _mm_loadl_epi64((const __m128i *)(input + 2 * stride))); 78 *in0 = _mm_unpacklo_epi64( 79 *in0, _mm_loadl_epi64((const __m128i *)(input + 3 * stride))); 80 // in0 = [i0 i1 i2 i3 iC iD iE iF] 81 // in1 = [i4 i5 i6 i7 i8 i9 iA iB] 82 // multiply by 16 to give some extra precision 83 *in0 = _mm_slli_epi16(*in0, 4); 84 *in1 = _mm_slli_epi16(*in1, 4); 85 // if (i == 0 && input[0]) input[0] += 1; 86 // add 1 to the upper left pixel if it is non-zero, which helps reduce 87 // the round-trip error 88 { 89 // The mask will only contain whether the first value is zero, all 90 // other comparison will fail as something shifted by 4 (above << 4) 91 // can never be equal to one. To increment in the non-zero case, we 92 // add the mask and one for the first element: 93 // - if zero, mask = -1, v = v - 1 + 1 = v 94 // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1 95 __m128i mask = _mm_cmpeq_epi16(*in0, k__nonzero_bias_a); 96 *in0 = _mm_add_epi16(*in0, mask); 97 *in0 = _mm_add_epi16(*in0, k__nonzero_bias_b); 98 } 99 // There are 4 total stages, alternating between an add/subtract stage 100 // followed by an multiply-and-add stage. 101 { 102 // Stage 1: Add/subtract 103 104 // in0 = [i0 i1 i2 i3 iC iD iE iF] 105 // in1 = [i4 i5 i6 i7 i8 i9 iA iB] 106 const __m128i r0 = _mm_unpacklo_epi16(*in0, *in1); 107 const __m128i r1 = _mm_unpackhi_epi16(*in0, *in1); 108 // r0 = [i0 i4 i1 i5 i2 i6 i3 i7] 109 // r1 = [iC i8 iD i9 iE iA iF iB] 110 const __m128i r2 = _mm_shuffle_epi32(r0, 0xB4); 111 const __m128i r3 = _mm_shuffle_epi32(r1, 0xB4); 112 // r2 = [i0 i4 i1 i5 i3 i7 i2 i6] 113 // r3 = [iC i8 iD i9 iF iB iE iA] 114 115 const __m128i t0 = _mm_add_epi16(r2, r3); 116 const __m128i t1 = _mm_sub_epi16(r2, r3); 117 // t0 = [a0 a4 a1 a5 a3 a7 a2 a6] 118 // t1 = [aC a8 aD a9 aF aB aE aA] 119 120 // Stage 2: multiply by constants (which gets us into 32 bits). 121 // The constants needed here are: 122 // k__cospi_A = [p16 p16 p16 p16 p16 m16 p16 m16] 123 // k__cospi_B = [p16 m16 p16 m16 p16 p16 p16 p16] 124 // k__cospi_C = [p08 p24 p08 p24 p24 m08 p24 m08] 125 // k__cospi_D = [p24 m08 p24 m08 p08 p24 p08 p24] 126 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_A); 127 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_B); 128 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_C); 129 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_D); 130 // Then add and right-shift to get back to 16-bit range 131 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 132 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 133 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 134 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 135 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 136 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 137 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 138 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 139 // w0 = [b0 b1 b7 b6] 140 // w1 = [b8 b9 bF bE] 141 // w2 = [b4 b5 b3 b2] 142 // w3 = [bC bD bB bA] 143 const __m128i x0 = _mm_packs_epi32(w0, w1); 144 const __m128i x1 = _mm_packs_epi32(w2, w3); 145 146 // x0 = [b0 b1 b7 b6 b8 b9 bF bE] 147 // x1 = [b4 b5 b3 b2 bC bD bB bA] 148 *in0 = _mm_shuffle_epi32(x0, 0xD8); 149 *in1 = _mm_shuffle_epi32(x1, 0x8D); 150 // in0 = [b0 b1 b8 b9 b7 b6 bF bE] 151 // in1 = [b3 b2 bB bA b4 b5 bC bD] 152 } 153 { 154 // vertical DCTs finished. Now we do the horizontal DCTs. 155 // Stage 3: Add/subtract 156 157 const __m128i t0 = ADD_EPI16(*in0, *in1); 158 const __m128i t1 = SUB_EPI16(*in0, *in1); 159 160 // Stage 4: multiply by constants (which gets us into 32 bits). 161 { 162 // The constants needed here are: 163 // k__cospi_E = [p16 p16 p16 p16 p16 p16 p16 p16] 164 // k__cospi_F = [p16 m16 p16 m16 p16 m16 p16 m16] 165 // k__cospi_G = [p08 p24 p08 p24 m08 m24 m08 m24] 166 // k__cospi_H = [p24 m08 p24 m08 m24 p08 m24 p08] 167 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_E); 168 const __m128i u1 = _mm_madd_epi16(t0, k__cospi_F); 169 const __m128i u2 = _mm_madd_epi16(t1, k__cospi_G); 170 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_H); 171 // Then add and right-shift to get back to 16-bit range 172 // but this combines the final right-shift as well to save operations 173 // This unusual rounding operations is to maintain bit-accurate 174 // compatibility with the c version of this function which has two 175 // rounding steps in a row. 176 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING2); 177 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING2); 178 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING2); 179 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING2); 180 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS2); 181 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS2); 182 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS2); 183 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS2); 184 *in0 = _mm_packs_epi32(w0, w2); 185 *in1 = _mm_packs_epi32(w1, w3); 186 } 187 } 188 } 189 #endif // defined(FDCT4x4_2D_HELPER) 190 191 #if defined(FDCT4x4_2D) 192 void FDCT4x4_2D(const int16_t *input, tran_low_t *output, int stride) { 193 // This 2D transform implements 4 vertical 1D transforms followed 194 // by 4 horizontal 1D transforms. The multiplies and adds are as given 195 // by Chen, Smith and Fralick ('77). The commands for moving the data 196 // around have been minimized by hand. 197 // For the purposes of the comments, the 16 inputs are referred to at i0 198 // through iF (in raster order), intermediate variables are a0, b0, c0 199 // through f, and correspond to the in-place computations mapped to input 200 // locations. The outputs, o0 through oF are labeled according to the 201 // output locations. 202 __m128i in0, in1; 203 FDCT4x4_2D_HELPER(input, stride, &in0, &in1); 204 205 // Post-condition (v + 1) >> 2 is now incorporated into previous 206 // add and right-shift commands. Only 2 store instructions needed 207 // because we are using the fact that 1/3 are stored just after 0/2. 208 storeu_output(&in0, output + 0 * 4); 209 storeu_output(&in1, output + 2 * 4); 210 } 211 #endif // defined(FDCT4x4_2D) 212 213 #if defined(FDCT4x4_2D_LP) 214 void FDCT4x4_2D_LP(const int16_t *input, int16_t *output, int stride) { 215 __m128i in0, in1; 216 FDCT4x4_2D_HELPER(input, stride, &in0, &in1); 217 _mm_storeu_si128((__m128i *)(output + 0 * 4), in0); 218 _mm_storeu_si128((__m128i *)(output + 2 * 4), in1); 219 } 220 #endif // defined(FDCT4x4_2D_LP) 221 222 #if CONFIG_INTERNAL_STATS 223 void FDCT8x8_2D(const int16_t *input, tran_low_t *output, int stride) { 224 int pass; 225 // Constants 226 // When we use them, in one case, they are all the same. In all others 227 // it's a pair of them that we need to repeat four times. This is done 228 // by constructing the 32 bit constant corresponding to that pair. 229 const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64); 230 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 231 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); 232 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); 233 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); 234 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); 235 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); 236 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); 237 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 238 #if DCT_HIGH_BIT_DEPTH 239 int overflow; 240 #endif 241 // Load input 242 __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); 243 __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); 244 __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); 245 __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); 246 __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride)); 247 __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride)); 248 __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride)); 249 __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride)); 250 // Pre-condition input (shift by two) 251 in0 = _mm_slli_epi16(in0, 2); 252 in1 = _mm_slli_epi16(in1, 2); 253 in2 = _mm_slli_epi16(in2, 2); 254 in3 = _mm_slli_epi16(in3, 2); 255 in4 = _mm_slli_epi16(in4, 2); 256 in5 = _mm_slli_epi16(in5, 2); 257 in6 = _mm_slli_epi16(in6, 2); 258 in7 = _mm_slli_epi16(in7, 2); 259 260 // We do two passes, first the columns, then the rows. The results of the 261 // first pass are transposed so that the same column code can be reused. The 262 // results of the second pass are also transposed so that the rows (processed 263 // as columns) are put back in row positions. 264 for (pass = 0; pass < 2; pass++) { 265 // To store results of each pass before the transpose. 266 __m128i res0, res1, res2, res3, res4, res5, res6, res7; 267 // Add/subtract 268 const __m128i q0 = ADD_EPI16(in0, in7); 269 const __m128i q1 = ADD_EPI16(in1, in6); 270 const __m128i q2 = ADD_EPI16(in2, in5); 271 const __m128i q3 = ADD_EPI16(in3, in4); 272 const __m128i q4 = SUB_EPI16(in3, in4); 273 const __m128i q5 = SUB_EPI16(in2, in5); 274 const __m128i q6 = SUB_EPI16(in1, in6); 275 const __m128i q7 = SUB_EPI16(in0, in7); 276 #if DCT_HIGH_BIT_DEPTH 277 if (pass == 1) { 278 overflow = 279 check_epi16_overflow_x8(&q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7); 280 if (overflow) { 281 aom_highbd_fdct8x8_c(input, output, stride); 282 return; 283 } 284 } 285 #endif // DCT_HIGH_BIT_DEPTH 286 // Work on first four results 287 { 288 // Add/subtract 289 const __m128i r0 = ADD_EPI16(q0, q3); 290 const __m128i r1 = ADD_EPI16(q1, q2); 291 const __m128i r2 = SUB_EPI16(q1, q2); 292 const __m128i r3 = SUB_EPI16(q0, q3); 293 #if DCT_HIGH_BIT_DEPTH 294 overflow = check_epi16_overflow_x4(&r0, &r1, &r2, &r3); 295 if (overflow) { 296 aom_highbd_fdct8x8_c(input, output, stride); 297 return; 298 } 299 #endif // DCT_HIGH_BIT_DEPTH 300 // Interleave to do the multiply by constants which gets us into 32bits 301 { 302 const __m128i t0 = _mm_unpacklo_epi16(r0, r1); 303 const __m128i t1 = _mm_unpackhi_epi16(r0, r1); 304 const __m128i t2 = _mm_unpacklo_epi16(r2, r3); 305 const __m128i t3 = _mm_unpackhi_epi16(r2, r3); 306 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); 307 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); 308 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16); 309 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16); 310 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08); 311 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08); 312 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24); 313 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24); 314 // dct_const_round_shift 315 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 316 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 317 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 318 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 319 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 320 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 321 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 322 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 323 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 324 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 325 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 326 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 327 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 328 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 329 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 330 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 331 // Combine 332 res0 = _mm_packs_epi32(w0, w1); 333 res4 = _mm_packs_epi32(w2, w3); 334 res2 = _mm_packs_epi32(w4, w5); 335 res6 = _mm_packs_epi32(w6, w7); 336 #if DCT_HIGH_BIT_DEPTH 337 overflow = check_epi16_overflow_x4(&res0, &res4, &res2, &res6); 338 if (overflow) { 339 aom_highbd_fdct8x8_c(input, output, stride); 340 return; 341 } 342 #endif // DCT_HIGH_BIT_DEPTH 343 } 344 } 345 // Work on next four results 346 { 347 // Interleave to do the multiply by constants which gets us into 32bits 348 const __m128i d0 = _mm_unpacklo_epi16(q6, q5); 349 const __m128i d1 = _mm_unpackhi_epi16(q6, q5); 350 const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16); 351 const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16); 352 const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16); 353 const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16); 354 // dct_const_round_shift 355 const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING); 356 const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING); 357 const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING); 358 const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING); 359 const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS); 360 const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS); 361 const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS); 362 const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS); 363 // Combine 364 const __m128i r0 = _mm_packs_epi32(s0, s1); 365 const __m128i r1 = _mm_packs_epi32(s2, s3); 366 #if DCT_HIGH_BIT_DEPTH 367 overflow = check_epi16_overflow_x2(&r0, &r1); 368 if (overflow) { 369 aom_highbd_fdct8x8_c(input, output, stride); 370 return; 371 } 372 #endif // DCT_HIGH_BIT_DEPTH 373 { 374 // Add/subtract 375 const __m128i x0 = ADD_EPI16(q4, r0); 376 const __m128i x1 = SUB_EPI16(q4, r0); 377 const __m128i x2 = SUB_EPI16(q7, r1); 378 const __m128i x3 = ADD_EPI16(q7, r1); 379 #if DCT_HIGH_BIT_DEPTH 380 overflow = check_epi16_overflow_x4(&x0, &x1, &x2, &x3); 381 if (overflow) { 382 aom_highbd_fdct8x8_c(input, output, stride); 383 return; 384 } 385 #endif // DCT_HIGH_BIT_DEPTH 386 // Interleave to do the multiply by constants which gets us into 32bits 387 { 388 const __m128i t0 = _mm_unpacklo_epi16(x0, x3); 389 const __m128i t1 = _mm_unpackhi_epi16(x0, x3); 390 const __m128i t2 = _mm_unpacklo_epi16(x1, x2); 391 const __m128i t3 = _mm_unpackhi_epi16(x1, x2); 392 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04); 393 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04); 394 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28); 395 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28); 396 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20); 397 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20); 398 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12); 399 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12); 400 // dct_const_round_shift 401 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 402 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 403 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 404 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 405 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 406 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 407 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 408 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 409 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 410 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 411 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 412 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 413 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 414 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 415 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 416 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 417 // Combine 418 res1 = _mm_packs_epi32(w0, w1); 419 res7 = _mm_packs_epi32(w2, w3); 420 res5 = _mm_packs_epi32(w4, w5); 421 res3 = _mm_packs_epi32(w6, w7); 422 #if DCT_HIGH_BIT_DEPTH 423 overflow = check_epi16_overflow_x4(&res1, &res7, &res5, &res3); 424 if (overflow) { 425 aom_highbd_fdct8x8_c(input, output, stride); 426 return; 427 } 428 #endif // DCT_HIGH_BIT_DEPTH 429 } 430 } 431 } 432 // Transpose the 8x8. 433 { 434 // 00 01 02 03 04 05 06 07 435 // 10 11 12 13 14 15 16 17 436 // 20 21 22 23 24 25 26 27 437 // 30 31 32 33 34 35 36 37 438 // 40 41 42 43 44 45 46 47 439 // 50 51 52 53 54 55 56 57 440 // 60 61 62 63 64 65 66 67 441 // 70 71 72 73 74 75 76 77 442 const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1); 443 const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3); 444 const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1); 445 const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3); 446 const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5); 447 const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7); 448 const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5); 449 const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7); 450 // 00 10 01 11 02 12 03 13 451 // 20 30 21 31 22 32 23 33 452 // 04 14 05 15 06 16 07 17 453 // 24 34 25 35 26 36 27 37 454 // 40 50 41 51 42 52 43 53 455 // 60 70 61 71 62 72 63 73 456 // 54 54 55 55 56 56 57 57 457 // 64 74 65 75 66 76 67 77 458 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); 459 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); 460 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); 461 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); 462 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); 463 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); 464 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); 465 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); 466 // 00 10 20 30 01 11 21 31 467 // 40 50 60 70 41 51 61 71 468 // 02 12 22 32 03 13 23 33 469 // 42 52 62 72 43 53 63 73 470 // 04 14 24 34 05 15 21 36 471 // 44 54 64 74 45 55 61 76 472 // 06 16 26 36 07 17 27 37 473 // 46 56 66 76 47 57 67 77 474 in0 = _mm_unpacklo_epi64(tr1_0, tr1_4); 475 in1 = _mm_unpackhi_epi64(tr1_0, tr1_4); 476 in2 = _mm_unpacklo_epi64(tr1_2, tr1_6); 477 in3 = _mm_unpackhi_epi64(tr1_2, tr1_6); 478 in4 = _mm_unpacklo_epi64(tr1_1, tr1_5); 479 in5 = _mm_unpackhi_epi64(tr1_1, tr1_5); 480 in6 = _mm_unpacklo_epi64(tr1_3, tr1_7); 481 in7 = _mm_unpackhi_epi64(tr1_3, tr1_7); 482 // 00 10 20 30 40 50 60 70 483 // 01 11 21 31 41 51 61 71 484 // 02 12 22 32 42 52 62 72 485 // 03 13 23 33 43 53 63 73 486 // 04 14 24 34 44 54 64 74 487 // 05 15 25 35 45 55 65 75 488 // 06 16 26 36 46 56 66 76 489 // 07 17 27 37 47 57 67 77 490 } 491 } 492 // Post-condition output and store it 493 { 494 // Post-condition (division by two) 495 // division of two 16 bits signed numbers using shifts 496 // n / 2 = (n - (n >> 15)) >> 1 497 const __m128i sign_in0 = _mm_srai_epi16(in0, 15); 498 const __m128i sign_in1 = _mm_srai_epi16(in1, 15); 499 const __m128i sign_in2 = _mm_srai_epi16(in2, 15); 500 const __m128i sign_in3 = _mm_srai_epi16(in3, 15); 501 const __m128i sign_in4 = _mm_srai_epi16(in4, 15); 502 const __m128i sign_in5 = _mm_srai_epi16(in5, 15); 503 const __m128i sign_in6 = _mm_srai_epi16(in6, 15); 504 const __m128i sign_in7 = _mm_srai_epi16(in7, 15); 505 in0 = _mm_sub_epi16(in0, sign_in0); 506 in1 = _mm_sub_epi16(in1, sign_in1); 507 in2 = _mm_sub_epi16(in2, sign_in2); 508 in3 = _mm_sub_epi16(in3, sign_in3); 509 in4 = _mm_sub_epi16(in4, sign_in4); 510 in5 = _mm_sub_epi16(in5, sign_in5); 511 in6 = _mm_sub_epi16(in6, sign_in6); 512 in7 = _mm_sub_epi16(in7, sign_in7); 513 in0 = _mm_srai_epi16(in0, 1); 514 in1 = _mm_srai_epi16(in1, 1); 515 in2 = _mm_srai_epi16(in2, 1); 516 in3 = _mm_srai_epi16(in3, 1); 517 in4 = _mm_srai_epi16(in4, 1); 518 in5 = _mm_srai_epi16(in5, 1); 519 in6 = _mm_srai_epi16(in6, 1); 520 in7 = _mm_srai_epi16(in7, 1); 521 // store results 522 store_output(&in0, (output + 0 * 8)); 523 store_output(&in1, (output + 1 * 8)); 524 store_output(&in2, (output + 2 * 8)); 525 store_output(&in3, (output + 3 * 8)); 526 store_output(&in4, (output + 4 * 8)); 527 store_output(&in5, (output + 5 * 8)); 528 store_output(&in6, (output + 6 * 8)); 529 store_output(&in7, (output + 7 * 8)); 530 } 531 } 532 #endif // CONFIG_INTERNAL_STATS 533 534 #undef ADD_EPI16 535 #undef SUB_EPI16