highbd_txfm_utility_sse4.h (5597B)
1 /* 2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #ifndef AOM_AV1_COMMON_X86_HIGHBD_TXFM_UTILITY_SSE4_H_ 13 #define AOM_AV1_COMMON_X86_HIGHBD_TXFM_UTILITY_SSE4_H_ 14 15 #include <smmintrin.h> /* SSE4.1 */ 16 17 #define TRANSPOSE_4X4(x0, x1, x2, x3, y0, y1, y2, y3) \ 18 do { \ 19 __m128i u0, u1, u2, u3; \ 20 u0 = _mm_unpacklo_epi32(x0, x1); \ 21 u1 = _mm_unpackhi_epi32(x0, x1); \ 22 u2 = _mm_unpacklo_epi32(x2, x3); \ 23 u3 = _mm_unpackhi_epi32(x2, x3); \ 24 y0 = _mm_unpacklo_epi64(u0, u2); \ 25 y1 = _mm_unpackhi_epi64(u0, u2); \ 26 y2 = _mm_unpacklo_epi64(u1, u3); \ 27 y3 = _mm_unpackhi_epi64(u1, u3); \ 28 } while (0) 29 30 static inline void transpose_8x8(const __m128i *in, __m128i *out) { 31 TRANSPOSE_4X4(in[0], in[2], in[4], in[6], out[0], out[2], out[4], out[6]); 32 TRANSPOSE_4X4(in[1], in[3], in[5], in[7], out[8], out[10], out[12], out[14]); 33 TRANSPOSE_4X4(in[8], in[10], in[12], in[14], out[1], out[3], out[5], out[7]); 34 TRANSPOSE_4X4(in[9], in[11], in[13], in[15], out[9], out[11], out[13], 35 out[15]); 36 } 37 38 static inline void transpose_16x16(const __m128i *in, __m128i *out) { 39 // Upper left 8x8 40 TRANSPOSE_4X4(in[0], in[4], in[8], in[12], out[0], out[4], out[8], out[12]); 41 TRANSPOSE_4X4(in[1], in[5], in[9], in[13], out[16], out[20], out[24], 42 out[28]); 43 TRANSPOSE_4X4(in[16], in[20], in[24], in[28], out[1], out[5], out[9], 44 out[13]); 45 TRANSPOSE_4X4(in[17], in[21], in[25], in[29], out[17], out[21], out[25], 46 out[29]); 47 48 // Upper right 8x8 49 TRANSPOSE_4X4(in[2], in[6], in[10], in[14], out[32], out[36], out[40], 50 out[44]); 51 TRANSPOSE_4X4(in[3], in[7], in[11], in[15], out[48], out[52], out[56], 52 out[60]); 53 TRANSPOSE_4X4(in[18], in[22], in[26], in[30], out[33], out[37], out[41], 54 out[45]); 55 TRANSPOSE_4X4(in[19], in[23], in[27], in[31], out[49], out[53], out[57], 56 out[61]); 57 58 // Lower left 8x8 59 TRANSPOSE_4X4(in[32], in[36], in[40], in[44], out[2], out[6], out[10], 60 out[14]); 61 TRANSPOSE_4X4(in[33], in[37], in[41], in[45], out[18], out[22], out[26], 62 out[30]); 63 TRANSPOSE_4X4(in[48], in[52], in[56], in[60], out[3], out[7], out[11], 64 out[15]); 65 TRANSPOSE_4X4(in[49], in[53], in[57], in[61], out[19], out[23], out[27], 66 out[31]); 67 // Lower right 8x8 68 TRANSPOSE_4X4(in[34], in[38], in[42], in[46], out[34], out[38], out[42], 69 out[46]); 70 TRANSPOSE_4X4(in[35], in[39], in[43], in[47], out[50], out[54], out[58], 71 out[62]); 72 TRANSPOSE_4X4(in[50], in[54], in[58], in[62], out[35], out[39], out[43], 73 out[47]); 74 TRANSPOSE_4X4(in[51], in[55], in[59], in[63], out[51], out[55], out[59], 75 out[63]); 76 } 77 78 static inline void transpose_8nx8n(const __m128i *input, __m128i *output, 79 const int width, const int height) { 80 const int numcol = height >> 2; 81 const int numrow = width >> 2; 82 for (int j = 0; j < numrow; j++) { 83 for (int i = 0; i < numcol; i++) { 84 TRANSPOSE_4X4(input[i * width + j + (numrow * 0)], 85 input[i * width + j + (numrow * 1)], 86 input[i * width + j + (numrow * 2)], 87 input[i * width + j + (numrow * 3)], 88 output[j * height + i + (numcol * 0)], 89 output[j * height + i + (numcol * 1)], 90 output[j * height + i + (numcol * 2)], 91 output[j * height + i + (numcol * 3)]); 92 } 93 } 94 } 95 96 // Note: 97 // rounding = 1 << (bit - 1) 98 static inline __m128i half_btf_sse4_1(const __m128i *w0, const __m128i *n0, 99 const __m128i *w1, const __m128i *n1, 100 const __m128i *rounding, int bit) { 101 __m128i x, y; 102 103 x = _mm_mullo_epi32(*w0, *n0); 104 y = _mm_mullo_epi32(*w1, *n1); 105 x = _mm_add_epi32(x, y); 106 x = _mm_add_epi32(x, *rounding); 107 x = _mm_srai_epi32(x, bit); 108 return x; 109 } 110 111 static inline __m128i half_btf_0_sse4_1(const __m128i *w0, const __m128i *n0, 112 const __m128i *rounding, int bit) { 113 __m128i x; 114 115 x = _mm_mullo_epi32(*w0, *n0); 116 x = _mm_add_epi32(x, *rounding); 117 x = _mm_srai_epi32(x, bit); 118 return x; 119 } 120 121 typedef void (*transform_1d_sse4_1)(__m128i *in, __m128i *out, int bit, 122 int do_cols, int bd, int out_shift); 123 124 typedef void (*fwd_transform_1d_sse4_1)(__m128i *in, __m128i *out, int bit, 125 const int num_cols); 126 127 void av1_highbd_inv_txfm2d_add_universe_sse4_1(const int32_t *input, 128 uint8_t *output, int stride, 129 TX_TYPE tx_type, TX_SIZE tx_size, 130 int eob, const int bd); 131 132 #endif // AOM_AV1_COMMON_X86_HIGHBD_TXFM_UTILITY_SSE4_H_