variance_impl_ssse3.c (4668B)
1 /* 2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <tmmintrin.h> 13 14 #include "config/aom_config.h" 15 #include "config/aom_dsp_rtcd.h" 16 17 #include "aom_dsp/x86/synonyms.h" 18 #include "aom_dsp/x86/variance_impl_ssse3.h" 19 20 void aom_var_filter_block2d_bil_first_pass_ssse3( 21 const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line, 22 unsigned int pixel_step, unsigned int output_height, 23 unsigned int output_width, const uint8_t *filter) { 24 // Note: filter[0], filter[1] could be {128, 0}, where 128 will overflow 25 // in computation using _mm_maddubs_epi16. 26 // Change {128, 0} to {64, 0} and reduce FILTER_BITS by 1 to avoid overflow. 27 const int16_t round = (1 << (FILTER_BITS - 1)) >> 1; 28 const __m128i r = _mm_set1_epi16(round); 29 const int8_t f0 = (int8_t)(filter[0] >> 1); 30 const int8_t f1 = (int8_t)(filter[1] >> 1); 31 const __m128i filters = _mm_setr_epi8(f0, f1, f0, f1, f0, f1, f0, f1, f0, f1, 32 f0, f1, f0, f1, f0, f1); 33 unsigned int i, j; 34 (void)pixel_step; 35 36 if (output_width >= 8) { 37 for (i = 0; i < output_height; ++i) { 38 for (j = 0; j < output_width; j += 8) { 39 // load source 40 __m128i source_low = xx_loadl_64(a); 41 __m128i source_hi = xx_loadl_64(a + 1); 42 43 // unpack to: 44 // { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4], 45 // a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] } 46 __m128i source = _mm_unpacklo_epi8(source_low, source_hi); 47 48 // b[i] = a[i] * filter[0] + a[i + 1] * filter[1] 49 __m128i res = _mm_maddubs_epi16(source, filters); 50 51 // round 52 res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1); 53 54 xx_storeu_128(b, res); 55 56 a += 8; 57 b += 8; 58 } 59 60 a += src_pixels_per_line - output_width; 61 } 62 } else { 63 const __m128i shuffle_mask = 64 _mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8); 65 for (i = 0; i < output_height; ++i) { 66 // load source, only first 5 values are meaningful: 67 // { a[0], a[1], a[2], a[3], a[4], xxxx } 68 __m128i source = xx_loadl_64(a); 69 70 // shuffle, up to the first 8 are useful 71 // { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4], 72 // a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] } 73 __m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask); 74 75 __m128i res = _mm_maddubs_epi16(source_shuffle, filters); 76 res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1); 77 78 xx_storel_64(b, res); 79 80 a += src_pixels_per_line; 81 b += output_width; 82 } 83 } 84 } 85 86 void aom_var_filter_block2d_bil_second_pass_ssse3( 87 const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line, 88 unsigned int pixel_step, unsigned int output_height, 89 unsigned int output_width, const uint8_t *filter) { 90 const int16_t round = (1 << FILTER_BITS) >> 1; 91 const __m128i r = _mm_set1_epi32(round); 92 const __m128i filters = 93 _mm_setr_epi16(filter[0], filter[1], filter[0], filter[1], filter[0], 94 filter[1], filter[0], filter[1]); 95 const __m128i shuffle_mask = 96 _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); 97 const __m128i mask = 98 _mm_setr_epi8(0, 4, 8, 12, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); 99 unsigned int i, j; 100 101 for (i = 0; i < output_height; ++i) { 102 for (j = 0; j < output_width; j += 4) { 103 // load source as: 104 // { a[0], a[1], a[2], a[3], a[w], a[w+1], a[w+2], a[w+3] } 105 __m128i source1 = xx_loadl_64(a); 106 __m128i source2 = xx_loadl_64(a + pixel_step); 107 __m128i source = _mm_unpacklo_epi64(source1, source2); 108 109 // shuffle source to: 110 // { a[0], a[w], a[1], a[w+1], a[2], a[w+2], a[3], a[w+3] } 111 __m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask); 112 113 // b[i] = a[i] * filter[0] + a[w + i] * filter[1] 114 __m128i res = _mm_madd_epi16(source_shuffle, filters); 115 116 // round 117 res = _mm_srai_epi32(_mm_add_epi32(res, r), FILTER_BITS); 118 119 // shuffle to get each lower 8 bit of every 32 bit 120 res = _mm_shuffle_epi8(res, mask); 121 122 xx_storel_32(b, res); 123 124 a += 4; 125 b += 4; 126 } 127 128 a += src_pixels_per_line - output_width; 129 } 130 }