ConvolutionFilterAVX2.cpp (4381B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 // Copyright (c) 2011-2016 Google Inc. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the gfx/skia/LICENSE file. 6 7 #include "SkConvolver.h" 8 #include <immintrin.h> 9 10 namespace skia { 11 12 void convolve_vertically_avx2( 13 const SkConvolutionFilter1D::ConvolutionFixed* filter, int filterLen, 14 unsigned char* const* srcRows, int width, unsigned char* out, 15 bool hasAlpha) { 16 // It's simpler to work with the output array in terms of 4-byte pixels. 17 auto* dst = (int*)out; 18 19 // Output up to eight pixels per iteration. 20 for (int x = 0; x < width; x += 8) { 21 // Accumulated result for 4 (non-adjacent) pairs of pixels, 22 // with each channel in signed 17.14 fixed point. 23 auto accum04 = _mm256_setzero_si256(), accum15 = _mm256_setzero_si256(), 24 accum26 = _mm256_setzero_si256(), accum37 = _mm256_setzero_si256(); 25 26 // Convolve with the filter. (This inner loop is where we spend ~all our 27 // time.) While we can, we consume 2 filter coefficients and 2 rows of 8 28 // pixels each at a time. 29 auto convolve_16_pixels = [&](__m256i interlaced_coeffs, 30 __m256i pixels_01234567, 31 __m256i pixels_89ABCDEF) { 32 // Interlaced R0R8 G0G8 B0B8 A0A8 R1R9 G1G9... 32 8-bit values each. 33 auto _08194C5D = _mm256_unpacklo_epi8(pixels_01234567, pixels_89ABCDEF), 34 _2A3B6E7F = _mm256_unpackhi_epi8(pixels_01234567, pixels_89ABCDEF); 35 36 // Still interlaced R0R8 G0G8... as above, each channel expanded to 16-bit 37 // lanes. 38 auto _084C = _mm256_unpacklo_epi8(_08194C5D, _mm256_setzero_si256()), 39 _195D = _mm256_unpackhi_epi8(_08194C5D, _mm256_setzero_si256()), 40 _2A6E = _mm256_unpacklo_epi8(_2A3B6E7F, _mm256_setzero_si256()), 41 _3B7F = _mm256_unpackhi_epi8(_2A3B6E7F, _mm256_setzero_si256()); 42 43 // accum0_R += R0*coeff0 + R8*coeff1, etc. 44 accum04 = _mm256_add_epi32(accum04, 45 _mm256_madd_epi16(_084C, interlaced_coeffs)); 46 accum15 = _mm256_add_epi32(accum15, 47 _mm256_madd_epi16(_195D, interlaced_coeffs)); 48 accum26 = _mm256_add_epi32(accum26, 49 _mm256_madd_epi16(_2A6E, interlaced_coeffs)); 50 accum37 = _mm256_add_epi32(accum37, 51 _mm256_madd_epi16(_3B7F, interlaced_coeffs)); 52 }; 53 54 int i = 0; 55 for (; i < filterLen / 2 * 2; i += 2) { 56 convolve_16_pixels( 57 _mm256_set1_epi32(*(const int32_t*)(filter + i)), 58 _mm256_loadu_si256((const __m256i*)(srcRows[i + 0] + x * 4)), 59 _mm256_loadu_si256((const __m256i*)(srcRows[i + 1] + x * 4))); 60 } 61 if (i < filterLen) { 62 convolve_16_pixels( 63 _mm256_set1_epi32(*(const int16_t*)(filter + i)), 64 _mm256_loadu_si256((const __m256i*)(srcRows[i] + x * 4)), 65 _mm256_setzero_si256()); 66 } 67 68 // Trim the fractional parts off the accumulators. 69 accum04 = _mm256_srai_epi32(accum04, 14); 70 accum15 = _mm256_srai_epi32(accum15, 14); 71 accum26 = _mm256_srai_epi32(accum26, 14); 72 accum37 = _mm256_srai_epi32(accum37, 14); 73 74 // Pack back down to 8-bit channels. 75 auto pixels = _mm256_packus_epi16(_mm256_packs_epi32(accum04, accum15), 76 _mm256_packs_epi32(accum26, accum37)); 77 78 if (hasAlpha) { 79 // Clamp alpha to the max of r,g,b to make sure we stay premultiplied. 80 __m256i max_rg = _mm256_max_epu8(pixels, _mm256_srli_epi32(pixels, 8)), 81 max_rgb = _mm256_max_epu8(max_rg, _mm256_srli_epi32(pixels, 16)); 82 pixels = _mm256_max_epu8(pixels, _mm256_slli_epi32(max_rgb, 24)); 83 } else { 84 // Force opaque. 85 pixels = _mm256_or_si256(pixels, _mm256_set1_epi32(0xff000000)); 86 } 87 88 // Normal path to store 8 pixels. 89 if (x + 8 <= width) { 90 _mm256_storeu_si256((__m256i*)dst, pixels); 91 dst += 8; 92 continue; 93 } 94 95 // Store one pixel at a time on the last iteration. 96 for (int i = x; i < width; i++) { 97 *dst++ = _mm_cvtsi128_si32(_mm256_castsi256_si128(pixels)); 98 pixels = _mm256_permutevar8x32_epi32( 99 pixels, _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0)); 100 } 101 } 102 } 103 104 } // namespace skia