rescaler_sse2.c (14579B)
1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // SSE2 Rescaling functions 11 // 12 // Author: Skal (pascal.massimino@gmail.com) 13 14 #include "src/dsp/dsp.h" 15 16 #if defined(WEBP_USE_SSE2) && !defined(WEBP_REDUCE_SIZE) 17 #include <emmintrin.h> 18 19 #include <assert.h> 20 #include <stddef.h> 21 22 #include "src/dsp/cpu.h" 23 #include "src/utils/rescaler_utils.h" 24 #include "src/utils/utils.h" 25 #include "src/webp/types.h" 26 27 //------------------------------------------------------------------------------ 28 // Implementations of critical functions ImportRow / ExportRow 29 30 #define ROUNDER (WEBP_RESCALER_ONE >> 1) 31 #define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) 32 #define MULT_FIX_FLOOR(x, y) (((uint64_t)(x) * (y)) >> WEBP_RESCALER_RFIX) 33 34 // input: 8 bytes ABCDEFGH -> output: A0E0B0F0C0G0D0H0 35 static void LoadTwoPixels_SSE2(const uint8_t* const src, __m128i* out) { 36 const __m128i zero = _mm_setzero_si128(); 37 const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH 38 const __m128i B = _mm_unpacklo_epi8(A, zero); // A0B0C0D0E0F0G0H0 39 const __m128i C = _mm_srli_si128(B, 8); // E0F0G0H0 40 *out = _mm_unpacklo_epi16(B, C); 41 } 42 43 // input: 8 bytes ABCDEFGH -> output: A0B0C0D0E0F0G0H0 44 static void LoadEightPixels_SSE2(const uint8_t* const src, __m128i* out) { 45 const __m128i zero = _mm_setzero_si128(); 46 const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH 47 *out = _mm_unpacklo_epi8(A, zero); 48 } 49 50 static void RescalerImportRowExpand_SSE2(WebPRescaler* WEBP_RESTRICT const wrk, 51 const uint8_t* WEBP_RESTRICT src) { 52 rescaler_t* frow = wrk->frow; 53 const rescaler_t* const frow_end = frow + wrk->dst_width * wrk->num_channels; 54 const int x_add = wrk->x_add; 55 int accum = x_add; 56 __m128i cur_pixels; 57 58 // SSE2 implementation only works with 16b signed arithmetic at max. 59 if (wrk->src_width < 8 || accum >= (1 << 15)) { 60 WebPRescalerImportRowExpand_C(wrk, src); 61 return; 62 } 63 64 assert(!WebPRescalerInputDone(wrk)); 65 assert(wrk->x_expand); 66 if (wrk->num_channels == 4) { 67 LoadTwoPixels_SSE2(src, &cur_pixels); 68 src += 4; 69 while (1) { 70 const __m128i mult = _mm_set1_epi32(((x_add - accum) << 16) | accum); 71 const __m128i out = _mm_madd_epi16(cur_pixels, mult); 72 _mm_storeu_si128((__m128i*)frow, out); 73 frow += 4; 74 if (frow >= frow_end) break; 75 accum -= wrk->x_sub; 76 if (accum < 0) { 77 LoadTwoPixels_SSE2(src, &cur_pixels); 78 src += 4; 79 accum += x_add; 80 } 81 } 82 } else { 83 int left; 84 const uint8_t* const src_limit = src + wrk->src_width - 8; 85 LoadEightPixels_SSE2(src, &cur_pixels); 86 src += 7; 87 left = 7; 88 while (1) { 89 const __m128i mult = _mm_cvtsi32_si128(((x_add - accum) << 16) | accum); 90 const __m128i out = _mm_madd_epi16(cur_pixels, mult); 91 assert(sizeof(*frow) == sizeof(uint32_t)); 92 WebPInt32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); 93 frow += 1; 94 if (frow >= frow_end) break; 95 accum -= wrk->x_sub; 96 if (accum < 0) { 97 if (--left) { 98 cur_pixels = _mm_srli_si128(cur_pixels, 2); 99 } else if (src <= src_limit) { 100 LoadEightPixels_SSE2(src, &cur_pixels); 101 src += 7; 102 left = 7; 103 } else { // tail 104 cur_pixels = _mm_srli_si128(cur_pixels, 2); 105 cur_pixels = _mm_insert_epi16(cur_pixels, src[1], 1); 106 src += 1; 107 left = 1; 108 } 109 accum += x_add; 110 } 111 } 112 } 113 assert(accum == 0); 114 } 115 116 static void RescalerImportRowShrink_SSE2(WebPRescaler* WEBP_RESTRICT const wrk, 117 const uint8_t* WEBP_RESTRICT src) { 118 const int x_sub = wrk->x_sub; 119 int accum = 0; 120 const __m128i zero = _mm_setzero_si128(); 121 const __m128i mult0 = _mm_set1_epi16(x_sub); 122 const __m128i mult1 = _mm_set1_epi32(wrk->fx_scale); 123 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 124 __m128i sum = zero; 125 rescaler_t* frow = wrk->frow; 126 const rescaler_t* const frow_end = wrk->frow + 4 * wrk->dst_width; 127 128 if (wrk->num_channels != 4 || wrk->x_add > (x_sub << 7)) { 129 WebPRescalerImportRowShrink_C(wrk, src); 130 return; 131 } 132 assert(!WebPRescalerInputDone(wrk)); 133 assert(!wrk->x_expand); 134 135 for (; frow < frow_end; frow += 4) { 136 __m128i base = zero; 137 accum += wrk->x_add; 138 while (accum > 0) { 139 const __m128i A = _mm_cvtsi32_si128(WebPMemToInt32(src)); 140 src += 4; 141 base = _mm_unpacklo_epi8(A, zero); 142 // To avoid overflow, we need: base * x_add / x_sub < 32768 143 // => x_add < x_sub << 7. That's a 1/128 reduction ratio limit. 144 sum = _mm_add_epi16(sum, base); 145 accum -= x_sub; 146 } 147 { // Emit next horizontal pixel. 148 const __m128i mult = _mm_set1_epi16(-accum); 149 const __m128i frac0 = _mm_mullo_epi16(base, mult); // 16b x 16b -> 32b 150 const __m128i frac1 = _mm_mulhi_epu16(base, mult); 151 const __m128i frac = _mm_unpacklo_epi16(frac0, frac1); // frac is 32b 152 const __m128i A0 = _mm_mullo_epi16(sum, mult0); 153 const __m128i A1 = _mm_mulhi_epu16(sum, mult0); 154 const __m128i B0 = _mm_unpacklo_epi16(A0, A1); // sum * x_sub 155 const __m128i frow_out = _mm_sub_epi32(B0, frac); // sum * x_sub - frac 156 const __m128i D0 = _mm_srli_epi64(frac, 32); 157 const __m128i D1 = _mm_mul_epu32(frac, mult1); // 32b x 16b -> 64b 158 const __m128i D2 = _mm_mul_epu32(D0, mult1); 159 const __m128i E1 = _mm_add_epi64(D1, rounder); 160 const __m128i E2 = _mm_add_epi64(D2, rounder); 161 const __m128i F1 = _mm_shuffle_epi32(E1, 1 | (3 << 2)); 162 const __m128i F2 = _mm_shuffle_epi32(E2, 1 | (3 << 2)); 163 const __m128i G = _mm_unpacklo_epi32(F1, F2); 164 sum = _mm_packs_epi32(G, zero); 165 _mm_storeu_si128((__m128i*)frow, frow_out); 166 } 167 } 168 assert(accum == 0); 169 } 170 171 //------------------------------------------------------------------------------ 172 // Row export 173 174 // load *src as epi64, multiply by mult and store result in [out0 ... out3] 175 static WEBP_INLINE void LoadDispatchAndMult_SSE2( 176 const rescaler_t* WEBP_RESTRICT const src, const __m128i* const mult, 177 __m128i* const out0, __m128i* const out1, __m128i* const out2, 178 __m128i* const out3) { 179 const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + 0)); 180 const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + 4)); 181 const __m128i A2 = _mm_srli_epi64(A0, 32); 182 const __m128i A3 = _mm_srli_epi64(A1, 32); 183 if (mult != NULL) { 184 *out0 = _mm_mul_epu32(A0, *mult); 185 *out1 = _mm_mul_epu32(A1, *mult); 186 *out2 = _mm_mul_epu32(A2, *mult); 187 *out3 = _mm_mul_epu32(A3, *mult); 188 } else { 189 *out0 = A0; 190 *out1 = A1; 191 *out2 = A2; 192 *out3 = A3; 193 } 194 } 195 196 static WEBP_INLINE void ProcessRow_SSE2(const __m128i* const A0, 197 const __m128i* const A1, 198 const __m128i* const A2, 199 const __m128i* const A3, 200 const __m128i* const mult, 201 uint8_t* const dst) { 202 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 203 const __m128i mask = _mm_set_epi32(~0, 0, ~0, 0); 204 const __m128i B0 = _mm_mul_epu32(*A0, *mult); 205 const __m128i B1 = _mm_mul_epu32(*A1, *mult); 206 const __m128i B2 = _mm_mul_epu32(*A2, *mult); 207 const __m128i B3 = _mm_mul_epu32(*A3, *mult); 208 const __m128i C0 = _mm_add_epi64(B0, rounder); 209 const __m128i C1 = _mm_add_epi64(B1, rounder); 210 const __m128i C2 = _mm_add_epi64(B2, rounder); 211 const __m128i C3 = _mm_add_epi64(B3, rounder); 212 const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); 213 const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); 214 #if (WEBP_RESCALER_RFIX < 32) 215 const __m128i D2 = 216 _mm_and_si128(_mm_slli_epi64(C2, 32 - WEBP_RESCALER_RFIX), mask); 217 const __m128i D3 = 218 _mm_and_si128(_mm_slli_epi64(C3, 32 - WEBP_RESCALER_RFIX), mask); 219 #else 220 const __m128i D2 = _mm_and_si128(C2, mask); 221 const __m128i D3 = _mm_and_si128(C3, mask); 222 #endif 223 const __m128i E0 = _mm_or_si128(D0, D2); 224 const __m128i E1 = _mm_or_si128(D1, D3); 225 const __m128i F = _mm_packs_epi32(E0, E1); 226 const __m128i G = _mm_packus_epi16(F, F); 227 _mm_storel_epi64((__m128i*)dst, G); 228 } 229 230 static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) { 231 int x_out; 232 uint8_t* const dst = wrk->dst; 233 rescaler_t* const irow = wrk->irow; 234 const int x_out_max = wrk->dst_width * wrk->num_channels; 235 const rescaler_t* const frow = wrk->frow; 236 const __m128i mult = _mm_set_epi32(0, wrk->fy_scale, 0, wrk->fy_scale); 237 238 assert(!WebPRescalerOutputDone(wrk)); 239 assert(wrk->y_accum <= 0 && wrk->y_sub + wrk->y_accum >= 0); 240 assert(wrk->y_expand); 241 if (wrk->y_accum == 0) { 242 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 243 __m128i A0, A1, A2, A3; 244 LoadDispatchAndMult_SSE2(frow + x_out, NULL, &A0, &A1, &A2, &A3); 245 ProcessRow_SSE2(&A0, &A1, &A2, &A3, &mult, dst + x_out); 246 } 247 for (; x_out < x_out_max; ++x_out) { 248 const uint32_t J = frow[x_out]; 249 const int v = (int)MULT_FIX(J, wrk->fy_scale); 250 dst[x_out] = (v > 255) ? 255u : (uint8_t)v; 251 } 252 } else { 253 const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); 254 const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); 255 const __m128i mA = _mm_set_epi32(0, A, 0, A); 256 const __m128i mB = _mm_set_epi32(0, B, 0, B); 257 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 258 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 259 __m128i A0, A1, A2, A3, B0, B1, B2, B3; 260 LoadDispatchAndMult_SSE2(frow + x_out, &mA, &A0, &A1, &A2, &A3); 261 LoadDispatchAndMult_SSE2(irow + x_out, &mB, &B0, &B1, &B2, &B3); 262 { 263 const __m128i C0 = _mm_add_epi64(A0, B0); 264 const __m128i C1 = _mm_add_epi64(A1, B1); 265 const __m128i C2 = _mm_add_epi64(A2, B2); 266 const __m128i C3 = _mm_add_epi64(A3, B3); 267 const __m128i D0 = _mm_add_epi64(C0, rounder); 268 const __m128i D1 = _mm_add_epi64(C1, rounder); 269 const __m128i D2 = _mm_add_epi64(C2, rounder); 270 const __m128i D3 = _mm_add_epi64(C3, rounder); 271 const __m128i E0 = _mm_srli_epi64(D0, WEBP_RESCALER_RFIX); 272 const __m128i E1 = _mm_srli_epi64(D1, WEBP_RESCALER_RFIX); 273 const __m128i E2 = _mm_srli_epi64(D2, WEBP_RESCALER_RFIX); 274 const __m128i E3 = _mm_srli_epi64(D3, WEBP_RESCALER_RFIX); 275 ProcessRow_SSE2(&E0, &E1, &E2, &E3, &mult, dst + x_out); 276 } 277 } 278 for (; x_out < x_out_max; ++x_out) { 279 const uint64_t I = (uint64_t)A * frow[x_out] 280 + (uint64_t)B * irow[x_out]; 281 const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); 282 const int v = (int)MULT_FIX(J, wrk->fy_scale); 283 dst[x_out] = (v > 255) ? 255u : (uint8_t)v; 284 } 285 } 286 } 287 288 static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) { 289 int x_out; 290 uint8_t* const dst = wrk->dst; 291 rescaler_t* const irow = wrk->irow; 292 const int x_out_max = wrk->dst_width * wrk->num_channels; 293 const rescaler_t* const frow = wrk->frow; 294 const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); 295 assert(!WebPRescalerOutputDone(wrk)); 296 assert(wrk->y_accum <= 0); 297 assert(!wrk->y_expand); 298 if (yscale) { 299 const int scale_xy = wrk->fxy_scale; 300 const __m128i mult_xy = _mm_set_epi32(0, scale_xy, 0, scale_xy); 301 const __m128i mult_y = _mm_set_epi32(0, yscale, 0, yscale); 302 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 303 __m128i A0, A1, A2, A3, B0, B1, B2, B3; 304 LoadDispatchAndMult_SSE2(irow + x_out, NULL, &A0, &A1, &A2, &A3); 305 LoadDispatchAndMult_SSE2(frow + x_out, &mult_y, &B0, &B1, &B2, &B3); 306 { 307 const __m128i D0 = _mm_srli_epi64(B0, WEBP_RESCALER_RFIX); // = frac 308 const __m128i D1 = _mm_srli_epi64(B1, WEBP_RESCALER_RFIX); 309 const __m128i D2 = _mm_srli_epi64(B2, WEBP_RESCALER_RFIX); 310 const __m128i D3 = _mm_srli_epi64(B3, WEBP_RESCALER_RFIX); 311 const __m128i E0 = _mm_sub_epi64(A0, D0); // irow[x] - frac 312 const __m128i E1 = _mm_sub_epi64(A1, D1); 313 const __m128i E2 = _mm_sub_epi64(A2, D2); 314 const __m128i E3 = _mm_sub_epi64(A3, D3); 315 const __m128i F2 = _mm_slli_epi64(D2, 32); 316 const __m128i F3 = _mm_slli_epi64(D3, 32); 317 const __m128i G0 = _mm_or_si128(D0, F2); 318 const __m128i G1 = _mm_or_si128(D1, F3); 319 _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0); 320 _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1); 321 ProcessRow_SSE2(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out); 322 } 323 } 324 for (; x_out < x_out_max; ++x_out) { 325 const uint32_t frac = (int)MULT_FIX_FLOOR(frow[x_out], yscale); 326 const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); 327 dst[x_out] = (v > 255) ? 255u : (uint8_t)v; 328 irow[x_out] = frac; // new fractional start 329 } 330 } else { 331 const uint32_t scale = wrk->fxy_scale; 332 const __m128i mult = _mm_set_epi32(0, scale, 0, scale); 333 const __m128i zero = _mm_setzero_si128(); 334 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 335 __m128i A0, A1, A2, A3; 336 LoadDispatchAndMult_SSE2(irow + x_out, NULL, &A0, &A1, &A2, &A3); 337 _mm_storeu_si128((__m128i*)(irow + x_out + 0), zero); 338 _mm_storeu_si128((__m128i*)(irow + x_out + 4), zero); 339 ProcessRow_SSE2(&A0, &A1, &A2, &A3, &mult, dst + x_out); 340 } 341 for (; x_out < x_out_max; ++x_out) { 342 const int v = (int)MULT_FIX(irow[x_out], scale); 343 dst[x_out] = (v > 255) ? 255u : (uint8_t)v; 344 irow[x_out] = 0; 345 } 346 } 347 } 348 349 #undef MULT_FIX_FLOOR 350 #undef MULT_FIX 351 #undef ROUNDER 352 353 //------------------------------------------------------------------------------ 354 355 extern void WebPRescalerDspInitSSE2(void); 356 357 WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitSSE2(void) { 358 WebPRescalerImportRowExpand = RescalerImportRowExpand_SSE2; 359 WebPRescalerImportRowShrink = RescalerImportRowShrink_SSE2; 360 WebPRescalerExportRowExpand = RescalerExportRowExpand_SSE2; 361 WebPRescalerExportRowShrink = RescalerExportRowShrink_SSE2; 362 } 363 364 #else // !WEBP_USE_SSE2 365 366 WEBP_DSP_INIT_STUB(WebPRescalerDspInitSSE2) 367 368 #endif // WEBP_USE_SSE2