jidctint-mmi.c (23014B)
1 /* 2 * Loongson MMI optimizations for libjpeg-turbo 3 * 4 * Copyright (C) 2014-2015, 2018, 2020, D. R. Commander. All Rights Reserved. 5 * Copyright (C) 2016-2017, Loongson Technology Corporation Limited, BeiJing. 6 * All Rights Reserved. 7 * Authors: ZhuChen <zhuchen@loongson.cn> 8 * CaiWanwei <caiwanwei@loongson.cn> 9 * SunZhangzhi <sunzhangzhi-cq@loongson.cn> 10 * 11 * Based on the x86 SIMD extension for IJG JPEG library 12 * Copyright (C) 1999-2006, MIYASAKA Masaru. 13 * 14 * This software is provided 'as-is', without any express or implied 15 * warranty. In no event will the authors be held liable for any damages 16 * arising from the use of this software. 17 * 18 * Permission is granted to anyone to use this software for any purpose, 19 * including commercial applications, and to alter it and redistribute it 20 * freely, subject to the following restrictions: 21 * 22 * 1. The origin of this software must not be misrepresented; you must not 23 * claim that you wrote the original software. If you use this software 24 * in a product, an acknowledgment in the product documentation would be 25 * appreciated but is not required. 26 * 2. Altered source versions must be plainly marked as such, and must not be 27 * misrepresented as being the original software. 28 * 3. This notice may not be removed or altered from any source distribution. 29 */ 30 31 /* ACCUATE INTEGER INVERSE DCT */ 32 33 #include "jsimd_mmi.h" 34 35 36 #define CONST_BITS 13 37 #define PASS1_BITS 2 38 #define DESCALE_P1 (CONST_BITS - PASS1_BITS) 39 #define DESCALE_P2 (CONST_BITS + PASS1_BITS + 3) 40 #define CENTERJSAMPLE 128 41 42 #define FIX_0_298 ((short)2446) /* FIX(0.298631336) */ 43 #define FIX_0_390 ((short)3196) /* FIX(0.390180644) */ 44 #define FIX_0_899 ((short)7373) /* FIX(0.899976223) */ 45 #define FIX_0_541 ((short)4433) /* FIX(0.541196100) */ 46 #define FIX_0_765 ((short)6270) /* FIX(0.765366865) */ 47 #define FIX_1_175 ((short)9633) /* FIX(1.175875602) */ 48 #define FIX_1_501 ((short)12299) /* FIX(1.501321110) */ 49 #define FIX_1_847 ((short)15137) /* FIX(1.847759065) */ 50 #define FIX_1_961 ((short)16069) /* FIX(1.961570560) */ 51 #define FIX_2_053 ((short)16819) /* FIX(2.053119869) */ 52 #define FIX_2_562 ((short)20995) /* FIX(2.562915447) */ 53 #define FIX_3_072 ((short)25172) /* FIX(3.072711026) */ 54 55 enum const_index { 56 index_PW_F130_F054, 57 index_PW_F054_MF130, 58 index_PW_MF078_F117, 59 index_PW_F117_F078, 60 index_PW_MF060_MF089, 61 index_PW_MF089_F060, 62 index_PW_MF050_MF256, 63 index_PW_MF256_F050, 64 index_PD_DESCALE_P1, 65 index_PD_DESCALE_P2, 66 index_PB_CENTERJSAMP 67 }; 68 69 static uint64_t const_value[] = { 70 _uint64_set_pi16(FIX_0_541, (FIX_0_541 + FIX_0_765), 71 FIX_0_541, (FIX_0_541 + FIX_0_765)), 72 _uint64_set_pi16((FIX_0_541 - FIX_1_847), FIX_0_541, 73 (FIX_0_541 - FIX_1_847), FIX_0_541), 74 _uint64_set_pi16(FIX_1_175, (FIX_1_175 - FIX_1_961), 75 FIX_1_175, (FIX_1_175 - FIX_1_961)), 76 _uint64_set_pi16((FIX_1_175 - FIX_0_390), FIX_1_175, 77 (FIX_1_175 - FIX_0_390), FIX_1_175), 78 _uint64_set_pi16(-FIX_0_899, (FIX_0_298 - FIX_0_899), 79 -FIX_0_899, (FIX_0_298 - FIX_0_899)), 80 _uint64_set_pi16((FIX_1_501 - FIX_0_899), -FIX_0_899, 81 (FIX_1_501 - FIX_0_899), -FIX_0_899), 82 _uint64_set_pi16(-FIX_2_562, (FIX_2_053 - FIX_2_562), 83 -FIX_2_562, (FIX_2_053 - FIX_2_562)), 84 _uint64_set_pi16((FIX_3_072 - FIX_2_562), -FIX_2_562, 85 (FIX_3_072 - FIX_2_562), -FIX_2_562), 86 _uint64_set_pi32((1 << (DESCALE_P1 - 1)), (1 << (DESCALE_P1 - 1))), 87 _uint64_set_pi32((1 << (DESCALE_P2 - 1)), (1 << (DESCALE_P2 - 1))), 88 _uint64_set_pi8(CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE, 89 CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE, CENTERJSAMPLE) 90 }; 91 92 #define PW_F130_F054 get_const_value(index_PW_F130_F054) 93 #define PW_F054_MF130 get_const_value(index_PW_F054_MF130) 94 #define PW_MF078_F117 get_const_value(index_PW_MF078_F117) 95 #define PW_F117_F078 get_const_value(index_PW_F117_F078) 96 #define PW_MF060_MF089 get_const_value(index_PW_MF060_MF089) 97 #define PW_MF089_F060 get_const_value(index_PW_MF089_F060) 98 #define PW_MF050_MF256 get_const_value(index_PW_MF050_MF256) 99 #define PW_MF256_F050 get_const_value(index_PW_MF256_F050) 100 #define PD_DESCALE_P1 get_const_value(index_PD_DESCALE_P1) 101 #define PD_DESCALE_P2 get_const_value(index_PD_DESCALE_P2) 102 #define PB_CENTERJSAMP get_const_value(index_PB_CENTERJSAMP) 103 104 105 #define test_m32_zero(mm32) (!(*(uint32_t *)&mm32)) 106 #define test_m64_zero(mm64) (!(*(uint64_t *)&mm64)) 107 108 109 #define DO_IDCT_COMMON(PASS) { \ 110 __m64 tmp0_3l, tmp0_3h, tmp1_2l, tmp1_2h; \ 111 __m64 tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h; \ 112 __m64 z34l, z34h, z3l, z3h, z4l, z4h, z3, z4; \ 113 __m64 out0l, out0h, out1l, out1h, out2l, out2h, out3l, out3h; \ 114 __m64 out4l, out4h, out5l, out5h, out6l, out6h, out7l, out7h; \ 115 \ 116 z3 = _mm_add_pi16(tmp0, tmp2); \ 117 z4 = _mm_add_pi16(tmp1, tmp3); \ 118 \ 119 /* (Original) \ 120 * z5 = (z3 + z4) * 1.175875602; \ 121 * z3 = z3 * -1.961570560; z4 = z4 * -0.390180644; \ 122 * z3 += z5; z4 += z5; \ 123 * \ 124 * (This implementation) \ 125 * z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602; \ 126 * z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644); \ 127 */ \ 128 \ 129 z34l = _mm_unpacklo_pi16(z3, z4); \ 130 z34h = _mm_unpackhi_pi16(z3, z4); \ 131 z3l = _mm_madd_pi16(z34l, PW_MF078_F117); \ 132 z3h = _mm_madd_pi16(z34h, PW_MF078_F117); \ 133 z4l = _mm_madd_pi16(z34l, PW_F117_F078); \ 134 z4h = _mm_madd_pi16(z34h, PW_F117_F078); \ 135 \ 136 /* (Original) \ 137 * z1 = tmp0 + tmp3; z2 = tmp1 + tmp2; \ 138 * tmp0 = tmp0 * 0.298631336; tmp1 = tmp1 * 2.053119869; \ 139 * tmp2 = tmp2 * 3.072711026; tmp3 = tmp3 * 1.501321110; \ 140 * z1 = z1 * -0.899976223; z2 = z2 * -2.562915447; \ 141 * tmp0 += z1 + z3; tmp1 += z2 + z4; \ 142 * tmp2 += z2 + z3; tmp3 += z1 + z4; \ 143 * \ 144 * (This implementation) \ 145 * tmp0 = tmp0 * (0.298631336 - 0.899976223) + tmp3 * -0.899976223; \ 146 * tmp1 = tmp1 * (2.053119869 - 2.562915447) + tmp2 * -2.562915447; \ 147 * tmp2 = tmp1 * -2.562915447 + tmp2 * (3.072711026 - 2.562915447); \ 148 * tmp3 = tmp0 * -0.899976223 + tmp3 * (1.501321110 - 0.899976223); \ 149 * tmp0 += z3; tmp1 += z4; \ 150 * tmp2 += z3; tmp3 += z4; \ 151 */ \ 152 \ 153 tmp0_3l = _mm_unpacklo_pi16(tmp0, tmp3); \ 154 tmp0_3h = _mm_unpackhi_pi16(tmp0, tmp3); \ 155 \ 156 tmp0l = _mm_madd_pi16(tmp0_3l, PW_MF060_MF089); \ 157 tmp0h = _mm_madd_pi16(tmp0_3h, PW_MF060_MF089); \ 158 tmp3l = _mm_madd_pi16(tmp0_3l, PW_MF089_F060); \ 159 tmp3h = _mm_madd_pi16(tmp0_3h, PW_MF089_F060); \ 160 \ 161 tmp0l = _mm_add_pi32(tmp0l, z3l); \ 162 tmp0h = _mm_add_pi32(tmp0h, z3h); \ 163 tmp3l = _mm_add_pi32(tmp3l, z4l); \ 164 tmp3h = _mm_add_pi32(tmp3h, z4h); \ 165 \ 166 tmp1_2l = _mm_unpacklo_pi16(tmp1, tmp2); \ 167 tmp1_2h = _mm_unpackhi_pi16(tmp1, tmp2); \ 168 \ 169 tmp1l = _mm_madd_pi16(tmp1_2l, PW_MF050_MF256); \ 170 tmp1h = _mm_madd_pi16(tmp1_2h, PW_MF050_MF256); \ 171 tmp2l = _mm_madd_pi16(tmp1_2l, PW_MF256_F050); \ 172 tmp2h = _mm_madd_pi16(tmp1_2h, PW_MF256_F050); \ 173 \ 174 tmp1l = _mm_add_pi32(tmp1l, z4l); \ 175 tmp1h = _mm_add_pi32(tmp1h, z4h); \ 176 tmp2l = _mm_add_pi32(tmp2l, z3l); \ 177 tmp2h = _mm_add_pi32(tmp2h, z3h); \ 178 \ 179 /* Final output stage */ \ 180 \ 181 out0l = _mm_add_pi32(tmp10l, tmp3l); \ 182 out0h = _mm_add_pi32(tmp10h, tmp3h); \ 183 out7l = _mm_sub_pi32(tmp10l, tmp3l); \ 184 out7h = _mm_sub_pi32(tmp10h, tmp3h); \ 185 \ 186 out0l = _mm_add_pi32(out0l, PD_DESCALE_P##PASS); \ 187 out0h = _mm_add_pi32(out0h, PD_DESCALE_P##PASS); \ 188 out0l = _mm_srai_pi32(out0l, DESCALE_P##PASS); \ 189 out0h = _mm_srai_pi32(out0h, DESCALE_P##PASS); \ 190 \ 191 out7l = _mm_add_pi32(out7l, PD_DESCALE_P##PASS); \ 192 out7h = _mm_add_pi32(out7h, PD_DESCALE_P##PASS); \ 193 out7l = _mm_srai_pi32(out7l, DESCALE_P##PASS); \ 194 out7h = _mm_srai_pi32(out7h, DESCALE_P##PASS); \ 195 \ 196 out0 = _mm_packs_pi32(out0l, out0h); \ 197 out7 = _mm_packs_pi32(out7l, out7h); \ 198 \ 199 out1l = _mm_add_pi32(tmp11l, tmp2l); \ 200 out1h = _mm_add_pi32(tmp11h, tmp2h); \ 201 out6l = _mm_sub_pi32(tmp11l, tmp2l); \ 202 out6h = _mm_sub_pi32(tmp11h, tmp2h); \ 203 \ 204 out1l = _mm_add_pi32(out1l, PD_DESCALE_P##PASS); \ 205 out1h = _mm_add_pi32(out1h, PD_DESCALE_P##PASS); \ 206 out1l = _mm_srai_pi32(out1l, DESCALE_P##PASS); \ 207 out1h = _mm_srai_pi32(out1h, DESCALE_P##PASS); \ 208 \ 209 out6l = _mm_add_pi32(out6l, PD_DESCALE_P##PASS); \ 210 out6h = _mm_add_pi32(out6h, PD_DESCALE_P##PASS); \ 211 out6l = _mm_srai_pi32(out6l, DESCALE_P##PASS); \ 212 out6h = _mm_srai_pi32(out6h, DESCALE_P##PASS); \ 213 \ 214 out1 = _mm_packs_pi32(out1l, out1h); \ 215 out6 = _mm_packs_pi32(out6l, out6h); \ 216 \ 217 out2l = _mm_add_pi32(tmp12l, tmp1l); \ 218 out2h = _mm_add_pi32(tmp12h, tmp1h); \ 219 out5l = _mm_sub_pi32(tmp12l, tmp1l); \ 220 out5h = _mm_sub_pi32(tmp12h, tmp1h); \ 221 \ 222 out2l = _mm_add_pi32(out2l, PD_DESCALE_P##PASS); \ 223 out2h = _mm_add_pi32(out2h, PD_DESCALE_P##PASS); \ 224 out2l = _mm_srai_pi32(out2l, DESCALE_P##PASS); \ 225 out2h = _mm_srai_pi32(out2h, DESCALE_P##PASS); \ 226 \ 227 out5l = _mm_add_pi32(out5l, PD_DESCALE_P##PASS); \ 228 out5h = _mm_add_pi32(out5h, PD_DESCALE_P##PASS); \ 229 out5l = _mm_srai_pi32(out5l, DESCALE_P##PASS); \ 230 out5h = _mm_srai_pi32(out5h, DESCALE_P##PASS); \ 231 \ 232 out2 = _mm_packs_pi32(out2l, out2h); \ 233 out5 = _mm_packs_pi32(out5l, out5h); \ 234 \ 235 out3l = _mm_add_pi32(tmp13l, tmp0l); \ 236 out3h = _mm_add_pi32(tmp13h, tmp0h); \ 237 \ 238 out4l = _mm_sub_pi32(tmp13l, tmp0l); \ 239 out4h = _mm_sub_pi32(tmp13h, tmp0h); \ 240 \ 241 out3l = _mm_add_pi32(out3l, PD_DESCALE_P##PASS); \ 242 out3h = _mm_add_pi32(out3h, PD_DESCALE_P##PASS); \ 243 out3l = _mm_srai_pi32(out3l, DESCALE_P##PASS); \ 244 out3h = _mm_srai_pi32(out3h, DESCALE_P##PASS); \ 245 \ 246 out4l = _mm_add_pi32(out4l, PD_DESCALE_P##PASS); \ 247 out4h = _mm_add_pi32(out4h, PD_DESCALE_P##PASS); \ 248 out4l = _mm_srai_pi32(out4l, DESCALE_P##PASS); \ 249 out4h = _mm_srai_pi32(out4h, DESCALE_P##PASS); \ 250 \ 251 out3 = _mm_packs_pi32(out3l, out3h); \ 252 out4 = _mm_packs_pi32(out4l, out4h); \ 253 } 254 255 #define DO_IDCT_PASS1(iter) { \ 256 __m64 col0l, col1l, col2l, col3l, col4l, col5l, col6l, col7l; \ 257 __m64 quant0l, quant1l, quant2l, quant3l; \ 258 __m64 quant4l, quant5l, quant6l, quant7l; \ 259 __m64 z23, z2, z3, z23l, z23h; \ 260 __m64 row01a, row01b, row01c, row01d, row23a, row23b, row23c, row23d; \ 261 __m64 row0l, row0h, row1l, row1h, row2l, row2h, row3l, row3h; \ 262 __m64 tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h; \ 263 __m64 tmp10l, tmp10h, tmp11l, tmp11h, tmp12l, tmp12h, tmp13l, tmp13h; \ 264 __m32 col0a, col1a, mm0; \ 265 \ 266 col0a = _mm_load_si32((__m32 *)&inptr[DCTSIZE * 1]); \ 267 col1a = _mm_load_si32((__m32 *)&inptr[DCTSIZE * 2]); \ 268 mm0 = _mm_or_si32(col0a, col1a); \ 269 \ 270 if (test_m32_zero(mm0)) { \ 271 __m64 mm1, mm2; \ 272 \ 273 col0l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 0]); \ 274 col1l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 1]); \ 275 col2l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 2]); \ 276 col3l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 3]); \ 277 col4l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 4]); \ 278 col5l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 5]); \ 279 col6l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 6]); \ 280 col7l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 7]); \ 281 \ 282 mm1 = _mm_or_si64(col1l, col3l); \ 283 mm2 = _mm_or_si64(col2l, col4l); \ 284 mm1 = _mm_or_si64(mm1, col5l); \ 285 mm2 = _mm_or_si64(mm2, col6l); \ 286 mm1 = _mm_or_si64(mm1, col7l); \ 287 mm1 = _mm_or_si64(mm1, mm2); \ 288 \ 289 if (test_m64_zero(mm1)) { \ 290 __m64 dcval, dcvall, dcvalh, row0, row1, row2, row3; \ 291 \ 292 /* AC terms all zero */ \ 293 \ 294 quant0l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 0]); \ 295 \ 296 dcval = _mm_mullo_pi16(col0l, quant0l); \ 297 dcval = _mm_slli_pi16(dcval, PASS1_BITS); /* dcval=(00 10 20 30) */ \ 298 \ 299 dcvall = _mm_unpacklo_pi16(dcval, dcval); /* dcvall=(00 00 10 10) */ \ 300 dcvalh = _mm_unpackhi_pi16(dcval, dcval); /* dcvalh=(20 20 30 30) */ \ 301 \ 302 row0 = _mm_unpacklo_pi32(dcvall, dcvall); /* row0=(00 00 00 00) */ \ 303 row1 = _mm_unpackhi_pi32(dcvall, dcvall); /* row1=(10 10 10 10) */ \ 304 row2 = _mm_unpacklo_pi32(dcvalh, dcvalh); /* row2=(20 20 20 20) */ \ 305 row3 = _mm_unpackhi_pi32(dcvalh, dcvalh); /* row3=(30 30 30 30) */ \ 306 \ 307 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0], row0); \ 308 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0 + 4], row0); \ 309 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1], row1); \ 310 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1 + 4], row1); \ 311 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2], row2); \ 312 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2 + 4], row2); \ 313 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3], row3); \ 314 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3 + 4], row3); \ 315 \ 316 goto nextcolumn##iter; \ 317 } \ 318 } \ 319 \ 320 /* Even part \ 321 * \ 322 * (Original) \ 323 * z1 = (z2 + z3) * 0.541196100; \ 324 * tmp2 = z1 + z3 * -1.847759065; \ 325 * tmp3 = z1 + z2 * 0.765366865; \ 326 * \ 327 * (This implementation) \ 328 * tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065); \ 329 * tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100; \ 330 */ \ 331 \ 332 col0l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 0]); /* (00 10 20 30) */ \ 333 col2l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 2]); /* (02 12 22 32) */ \ 334 col4l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 4]); /* (04 14 24 34) */ \ 335 col6l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 6]); /* (06 16 26 36) */ \ 336 \ 337 quant0l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 0]); \ 338 quant2l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 2]); \ 339 quant4l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 4]); \ 340 quant6l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 6]); \ 341 \ 342 z2 = _mm_mullo_pi16(col2l, quant2l); \ 343 z3 = _mm_mullo_pi16(col6l, quant6l); \ 344 \ 345 z23l = _mm_unpacklo_pi16(z2, z3); \ 346 z23h = _mm_unpackhi_pi16(z2, z3); \ 347 tmp3l = _mm_madd_pi16(z23l, PW_F130_F054); \ 348 tmp3h = _mm_madd_pi16(z23h, PW_F130_F054); \ 349 tmp2l = _mm_madd_pi16(z23l, PW_F054_MF130); \ 350 tmp2h = _mm_madd_pi16(z23h, PW_F054_MF130); \ 351 \ 352 z2 = _mm_mullo_pi16(col0l, quant0l); \ 353 z3 = _mm_mullo_pi16(col4l, quant4l); \ 354 \ 355 z23 = _mm_add_pi16(z2, z3); \ 356 tmp0l = _mm_loadlo_pi16_f(z23); \ 357 tmp0h = _mm_loadhi_pi16_f(z23); \ 358 tmp0l = _mm_srai_pi32(tmp0l, (16 - CONST_BITS)); \ 359 tmp0h = _mm_srai_pi32(tmp0h, (16 - CONST_BITS)); \ 360 \ 361 tmp10l = _mm_add_pi32(tmp0l, tmp3l); \ 362 tmp10h = _mm_add_pi32(tmp0h, tmp3h); \ 363 tmp13l = _mm_sub_pi32(tmp0l, tmp3l); \ 364 tmp13h = _mm_sub_pi32(tmp0h, tmp3h); \ 365 \ 366 z23 = _mm_sub_pi16(z2, z3); \ 367 tmp1l = _mm_loadlo_pi16_f(z23); \ 368 tmp1h = _mm_loadhi_pi16_f(z23); \ 369 tmp1l = _mm_srai_pi32(tmp1l, (16 - CONST_BITS)); \ 370 tmp1h = _mm_srai_pi32(tmp1h, (16 - CONST_BITS)); \ 371 \ 372 tmp11l = _mm_add_pi32(tmp1l, tmp2l); \ 373 tmp11h = _mm_add_pi32(tmp1h, tmp2h); \ 374 tmp12l = _mm_sub_pi32(tmp1l, tmp2l); \ 375 tmp12h = _mm_sub_pi32(tmp1h, tmp2h); \ 376 \ 377 /* Odd part */ \ 378 \ 379 col1l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 1]); /* (01 11 21 31) */ \ 380 col3l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 3]); /* (03 13 23 33) */ \ 381 col5l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 5]); /* (05 15 25 35) */ \ 382 col7l = _mm_load_si64((__m64 *)&inptr[DCTSIZE * 7]); /* (07 17 27 37) */ \ 383 \ 384 quant1l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 1]); \ 385 quant3l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 3]); \ 386 quant5l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 5]); \ 387 quant7l = _mm_load_si64((__m64 *)&quantptr[DCTSIZE * 7]); \ 388 \ 389 tmp0 = _mm_mullo_pi16(col7l, quant7l); \ 390 tmp1 = _mm_mullo_pi16(col5l, quant5l); \ 391 tmp2 = _mm_mullo_pi16(col3l, quant3l); \ 392 tmp3 = _mm_mullo_pi16(col1l, quant1l); \ 393 \ 394 DO_IDCT_COMMON(1) \ 395 \ 396 /* out0=(00 10 20 30), out1=(01 11 21 31) */ \ 397 /* out2=(02 12 22 32), out3=(03 13 23 33) */ \ 398 /* out4=(04 14 24 34), out5=(05 15 25 35) */ \ 399 /* out6=(06 16 26 36), out7=(07 17 27 37) */ \ 400 \ 401 /* Transpose coefficients */ \ 402 \ 403 row01a = _mm_unpacklo_pi16(out0, out1); /* row01a=(00 01 10 11) */ \ 404 row23a = _mm_unpackhi_pi16(out0, out1); /* row23a=(20 21 30 31) */ \ 405 row01d = _mm_unpacklo_pi16(out6, out7); /* row01d=(06 07 16 17) */ \ 406 row23d = _mm_unpackhi_pi16(out6, out7); /* row23d=(26 27 36 37) */ \ 407 \ 408 row01b = _mm_unpacklo_pi16(out2, out3); /* row01b=(02 03 12 13) */ \ 409 row23b = _mm_unpackhi_pi16(out2, out3); /* row23b=(22 23 32 33) */ \ 410 row01c = _mm_unpacklo_pi16(out4, out5); /* row01c=(04 05 14 15) */ \ 411 row23c = _mm_unpackhi_pi16(out4, out5); /* row23c=(24 25 34 35) */ \ 412 \ 413 row0l = _mm_unpacklo_pi32(row01a, row01b); /* row0l=(00 01 02 03) */ \ 414 row1l = _mm_unpackhi_pi32(row01a, row01b); /* row1l=(10 11 12 13) */ \ 415 row2l = _mm_unpacklo_pi32(row23a, row23b); /* row2l=(20 21 22 23) */ \ 416 row3l = _mm_unpackhi_pi32(row23a, row23b); /* row3l=(30 31 32 33) */ \ 417 \ 418 row0h = _mm_unpacklo_pi32(row01c, row01d); /* row0h=(04 05 06 07) */ \ 419 row1h = _mm_unpackhi_pi32(row01c, row01d); /* row1h=(14 15 16 17) */ \ 420 row2h = _mm_unpacklo_pi32(row23c, row23d); /* row2h=(24 25 26 27) */ \ 421 row3h = _mm_unpackhi_pi32(row23c, row23d); /* row3h=(34 35 36 37) */ \ 422 \ 423 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0], row0l); \ 424 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 0 + 4], row0h); \ 425 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1], row1l); \ 426 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 1 + 4], row1h); \ 427 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2], row2l); \ 428 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 2 + 4], row2h); \ 429 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3], row3l); \ 430 _mm_store_si64((__m64 *)&wsptr[DCTSIZE * 3 + 4], row3h); \ 431 } 432 433 #define DO_IDCT_PASS2(ctr) { \ 434 __m64 row0l, row1l, row2l, row3l, row4l, row5l, row6l, row7l; \ 435 __m64 z23, z23l, z23h; \ 436 __m64 col0123a, col0123b, col0123c, col0123d; \ 437 __m64 col01l, col01h, col23l, col23h, row06, row17, row24, row35; \ 438 __m64 col0, col1, col2, col3; \ 439 __m64 tmp0l, tmp0h, tmp1l, tmp1h, tmp2l, tmp2h, tmp3l, tmp3h; \ 440 __m64 tmp10l, tmp10h, tmp11l, tmp11h, tmp12l, tmp12h, tmp13l, tmp13h; \ 441 \ 442 row0l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 0]); /* (00 01 02 03) */ \ 443 row1l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 1]); /* (10 11 12 13) */ \ 444 row2l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 2]); /* (20 21 22 23) */ \ 445 row3l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 3]); /* (30 31 32 33) */ \ 446 row4l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 4]); /* (40 41 42 43) */ \ 447 row5l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 5]); /* (50 51 52 53) */ \ 448 row6l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 6]); /* (60 61 62 63) */ \ 449 row7l = _mm_load_si64((__m64 *)&wsptr[DCTSIZE * 7]); /* (70 71 72 73) */ \ 450 \ 451 /* Even part \ 452 * \ 453 * (Original) \ 454 * z1 = (z2 + z3) * 0.541196100; \ 455 * tmp2 = z1 + z3 * -1.847759065; \ 456 * tmp3 = z1 + z2 * 0.765366865; \ 457 * \ 458 * (This implementation) \ 459 * tmp2 = z2 * 0.541196100 + z3 * (0.541196100 - 1.847759065); \ 460 * tmp3 = z2 * (0.541196100 + 0.765366865) + z3 * 0.541196100; \ 461 */ \ 462 \ 463 z23l = _mm_unpacklo_pi16(row2l, row6l); \ 464 z23h = _mm_unpackhi_pi16(row2l, row6l); \ 465 \ 466 tmp3l = _mm_madd_pi16(z23l, PW_F130_F054); \ 467 tmp3h = _mm_madd_pi16(z23h, PW_F130_F054); \ 468 tmp2l = _mm_madd_pi16(z23l, PW_F054_MF130); \ 469 tmp2h = _mm_madd_pi16(z23h, PW_F054_MF130); \ 470 \ 471 z23 = _mm_add_pi16(row0l, row4l); \ 472 tmp0l = _mm_loadlo_pi16_f(z23); \ 473 tmp0h = _mm_loadhi_pi16_f(z23); \ 474 tmp0l = _mm_srai_pi32(tmp0l, (16 - CONST_BITS)); \ 475 tmp0h = _mm_srai_pi32(tmp0h, (16 - CONST_BITS)); \ 476 \ 477 tmp10l = _mm_add_pi32(tmp0l, tmp3l); \ 478 tmp10h = _mm_add_pi32(tmp0h, tmp3h); \ 479 tmp13l = _mm_sub_pi32(tmp0l, tmp3l); \ 480 tmp13h = _mm_sub_pi32(tmp0h, tmp3h); \ 481 \ 482 z23 = _mm_sub_pi16(row0l, row4l); \ 483 tmp1l = _mm_loadlo_pi16_f(z23); \ 484 tmp1h = _mm_loadhi_pi16_f(z23); \ 485 tmp1l = _mm_srai_pi32(tmp1l, (16 - CONST_BITS)); \ 486 tmp1h = _mm_srai_pi32(tmp1h, (16 - CONST_BITS)); \ 487 \ 488 tmp11l = _mm_add_pi32(tmp1l, tmp2l); \ 489 tmp11h = _mm_add_pi32(tmp1h, tmp2h); \ 490 tmp12l = _mm_sub_pi32(tmp1l, tmp2l); \ 491 tmp12h = _mm_sub_pi32(tmp1h, tmp2h); \ 492 \ 493 /* Odd part */ \ 494 \ 495 tmp0 = row7l; \ 496 tmp1 = row5l; \ 497 tmp2 = row3l; \ 498 tmp3 = row1l; \ 499 \ 500 DO_IDCT_COMMON(2) \ 501 \ 502 /* out0=(00 01 02 03), out1=(10 11 12 13) */ \ 503 /* out2=(20 21 22 23), out3=(30 31 32 33) */ \ 504 /* out4=(40 41 42 43), out5=(50 51 52 53) */ \ 505 /* out6=(60 61 62 63), out7=(70 71 72 73) */ \ 506 \ 507 row06 = _mm_packs_pi16(out0, out6); /* row06=(00 01 02 03 60 61 62 63) */ \ 508 row17 = _mm_packs_pi16(out1, out7); /* row17=(10 11 12 13 70 71 72 73) */ \ 509 row24 = _mm_packs_pi16(out2, out4); /* row24=(20 21 22 23 40 41 42 43) */ \ 510 row35 = _mm_packs_pi16(out3, out5); /* row35=(30 31 32 33 50 51 52 53) */ \ 511 \ 512 row06 = _mm_add_pi8(row06, PB_CENTERJSAMP); \ 513 row17 = _mm_add_pi8(row17, PB_CENTERJSAMP); \ 514 row24 = _mm_add_pi8(row24, PB_CENTERJSAMP); \ 515 row35 = _mm_add_pi8(row35, PB_CENTERJSAMP); \ 516 \ 517 /* Transpose coefficients */ \ 518 \ 519 col0123a = _mm_unpacklo_pi8(row06, row17); /* col0123a=(00 10 01 11 02 12 03 13) */ \ 520 col0123d = _mm_unpackhi_pi8(row06, row17); /* col0123d=(60 70 61 71 62 72 63 73) */ \ 521 col0123b = _mm_unpacklo_pi8(row24, row35); /* col0123b=(20 30 21 31 22 32 23 33) */ \ 522 col0123c = _mm_unpackhi_pi8(row24, row35); /* col0123c=(40 50 41 51 42 52 43 53) */ \ 523 \ 524 col01l = _mm_unpacklo_pi16(col0123a, col0123b); /* col01l=(00 10 20 30 01 11 21 31) */ \ 525 col23l = _mm_unpackhi_pi16(col0123a, col0123b); /* col23l=(02 12 22 32 03 13 23 33) */ \ 526 col01h = _mm_unpacklo_pi16(col0123c, col0123d); /* col01h=(40 50 60 70 41 51 61 71) */ \ 527 col23h = _mm_unpackhi_pi16(col0123c, col0123d); /* col23h=(42 52 62 72 43 53 63 73) */ \ 528 \ 529 col0 = _mm_unpacklo_pi32(col01l, col01h); /* col0=(00 10 20 30 40 50 60 70) */ \ 530 col1 = _mm_unpackhi_pi32(col01l, col01h); /* col1=(01 11 21 31 41 51 61 71) */ \ 531 col2 = _mm_unpacklo_pi32(col23l, col23h); /* col2=(02 12 22 32 42 52 62 72) */ \ 532 col3 = _mm_unpackhi_pi32(col23l, col23h); /* col3=(03 13 23 33 43 53 63 73) */ \ 533 \ 534 _mm_store_si64((__m64 *)(output_buf[ctr + 0] + output_col), col0); \ 535 _mm_store_si64((__m64 *)(output_buf[ctr + 1] + output_col), col1); \ 536 _mm_store_si64((__m64 *)(output_buf[ctr + 2] + output_col), col2); \ 537 _mm_store_si64((__m64 *)(output_buf[ctr + 3] + output_col), col3); \ 538 } 539 540 void jsimd_idct_islow_mmi(void *dct_table, JCOEFPTR coef_block, 541 JSAMPARRAY output_buf, JDIMENSION output_col) 542 { 543 __m64 tmp0, tmp1, tmp2, tmp3; 544 __m64 out0, out1, out2, out3, out4, out5, out6, out7; 545 JCOEFPTR inptr; 546 ISLOW_MULT_TYPE *quantptr; 547 JCOEF *wsptr; 548 JCOEF workspace[DCTSIZE2]; /* buffers data between passes */ 549 550 /* Pass 1: process columns. */ 551 552 inptr = coef_block; 553 quantptr = (ISLOW_MULT_TYPE *)dct_table; 554 wsptr = workspace; 555 556 DO_IDCT_PASS1(1) 557 nextcolumn1: 558 inptr += 4; 559 quantptr += 4; 560 wsptr += DCTSIZE * 4; 561 DO_IDCT_PASS1(2) 562 nextcolumn2: 563 564 /* Pass 2: process rows. */ 565 566 wsptr = workspace; 567 568 DO_IDCT_PASS2(0) 569 wsptr += 4; 570 DO_IDCT_PASS2(4) 571 }