itx_sse.asm (264136B)
1 ; Copyright © 2018-2021, VideoLAN and dav1d authors 2 ; Copyright © 2018, Two Orioles, LLC 3 ; All rights reserved. 4 ; 5 ; Redistribution and use in source and binary forms, with or without 6 ; modification, are permitted provided that the following conditions are met: 7 ; 8 ; 1. Redistributions of source code must retain the above copyright notice, this 9 ; list of conditions and the following disclaimer. 10 ; 11 ; 2. Redistributions in binary form must reproduce the above copyright notice, 12 ; this list of conditions and the following disclaimer in the documentation 13 ; and/or other materials provided with the distribution. 14 ; 15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 26 %include "config.asm" 27 %include "ext/x86/x86inc.asm" 28 29 30 SECTION_RODATA 16 31 32 deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 33 34 deint_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 35 deint_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 36 37 %macro COEF_PAIR 2-3 0 ; !0 = m%1_m%2, 2 = no %2_%1 38 pw_%1_m%2: times 4 dw %1, -%2 39 %if %3 != 2 40 pw_%2_%1: times 4 dw %2, %1 41 %endif 42 %if %3 43 pw_m%1_m%2: times 4 dw -%1, -%2 44 %endif 45 %endmacro 46 47 ;adst4 48 pw_1321_3803: times 4 dw 1321, 3803 49 pw_2482_m1321: times 4 dw 2482, -1321 50 pw_3344_2482: times 4 dw 3344, 2482 51 pw_3344_m3803: times 4 dw 3344, -3803 52 pw_3344_m3344: times 4 dw 3344, -3344 53 pw_0_3344 times 4 dw 0, 3344 54 pw_m6688_m3803: times 4 dw -6688, -3803 55 56 COEF_PAIR 2896, 2896 57 COEF_PAIR 1567, 3784 58 COEF_PAIR 799, 4017 59 COEF_PAIR 3406, 2276 60 COEF_PAIR 401, 4076 61 COEF_PAIR 1931, 3612 62 COEF_PAIR 3166, 2598 63 COEF_PAIR 3920, 1189 64 COEF_PAIR 3784, 1567, 1 65 COEF_PAIR 995, 3973 66 COEF_PAIR 1751, 3703 67 COEF_PAIR 3513, 2106 68 COEF_PAIR 3857, 1380 69 COEF_PAIR 4017, 799, 1 70 COEF_PAIR 201, 4091 71 COEF_PAIR 2440, 3290 72 COEF_PAIR 3035, 2751 73 COEF_PAIR 4052, 601 74 COEF_PAIR 2276, 3406, 1 75 COEF_PAIR 4076, 401, 2 76 COEF_PAIR 2598, 3166, 2 77 COEF_PAIR 3612, 1931, 2 78 COEF_PAIR 1189, 3920, 2 79 80 pd_2048: times 4 dd 2048 81 pw_2048: times 8 dw 2048 82 pw_m2048: times 8 dw -2048 83 pw_4096: times 8 dw 4096 84 pw_16384: times 8 dw 16384 85 pw_m16384: times 8 dw -16384 86 pw_1697x16: times 8 dw 1697*16 87 pw_1697x8: times 8 dw 1697*8 88 pw_2896x8: times 8 dw 2896*8 89 pw_3344x8: times 8 dw 3344*8 90 pw_8192: times 8 dw 8192 91 pw_m8192: times 8 dw -8192 92 pw_5: times 8 dw 5 93 pw_201x8: times 8 dw 201*8 94 pw_4091x8: times 8 dw 4091*8 95 pw_m2751x8: times 8 dw -2751*8 96 pw_3035x8: times 8 dw 3035*8 97 pw_1751x8: times 8 dw 1751*8 98 pw_3703x8: times 8 dw 3703*8 99 pw_m1380x8: times 8 dw -1380*8 100 pw_3857x8: times 8 dw 3857*8 101 pw_995x8: times 8 dw 995*8 102 pw_3973x8: times 8 dw 3973*8 103 pw_m2106x8: times 8 dw -2106*8 104 pw_3513x8: times 8 dw 3513*8 105 pw_2440x8: times 8 dw 2440*8 106 pw_3290x8: times 8 dw 3290*8 107 pw_m601x8: times 8 dw -601*8 108 pw_4052x8: times 8 dw 4052*8 109 110 pw_4095x8: times 8 dw 4095*8 111 pw_101x8: times 8 dw 101*8 112 pw_2967x8: times 8 dw 2967*8 113 pw_m2824x8: times 8 dw -2824*8 114 pw_3745x8: times 8 dw 3745*8 115 pw_1660x8: times 8 dw 1660*8 116 pw_3822x8: times 8 dw 3822*8 117 pw_m1474x8: times 8 dw -1474*8 118 pw_3996x8: times 8 dw 3996*8 119 pw_897x8: times 8 dw 897*8 120 pw_3461x8: times 8 dw 3461*8 121 pw_m2191x8: times 8 dw -2191*8 122 pw_3349x8: times 8 dw 3349*8 123 pw_2359x8: times 8 dw 2359*8 124 pw_4036x8: times 8 dw 4036*8 125 pw_m700x8: times 8 dw -700*8 126 pw_4065x8: times 8 dw 4065*8 127 pw_501x8: times 8 dw 501*8 128 pw_3229x8: times 8 dw 3229*8 129 pw_m2520x8: times 8 dw -2520*8 130 pw_3564x8: times 8 dw 3564*8 131 pw_2019x8: times 8 dw 2019*8 132 pw_3948x8: times 8 dw 3948*8 133 pw_m1092x8: times 8 dw -1092*8 134 pw_3889x8: times 8 dw 3889*8 135 pw_1285x8: times 8 dw 1285*8 136 pw_3659x8: times 8 dw 3659*8 137 pw_m1842x8: times 8 dw -1842*8 138 pw_3102x8: times 8 dw 3102*8 139 pw_2675x8: times 8 dw 2675*8 140 pw_4085x8: times 8 dw 4085*8 141 pw_m301x8: times 8 dw -301*8 142 143 SECTION .text 144 145 %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) 146 147 %if ARCH_X86_64 148 %define o(x) x 149 %else 150 %define o(x) r5-$$+x ; PIC 151 %endif 152 153 %macro WRITE_4X4 9 ;src[1-2], tmp[1-3], row[1-4] 154 lea r2, [dstq+strideq*2] 155 %assign %%i 1 156 %rotate 5 157 %rep 4 158 %if %1 & 2 159 CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1) 160 %else 161 CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1) 162 %endif 163 %assign %%i %%i + 1 164 %rotate 1 165 %endrep 166 167 movd m%3, [%%row_adr1] ;dst0 168 movd m%5, [%%row_adr2] ;dst1 169 punpckldq m%3, m%5 ;high: dst1 :low: dst0 170 movd m%4, [%%row_adr3] ;dst2 171 movd m%5, [%%row_adr4] ;dst3 172 punpckldq m%4, m%5 ;high: dst3 :low: dst2 173 174 pxor m%5, m%5 175 punpcklbw m%3, m%5 ;extend byte to word 176 punpcklbw m%4, m%5 ;extend byte to word 177 178 paddw m%3, m%1 ;high: dst1 + out1 ;low: dst0 + out0 179 paddw m%4, m%2 ;high: dst3 + out3 ;low: dst2 + out2 180 181 packuswb m%3, m%4 ;high->low: dst3 + out3, dst2 + out2, dst1 + out1, dst0 + out0 182 183 movd [%%row_adr1], m%3 ;store dst0 + out0 184 pshuflw m%4, m%3, q1032 185 movd [%%row_adr2], m%4 ;store dst1 + out1 186 punpckhqdq m%3, m%3 187 movd [%%row_adr3], m%3 ;store dst2 + out2 188 psrlq m%3, 32 189 movd [%%row_adr4], m%3 ;store dst3 + out3 190 %endmacro 191 192 %macro ITX4_END 4-5 2048 ; row[1-4], rnd 193 %if %5 194 mova m2, [o(pw_%5)] 195 pmulhrsw m0, m2 196 pmulhrsw m1, m2 197 %endif 198 199 WRITE_4X4 0, 1, 2, 3, 4, %1, %2, %3, %4 200 ret 201 %endmacro 202 203 ; flags: 1 = swap, 2: coef_regs, 4: no_pack 204 %macro ITX_MUL2X_PACK 5-6 0 ; dst/src, tmp[1], rnd, coef[1-2], flags 205 %if %6 & 2 206 pmaddwd m%2, m%4, m%1 207 pmaddwd m%1, m%5 208 %elif %6 & 1 209 pmaddwd m%2, m%1, [o(pw_%5_%4)] 210 pmaddwd m%1, [o(pw_%4_m%5)] 211 %else 212 pmaddwd m%2, m%1, [o(pw_%4_m%5)] 213 pmaddwd m%1, [o(pw_%5_%4)] 214 %endif 215 paddd m%2, m%3 216 paddd m%1, m%3 217 psrad m%2, 12 218 psrad m%1, 12 219 %if %6 & 4 == 0 220 packssdw m%1, m%2 221 %endif 222 %endmacro 223 224 %macro IDCT4_1D_PACKED 0-1 ;pw_2896x8 225 mova m3, [o(pd_2048)] 226 punpckhwd m2, m0, m1 ;unpacked in1 in3 227 punpcklwd m0, m1 ;unpacked in0 in2 228 ITX_MUL2X_PACK 2, 1, 3, 1567, 3784 229 ITX_MUL2X_PACK 0, 1, 3, 2896, 2896 230 psubsw m1, m0, m2 ;high: out2 ;low: out3 231 paddsw m0, m2 ;high: out1 ;low: out0 232 %endmacro 233 234 %macro INV_TXFM_FN 4+ ; type1, type2, size, xmm/stack 235 cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 6, %4, dst, stride, coeff, eob, tx2 236 %define %%p1 m(i%1_%3_internal_8bpc) 237 %if ARCH_X86_32 238 LEA r5, $$ 239 %endif 240 %if has_epilogue 241 %ifidn %1_%2, dct_dct 242 test eobd, eobd 243 jz %%end 244 %endif 245 lea tx2q, [o(m(i%2_%3_internal_8bpc).pass2)] 246 call %%p1 247 RET 248 %%end: 249 %else 250 lea tx2q, [o(m(i%2_%3_internal_8bpc).pass2)] 251 %ifidn %1_%2, dct_dct 252 test eobd, eobd 253 jnz %%p1 254 %else 255 times ((%%end - %%p1) >> 31) & 1 jmp %%p1 256 ALIGN function_align 257 %%end: 258 %endif 259 %endif 260 %endmacro 261 262 %macro INV_TXFM_4X4_FN 2 ; type1, type2 263 INV_TXFM_FN %1, %2, 4x4, 6 264 %ifidn %1_%2, dct_dct 265 pshuflw m0, [coeffq], q0000 266 punpcklqdq m0, m0 267 mova m1, [o(pw_2896x8)] 268 pmulhrsw m0, m1 269 mov [coeffq], eobd ;0 270 pmulhrsw m0, m1 271 mova m1, m0 272 TAIL_CALL m(iadst_4x4_internal_8bpc).end2 273 %endif 274 %endmacro 275 276 INIT_XMM ssse3 277 ; itx16 relies on dct_dct being the first function. If you change the order, adjust `itx8_start` in itx16. 278 279 INV_TXFM_4X4_FN dct, dct 280 INV_TXFM_4X4_FN dct, adst 281 INV_TXFM_4X4_FN dct, flipadst 282 INV_TXFM_4X4_FN dct, identity 283 284 cglobal idct_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 285 mova m0, [coeffq+16*0] ;high: in1 ;low: in0 286 mova m1, [coeffq+16*1] ;high: in3 ;low in2 287 288 IDCT4_1D_PACKED 289 290 mova m2, [o(deint_shuf)] 291 shufps m3, m0, m1, q1331 292 shufps m0, m1, q0220 293 pshufb m0, m2 ;high: in1 ;low: in0 294 pshufb m1, m3, m2 ;high: in3 ;low :in2 295 jmp tx2q 296 297 .pass2: 298 IDCT4_1D_PACKED 299 300 pxor m2, m2 301 mova [coeffq+16*0], m2 302 mova [coeffq+16*1], m2 ;memset(coeff, 0, sizeof(*coeff) * sh * sw); 303 304 ITX4_END 0, 1, 3, 2 305 306 INV_TXFM_4X4_FN adst, dct 307 INV_TXFM_4X4_FN adst, adst 308 INV_TXFM_4X4_FN adst, flipadst 309 INV_TXFM_4X4_FN adst, identity 310 311 cglobal iadst_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 312 mova m0, [coeffq+16*0] 313 mova m1, [coeffq+16*1] 314 call .main 315 punpckhwd m2, m0, m1 316 punpcklwd m0, m1 317 punpckhwd m1, m0, m2 ;high: in3 ;low :in2 318 punpcklwd m0, m2 ;high: in1 ;low: in0 319 jmp tx2q 320 321 .pass2: 322 call .main 323 324 .end: 325 pxor m2, m2 326 mova [coeffq+16*0], m2 327 mova [coeffq+16*1], m2 328 329 .end2: 330 ITX4_END 0, 1, 2, 3 331 332 ALIGN function_align 333 cglobal_label .main 334 punpcklwd m2, m0, m1 ;unpacked in0 in2 335 punpckhwd m0, m1 ;unpacked in1 in3 336 mova m3, m0 337 pmaddwd m1, m2, [o(pw_3344_m3344)];3344 * in0 - 3344 * in2 338 pmaddwd m0, [o(pw_0_3344)] ;3344 * in3 339 paddd m1, m0 ;t2 340 pmaddwd m0, m2, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 341 pmaddwd m2, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 342 pmaddwd m4, m3, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 343 pmaddwd m5, m3, [o(pw_3344_m3803)];3344 * in1 - 3803 * in3 344 paddd m4, m0 ;t0 + t3 345 pmaddwd m3, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 346 mova m0, [o(pd_2048)] 347 paddd m1, m0 ;t2 + 2048 348 paddd m2, m0 349 paddd m0, m4 ;t0 + t3 + 2048 350 paddd m5, m2 ;t1 + t3 + 2048 351 paddd m2, m4 352 paddd m2, m3 ;t0 + t1 - t3 + 2048 353 REPX {psrad x, 12}, m1, m0, m5, m2 354 packssdw m0, m5 ;high: out1 ;low: out0 355 packssdw m1, m2 ;high: out3 ;low: out3 356 ret 357 358 INV_TXFM_4X4_FN flipadst, dct 359 INV_TXFM_4X4_FN flipadst, adst 360 INV_TXFM_4X4_FN flipadst, flipadst 361 INV_TXFM_4X4_FN flipadst, identity 362 363 cglobal iflipadst_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 364 mova m0, [coeffq+16*0] 365 mova m1, [coeffq+16*1] 366 call m(iadst_4x4_internal_8bpc).main 367 punpcklwd m2, m1, m0 368 punpckhwd m1, m0 369 punpcklwd m0, m1, m2 ;high: in3 ;low :in2 370 punpckhwd m1, m2 ;high: in1 ;low: in0 371 jmp tx2q 372 373 .pass2: 374 call m(iadst_4x4_internal_8bpc).main 375 376 .end: 377 pxor m2, m2 378 mova [coeffq+16*0], m2 379 mova [coeffq+16*1], m2 380 381 .end2: 382 ITX4_END 3, 2, 1, 0 383 384 INV_TXFM_4X4_FN identity, dct 385 INV_TXFM_4X4_FN identity, adst 386 INV_TXFM_4X4_FN identity, flipadst 387 INV_TXFM_4X4_FN identity, identity 388 389 cglobal iidentity_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 390 mova m0, [coeffq+16*0] 391 mova m1, [coeffq+16*1] 392 mova m3, [o(pw_1697x8)] 393 pmulhrsw m2, m0, m3 394 pmulhrsw m3, m1 395 paddsw m0, m2 396 paddsw m1, m3 397 punpckhwd m2, m0, m1 398 punpcklwd m0, m1 399 punpckhwd m1, m0, m2 ;high: in3 ;low :in2 400 punpcklwd m0, m2 ;high: in1 ;low: in0 401 jmp tx2q 402 403 .pass2: 404 mova m3, [o(pw_1697x8)] 405 pmulhrsw m2, m3, m0 406 pmulhrsw m3, m1 407 paddsw m0, m2 408 paddsw m1, m3 409 jmp m(iadst_4x4_internal_8bpc).end 410 411 %macro IWHT4_1D_PACKED 0 412 punpckhqdq m3, m0, m1 ;low: in1 high: in3 413 punpcklqdq m0, m1 ;low: in0 high: in2 414 psubw m2, m0, m3 ;low: in0 - in1 high: in2 - in3 415 paddw m0, m3 ;low: in0 + in1 high: in2 + in3 416 punpckhqdq m2, m2 ;t2 t2 417 punpcklqdq m0, m0 ;t0 t0 418 psubw m1, m0, m2 419 psraw m1, 1 ;t4 t4 420 psubw m1, m3 ;low: t1/out2 high: t3/out1 421 psubw m0, m1 ;high: out0 422 paddw m2, m1 ;low: out3 423 %endmacro 424 425 INIT_XMM sse2 426 cglobal inv_txfm_add_wht_wht_4x4_8bpc, 3, 3, 4, dst, stride, coeff 427 mova m0, [coeffq+16*0] 428 mova m1, [coeffq+16*1] 429 pxor m2, m2 430 mova [coeffq+16*0], m2 431 mova [coeffq+16*1], m2 432 psraw m0, 2 433 psraw m1, 2 434 IWHT4_1D_PACKED 435 punpckhwd m0, m1 436 punpcklwd m3, m1, m2 437 punpckhdq m1, m0, m3 438 punpckldq m0, m3 439 IWHT4_1D_PACKED 440 shufpd m0, m2, 0x01 441 ITX4_END 0, 3, 2, 1, 0 442 443 %macro IDCT8_1D_PACKED 0 444 mova m6, [o(pd_2048)] 445 punpckhwd m4, m0, m3 ;unpacked in1 in7 446 punpcklwd m0, m2 ;unpacked in0 in4 447 punpckhwd m2, m1 ;unpacked in5 in3 448 punpcklwd m1, m3 ;unpacked in2 in6 449 ITX_MUL2X_PACK 4, 3, 6, 799, 4017 ;low: t7a high: t4a 450 ITX_MUL2X_PACK 2, 3, 6, 3406, 2276 ;low: t6a high: t5a 451 ITX_MUL2X_PACK 1, 3, 6, 1567, 3784 ;low: t3 high: t2 452 psubsw m3, m4, m2 ;low: t6a high: t5a 453 paddsw m4, m2 ;low: t7 high: t4 454 pshufb m3, [o(deint_shuf1)] 455 ITX_MUL2X_PACK 0, 2, 6, 2896, 2896 ;low: t0 high: t1 456 ITX_MUL2X_PACK 3, 2, 6, 2896, 2896 ;low: t6 high: t5 457 psubsw m2, m0, m1 ;low: tmp3 high: tmp2 458 paddsw m0, m1 ;low: tmp0 high: tmp1 459 punpcklqdq m1, m4, m3 ;low: t7 high: t6 460 punpckhqdq m4, m3 ;low: t4 high: t5 461 psubsw m3, m0, m1 ;low: out7 high: out6 462 paddsw m0, m1 ;low: out0 high: out1 463 paddsw m1, m2, m4 ;low: out3 high: out2 464 psubsw m2, m4 ;low: out4 high: out5 465 %endmacro 466 467 ;dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 468 ;dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 469 %macro ITX_MULSUB_2W 7-8 0 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2], dst2_in_tmp1 470 punpckhwd m%4, m%1, m%2 471 punpcklwd m%1, m%2 472 %if %7 < 8 473 pmaddwd m%2, m%7, m%1 474 pmaddwd m%3, m%7, m%4 475 %else 476 mova m%2, [o(pw_%7_%6)] 477 %if %8 478 pmaddwd m%3, m%1, m%2 479 pmaddwd m%2, m%4 480 %else 481 pmaddwd m%3, m%4, m%2 482 pmaddwd m%2, m%1 483 %endif 484 %endif 485 paddd m%3, m%5 486 paddd m%2, m%5 487 psrad m%3, 12 488 psrad m%2, 12 489 %if %8 490 packssdw m%3, m%2 491 %else 492 packssdw m%2, m%3 ;dst2 493 %endif 494 %if %7 < 8 495 pmaddwd m%4, m%6 496 pmaddwd m%1, m%6 497 %elif %8 498 mova m%2, [o(pw_%6_m%7)] 499 pmaddwd m%4, m%2 500 pmaddwd m%1, m%2 501 %else 502 mova m%3, [o(pw_%6_m%7)] 503 pmaddwd m%4, m%3 504 pmaddwd m%1, m%3 505 %endif 506 paddd m%4, m%5 507 paddd m%1, m%5 508 psrad m%4, 12 509 psrad m%1, 12 510 packssdw m%1, m%4 ;dst1 511 %endmacro 512 513 %macro IDCT4_1D 7 ; src[1-4], tmp[1-2], pd_2048 514 ITX_MULSUB_2W %2, %4, %5, %6, %7, 1567, 3784, 1 ;t2, t3 515 ITX_MULSUB_2W %1, %3, %4, %6, %7, 2896, 2896, 1 ;t1, t0 516 psubsw m%3, m%1, m%2 ;out2 517 paddsw m%2, m%1 ;out1 518 paddsw m%1, m%5, m%4 ;out0 519 psubsw m%4, m%5 ;out3 520 %endmacro 521 522 %macro WRITE_4X8 4 ;row[1-4] 523 WRITE_4X4 0, 1, 4, 5, 6, %1, %2, %3, %4 524 lea dstq, [dstq+strideq*4] 525 WRITE_4X4 2, 3, 4, 5, 6, %1, %2, %3, %4 526 %endmacro 527 528 %macro INV_4X8 0 529 punpckhwd m4, m2, m3 530 punpcklwd m2, m3 531 punpckhwd m3, m0, m1 532 punpcklwd m0, m1 533 punpckhdq m1, m0, m2 ;low: in2 high: in3 534 punpckldq m0, m2 ;low: in0 high: in1 535 punpckldq m2, m3, m4 ;low: in4 high: in5 536 punpckhdq m3, m4 ;low: in6 high: in7 537 %endmacro 538 539 %macro INV_TXFM_4X8_FN 2 ; type1, type2 540 INV_TXFM_FN %1, %2, 4x8, 8 541 %ifidn %1_%2, dct_dct 542 pshuflw m0, [coeffq], q0000 543 punpcklqdq m0, m0 544 mova m1, [o(pw_2896x8)] 545 pmulhrsw m0, m1 546 mov [coeffq], eobd 547 pmulhrsw m0, m1 548 pmulhrsw m0, m1 549 pmulhrsw m0, [o(pw_2048)] 550 mova m1, m0 551 mova m2, m0 552 mova m3, m0 553 TAIL_CALL m(iadst_4x8_internal_8bpc).end3 554 %endif 555 %endmacro 556 557 INIT_XMM ssse3 558 INV_TXFM_4X8_FN dct, dct 559 INV_TXFM_4X8_FN dct, adst 560 INV_TXFM_4X8_FN dct, flipadst 561 INV_TXFM_4X8_FN dct, identity 562 563 cglobal idct_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 564 mova m3, [o(pw_2896x8)] 565 pmulhrsw m0, m3, [coeffq+16*0] 566 pmulhrsw m1, m3, [coeffq+16*1] 567 pmulhrsw m2, m3, [coeffq+16*2] 568 pmulhrsw m3, [coeffq+16*3] 569 570 .pass1: 571 call m(idct_8x4_internal_8bpc).main 572 jmp m(iadst_4x8_internal_8bpc).pass1_end 573 574 .pass2: 575 call .main 576 shufps m1, m1, q1032 577 shufps m3, m3, q1032 578 mova m4, [o(pw_2048)] 579 jmp m(iadst_4x8_internal_8bpc).end2 580 581 ALIGN function_align 582 cglobal_label .main 583 IDCT8_1D_PACKED 584 ret 585 586 587 INV_TXFM_4X8_FN adst, dct 588 INV_TXFM_4X8_FN adst, adst 589 INV_TXFM_4X8_FN adst, flipadst 590 INV_TXFM_4X8_FN adst, identity 591 592 cglobal iadst_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 593 mova m3, [o(pw_2896x8)] 594 pmulhrsw m0, m3, [coeffq+16*0] 595 pmulhrsw m1, m3, [coeffq+16*1] 596 pmulhrsw m2, m3, [coeffq+16*2] 597 pmulhrsw m3, [coeffq+16*3] 598 599 .pass1: 600 call m(iadst_8x4_internal_8bpc).main 601 602 .pass1_end: 603 INV_4X8 604 jmp tx2q 605 606 .pass2: 607 shufps m0, m0, q1032 608 shufps m1, m1, q1032 609 call .main 610 mova m4, [o(pw_2048)] 611 pxor m5, m5 612 psubw m5, m4 613 614 .end: 615 punpcklqdq m4, m5 616 617 .end2: 618 pmulhrsw m0, m4 619 pmulhrsw m1, m4 620 pmulhrsw m2, m4 621 pmulhrsw m3, m4 622 pxor m5, m5 623 mova [coeffq+16*0], m5 624 mova [coeffq+16*1], m5 625 mova [coeffq+16*2], m5 626 mova [coeffq+16*3], m5 627 628 .end3: 629 WRITE_4X8 0, 1, 2, 3 630 RET 631 632 ALIGN function_align 633 cglobal_label .main 634 mova m6, [o(pd_2048)] 635 punpckhwd m4, m3, m0 ;unpacked in7 in0 636 punpckhwd m5, m2, m1 ;unpacked in5 in2 637 punpcklwd m1, m2 ;unpacked in3 in4 638 punpcklwd m0, m3 ;unpacked in1 in6 639 ITX_MUL2X_PACK 4, 2, 6, 401, 4076 ;low: t0a high: t1a 640 ITX_MUL2X_PACK 5, 2, 6, 1931, 3612 ;low: t2a high: t3a 641 ITX_MUL2X_PACK 1, 2, 6, 3166, 2598 ;low: t4a high: t5a 642 ITX_MUL2X_PACK 0, 2, 6, 3920, 1189 ;low: t6a high: t7a 643 644 psubsw m3, m4, m1 ;low: t4 high: t5 645 paddsw m4, m1 ;low: t0 high: t1 646 psubsw m2, m5, m0 ;low: t6 high: t7 647 paddsw m5, m0 ;low: t2 high: t3 648 649 shufps m1, m3, m2, q1032 650 punpckhwd m2, m1 651 punpcklwd m3, m1 652 ITX_MUL2X_PACK 3, 0, 6, 1567, 3784, 1 ;low: t5a high: t4a 653 ITX_MUL2X_PACK 2, 0, 6, 3784, 1567 ;low: t7a high: t6a 654 655 psubsw m1, m4, m5 ;low: t2 high: t3 656 paddsw m4, m5 ;low: out0 high: -out7 657 psubsw m5, m3, m2 ;low: t7 high: t6 658 paddsw m3, m2 ;low: out6 high: -out1 659 shufps m0, m4, m3, q3210 ;low: out0 high: -out1 660 shufps m3, m4, q3210 ;low: out6 high: -out7 661 662 mova m2, [o(pw_2896_m2896)] 663 mova m7, [o(pw_2896_2896)] 664 shufps m4, m1, m5, q1032 ;low: t3 high: t7 665 shufps m1, m5, q3210 ;low: t2 high: t6 666 punpcklwd m5, m1, m4 667 punpckhwd m1, m4 668 pmaddwd m4, m2, m1 ;-out5 669 pmaddwd m2, m5 ; out4 670 pmaddwd m1, m7 ; out2 671 pmaddwd m5, m7 ;-out3 672 REPX {paddd x, m6}, m4, m2, m1, m5 673 REPX {psrad x, 12}, m4, m2, m1, m5 674 packssdw m1, m5 ;low: out2 high: -out3 675 packssdw m2, m4 ;low: out4 high: -out5 676 ret 677 678 INV_TXFM_4X8_FN flipadst, dct 679 INV_TXFM_4X8_FN flipadst, adst 680 INV_TXFM_4X8_FN flipadst, flipadst 681 INV_TXFM_4X8_FN flipadst, identity 682 683 cglobal iflipadst_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 684 mova m3, [o(pw_2896x8)] 685 pmulhrsw m0, m3, [coeffq+16*0] 686 pmulhrsw m1, m3, [coeffq+16*1] 687 pmulhrsw m2, m3, [coeffq+16*2] 688 pmulhrsw m3, [coeffq+16*3] 689 690 .pass1: 691 call m(iadst_8x4_internal_8bpc).main 692 693 punpcklwd m4, m3, m2 694 punpckhwd m3, m2 695 punpcklwd m5, m1, m0 696 punpckhwd m1, m0 697 punpckldq m2, m3, m1 ;low: in4 high: in5 698 punpckhdq m3, m1 ;low: in6 high: in7 699 punpckldq m0, m4, m5 ;low: in0 high: in1 700 punpckhdq m1, m4, m5 ;low: in2 high: in3 701 jmp tx2q 702 703 .pass2: 704 shufps m0, m0, q1032 705 shufps m1, m1, q1032 706 call m(iadst_4x8_internal_8bpc).main 707 708 mova m4, m0 709 mova m5, m1 710 pshufd m0, m3, q1032 711 pshufd m1, m2, q1032 712 pshufd m2, m5, q1032 713 pshufd m3, m4, q1032 714 mova m5, [o(pw_2048)] 715 pxor m4, m4 716 psubw m4, m5 717 jmp m(iadst_4x8_internal_8bpc).end 718 719 INV_TXFM_4X8_FN identity, dct 720 INV_TXFM_4X8_FN identity, adst 721 INV_TXFM_4X8_FN identity, flipadst 722 INV_TXFM_4X8_FN identity, identity 723 724 cglobal iidentity_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 725 mova m3, [o(pw_2896x8)] 726 pmulhrsw m0, m3, [coeffq+16*0] 727 pmulhrsw m1, m3, [coeffq+16*1] 728 pmulhrsw m2, m3, [coeffq+16*2] 729 pmulhrsw m3, [coeffq+16*3] 730 731 .pass1: 732 mova m7, [o(pw_1697x8)] 733 pmulhrsw m4, m7, m0 734 pmulhrsw m5, m7, m1 735 pmulhrsw m6, m7, m2 736 pmulhrsw m7, m3 737 paddsw m0, m4 738 paddsw m1, m5 739 paddsw m2, m6 740 paddsw m3, m7 741 jmp m(iadst_4x8_internal_8bpc).pass1_end 742 743 .pass2: 744 mova m4, [o(pw_4096)] 745 jmp m(iadst_4x8_internal_8bpc).end2 746 747 748 %macro WRITE_8X2 5 ;coefs[1-2], tmp[1-3] 749 movq m%3, [dstq ] 750 movq m%4, [dstq+strideq] 751 pxor m%5, m%5 752 punpcklbw m%3, m%5 ;extend byte to word 753 punpcklbw m%4, m%5 ;extend byte to word 754 %ifnum %1 755 paddw m%3, m%1 756 %else 757 paddw m%3, %1 758 %endif 759 %ifnum %2 760 paddw m%4, m%2 761 %else 762 paddw m%4, %2 763 %endif 764 packuswb m%3, m%4 765 movq [dstq ], m%3 766 punpckhqdq m%3, m%3 767 movq [dstq+strideq], m%3 768 %endmacro 769 770 %macro WRITE_8X4 7 ;coefs[1-4], tmp[1-3] 771 WRITE_8X2 %1, %2, %5, %6, %7 772 lea dstq, [dstq+strideq*2] 773 WRITE_8X2 %3, %4, %5, %6, %7 774 %endmacro 775 776 %macro INV_TXFM_8X4_FN 2 ; type1, type2 777 INV_TXFM_FN %1, %2, 8x4, 8 778 %ifidn %1_%2, dct_dct 779 pshuflw m0, [coeffq], q0000 780 punpcklqdq m0, m0 781 mova m1, [o(pw_2896x8)] 782 pmulhrsw m0, m1 783 pmulhrsw m0, m1 784 mova m2, [o(pw_2048)] 785 pmulhrsw m0, m1 786 pmulhrsw m0, m2 787 mova m1, m0 788 mova m2, m0 789 mova m3, m0 790 TAIL_CALL m(iadst_8x4_internal_8bpc).end2 791 %endif 792 %endmacro 793 794 INV_TXFM_8X4_FN dct, dct 795 INV_TXFM_8X4_FN dct, adst 796 INV_TXFM_8X4_FN dct, flipadst 797 INV_TXFM_8X4_FN dct, identity 798 799 cglobal idct_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 800 mova m3, [o(pw_2896x8)] 801 pmulhrsw m0, m3, [coeffq+16*0] 802 pmulhrsw m1, m3, [coeffq+16*1] 803 pmulhrsw m2, m3, [coeffq+16*2] 804 pmulhrsw m3, [coeffq+16*3] 805 806 call m(idct_4x8_internal_8bpc).main 807 808 mova m4, [o(deint_shuf1)] 809 mova m5, [o(deint_shuf2)] 810 pshufb m0, m4 811 pshufb m1, m5 812 pshufb m2, m4 813 pshufb m3, m5 814 punpckhdq m4, m0, m1 815 punpckldq m0, m1 816 punpckhdq m5, m2, m3 817 punpckldq m2, m3 818 punpckhqdq m1, m0, m2 ;in1 819 punpcklqdq m0, m2 ;in0 820 punpckhqdq m3, m4, m5 ;in3 821 punpcklqdq m2 ,m4, m5 ;in2 822 jmp tx2q 823 824 .pass2: 825 call .main 826 jmp m(iadst_8x4_internal_8bpc).end 827 828 ALIGN function_align 829 cglobal_label .main 830 mova m6, [o(pd_2048)] 831 IDCT4_1D 0, 1, 2, 3, 4, 5, 6 832 ret 833 834 INV_TXFM_8X4_FN adst, dct 835 INV_TXFM_8X4_FN adst, adst 836 INV_TXFM_8X4_FN adst, flipadst 837 INV_TXFM_8X4_FN adst, identity 838 839 cglobal iadst_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 840 mova m3, [o(pw_2896x8)] 841 pmulhrsw m0, m3, [coeffq+16*0] 842 pmulhrsw m1, m3, [coeffq+16*1] 843 pmulhrsw m2, m3, [coeffq+16*2] 844 pmulhrsw m3, [coeffq+16*3] 845 846 shufps m0, m0, q1032 847 shufps m1, m1, q1032 848 call m(iadst_4x8_internal_8bpc).main 849 850 punpckhwd m4, m0, m1 851 punpcklwd m0, m1 852 punpckhwd m1, m2, m3 853 punpcklwd m2, m3 854 pxor m5, m5 855 psubsw m3, m5, m1 856 psubsw m5, m4 857 punpckhdq m4, m5, m3 858 punpckldq m5, m3 859 punpckhdq m3, m0, m2 860 punpckldq m0, m2 861 punpckhwd m1, m0, m5 ;in1 862 punpcklwd m0, m5 ;in0 863 punpcklwd m2, m3, m4 ;in2 864 punpckhwd m3, m4 ;in3 865 jmp tx2q 866 867 .pass2: 868 call .main 869 870 .end: 871 mova m4, [o(pw_2048)] 872 pmulhrsw m0, m4 873 pmulhrsw m1, m4 874 pmulhrsw m2, m4 875 pmulhrsw m3, m4 876 877 .end2: 878 pxor m6, m6 879 mova [coeffq+16*0], m6 880 mova [coeffq+16*1], m6 881 mova [coeffq+16*2], m6 882 mova [coeffq+16*3], m6 883 .end3: 884 WRITE_8X4 0, 1, 2, 3, 4, 5, 6 885 RET 886 887 ALIGN function_align 888 cglobal_label .main 889 punpckhwd m6, m0, m2 ;unpacked in0 in2 890 punpcklwd m0, m2 ;unpacked in0 in2 891 punpckhwd m7, m1, m3 ;unpacked in1 in3 892 punpcklwd m1, m3 ;unpacked in1 in3 893 894 mova m2, [o(pw_3344_m3344)] 895 mova m4, [o(pw_0_3344)] 896 pmaddwd m3, m2, m6 ;3344 * in0 - 3344 * in2 897 pmaddwd m5, m4, m7 ;3344 * in3 898 pmaddwd m2, m0 899 pmaddwd m4, m1 900 paddd m3, m5 901 paddd m2, m4 902 mova m4, [o(pd_2048)] 903 paddd m3, m4 ;t2 + 2048 904 paddd m2, m4 905 psrad m3, 12 906 psrad m2, 12 907 packssdw m2, m3 ;out2 908 909 pmaddwd m4, m0, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 910 pmaddwd m0, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 911 pmaddwd m3, m1, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 912 pmaddwd m5, m1, [o(pw_3344_m3803)] ;3344 * in1 - 3803 * in3 913 paddd m3, m4 ;t0 + t3 914 915 pmaddwd m1, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 916 mova m4, [o(pd_2048)] 917 paddd m0, m4 918 paddd m4, m3 ;t0 + t3 + 2048 919 paddd m5, m0 ;t1 + t3 + 2048 920 paddd m3, m0 921 paddd m3, m1 ;t0 + t1 - t3 + 2048 922 923 psrad m4, 12 ;out0 924 psrad m5, 12 ;out1 925 psrad m3, 12 ;out3 926 packssdw m0, m4, m5 ;low: out0 high: out1 927 928 pmaddwd m4, m6, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 929 pmaddwd m6, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 930 pmaddwd m1, m7, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 931 pmaddwd m5, m7, [o(pw_3344_m3803)] ;3344 * in1 - 3803 * in3 932 paddd m1, m4 ;t0 + t3 933 pmaddwd m7, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 934 935 mova m4, [o(pd_2048)] 936 paddd m6, m4 937 paddd m4, m1 ;t0 + t3 + 2048 938 paddd m5, m6 ;t1 + t3 + 2048 939 paddd m1, m6 940 paddd m1, m7 ;t0 + t1 - t3 + 2048 941 942 psrad m4, 12 ;out0 943 psrad m5, 12 ;out1 944 psrad m1, 12 ;out3 945 packssdw m3, m1 ;out3 946 packssdw m4, m5 ;low: out0 high: out1 947 948 punpckhqdq m1, m0, m4 ;out1 949 punpcklqdq m0, m4 ;out0 950 ret 951 952 INV_TXFM_8X4_FN flipadst, dct 953 INV_TXFM_8X4_FN flipadst, adst 954 INV_TXFM_8X4_FN flipadst, flipadst 955 INV_TXFM_8X4_FN flipadst, identity 956 957 cglobal iflipadst_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 958 mova m3, [o(pw_2896x8)] 959 pmulhrsw m0, m3, [coeffq+16*0] 960 pmulhrsw m1, m3, [coeffq+16*1] 961 pmulhrsw m2, m3, [coeffq+16*2] 962 pmulhrsw m3, [coeffq+16*3] 963 964 shufps m0, m0, q1032 965 shufps m1, m1, q1032 966 call m(iadst_4x8_internal_8bpc).main 967 968 punpckhwd m5, m3, m2 969 punpcklwd m3, m2 970 punpckhwd m2, m1, m0 971 punpcklwd m1, m0 972 973 pxor m0, m0 974 psubsw m4, m0, m2 975 psubsw m0, m5 976 punpckhdq m2, m0, m4 977 punpckldq m0, m4 978 punpckhdq m4, m3, m1 979 punpckldq m3, m1 980 punpckhwd m1, m0, m3 ;in1 981 punpcklwd m0, m3 ;in0 982 punpckhwd m3, m2, m4 ;in3 983 punpcklwd m2, m4 ;in2 984 jmp tx2q 985 986 .pass2: 987 call m(iadst_8x4_internal_8bpc).main 988 mova m4, m0 989 mova m5, m1 990 mova m0, m3 991 mova m1, m2 992 mova m2, m5 993 mova m3, m4 994 jmp m(iadst_8x4_internal_8bpc).end 995 996 INV_TXFM_8X4_FN identity, dct 997 INV_TXFM_8X4_FN identity, adst 998 INV_TXFM_8X4_FN identity, flipadst 999 INV_TXFM_8X4_FN identity, identity 1000 1001 cglobal iidentity_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1002 mova m3, [o(pw_2896x8)] 1003 pmulhrsw m0, m3, [coeffq+16*0] 1004 pmulhrsw m1, m3, [coeffq+16*1] 1005 pmulhrsw m2, m3, [coeffq+16*2] 1006 pmulhrsw m3, [coeffq+16*3] 1007 paddsw m0, m0 1008 paddsw m1, m1 1009 paddsw m2, m2 1010 paddsw m3, m3 1011 1012 punpckhwd m4, m0, m1 1013 punpcklwd m0, m1 1014 punpckhwd m1, m2, m3 1015 punpcklwd m2, m3 1016 punpckhdq m5, m4, m1 1017 punpckldq m4, m1 1018 punpckhdq m3, m0, m2 1019 punpckldq m0, m2 1020 punpckhwd m1, m0, m4 ;in1 1021 punpcklwd m0, m4 ;in0 1022 punpcklwd m2, m3, m5 ;in2 1023 punpckhwd m3, m5 ;in3 1024 jmp tx2q 1025 1026 .pass2: 1027 mova m7, [o(pw_1697x8)] 1028 pmulhrsw m4, m7, m0 1029 pmulhrsw m5, m7, m1 1030 pmulhrsw m6, m7, m2 1031 pmulhrsw m7, m3 1032 paddsw m0, m4 1033 paddsw m1, m5 1034 paddsw m2, m6 1035 paddsw m3, m7 1036 jmp m(iadst_8x4_internal_8bpc).end 1037 1038 %macro INV_TXFM_8X8_FN 2 ; type1, type2 1039 INV_TXFM_FN %1, %2, 8x8, 8, 16*4 1040 %ifidn %1_%2, dct_dct 1041 pshuflw m0, [coeffq], q0000 1042 punpcklwd m0, m0 1043 mova m1, [o(pw_2896x8)] 1044 pmulhrsw m0, m1 1045 mova m2, [o(pw_16384)] 1046 mov [coeffq], eobd 1047 pmulhrsw m0, m2 1048 psrlw m2, 3 1049 pmulhrsw m0, m1 1050 pmulhrsw m0, m2 1051 .end: 1052 mov r3d, 2 1053 lea tx2q, [o(m(inv_txfm_add_dct_dct_8x8_8bpc).end3)] 1054 .loop: 1055 WRITE_8X4 0, 0, 0, 0, 1, 2, 3 1056 lea dstq, [dstq+strideq*2] 1057 dec r3d 1058 jg .loop 1059 jmp tx2q 1060 .end3: 1061 RET 1062 %endif 1063 %endmacro 1064 1065 %macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 1066 %if %3 1067 mova m7, [o(pw_2896x8)] 1068 pmulhrsw m0, m7, [%1+%2*0] 1069 pmulhrsw m1, m7, [%1+%2*1] 1070 pmulhrsw m2, m7, [%1+%2*2] 1071 pmulhrsw m3, m7, [%1+%2*3] 1072 pmulhrsw m4, m7, [%1+%2*4] 1073 pmulhrsw m5, m7, [%1+%2*5] 1074 pmulhrsw m6, m7, [%1+%2*6] 1075 pmulhrsw m7, [%1+%2*7] 1076 %else 1077 mova m0, [%1+%2*0] 1078 mova m1, [%1+%2*1] 1079 mova m2, [%1+%2*2] 1080 mova m3, [%1+%2*3] 1081 mova m4, [%1+%2*4] 1082 mova m5, [%1+%2*5] 1083 mova m6, [%1+%2*6] 1084 mova m7, [%1+%2*7] 1085 %endif 1086 %endmacro 1087 1088 %macro IDCT8_1D_ODDHALF 7 ; src[1-4], tmp[1-2], pd_2048 1089 ITX_MULSUB_2W %1, %4, %5, %6, %7, 799, 4017 ;t4a, t7a 1090 ITX_MULSUB_2W %3, %2, %5, %6, %7, 3406, 2276, 1 ;t5a, t6a 1091 psubsw m%2, m%4, m%5 ;t6a 1092 paddsw m%4, m%5 ;t7 1093 psubsw m%5, m%1, m%3 ;t5a 1094 paddsw m%1, m%3 ;t4 1095 ITX_MULSUB_2W %2, %5, %3, %6, %7, 2896, 2896, 1 ;t5, t6 1096 %endmacro 1097 1098 INV_TXFM_8X8_FN dct, dct 1099 INV_TXFM_8X8_FN dct, adst 1100 INV_TXFM_8X8_FN dct, flipadst 1101 INV_TXFM_8X8_FN dct, identity 1102 1103 cglobal idct_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1104 LOAD_8ROWS coeffq, 16 1105 1106 .pass1: 1107 call .main 1108 1109 .pass1_end: 1110 mova m7, [o(pw_16384)] 1111 1112 .pass1_end1: 1113 REPX {pmulhrsw x, m7}, m0, m2, m4, m6 1114 mova [rsp+gprsize+16*1], m6 1115 1116 .pass1_end2: 1117 REPX {pmulhrsw x, m7}, m1, m3, m5 1118 pmulhrsw m7, [rsp+gprsize+16*0] 1119 1120 cglobal_label .pass1_end3 1121 punpcklwd m6, m1, m5 ;10 50 11 51 12 52 13 53 1122 punpckhwd m1, m5 ;14 54 15 55 16 56 17 57 1123 punpckhwd m5, m0, m4 ;04 44 05 45 06 46 07 47 1124 punpcklwd m0, m4 ;00 40 01 41 02 42 03 43 1125 punpckhwd m4, m3, m7 ;34 74 35 75 36 76 37 77 1126 punpcklwd m3, m7 ;30 70 31 71 32 72 33 73 1127 punpckhwd m7, m1, m4 ;16 36 56 76 17 37 57 77 1128 punpcklwd m1, m4 ;14 34 54 74 15 35 55 75 1129 punpckhwd m4, m6, m3 ;12 32 52 72 13 33 53 73 1130 punpcklwd m6, m3 ;10 30 50 70 11 31 51 71 1131 mova [rsp+gprsize+16*2], m6 1132 mova m6, [rsp+gprsize+16*1] 1133 punpckhwd m3, m2, m6 ;24 64 25 65 26 66 27 67 1134 punpcklwd m2, m6 ;20 60 21 61 22 62 23 63 1135 punpckhwd m6, m5, m3 ;06 26 46 66 07 27 47 67 1136 punpcklwd m5, m3 ;04 24 44 64 05 25 45 65 1137 punpckhwd m3, m0, m2 ;02 22 42 62 03 23 43 63 1138 punpcklwd m0, m2 ;00 20 40 60 01 21 41 61 1139 1140 punpckhwd m2, m6, m7 ;07 17 27 37 47 57 67 77 1141 punpcklwd m6, m7 ;06 16 26 36 46 56 66 76 1142 mova [rsp+gprsize+16*0], m2 1143 punpcklwd m2, m3, m4 ;02 12 22 32 42 52 62 72 1144 punpckhwd m3, m4 ;03 13 23 33 43 53 63 73 1145 punpcklwd m4, m5, m1 ;04 14 24 34 44 54 64 74 1146 punpckhwd m5, m1 ;05 15 25 35 45 55 65 75 1147 mova m7, [rsp+gprsize+16*2] 1148 punpckhwd m1, m0, m7 ;01 11 21 31 41 51 61 71 1149 punpcklwd m0, m7 ;00 10 20 30 40 50 60 70 1150 mova m7, [rsp+gprsize+16*0] 1151 jmp tx2q 1152 1153 .pass2: 1154 lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] 1155 1156 .pass2_main: 1157 call .main 1158 1159 .end: 1160 mova m7, [o(pw_2048)] 1161 REPX {pmulhrsw x, m7}, m0, m2, m4, m6 1162 mova [rsp+gprsize+16*1], m6 1163 1164 .end2: 1165 REPX {pmulhrsw x, m7}, m1, m3, m5 1166 pmulhrsw m7, [rsp+gprsize+16*0] 1167 mova [rsp+gprsize+16*2], m5 1168 mova [rsp+gprsize+16*0], m7 1169 1170 .end3: 1171 WRITE_8X4 0, 1, 2, 3, 5, 6, 7 1172 lea dstq, [dstq+strideq*2] 1173 WRITE_8X4 4, [rsp+gprsize+16*2], [rsp+gprsize+16*1], [rsp+gprsize+16*0], 5, 6, 7 1174 jmp tx2q 1175 1176 .end4: 1177 pxor m7, m7 1178 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 1179 ret 1180 1181 ALIGN function_align 1182 cglobal_label .main 1183 mova [rsp+gprsize*2+16*0], m7 1184 mova [rsp+gprsize*2+16*1], m3 1185 mova [rsp+gprsize*2+16*2], m1 1186 mova m7, [o(pd_2048)] 1187 IDCT4_1D 0, 2, 4, 6, 1, 3, 7 1188 mova m3, [rsp+gprsize*2+16*2] 1189 mova [rsp+gprsize*2+16*2], m2 1190 mova m2, [rsp+gprsize*2+16*1] 1191 mova [rsp+gprsize*2+16*1], m4 1192 mova m4, [rsp+gprsize*2+16*0] 1193 mova [rsp+gprsize*2+16*0], m6 1194 IDCT8_1D_ODDHALF 3, 2, 5, 4, 1, 6, 7 1195 mova m6, [rsp+gprsize*2+16*0] 1196 psubsw m7, m0, m4 ;out7 1197 paddsw m0, m4 ;out0 1198 mova [rsp+gprsize*2+16*0], m7 1199 mova m1, [rsp+gprsize*2+16*2] 1200 psubsw m4, m6, m3 ;out4 1201 paddsw m3, m6 ;out3 1202 mova m7, [rsp+gprsize*2+16*1] 1203 psubsw m6, m1, m5 ;out6 1204 paddsw m1, m5 ;out1 1205 psubsw m5, m7, m2 ;out5 1206 paddsw m2, m7 ;out2 1207 ret 1208 1209 1210 INV_TXFM_8X8_FN adst, dct 1211 INV_TXFM_8X8_FN adst, adst 1212 INV_TXFM_8X8_FN adst, flipadst 1213 INV_TXFM_8X8_FN adst, identity 1214 1215 cglobal iadst_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1216 LOAD_8ROWS coeffq, 16 1217 1218 .pass1: 1219 call .main 1220 call .main_pass1_end 1221 1222 .pass1_end: 1223 mova m7, [o(pw_16384)] 1224 1225 .pass1_end1: 1226 REPX {pmulhrsw x, m7}, m0, m2, m4, m6 1227 mova [rsp+gprsize+16*1], m6 1228 pxor m6, m6 1229 psubw m6, m7 1230 mova m7, m6 1231 jmp m(idct_8x8_internal_8bpc).pass1_end2 1232 1233 ALIGN function_align 1234 .pass2: 1235 lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] 1236 1237 .pass2_main: 1238 call .main 1239 call .main_pass2_end 1240 1241 .end: 1242 mova m7, [o(pw_2048)] 1243 REPX {pmulhrsw x, m7}, m0, m2, m4, m6 1244 mova [rsp+gprsize+16*1], m6 1245 pxor m6, m6 1246 psubw m6, m7 1247 mova m7, m6 1248 jmp m(idct_8x8_internal_8bpc).end2 1249 1250 ALIGN function_align 1251 cglobal_label .main 1252 mova [rsp+gprsize*2+16*0], m7 1253 mova [rsp+gprsize*2+16*1], m3 1254 mova [rsp+gprsize*2+16*2], m4 1255 mova m7, [o(pd_2048)] 1256 ITX_MULSUB_2W 5, 2, 3, 4, 7, 1931, 3612 ;t3a, t2a 1257 ITX_MULSUB_2W 1, 6, 3, 4, 7, 3920, 1189 ;t7a, t6a 1258 paddsw m3, m2, m6 ;t2 1259 psubsw m2, m6 ;t6 1260 paddsw m4, m5, m1 ;t3 1261 psubsw m5, m1 ;t7 1262 ITX_MULSUB_2W 5, 2, 1, 6, 7, 3784, 1567 ;t6a, t7a 1263 1264 mova m6, [rsp+gprsize*2+16*2] 1265 mova [rsp+gprsize*2+16*2], m5 1266 mova m1, [rsp+gprsize*2+16*1] 1267 mova [rsp+gprsize*2+16*1], m2 1268 mova m5, [rsp+gprsize*2+16*0] 1269 mova [rsp+gprsize*2+16*0], m3 1270 ITX_MULSUB_2W 5, 0, 2, 3, 7, 401, 4076 ;t1a, t0a 1271 ITX_MULSUB_2W 1, 6, 2, 3, 7, 3166, 2598 ;t5a, t4a 1272 psubsw m2, m0, m6 ;t4 1273 paddsw m0, m6 ;t0 1274 paddsw m3, m5, m1 ;t1 1275 psubsw m5, m1 ;t5 1276 ITX_MULSUB_2W 2, 5, 1, 6, 7, 1567, 3784 ;t5a, t4a 1277 1278 mova m7, [rsp+gprsize*2+16*0] 1279 paddsw m1, m3, m4 ;-out7 1280 psubsw m3, m4 ;t3 1281 mova [rsp+gprsize*2+16*0], m1 1282 psubsw m4, m0, m7 ;t2 1283 paddsw m0, m7 ;out0 1284 mova m6, [rsp+gprsize*2+16*2] 1285 mova m7, [rsp+gprsize*2+16*1] 1286 paddsw m1, m5, m6 ;-out1 1287 psubsw m5, m6 ;t6 1288 paddsw m6, m2, m7 ;out6 1289 psubsw m2, m7 ;t7 1290 ret 1291 ALIGN function_align 1292 .main_pass1_end: 1293 mova [rsp+gprsize*2+16*1], m1 1294 mova [rsp+gprsize*2+16*2], m6 1295 punpckhwd m1, m4, m3 1296 punpcklwd m4, m3 1297 punpckhwd m7, m5, m2 1298 punpcklwd m5, m2 1299 mova m2, [o(pw_2896_2896)] 1300 mova m6, [o(pd_2048)] 1301 pmaddwd m3, m2, m7 1302 pmaddwd m2, m5 1303 paddd m3, m6 1304 paddd m2, m6 1305 psrad m3, 12 1306 psrad m2, 12 1307 packssdw m2, m3 ;out2 1308 mova m3, [o(pw_2896_m2896)] 1309 pmaddwd m7, m3 1310 pmaddwd m5, m3 1311 paddd m7, m6 1312 paddd m5, m6 1313 psrad m7, 12 1314 psrad m5, 12 1315 packssdw m5, m7 ;-out5 1316 mova m3, [o(pw_2896_2896)] 1317 pmaddwd m7, m3, m1 1318 pmaddwd m3, m4 1319 paddd m7, m6 1320 paddd m3, m6 1321 psrad m7, 12 1322 psrad m3, 12 1323 packssdw m3, m7 ;-out3 1324 mova m7, [o(pw_2896_m2896)] 1325 pmaddwd m1, m7 1326 pmaddwd m4, m7 1327 paddd m1, m6 1328 paddd m4, m6 1329 psrad m1, 12 1330 psrad m4, 12 1331 packssdw m4, m1 ;-out5 1332 mova m1, [rsp+gprsize*2+16*1] 1333 mova m6, [rsp+gprsize*2+16*2] 1334 ret 1335 ALIGN function_align 1336 cglobal_label .main_pass2_end 1337 paddsw m7, m4, m3 ;t2 + t3 1338 psubsw m4, m3 ;t2 - t3 1339 paddsw m3, m5, m2 ;t6 + t7 1340 psubsw m5, m2 ;t6 - t7 1341 mova m2, [o(pw_2896x8)] 1342 pmulhrsw m4, m2 ;out4 1343 pmulhrsw m5, m2 ;-out5 1344 pmulhrsw m7, m2 ;-out3 1345 pmulhrsw m2, m3 ;out2 1346 mova m3, m7 1347 ret 1348 1349 INV_TXFM_8X8_FN flipadst, dct 1350 INV_TXFM_8X8_FN flipadst, adst 1351 INV_TXFM_8X8_FN flipadst, flipadst 1352 INV_TXFM_8X8_FN flipadst, identity 1353 1354 cglobal iflipadst_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1355 LOAD_8ROWS coeffq, 16 1356 1357 .pass1: 1358 call m(iadst_8x8_internal_8bpc).main 1359 call m(iadst_8x8_internal_8bpc).main_pass1_end 1360 1361 .pass1_end: 1362 mova m7, [o(pw_m16384)] 1363 1364 .pass1_end1: 1365 pmulhrsw m1, m7 1366 mova [rsp+gprsize+16*1], m1 1367 mova m1, m6 1368 mova m6, m2 1369 pmulhrsw m2, m5, m7 1370 mova m5, m6 1371 mova m6, m4 1372 pmulhrsw m4, m3, m7 1373 mova m3, m6 1374 mova m6, m0 1375 mova m0, m7 1376 pxor m7, m7 1377 psubw m7, m0 1378 pmulhrsw m0, [rsp+gprsize+16*0] 1379 REPX {pmulhrsw x, m7}, m1, m3, m5 1380 pmulhrsw m7, m6 1381 jmp m(idct_8x8_internal_8bpc).pass1_end3 1382 1383 ALIGN function_align 1384 .pass2: 1385 lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] 1386 1387 .pass2_main: 1388 call m(iadst_8x8_internal_8bpc).main 1389 call m(iadst_8x8_internal_8bpc).main_pass2_end 1390 1391 .end: 1392 mova m7, [o(pw_2048)] 1393 REPX {pmulhrsw x, m7}, m0, m2, m4, m6 1394 mova [rsp+gprsize+16*2], m2 1395 mova m2, m0 1396 pxor m0, m0 1397 psubw m0, m7 1398 mova m7, m2 1399 pmulhrsw m1, m0 1400 pmulhrsw m2, m5, m0 1401 mova [rsp+gprsize+16*1], m1 1402 mova m5, m4 1403 mova m1, m6 1404 pmulhrsw m4, m3, m0 1405 pmulhrsw m0, [rsp+gprsize+16*0] 1406 mova m3, m5 1407 mova [rsp+gprsize+16*0], m7 1408 jmp m(idct_8x8_internal_8bpc).end3 1409 1410 INV_TXFM_8X8_FN identity, dct 1411 INV_TXFM_8X8_FN identity, adst 1412 INV_TXFM_8X8_FN identity, flipadst 1413 INV_TXFM_8X8_FN identity, identity 1414 1415 cglobal iidentity_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1416 LOAD_8ROWS coeffq, 16 1417 mova [rsp+gprsize+16*1], m6 1418 jmp m(idct_8x8_internal_8bpc).pass1_end3 1419 1420 ALIGN function_align 1421 .pass2: 1422 lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] 1423 1424 .end: 1425 pmulhrsw m7, [o(pw_4096)] 1426 mova [rsp+gprsize+16*0], m7 1427 mova m7, [o(pw_4096)] 1428 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 1429 mova [rsp+gprsize+16*2], m5 1430 mova [rsp+gprsize+16*1], m6 1431 jmp m(idct_8x8_internal_8bpc).end3 1432 1433 1434 %macro INV_TXFM_4X16_FN 2 ; type1, type2 1435 INV_TXFM_FN %1, %2, 4x16, 8 1436 %ifidn %1_%2, dct_dct 1437 pshuflw m0, [coeffq], q0000 1438 punpcklwd m0, m0 1439 mova m1, [o(pw_2896x8)] 1440 pmulhrsw m0, m1 1441 mov [coeffq], eobd 1442 pmulhrsw m0, [o(pw_16384)] 1443 pmulhrsw m0, m1 1444 pmulhrsw m0, [o(pw_2048)] 1445 .end: 1446 WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 1447 lea dstq, [dstq+strideq*4] 1448 WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 1449 lea dstq, [dstq+strideq*4] 1450 WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 1451 lea dstq, [dstq+strideq*4] 1452 WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 1453 RET 1454 %endif 1455 %endmacro 1456 1457 INV_TXFM_4X16_FN dct, dct 1458 INV_TXFM_4X16_FN dct, adst 1459 INV_TXFM_4X16_FN dct, flipadst 1460 INV_TXFM_4X16_FN dct, identity 1461 1462 cglobal idct_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1463 lea r3, [o(m(idct_4x8_internal_8bpc).pass1)] 1464 1465 .pass1: 1466 mova m0, [coeffq+16*1] 1467 mova m1, [coeffq+16*3] 1468 mova m2, [coeffq+16*5] 1469 mova m3, [coeffq+16*7] 1470 push tx2q 1471 lea tx2q, [o(m(idct_4x16_internal_8bpc).pass1_2)] 1472 jmp r3 1473 1474 .pass1_2: 1475 mova [coeffq+16*1], m0 1476 mova [coeffq+16*3], m1 1477 mova [coeffq+16*5], m2 1478 mova [coeffq+16*7], m3 1479 mova m0, [coeffq+16*0] 1480 mova m1, [coeffq+16*2] 1481 mova m2, [coeffq+16*4] 1482 mova m3, [coeffq+16*6] 1483 lea tx2q, [o(m(idct_4x16_internal_8bpc).pass1_end)] 1484 jmp r3 1485 1486 .pass1_end: 1487 pop tx2q 1488 1489 mova m4, [coeffq+16*1] 1490 mova m5, [coeffq+16*3] 1491 mova m6, [coeffq+16*5] 1492 mova m7, [o(pw_16384)] 1493 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 1494 1495 pmulhrsw m7, [coeffq+16*7] 1496 mova [coeffq+16*7], m7 1497 jmp tx2q 1498 1499 .pass2: 1500 call m(idct_16x4_internal_8bpc).main 1501 1502 .end: 1503 mova m7, [o(pw_2048)] 1504 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 1505 pmulhrsw m7, [coeffq+16*7] 1506 mova [coeffq+16*4], m4 1507 1508 .end1: 1509 mova [coeffq+16*5], m5 1510 mova [coeffq+16*6], m6 1511 mov r3, coeffq 1512 WRITE_4X8 0, 1, 3, 2 1513 1514 mova m0, [r3+16*4] 1515 mova m1, [r3+16*5] 1516 mova m2, [r3+16*6] 1517 mova m3, m7 1518 lea dstq, [dstq+strideq*4] 1519 WRITE_4X8 0, 1, 3, 2 1520 1521 .end2: 1522 pxor m7, m7 1523 REPX {mova [r3+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 1524 ret 1525 1526 INV_TXFM_4X16_FN adst, dct 1527 INV_TXFM_4X16_FN adst, adst 1528 INV_TXFM_4X16_FN adst, flipadst 1529 INV_TXFM_4X16_FN adst, identity 1530 1531 cglobal iadst_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1532 lea r3, [o(m(iadst_4x8_internal_8bpc).pass1)] 1533 jmp m(idct_4x16_internal_8bpc).pass1 1534 1535 .pass2: 1536 call m(iadst_16x4_internal_8bpc).main 1537 call m(iadst_16x4_internal_8bpc).main_pass2_end 1538 1539 punpcklqdq m6, m5, m4 ;low: -out5 high: -out7 1540 punpckhqdq m4, m5 ;low: out8 high: out10 1541 punpcklqdq m5, m7, m2 ;low: out4 high: out6 1542 punpckhqdq m2, m7 ;low: -out9 high: -out11 1543 mova [coeffq+16*4], m2 1544 mova [coeffq+16*5], m6 1545 mova m2, [coeffq+16*6] 1546 mova m6, [coeffq+16*7] 1547 punpckhqdq m1, m6, m0 ;low: -out13 high: -out15 1548 punpcklqdq m0, m6 ;low: out0 high: out2 1549 punpckhqdq m6, m3, m2 ;low: out12 high: out14 1550 punpcklqdq m2, m3 ;low: -out1 high: -out3 1551 1552 mova m7, [o(pw_2048)] 1553 1554 .end1: 1555 REPX {pmulhrsw x, m7}, m0, m5, m4, m6 1556 pxor m3, m3 1557 psubw m3, m7 1558 mova m7, [coeffq+16*4] 1559 REPX {pmulhrsw x, m3}, m2, m7, m1 1560 pmulhrsw m3, [coeffq+16*5] 1561 mova [coeffq+16*7], m5 1562 1563 punpckhqdq m5, m4, m7 ;low: out10 high: out11 1564 punpcklqdq m4, m7 ;low: out8 high: out9 1565 punpckhqdq m7, m6, m1 ;low: out14 high: out15 1566 punpcklqdq m6, m1 ;low: out12 high: out13 1567 punpckhqdq m1, m0, m2 ;low: out2 high: out3 1568 punpcklqdq m0, m2 ;low: out0 high: out1 1569 mova [coeffq+16*4], m4 1570 mova m4, [coeffq+16*7] 1571 punpcklqdq m2, m4, m3 ;low: out4 high: out5 1572 punpckhqdq m4, m3 ;low: out6 high: out7 1573 mova m3, m4 1574 1575 .end2: 1576 mova [coeffq+16*5], m5 1577 mova [coeffq+16*6], m6 1578 mov r3, coeffq 1579 WRITE_4X8 0, 1, 2, 3 1580 1581 mova m0, [r3+16*4] 1582 mova m1, [r3+16*5] 1583 mova m2, [r3+16*6] 1584 mova m3, m7 1585 lea dstq, [dstq+strideq*4] 1586 WRITE_4X8 0, 1, 2, 3 1587 1588 .end3: 1589 pxor m7, m7 1590 REPX {mova [r3+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 1591 ret 1592 1593 1594 INV_TXFM_4X16_FN flipadst, dct 1595 INV_TXFM_4X16_FN flipadst, adst 1596 INV_TXFM_4X16_FN flipadst, flipadst 1597 INV_TXFM_4X16_FN flipadst, identity 1598 1599 cglobal iflipadst_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1600 lea r3, [o(m(iflipadst_4x8_internal_8bpc).pass1)] 1601 jmp m(idct_4x16_internal_8bpc).pass1 1602 1603 .pass2: 1604 call m(iadst_16x4_internal_8bpc).main 1605 call m(iadst_16x4_internal_8bpc).main_pass2_end 1606 1607 punpckhqdq m6, m5, m4 ;low: out5 high: out7 1608 punpcklqdq m4, m5 ;low: -out8 high: -out10 1609 punpckhqdq m5, m7, m2 ;low: -out4 high: -out6 1610 punpcklqdq m2, m7 ;low: out9 high: out11 1611 mova [coeffq+16*4], m2 1612 mova [coeffq+16*5], m6 1613 mova m2, [coeffq+16*6] 1614 mova m6, [coeffq+16*7] 1615 punpcklqdq m1, m6, m0 ;low: out13 high: out15 1616 punpckhqdq m0, m6 ;low: -out0 high: -out2 1617 punpcklqdq m6, m3, m2 ;low: -out12 high: -out14 1618 punpckhqdq m2, m3 ;low: out1 high: out3 1619 1620 mova m7, [o(pw_m2048)] 1621 jmp m(iadst_4x16_internal_8bpc).end1 1622 1623 1624 INV_TXFM_4X16_FN identity, dct 1625 INV_TXFM_4X16_FN identity, adst 1626 INV_TXFM_4X16_FN identity, flipadst 1627 INV_TXFM_4X16_FN identity, identity 1628 1629 %macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] 1630 pmulhrsw m%2, m%3, m%1 1631 %if %0 == 4 ; if downshifting by 1 1632 pmulhrsw m%2, m%4 1633 %else 1634 paddsw m%1, m%1 1635 %endif 1636 paddsw m%1, m%2 1637 %endmacro 1638 1639 cglobal iidentity_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1640 mova m0, [coeffq+16*1] 1641 mova m6, [o(pw_1697x8)] 1642 mova m1, [coeffq+16*3] 1643 mova m2, [coeffq+16*5] 1644 mova m3, [coeffq+16*7] 1645 pcmpeqw m7, m7 1646 mov r3, tx2q 1647 lea tx2q, [o(.pass1_2)] 1648 .pass1: 1649 pmulhrsw m4, m6, m0 1650 pmulhrsw m5, m6, m1 1651 pavgw m4, m0 1652 pcmpeqw m0, m7 1653 pavgw m5, m1 1654 pcmpeqw m1, m7 1655 pandn m0, m4 1656 pmulhrsw m4, m6, m2 1657 pandn m1, m5 1658 pmulhrsw m5, m6, m3 1659 pavgw m4, m2 1660 pcmpeqw m2, m7 1661 pavgw m5, m3 1662 pcmpeqw m3, m7 1663 pandn m2, m4 1664 pandn m3, m5 1665 jmp m(iadst_4x8_internal_8bpc).pass1_end 1666 .pass1_2: 1667 mova [coeffq+16*1], m0 1668 mova [coeffq+16*3], m1 1669 mova [coeffq+16*5], m2 1670 mova [coeffq+16*7], m3 1671 mova m0, [coeffq+16*0] 1672 mova m1, [coeffq+16*2] 1673 mova m2, [coeffq+16*4] 1674 mova m3, [coeffq+16*6] 1675 lea tx2q, [o(.pass1_end)] 1676 jmp .pass1 1677 .pass1_end: 1678 mova m4, [coeffq+16*1] 1679 mova m5, [coeffq+16*3] 1680 mova m6, [coeffq+16*5] 1681 jmp r3 1682 .pass2: 1683 mova m7, [o(pw_1697x16)] 1684 mova [coeffq+16*6], m6 1685 REPX {IDTX16 x, 6, 7}, 0, 1, 2, 3, 4, 5 1686 mova m6, [coeffq+16*7] 1687 IDTX16 6, 7, 7 1688 mova [coeffq+16*7], m6 1689 mova m6, [coeffq+16*6] 1690 pmulhrsw m7, m6, [o(pw_1697x16)] 1691 paddsw m6, m6 1692 paddsw m6, m7 1693 mova m7, [o(pw_2048)] 1694 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 1695 pmulhrsw m7, [coeffq+16*7] 1696 mova [coeffq+16*4], m4 1697 jmp m(iadst_4x16_internal_8bpc).end2 1698 1699 1700 %macro INV_TXFM_16X4_FN 2 ; type1, type2 1701 INV_TXFM_FN %1, %2, 16x4, 8 1702 %ifidn %1_%2, dct_dct 1703 movd m1, [o(pw_2896x8)] 1704 pmulhrsw m0, m1, [coeffq] 1705 movd m2, [o(pw_16384)] 1706 mov [coeffq], eobd 1707 mov r2d, 2 1708 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x4_8bpc).end)] 1709 .dconly: 1710 pmulhrsw m0, m2 1711 movd m2, [o(pw_2048)] ;intentionally rip-relative 1712 pmulhrsw m0, m1 1713 pmulhrsw m0, m2 1714 pshuflw m0, m0, q0000 1715 punpcklwd m0, m0 1716 pxor m5, m5 1717 .dconly_loop: 1718 mova m1, [dstq] 1719 mova m3, [dstq+strideq] 1720 punpckhbw m2, m1, m5 1721 punpcklbw m1, m5 1722 punpckhbw m4, m3, m5 1723 punpcklbw m3, m5 1724 paddw m2, m0 1725 paddw m1, m0 1726 paddw m4, m0 1727 paddw m3, m0 1728 packuswb m1, m2 1729 packuswb m3, m4 1730 mova [dstq], m1 1731 mova [dstq+strideq], m3 1732 lea dstq, [dstq+strideq*2] 1733 dec r2d 1734 jg .dconly_loop 1735 jmp tx2q 1736 .end: 1737 RET 1738 %endif 1739 %endmacro 1740 1741 %macro LOAD_7ROWS 2 ;src, stride 1742 mova m0, [%1+%2*0] 1743 mova m1, [%1+%2*1] 1744 mova m2, [%1+%2*2] 1745 mova m3, [%1+%2*3] 1746 mova m4, [%1+%2*4] 1747 mova m5, [%1+%2*5] 1748 mova m6, [%1+%2*6] 1749 %endmacro 1750 1751 %macro SAVE_7ROWS 2 ;src, stride 1752 mova [%1+%2*0], m0 1753 mova [%1+%2*1], m1 1754 mova [%1+%2*2], m2 1755 mova [%1+%2*3], m3 1756 mova [%1+%2*4], m4 1757 mova [%1+%2*5], m5 1758 mova [%1+%2*6], m6 1759 %endmacro 1760 1761 %macro IDCT16_1D_PACKED_ODDHALF 7 ;src[1-4], tmp[1-3] 1762 punpckhwd m%5, m%4, m%1 ;packed in13 in3 1763 punpcklwd m%1, m%4 ;packed in1 in15 1764 punpcklwd m%4, m%3, m%2 ;packed in9 in7 1765 punpckhwd m%2, m%3 ;packed in5 in11 1766 mova m%7, [o(pd_2048)] 1767 ITX_MUL2X_PACK %1, %6, %7, 401, 4076, 1 ;low: t8a high: t15a 1768 ITX_MUL2X_PACK %4, %6, %7, 3166, 2598, 1 ;low: t9a high: t14a 1769 ITX_MUL2X_PACK %2, %6, %7, 1931, 3612, 1 ;low: t10a high: t13a 1770 ITX_MUL2X_PACK %5, %6, %7, 3920, 1189, 1 ;low: t11a high: t12a 1771 psubsw m%6, m%1, m%4 ;low: t9 high: t14 1772 paddsw m%1, m%4 ;low: t8 high: t15 1773 psubsw m%4, m%5, m%2 ;low: t10 high: t13 1774 paddsw m%5, m%2 ;low: t11 high: t12 1775 mova m%2, [o(deint_shuf2)] 1776 pshufb m%6, m%2 1777 pshufb m%4, m%2 1778 ITX_MUL2X_PACK %6, %3, %7, 1567, 3784, 1 ;low: t9a high: t14a 1779 ITX_MUL2X_PACK %4, %3, %7, m3784, 1567, 1 ;low: t10a high: t13a 1780 psubsw m%3, m%1, m%5 ;low: t11a high: t12a 1781 paddsw m%1, m%5 ;low: t8a high: t15a 1782 psubsw m%5, m%6, m%4 ;low: t10 high: t13 1783 paddsw m%6, m%4 ;low: t9 high: t14 1784 pshufb m%3, m%2 1785 pshufb m%5, m%2 1786 ITX_MUL2X_PACK %3, %2, %7, 2896, 2896, 4 ;t12, t11 1787 ITX_MUL2X_PACK %5, %4, %7, 2896, 2896, 4 ;t13a, t10a 1788 packssdw m%2, m%4 ;low: t11 high: t10a 1789 packssdw m%3, m%5 ;low: t12 high: t13a 1790 punpckhqdq m%4, m%1, m%6 ;low: t15a high: t14 1791 punpcklqdq m%1, m%6 ;low: t8a high: t9 1792 %endmacro 1793 1794 INV_TXFM_16X4_FN dct, dct 1795 INV_TXFM_16X4_FN dct, adst 1796 INV_TXFM_16X4_FN dct, flipadst 1797 INV_TXFM_16X4_FN dct, identity 1798 1799 cglobal idct_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1800 LOAD_7ROWS coeffq, 16 1801 call .main 1802 1803 .pass1_end: 1804 punpckhwd m7, m0, m2 ;packed out1, out5 1805 punpcklwd m0, m2 ;packed out0, out4 1806 punpcklwd m2, m1, m3 ;packed out3, out7 1807 punpckhwd m1, m3 ;packed out2, out6 1808 mova [coeffq+16*6], m7 1809 mova m7, [coeffq+16*7] 1810 punpckhwd m3, m4, m6 ;packed out9, out13 1811 punpcklwd m4, m6 ;packed out8, out12 1812 punpcklwd m6, m5, m7 ;packed out11, out15 1813 punpckhwd m5, m7 ;packed out10, out14 1814 1815 .pass1_end2: 1816 mova m7, [o(pw_16384)] 1817 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 1818 pmulhrsw m7, [coeffq+16*6] 1819 mova [coeffq+16*6], m7 1820 1821 .pass1_end3: 1822 punpckhwd m7, m3, m6 ;packed 9, 11, 13, 15 high 1823 punpcklwd m3, m6 ;packed 9, 10, 13, 15 low 1824 punpckhwd m6, m4, m5 ;packed 8, 10, 12, 14 high 1825 punpcklwd m4, m5 ;packed 8, 10, 12, 14 low 1826 punpckhwd m5, m4, m3 ;8, 9, 10, 11, 12, 13, 14, 15(1) 1827 punpcklwd m4, m3 ;8, 9, 10, 11, 12, 13, 14, 15(0) 1828 punpckhwd m3, m6, m7 ;8, 9, 10, 11, 12, 13, 14, 15(3) 1829 punpcklwd m6, m7 ;8, 9, 10, 11, 12, 13, 14, 15(2) 1830 mova [coeffq+16*7], m3 1831 mova m3, [coeffq+16*6] 1832 punpckhwd m7, m3, m2 ;packed 1, 3, 5, 7 high 1833 punpcklwd m3, m2 ;packed 1, 3, 5, 7 low 1834 punpckhwd m2, m0, m1 ;packed 0, 2, 4, 6 high 1835 punpcklwd m0, m1 ;packed 0, 2, 4, 6 low 1836 punpckhwd m1, m0, m3 ;0, 1, 2, 3, 4, 5, 6, 7(1) 1837 punpcklwd m0, m3 ;0, 1, 2, 3, 4, 5, 6, 7(0) 1838 punpckhwd m3, m2, m7 ;0, 1, 2, 3, 4, 5, 6, 7(3) 1839 punpcklwd m2, m7 ;0, 1, 2, 3, 4, 5, 6, 7(2) 1840 jmp tx2q 1841 1842 .pass2: 1843 lea tx2q, [o(m(idct_8x4_internal_8bpc).pass2)] 1844 1845 .pass2_end: 1846 mova [coeffq+16*4], m4 1847 mova [coeffq+16*5], m5 1848 mova [coeffq+16*6], m6 1849 lea r3, [dstq+8] 1850 call tx2q 1851 1852 add coeffq, 16*4 1853 mova m0, [coeffq+16*0] 1854 mova m1, [coeffq+16*1] 1855 mova m2, [coeffq+16*2] 1856 mova m3, [coeffq+16*3] 1857 mov dstq, r3 1858 jmp tx2q 1859 1860 ALIGN function_align 1861 cglobal_label .main 1862 punpckhqdq m7, m0, m1 ;low:in1 high:in3 1863 punpcklqdq m0, m1 1864 punpcklqdq m1, m2, m3 1865 punpckhqdq m3, m2 ;low:in7 high:in5 1866 mova [coeffq+16*4], m7 1867 mova [coeffq+16*5], m3 1868 mova m7, [coeffq+16*7] 1869 punpcklqdq m2, m4, m5 1870 punpckhqdq m4, m5 ;low:in9 high:in11 1871 punpcklqdq m3, m6, m7 1872 punpckhqdq m7, m6 ;low:in15 high:in13 1873 mova [coeffq+16*6], m4 1874 IDCT8_1D_PACKED 1875 mova m6, [coeffq+16*4] 1876 mova m4, [coeffq+16*5] 1877 mova m5, [coeffq+16*6] 1878 mova [coeffq+16*4], m1 1879 mova [coeffq+16*5], m2 1880 mova [coeffq+16*6], m3 1881 1882 IDCT16_1D_PACKED_ODDHALF 6, 4, 5, 7, 1, 2, 3 1883 1884 mova m1, [coeffq+16*4] 1885 psubsw m3, m0, m7 ;low:out15 high:out14 1886 paddsw m0, m7 ;low:out0 high:out1 1887 psubsw m7, m1, m5 ;low:out12 high:out13 1888 paddsw m1, m5 ;low:out3 high:out2 1889 mova [coeffq+16*7], m3 1890 mova m2, [coeffq+16*5] 1891 mova m3, [coeffq+16*6] 1892 psubsw m5, m2, m4 ;low:out11 high:out10 1893 paddsw m2, m4 ;low:out4 high:out5 1894 psubsw m4, m3, m6 ;low:out8 high:out9 1895 paddsw m3, m6 ;low:out7 high:out6 1896 mova m6, m7 1897 ret 1898 1899 INV_TXFM_16X4_FN adst, dct 1900 INV_TXFM_16X4_FN adst, adst 1901 INV_TXFM_16X4_FN adst, flipadst 1902 INV_TXFM_16X4_FN adst, identity 1903 1904 cglobal iadst_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 1905 LOAD_7ROWS coeffq, 16 1906 call .main 1907 call .main_pass1_end 1908 1909 punpckhwd m6, m7, m0 ;packed -out11, -out15 1910 punpcklwd m0, m7 ;packed out0, out4 1911 punpcklwd m7, m3, m4 ;packed -out3, -out7 1912 punpckhwd m4, m3 ;packed out8, out12 1913 mova m1, [coeffq+16*6] 1914 punpcklwd m3, m1, m5 ;packed -out1, -out5 1915 punpckhwd m5, m1 ;packed out10, out14 1916 mova m1, [coeffq+16*7] 1917 mova [coeffq+16*6], m3 1918 mova [coeffq+16*7], m7 1919 punpckhwd m3, m2, m1 ;packed -out9, -out13 1920 punpcklwd m1, m2 ;packed out2, out6 1921 1922 mova m7, [o(pw_16384)] 1923 1924 .pass1_end: 1925 REPX {pmulhrsw x, m7}, m0, m1, m4, m5 1926 pxor m2, m2 1927 psubw m2, m7 1928 mova m7, [coeffq+16*6] 1929 REPX {pmulhrsw x, m2}, m7, m3, m6 1930 pmulhrsw m2, [coeffq+16*7] 1931 mova [coeffq+16*6], m7 1932 jmp m(idct_16x4_internal_8bpc).pass1_end3 1933 1934 .pass2: 1935 lea tx2q, [o(m(iadst_8x4_internal_8bpc).pass2)] 1936 jmp m(idct_16x4_internal_8bpc).pass2_end 1937 1938 ALIGN function_align 1939 cglobal_label .main 1940 mova [coeffq+16*6], m0 1941 pshufd m0, m1, q1032 1942 pshufd m2, m2, q1032 1943 punpckhwd m1, m6, m0 ;packed in13, in2 1944 punpcklwd m0, m6 ;packed in3, in12 1945 punpckhwd m7, m5, m2 ;packed in11, in4 1946 punpcklwd m2, m5 ;packed in5, in10 1947 mova m6, [o(pd_2048)] 1948 ITX_MUL2X_PACK 1, 5, 6, 995, 3973 ;low:t2 high:t3 1949 ITX_MUL2X_PACK 7, 5, 6, 1751, 3703 ;low:t4 high:t5 1950 ITX_MUL2X_PACK 2, 5, 6, 3513, 2106 ;low:t10 high:t11 1951 ITX_MUL2X_PACK 0, 5, 6, 3857, 1380 ;low:t12 high:t13 1952 psubsw m5, m1, m2 ;low:t10a high:t11a 1953 paddsw m1, m2 ;low:t2a high:t3a 1954 psubsw m2, m7, m0 ;low:t12a high:t13a 1955 paddsw m7, m0 ;low:t4a high:t5a 1956 punpcklqdq m0, m5 1957 punpckhwd m0, m5 ;packed t10a, t11a 1958 punpcklqdq m5, m2 1959 punpckhwd m2, m5 ;packed t13a, t12a 1960 ITX_MUL2X_PACK 0, 5, 6, 3406, 2276 ;low:t10 high:t11 1961 ITX_MUL2X_PACK 2, 5, 6, 4017, 799, 1 ;low:t12 high:t13 1962 mova [coeffq+16*4], m1 1963 mova [coeffq+16*5], m7 1964 mova m1, [coeffq+16*6] 1965 mova m7, [coeffq+16*7] 1966 pshufd m1, m1, q1032 1967 pshufd m3, m3, q1032 1968 punpckhwd m5, m7, m1 ;packed in15, in0 1969 punpcklwd m1, m7 ;packed in1, in14 1970 punpckhwd m7, m4, m3 ;packed in9, in6 1971 punpcklwd m3, m4 ;packed in7, in8 1972 ITX_MUL2X_PACK 5, 4, 6, 201, 4091 ;low:t0 high:t1 1973 ITX_MUL2X_PACK 7, 4, 6, 2440, 3290 ;low:t6 high:t7 1974 ITX_MUL2X_PACK 3, 4, 6, 3035, 2751 ;low:t8 high:t9 1975 ITX_MUL2X_PACK 1, 4, 6, 4052, 601 ;low:t14 high:t15 1976 psubsw m4, m5, m3 ;low:t8a high:t9a 1977 paddsw m5, m3 ;low:t0a high:t1a 1978 psubsw m3, m7, m1 ;low:t14a high:t15a 1979 paddsw m7, m1 ;low:t6a high:t7a 1980 punpcklqdq m1, m4 1981 punpckhwd m1, m4 ;packed t8a, t9a 1982 punpcklqdq m4, m3 1983 punpckhwd m3, m4 ;packed t15a, t14a 1984 ITX_MUL2X_PACK 1, 4, 6, 799, 4017 ;low:t8 high:t9 1985 ITX_MUL2X_PACK 3, 4, 6, 2276, 3406, 1 ;low:t14 high:t15 1986 paddsw m4, m1, m2 ;low:t12a high:t13a 1987 psubsw m1, m2 ;low:t8a high:t9a 1988 psubsw m2, m0, m3 ;low:t14a high:t15a 1989 paddsw m0, m3 ;low:t10a high:t11a 1990 punpcklqdq m3, m1 1991 punpckhwd m3, m1 ;packed t12a, t13a 1992 punpcklqdq m1, m2 1993 punpckhwd m2, m1 ;packed t15a, t14a 1994 ITX_MUL2X_PACK 3, 1, 6, 1567, 3784 ;low:t12 high:t13 1995 ITX_MUL2X_PACK 2, 1, 6, 3784, 1567, 1 ;low:t14 high:t15 1996 psubsw m1, m3, m2 ;low:t14a high:t15a 1997 paddsw m3, m2 ;low:out2 high:-out13 1998 psubsw m2, m4, m0 ;low:t10 high:t11 1999 paddsw m0, m4 ;low:-out1 high:out14 2000 mova [coeffq+16*6], m0 2001 mova [coeffq+16*7], m3 2002 mova m0, [coeffq+16*4] 2003 mova m3, [coeffq+16*5] 2004 psubsw m4, m5, m3 ;low:t4 high:t5 2005 paddsw m5, m3 ;low:t0 high:t1 2006 psubsw m3, m0, m7 ;low:t6 high:t7 2007 paddsw m0, m7 ;low:t2 high:t3 2008 punpcklqdq m7, m4 2009 punpckhwd m7, m4 ;packed t4, t5 2010 punpcklqdq m4, m3 2011 punpckhwd m3, m4 ;packed t7, t6 2012 ITX_MUL2X_PACK 7, 4, 6, 1567, 3784 ;low:t4a high:t5a 2013 ITX_MUL2X_PACK 3, 4, 6, 3784, 1567, 1 ;low:t6a high:t7a 2014 psubsw m4, m5, m0 ;low:t2a high:t3a 2015 paddsw m0, m5 ;low:out0 high:-out15 2016 psubsw m5, m7, m3 ;low:t6 high:t7 2017 paddsw m3, m7 ;low:-out3 high:out12 2018 ret 2019 ALIGN function_align 2020 .main_pass1_end: 2021 mova m7, [o(deint_shuf1)] 2022 mova [coeffq+16*4], m0 2023 mova [coeffq+16*5], m3 2024 mova m0, [o(pw_2896_m2896)] 2025 mova m3, [o(pw_2896_2896)] 2026 pshufb m1, m7 ;t14a t15a 2027 pshufb m2, m7 ;t10 t11 2028 pshufb m4, m7 ;t2a t3a 2029 pshufb m5, m7 ;t6 t7 2030 pmaddwd m7, m0, m2 2031 pmaddwd m2, m3 2032 paddd m7, m6 2033 paddd m2, m6 2034 psrad m7, 12 2035 psrad m2, 12 2036 packssdw m2, m7 ;low:out6 high:-out9 2037 pmaddwd m7, m0, m4 2038 pmaddwd m4, m3 2039 paddd m7, m6 2040 paddd m4, m6 2041 psrad m7, 12 2042 psrad m4, 12 2043 packssdw m4, m7 ;low:-out7 high:out8 2044 pmaddwd m7, m3, m5 2045 pmaddwd m5, m0 2046 paddd m7, m6 2047 paddd m5, m6 2048 psrad m7, 12 2049 psrad m5, 12 2050 packssdw m7, m5 ;low:out4 high:-out11 2051 pmaddwd m5, m3, m1 2052 pmaddwd m1, m0 2053 paddd m5, m6 2054 paddd m1, m6 2055 psrad m5, 12 2056 psrad m1, 12 2057 packssdw m5, m1 ;low:-out5 high:out10 2058 mova m0, [coeffq+16*4] 2059 mova m3, [coeffq+16*5] 2060 ret 2061 ALIGN function_align 2062 cglobal_label .main_pass2_end 2063 mova m7, [o(pw_2896x8)] 2064 punpckhqdq m6, m2, m1 ;low:t11 high:t15a 2065 punpcklqdq m2, m1 ;low:t10 high:t14a 2066 psubsw m1, m2, m6 2067 paddsw m2, m6 2068 punpckhqdq m6, m4, m5 ;low:t3a high:t7 2069 punpcklqdq m4, m5 ;low:t2a high:t6 2070 psubsw m5, m4, m6 2071 paddsw m4, m6 2072 pmulhrsw m1, m7 ;low:-out9 high:out10 2073 pmulhrsw m2, m7 ;low:out6 high:-out5 2074 pmulhrsw m5, m7 ;low:out8 high:-out11 2075 pmulhrsw m4, m7 ;low:-out7 high:out4 2076 punpckhqdq m7, m4, m5 ;low:out4 high:-out11 2077 punpcklqdq m4, m5 ;low:-out7 high:out8 2078 punpckhqdq m5, m2, m1 ;low:-out5 high:out10 2079 punpcklqdq m2, m1 ;low:out6 high:-out9 2080 ret 2081 2082 2083 INV_TXFM_16X4_FN flipadst, dct 2084 INV_TXFM_16X4_FN flipadst, adst 2085 INV_TXFM_16X4_FN flipadst, flipadst 2086 INV_TXFM_16X4_FN flipadst, identity 2087 2088 cglobal iflipadst_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2089 LOAD_7ROWS coeffq, 16 2090 call m(iadst_16x4_internal_8bpc).main 2091 call m(iadst_16x4_internal_8bpc).main_pass1_end 2092 2093 punpcklwd m6, m7, m0 ;packed out11, out15 2094 punpckhwd m0, m7 ;packed -out0, -out4 2095 punpckhwd m7, m3, m4 ;packed out3, out7 2096 punpcklwd m4, m3 ;packed -out8, -out12 2097 mova m1, [coeffq+16*6] 2098 punpckhwd m3, m1, m5 ;packed out1, out5 2099 punpcklwd m5, m1 ;packed -out10, -out14 2100 mova m1, [coeffq+16*7] 2101 mova [coeffq+16*6], m3 2102 mova [coeffq+16*7], m7 2103 punpcklwd m3, m2, m1 ;packed out9, out13 2104 punpckhwd m1, m2 ;packed -out2, -out6 2105 2106 mova m7, [o(pw_m16384)] 2107 jmp m(iadst_16x4_internal_8bpc).pass1_end 2108 2109 .pass2: 2110 lea tx2q, [o(m(iflipadst_8x4_internal_8bpc).pass2)] 2111 jmp m(idct_16x4_internal_8bpc).pass2_end 2112 2113 2114 INV_TXFM_16X4_FN identity, dct 2115 INV_TXFM_16X4_FN identity, adst 2116 INV_TXFM_16X4_FN identity, flipadst 2117 INV_TXFM_16X4_FN identity, identity 2118 2119 cglobal iidentity_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2120 mova m1, [coeffq+16*6] 2121 mova m0, [coeffq+16*5] 2122 mova m2, [coeffq+16*7] 2123 mova m6, [o(pw_1697x16)] 2124 mova m7, [o(pw_16384)] 2125 pmulhrsw m4, m6, m1 2126 pmulhrsw m3, m6, m0 2127 pmulhrsw m5, m6, m2 2128 pmulhrsw m4, m7 2129 pmulhrsw m3, m7 2130 pmulhrsw m5, m7 2131 paddsw m1, m4 2132 paddsw m0, m3 2133 paddsw m5, m2 2134 mova m2, [coeffq+16*2] 2135 mova m3, [coeffq+16*3] 2136 mova m4, [coeffq+16*4] 2137 mova [coeffq+16*6], m1 2138 mova [coeffq+16*5], m0 2139 mova [coeffq+16*7], m5 2140 pmulhrsw m0, m6, m2 2141 pmulhrsw m1, m6, m3 2142 pmulhrsw m5, m6, m4 2143 pmulhrsw m0, m7 2144 pmulhrsw m1, m7 2145 pmulhrsw m5, m7 2146 paddsw m2, m0 2147 paddsw m3, m1 2148 paddsw m4, m5 2149 mova m0, [coeffq+16*0] 2150 mova m1, [coeffq+16*1] 2151 pmulhrsw m5, m6, m0 2152 pmulhrsw m6, m1 2153 pmulhrsw m5, m7 2154 pmulhrsw m6, m7 2155 paddsw m0, m5 2156 paddsw m1, m6 2157 mova m6, [coeffq+16*6] 2158 mova m5, [coeffq+16*5] 2159 punpckhwd m7, m0, m2 ;packed out1, out5 2160 punpcklwd m0, m2 ;packed out0, out4 2161 punpckhwd m2, m1, m3 ;packed out3, out7 2162 punpcklwd m1, m3 ;packed out2, out6 2163 mova [coeffq+16*6], m7 2164 mova m7, [coeffq+16*7] 2165 punpckhwd m3, m4, m6 ;packed out9, out13 2166 punpcklwd m4, m6 ;packed out8, out12 2167 punpckhwd m6, m5, m7 ;packed out11, out15 2168 punpcklwd m5, m7 ;packed out10, out14 2169 jmp m(idct_16x4_internal_8bpc).pass1_end3 2170 2171 .pass2: 2172 lea tx2q, [o(m(iidentity_8x4_internal_8bpc).pass2)] 2173 jmp m(idct_16x4_internal_8bpc).pass2_end 2174 2175 2176 %macro SAVE_8ROWS 2 ;src, stride 2177 mova [%1+%2*0], m0 2178 mova [%1+%2*1], m1 2179 mova [%1+%2*2], m2 2180 mova [%1+%2*3], m3 2181 mova [%1+%2*4], m4 2182 mova [%1+%2*5], m5 2183 mova [%1+%2*6], m6 2184 mova [%1+%2*7], m7 2185 %endmacro 2186 2187 %macro INV_TXFM_8X16_FN 2 ; type1, type2 2188 INV_TXFM_FN %1, %2, 8x16, 8, 16*16 2189 %ifidn %1_%2, dct_dct 2190 pshuflw m0, [coeffq], q0000 2191 punpcklwd m0, m0 2192 mova m1, [o(pw_2896x8)] 2193 pmulhrsw m0, m1 2194 mova m2, [o(pw_16384)] 2195 mov [coeffq], eobd 2196 pmulhrsw m0, m1 2197 pmulhrsw m0, m2 2198 psrlw m2, 3 ; pw_2048 2199 pmulhrsw m0, m1 2200 pmulhrsw m0, m2 2201 mov r3d, 4 2202 lea tx2q, [o(m(inv_txfm_add_dct_dct_8x16_8bpc).end)] 2203 jmp m(inv_txfm_add_dct_dct_8x8_8bpc).loop 2204 .end: 2205 RET 2206 %endif 2207 %endmacro 2208 2209 INV_TXFM_8X16_FN dct, dct 2210 INV_TXFM_8X16_FN dct, adst 2211 INV_TXFM_8X16_FN dct, flipadst 2212 INV_TXFM_8X16_FN dct, identity 2213 2214 cglobal idct_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2215 lea r3, [o(m(idct_8x8_internal_8bpc).pass1)] 2216 2217 .pass1: 2218 LOAD_8ROWS coeffq+16*1, 32, 1 2219 mov [rsp+gprsize+16*11], tx2q 2220 lea tx2q, [o(m(idct_8x16_internal_8bpc).pass1_end)] 2221 jmp r3 2222 2223 .pass1_end: 2224 SAVE_8ROWS coeffq+16*1, 32 2225 LOAD_8ROWS coeffq+16*0, 32, 1 2226 mov tx2q, [rsp+gprsize+16*11] 2227 jmp r3 2228 2229 .pass2: 2230 lea tx2q, [o(m(idct_8x16_internal_8bpc).end)] 2231 2232 .pass2_pre: 2233 mova [coeffq+16*2 ], m1 2234 mova [coeffq+16*6 ], m3 2235 mova [coeffq+16*10], m5 2236 mova [coeffq+16*14], m7 2237 mova m1, m2 2238 mova m2, m4 2239 mova m3, m6 2240 mova m4, [coeffq+16*1 ] 2241 mova m5, [coeffq+16*5 ] 2242 mova m6, [coeffq+16*9 ] 2243 mova m7, [coeffq+16*13] 2244 2245 .pass2_main: 2246 call m(idct_8x8_internal_8bpc).main 2247 2248 SAVE_7ROWS rsp+gprsize+16*3, 16 2249 mova m0, [coeffq+16*2 ] 2250 mova m1, [coeffq+16*6 ] 2251 mova m2, [coeffq+16*10] 2252 mova m3, [coeffq+16*14] 2253 mova m4, [coeffq+16*3 ] 2254 mova m5, [coeffq+16*7 ] 2255 mova m6, [coeffq+16*11] 2256 mova m7, [coeffq+16*15] 2257 call m(idct_16x8_internal_8bpc).main 2258 2259 mov r3, dstq 2260 lea dstq, [dstq+strideq*8] 2261 jmp m(idct_8x8_internal_8bpc).end 2262 2263 .end: 2264 LOAD_8ROWS rsp+gprsize+16*3, 16 2265 mova [rsp+gprsize+16*0], m7 2266 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2267 mov dstq, r3 2268 jmp m(idct_8x8_internal_8bpc).end 2269 2270 .end1: 2271 pxor m7, m7 2272 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 2273 ret 2274 2275 INV_TXFM_8X16_FN adst, dct 2276 INV_TXFM_8X16_FN adst, adst 2277 INV_TXFM_8X16_FN adst, flipadst 2278 INV_TXFM_8X16_FN adst, identity 2279 2280 cglobal iadst_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2281 lea r3, [o(m(iadst_8x8_internal_8bpc).pass1)] 2282 jmp m(idct_8x16_internal_8bpc).pass1 2283 2284 .pass2: 2285 lea tx2q, [o(m(iadst_8x16_internal_8bpc).end)] 2286 2287 .pass2_pre: 2288 mova [rsp+gprsize+16*7], m0 2289 mova [rsp+gprsize+16*8], m1 2290 mova [rsp+gprsize+16*5], m6 2291 mova [rsp+gprsize+16*6], m7 2292 mova m0, m2 2293 mova m1, m3 2294 mova m2, m4 2295 mova m3, m5 2296 2297 .pass2_main: 2298 mova m4, [coeffq+16*1 ] 2299 mova m5, [coeffq+16*3 ] 2300 mova m6, [coeffq+16*13] 2301 mova m7, [coeffq+16*15] 2302 mova [rsp+gprsize+16*3], m4 2303 mova [rsp+gprsize+16*4], m5 2304 mova [rsp+gprsize+16*9], m6 2305 mova [rsp+gprsize+32*5], m7 2306 mova m4, [coeffq+16*5 ] 2307 mova m5, [coeffq+16*7 ] 2308 mova m6, [coeffq+16*9 ] 2309 mova m7, [coeffq+16*11] 2310 2311 call m(iadst_16x8_internal_8bpc).main 2312 call m(iadst_16x8_internal_8bpc).main_pass2_end 2313 2314 mov r3, dstq 2315 lea dstq, [dstq+strideq*8] 2316 jmp m(iadst_8x8_internal_8bpc).end 2317 2318 .end: 2319 LOAD_8ROWS rsp+gprsize+16*3, 16 2320 mova [rsp+gprsize+16*0], m7 2321 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2322 mov dstq, r3 2323 jmp m(iadst_8x8_internal_8bpc).end 2324 2325 2326 INV_TXFM_8X16_FN flipadst, dct 2327 INV_TXFM_8X16_FN flipadst, adst 2328 INV_TXFM_8X16_FN flipadst, flipadst 2329 INV_TXFM_8X16_FN flipadst, identity 2330 2331 cglobal iflipadst_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2332 lea r3, [o(m(iflipadst_8x8_internal_8bpc).pass1)] 2333 jmp m(idct_8x16_internal_8bpc).pass1 2334 2335 .pass2: 2336 lea tx2q, [o(m(iflipadst_8x16_internal_8bpc).end)] 2337 lea r3, [dstq+strideq*8] 2338 2339 .pass2_pre: 2340 mova [rsp+gprsize+16*7], m0 2341 mova [rsp+gprsize+16*8], m1 2342 mova [rsp+gprsize+16*5], m6 2343 mova [rsp+gprsize+16*6], m7 2344 mova m0, m2 2345 mova m1, m3 2346 mova m2, m4 2347 mova m3, m5 2348 2349 .pass2_main: 2350 mova m4, [coeffq+16*1 ] 2351 mova m5, [coeffq+16*3 ] 2352 mova m6, [coeffq+16*13] 2353 mova m7, [coeffq+16*15] 2354 mova [rsp+gprsize+16*3], m4 2355 mova [rsp+gprsize+16*4], m5 2356 mova [rsp+gprsize+16*9], m6 2357 mova [rsp+gprsize+32*5], m7 2358 mova m4, [coeffq+16*5 ] 2359 mova m5, [coeffq+16*7 ] 2360 mova m6, [coeffq+16*9 ] 2361 mova m7, [coeffq+16*11] 2362 2363 call m(iadst_16x8_internal_8bpc).main 2364 call m(iadst_16x8_internal_8bpc).main_pass2_end 2365 jmp m(iflipadst_8x8_internal_8bpc).end 2366 2367 .end: 2368 LOAD_8ROWS rsp+gprsize+16*3, 16 2369 mova [rsp+gprsize+16*0], m7 2370 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2371 mov dstq, r3 2372 jmp m(iflipadst_8x8_internal_8bpc).end 2373 2374 2375 INV_TXFM_8X16_FN identity, dct 2376 INV_TXFM_8X16_FN identity, adst 2377 INV_TXFM_8X16_FN identity, flipadst 2378 INV_TXFM_8X16_FN identity, identity 2379 2380 cglobal iidentity_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2381 LOAD_8ROWS coeffq+16*1, 32, 1 2382 mov r3, tx2q 2383 lea tx2q, [o(.pass1_end)] 2384 mova [rsp+gprsize+16*1], m6 2385 jmp m(idct_8x8_internal_8bpc).pass1_end3 2386 2387 .pass1_end: 2388 SAVE_8ROWS coeffq+16*1, 32 2389 LOAD_8ROWS coeffq+16*0, 32, 1 2390 mov tx2q, r3 2391 mova [rsp+gprsize+16*1], m6 2392 jmp m(idct_8x8_internal_8bpc).pass1_end3 2393 2394 .pass2: 2395 lea tx2q, [o(.end1)] 2396 2397 .end: 2398 mova [rsp+gprsize+16*0], m7 2399 mova [rsp+gprsize+16*1], m6 2400 mova m7, [o(pw_1697x16)] 2401 REPX {IDTX16 x, 6, 7}, 0, 1, 2, 3, 4, 5 2402 mova m6, [rsp+gprsize+16*1] 2403 mova [rsp+gprsize+16*2], m5 2404 IDTX16 6, 5, 7 2405 mova m5, [rsp+gprsize+16*0] 2406 IDTX16 5, 7, 7 2407 mova m7, [o(pw_2048)] 2408 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 2409 pmulhrsw m7, [rsp+gprsize+16*2] 2410 mova [rsp+gprsize+16*0], m5 2411 mova [rsp+gprsize+16*1], m6 2412 mova [rsp+gprsize+16*2], m7 2413 jmp m(idct_8x8_internal_8bpc).end3 2414 2415 .end1: 2416 LOAD_8ROWS coeffq+16*1, 32 2417 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2418 lea dstq, [dstq+strideq*2] 2419 jmp .end 2420 2421 2422 %macro INV_TXFM_16X8_FN 2 ; type1, type2 2423 INV_TXFM_FN %1, %2, 16x8, 8, 16*16 2424 %ifidn %1_%2, dct_dct 2425 movd m1, [o(pw_2896x8)] 2426 pmulhrsw m0, m1, [coeffq] 2427 movd m2, [o(pw_16384)] 2428 mov [coeffq], eobd 2429 pmulhrsw m0, m1 2430 mov r2d, 4 2431 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x8_8bpc).end)] 2432 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly 2433 .end: 2434 RET 2435 %endif 2436 %endmacro 2437 2438 INV_TXFM_16X8_FN dct, dct 2439 INV_TXFM_16X8_FN dct, adst 2440 INV_TXFM_16X8_FN dct, flipadst 2441 INV_TXFM_16X8_FN dct, identity 2442 2443 cglobal idct_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2444 LOAD_8ROWS coeffq+16*0, 32, 1 2445 call m(idct_8x8_internal_8bpc).main 2446 SAVE_7ROWS rsp+gprsize+16*3, 16 2447 2448 LOAD_8ROWS coeffq+16*1, 32, 1 2449 call .main 2450 mov r3, tx2q 2451 lea tx2q, [o(.pass1_end)] 2452 jmp m(idct_8x8_internal_8bpc).pass1_end 2453 2454 .pass1_end: 2455 SAVE_8ROWS coeffq+16*1, 32 2456 LOAD_8ROWS rsp+gprsize+16*3, 16 2457 mova [rsp+gprsize+16*0], m7 2458 mov tx2q, r3 2459 jmp m(idct_8x8_internal_8bpc).pass1_end 2460 2461 .pass2: 2462 lea tx2q, [o(.end)] 2463 lea r3, [dstq+8] 2464 jmp m(idct_8x8_internal_8bpc).pass2_main 2465 2466 .end: 2467 LOAD_8ROWS coeffq+16*1, 32 2468 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2469 mov dstq, r3 2470 jmp m(idct_8x8_internal_8bpc).pass2_main 2471 2472 2473 ALIGN function_align 2474 cglobal_label .main 2475 mova [rsp+gprsize*2+16*1], m2 2476 mova [rsp+gprsize*2+16*2], m6 2477 mova [rsp+gprsize*2+32*5], m5 2478 2479 mova m6, [o(pd_2048)] 2480 ITX_MULSUB_2W 0, 7, 2, 5, 6, 401, 4076 ;t8a, t15a 2481 ITX_MULSUB_2W 4, 3, 2, 5, 6, 3166, 2598 ;t9a, t14a 2482 psubsw m2, m0, m4 ;t9 2483 paddsw m0, m4 ;t8 2484 psubsw m4, m7, m3 ;t14 2485 paddsw m7, m3 ;t15 2486 ITX_MULSUB_2W 4, 2, 3, 5, 6, 1567, 3784 ;t9a, t14a 2487 mova m3, [rsp+gprsize*2+16*1] 2488 mova m5, [rsp+gprsize*2+32*5] 2489 mova [rsp+gprsize*2+16*1], m2 2490 mova [rsp+gprsize*2+32*5], m4 2491 mova m2, [rsp+gprsize*2+16*2] 2492 mova [rsp+gprsize*2+16*2], m7 2493 ITX_MULSUB_2W 3, 5, 7, 4, 6, 1931, 3612 ;t10a, t13a 2494 ITX_MULSUB_2W 2, 1, 7, 4, 6, 3920, 1189 ;t11a, t12a 2495 psubsw m4, m2, m3 ;t10 2496 paddsw m2, m3 ;t11 2497 psubsw m3, m1, m5 ;t13 2498 paddsw m1, m5 ;t12 2499 ITX_MULSUB_2W 3, 4, 7, 5, 6, m3784, 1567 ;t10a, t13a 2500 mova m7, [rsp+gprsize*2+32*5] 2501 psubsw m6, m0, m2 ;t11a 2502 paddsw m0, m2 ;t8a 2503 paddsw m2, m7, m3 ;t9 2504 psubsw m7, m3 ;t10 2505 mova m5, [rsp+gprsize*2+16*0] 2506 psubsw m3, m5, m0 ;out8 2507 paddsw m0, m5 ;out7 2508 mova [rsp+gprsize*2+32*5], m0 2509 mova m5, [rsp+gprsize*2+16*9] 2510 psubsw m0, m5, m2 ;out9 2511 paddsw m2, m5 ;out6 2512 mova [rsp+gprsize*2+16*0], m0 2513 mova [rsp+gprsize*2+16*9], m2 2514 mova m0, [rsp+gprsize*2+16*1] 2515 mova m2, [rsp+gprsize*2+16*2] 2516 mova [rsp+gprsize*2+16*1], m3 2517 psubsw m5, m0, m4 ;t13 2518 paddsw m0, m4 ;t14 2519 mova m3, [o(pd_2048)] 2520 psubsw m4, m2, m1 ;t12a 2521 paddsw m1, m2 ;t15a 2522 mova [rsp+gprsize*2+16*2], m1 2523 ITX_MULSUB_2W 5, 7, 1, 2, 3, 2896, 2896 ;t10a, t13a 2524 ITX_MULSUB_2W 4, 6, 1, 2, 3, 2896, 2896 ;t11, t12 2525 mova m3, [rsp+gprsize*2+16*8] 2526 psubsw m2, m3, m5 ;out10 2527 paddsw m3, m5 ;out5 2528 mova m5, [rsp+gprsize*2+16*7] 2529 mova [rsp+gprsize*2+16*8], m3 2530 psubsw m3, m5, m4 ;out11 2531 paddsw m5, m4 ;out4 2532 mova m4, [rsp+gprsize*2+16*6] 2533 mova [rsp+gprsize*2+16*7], m5 2534 paddsw m5, m4, m6 ;out3 2535 psubsw m4, m6 ;out12 2536 mova m6, [rsp+gprsize*2+16*5] 2537 mova [rsp+gprsize*2+16*6], m5 2538 psubsw m5, m6, m7 ;out13 2539 paddsw m6, m7 ;out2 2540 mova m7, [rsp+gprsize*2+16*4] 2541 mova [rsp+gprsize*2+16*5], m6 2542 psubsw m6, m7, m0 ;out14 2543 paddsw m7, m0 ;out1 2544 mova m1, [rsp+gprsize*2+16*2] 2545 mova m0, [rsp+gprsize*2+16*3] 2546 mova [rsp+gprsize*2+16*4], m7 2547 psubsw m7, m0, m1 ;out15 2548 paddsw m0, m1 ;out0 2549 mova [rsp+gprsize*2+16*3], m0 2550 mova m1, [rsp+gprsize*2+16*0] 2551 mova m0, [rsp+gprsize*2+16*1] 2552 mova [rsp+gprsize*2+16*0], m7 2553 ret 2554 2555 INV_TXFM_16X8_FN adst, dct 2556 INV_TXFM_16X8_FN adst, adst 2557 INV_TXFM_16X8_FN adst, flipadst 2558 INV_TXFM_16X8_FN adst, identity 2559 2560 cglobal iadst_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2561 mova m7, [o(pw_2896x8)] 2562 pmulhrsw m0, m7, [coeffq+16*0 ] 2563 pmulhrsw m1, m7, [coeffq+16*1 ] 2564 pmulhrsw m2, m7, [coeffq+16*14] 2565 pmulhrsw m3, m7, [coeffq+16*15] 2566 mova [rsp+gprsize+16*7], m0 2567 mova [rsp+gprsize+16*8], m1 2568 mova [rsp+gprsize+16*9], m2 2569 mova [rsp+gprsize+32*5], m3 2570 pmulhrsw m0, m7, [coeffq+16*6 ] 2571 pmulhrsw m1, m7, [coeffq+16*7 ] 2572 pmulhrsw m2, m7, [coeffq+16*8 ] 2573 pmulhrsw m3, m7, [coeffq+16*9 ] 2574 mova [rsp+gprsize+16*3], m2 2575 mova [rsp+gprsize+16*4], m3 2576 mova [rsp+gprsize+16*5], m0 2577 mova [rsp+gprsize+16*6], m1 2578 pmulhrsw m0, m7, [coeffq+16*2 ] 2579 pmulhrsw m1, m7, [coeffq+16*3 ] 2580 pmulhrsw m2, m7, [coeffq+16*4 ] 2581 pmulhrsw m3, m7, [coeffq+16*5 ] 2582 pmulhrsw m4, m7, [coeffq+16*10] 2583 pmulhrsw m5, m7, [coeffq+16*11] 2584 pmulhrsw m6, m7, [coeffq+16*12] 2585 pmulhrsw m7, [coeffq+16*13] 2586 2587 call .main 2588 call .main_pass1_end 2589 mov r3, tx2q 2590 lea tx2q, [o(.pass1_end)] 2591 jmp m(iadst_8x8_internal_8bpc).pass1_end 2592 2593 .pass1_end: 2594 SAVE_8ROWS coeffq+16*1, 32 2595 LOAD_8ROWS rsp+gprsize+16*3, 16 2596 mova [rsp+gprsize+16*0], m7 2597 mov tx2q, r3 2598 jmp m(iadst_8x8_internal_8bpc).pass1_end 2599 2600 .pass2: 2601 lea tx2q, [o(.end)] 2602 lea r3, [dstq+8] 2603 jmp m(iadst_8x8_internal_8bpc).pass2_main 2604 2605 .end: 2606 LOAD_8ROWS coeffq+16*1, 32 2607 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2608 mov dstq, r3 2609 jmp m(iadst_8x8_internal_8bpc).pass2_main 2610 2611 ALIGN function_align 2612 cglobal_label .main 2613 mova [rsp+gprsize*2+16*0], m1 2614 mova [rsp+gprsize*2+16*1], m2 2615 mova [rsp+gprsize*2+16*2], m6 2616 2617 mova m6, [o(pd_2048)] 2618 ITX_MULSUB_2W 7, 0, 1, 2, 6, 995, 3973 ;t3, t2 2619 ITX_MULSUB_2W 3, 4, 1, 2, 6, 3513, 2106 ;t11, t10 2620 psubsw m1, m0, m4 ;t10a 2621 paddsw m0, m4 ;t2a 2622 psubsw m4, m7, m3 ;t11a 2623 paddsw m3, m7 ;t3a 2624 ITX_MULSUB_2W 1, 4, 7, 2, 6, 3406, 2276 ;t11, t10 2625 mova m2, [rsp+gprsize*2+16*0] ;in3 2626 mova m7, [rsp+gprsize*2+16*1] ;in4 2627 mova [rsp+gprsize*2+16*0], m1 ;t11 2628 mova [rsp+gprsize*2+16*1], m4 ;t10 2629 mova m1, [rsp+gprsize*2+16*2] ;in12 2630 mova [rsp+gprsize*2+16*2], m0 ;t2a 2631 ITX_MULSUB_2W 5, 7, 0, 4, 6, 1751, 3703 ;t5, t4 2632 ITX_MULSUB_2W 2, 1, 0, 4, 6, 3857, 1380 ;t13, t12 2633 psubsw m0, m7, m1 ;t12a 2634 paddsw m1, m7 ;t4a 2635 psubsw m4, m5, m2 ;t13a 2636 paddsw m5, m2 ;t5a 2637 ITX_MULSUB_2W 4, 0, 7, 2, 6, 4017, 799 ;t12, t13 2638 mova m2, [rsp+gprsize*2+16*8] ;in1 2639 mova m7, [rsp+gprsize*2+16*9] ;in14 2640 mova [rsp+gprsize*2+16*8], m4 ;t12 2641 mova [rsp+gprsize*2+16*9], m0 ;t13 2642 mova m4, [rsp+gprsize*2+16*4] ;in9 2643 mova m0, [rsp+gprsize*2+16*5] ;in6 2644 mova [rsp+gprsize*2+16*4], m1 ;t4a 2645 mova [rsp+gprsize*2+16*5], m5 ;t5a 2646 ITX_MULSUB_2W 2, 7, 1, 5, 6, 4052, 601 ;t15, t14 2647 ITX_MULSUB_2W 4, 0, 1, 5, 6, 2440, 3290 ;t7, t6 2648 psubsw m1, m0, m7 ;t14a 2649 paddsw m0, m7 ;t6a 2650 psubsw m5, m4, m2 ;t15a 2651 paddsw m4, m2 ;t7a 2652 ITX_MULSUB_2W 5, 1, 7, 2, 6, 2276, 3406 ;t14, t15 2653 mova m2, [rsp+gprsize*2+16*2] ;t2a 2654 mova [rsp+gprsize*2+16*2], m5 ;t14 2655 psubsw m7, m2, m0 ;t6 2656 paddsw m2, m0 ;t2 2657 psubsw m0, m3, m4 ;t7 2658 paddsw m3, m4 ;t3 2659 ITX_MULSUB_2W 0, 7, 4, 5, 6, 3784, 1567 ;t6a, t7a 2660 mova m4, [rsp+gprsize*2+16*7] ;in0 2661 mova m5, [rsp+gprsize*2+32*5] ;in15 2662 mova [rsp+gprsize*2+16*7], m3 ;t3 2663 mova [rsp+gprsize*2+32*5], m1 ;t15 2664 mova m1, [rsp+gprsize*2+16*6] ;in7 2665 mova m3, [rsp+gprsize*2+16*3] ;in8 2666 mova [rsp+gprsize*2+16*6], m7 ;t7a 2667 mova [rsp+gprsize*2+16*3], m0 ;t6a 2668 ITX_MULSUB_2W 5, 4, 0, 7, 6, 201, 4091 ;t1, t0 2669 ITX_MULSUB_2W 1, 3, 0, 7, 6, 3035, 2751 ;t9, t8 2670 psubsw m0, m4, m3 ;t8a 2671 paddsw m4, m3 ;t0a 2672 psubsw m3, m5, m1 ;t9a 2673 paddsw m5, m1 ;t1a 2674 ITX_MULSUB_2W 0, 3, 1, 7, 6, 799, 4017 ;t9, t8 2675 mova m1, [rsp+gprsize*2+16*4] ;t4a 2676 mova m7, [rsp+gprsize*2+16*5] ;t5a 2677 mova [rsp+gprsize*2+16*4], m3 ;t8 2678 mova [rsp+gprsize*2+16*5], m0 ;t9 2679 psubsw m0, m4, m1 ;t4 2680 paddsw m4, m1 ;t0 2681 psubsw m3, m5, m7 ;t5 2682 paddsw m5, m7 ;t1 2683 ITX_MULSUB_2W 0, 3, 1, 7, 6, 1567, 3784 ;t5a, t4a 2684 mova m7, [rsp+gprsize*2+16*3] ;t6a 2685 psubsw m1, m4, m2 ;t2a 2686 paddsw m4, m2 ;out0 2687 mova [rsp+gprsize*2+16*3], m4 ;out0 2688 mova m4, [rsp+gprsize*2+16*6] ;t7a 2689 psubsw m2, m3, m7 ;t6 2690 paddsw m3, m7 ;-out3 2691 mova [rsp+gprsize*2+16*6], m3 ;-out3 2692 psubsw m3, m0, m4 ;t7 2693 paddsw m0, m4 ;out12 2694 mova [rsp+gprsize*2+16*12], m3 2695 mova m3, [rsp+gprsize*2+16*7] ;t3 2696 mova [rsp+gprsize*2+16* 7], m2 ;out4 2697 psubsw m2, m5, m3 ;t3a 2698 paddsw m5, m3 ;-out15 2699 mova [rsp+gprsize*2+16*11], m2 2700 mova m2, [rsp+gprsize*2+32*5] ;t15 2701 mova [rsp+gprsize*2+16*10], m1 ;-out7 2702 mova m1, [rsp+gprsize*2+16*0] ;t11 2703 mova [rsp+gprsize*2+16*0 ], m5 ;-out15 2704 mova m3, [rsp+gprsize*2+16*1] ;t10 2705 mova [rsp+gprsize*2+16*1 ], m4 ;-out11 2706 mova m4, [rsp+gprsize*2+16*2] ;t14 2707 mova [rsp+gprsize*2+16*2 ], m0 ;out12 2708 psubsw m0, m3, m4 ;t14a 2709 paddsw m3, m4 ;t10a 2710 psubsw m5, m1, m2 ;t15a 2711 paddsw m1, m2 ;t11a 2712 ITX_MULSUB_2W 5, 0, 2, 4, 6, 3784, 1567 ;t14, t15 2713 mova m2, [rsp+gprsize*2+16*4] ;t8 2714 mova m4, [rsp+gprsize*2+16*5] ;t9 2715 mova [rsp+gprsize*2+16*4], m3 ;t10a 2716 mova [rsp+gprsize*2+16*5], m1 ;t11a 2717 mova m3, [rsp+gprsize*2+16*8] ;t12 2718 mova m1, [rsp+gprsize*2+16*9] ;t13 2719 mova [rsp+gprsize*2+16*8], m5 ;t14 2720 mova [rsp+gprsize*2+16*9], m0 ;t15 2721 psubsw m5, m2, m3 ;t12a 2722 paddsw m2, m3 ;t8a 2723 psubsw m0, m4, m1 ;t13a 2724 paddsw m4, m1 ;t9a 2725 ITX_MULSUB_2W 5, 0, 1, 3, 6, 1567, 3784 ;t13, t12 2726 mova m6, [rsp+gprsize*2+16*4] ;t10a 2727 mova m1, [rsp+gprsize*2+16*5] ;t11a 2728 psubsw m3, m2, m6 ;t10 2729 paddsw m2, m6 ;-out1 2730 paddsw m6, m4, m1 ;out14 2731 psubsw m4, m1 ;t11 2732 mova [rsp+gprsize*2+16*14], m4 2733 mova [rsp+gprsize*2+16* 4], m2 ;-out1 2734 mova m4, [rsp+gprsize*2+16*8] ;t14 2735 mova m2, [rsp+gprsize*2+16*9] ;t15 2736 mova [rsp+gprsize*2+16* 9], m3 ;out6 2737 psubsw m3, m0, m4 ;t14a 2738 paddsw m0, m4 ;out2 2739 psubsw m4, m5, m2 ;t15a 2740 paddsw m5, m2 ;-out13 2741 mova [rsp+gprsize*2+16* 5], m0 ;out2 2742 ret 2743 ALIGN function_align 2744 .main_pass1_end: 2745 mova m0, [rsp+gprsize*2+16*14] 2746 mova [rsp+gprsize*2+16*14], m5 2747 mova [rsp+gprsize*2+16*15], m6 2748 mova m5, [o(pw_2896_2896)] 2749 mova m6, [o(pw_2896_m2896)] 2750 mova m7, [o(pd_2048)] 2751 punpcklwd m2, m3, m4 2752 punpckhwd m3, m4 2753 pmaddwd m4, m5, m2 2754 pmaddwd m2, m6 2755 pmaddwd m1, m5, m3 2756 pmaddwd m3, m6 2757 REPX {paddd x, m7}, m4, m2, m1, m3 2758 REPX {psrad x, 12}, m4, m1, m2, m3 2759 packssdw m4, m1 ;-out5 2760 packssdw m2, m3 ;out10 2761 mova [rsp+gprsize*2+16* 8], m4 2762 mova m3, [rsp+gprsize*2+16* 9] 2763 punpcklwd m1, m3, m0 2764 punpckhwd m3, m0 2765 pmaddwd m0, m5, m1 2766 pmaddwd m1, m6 2767 pmaddwd m4, m5, m3 2768 pmaddwd m3, m6 2769 REPX {paddd x, m7}, m0, m1, m4, m3 2770 REPX {psrad x, 12}, m0, m4, m1, m3 2771 packssdw m0, m4 ;out6 2772 packssdw m1, m3 ;-out9 2773 mova [rsp+gprsize*2+16* 9], m0 2774 mova m0, [rsp+gprsize*2+16* 7] 2775 mova m4, [rsp+gprsize*2+16*12] 2776 punpcklwd m3, m0, m4 2777 punpckhwd m0, m4 2778 pmaddwd m4, m5, m3 2779 pmaddwd m3, m6 2780 pmaddwd m5, m0 2781 pmaddwd m0, m6 2782 REPX {paddd x, m7}, m4, m3, m5, m0 2783 REPX {psrad x, 12}, m4, m5, m3, m0 2784 packssdw m4, m5 ;out4 2785 packssdw m3, m0 ;-out11 2786 mova [rsp+gprsize*2+16* 7], m4 2787 mova m4, [rsp+gprsize*2+16*10] 2788 mova m5, [rsp+gprsize*2+16*11] 2789 punpcklwd m0, m4, m5 2790 punpckhwd m4, m5 2791 pmaddwd m5, m0, [o(pw_2896_2896)] 2792 pmaddwd m0, m6 2793 pmaddwd m6, m4 2794 pmaddwd m4, [o(pw_2896_2896)] 2795 REPX {paddd x, m7}, m5, m0, m6, m4 2796 REPX {psrad x, 12}, m0, m6, m5, m4 2797 packssdw m0, m6 ;out8 2798 packssdw m5, m4 ;-out7 2799 mova [rsp+gprsize*2+16*10], m5 2800 mova m4, [rsp+gprsize*2+16* 2] ;out12 2801 mova m5, [rsp+gprsize*2+16*14] ;-out13 2802 mova m6, [rsp+gprsize*2+16*15] ;out14 2803 ret 2804 ALIGN function_align 2805 cglobal_label .main_pass2_end 2806 mova m7, [o(pw_2896x8)] 2807 mova m1, [rsp+gprsize*2+16* 9] 2808 mova m2, [rsp+gprsize*2+16*14] 2809 paddsw m0, m1, m2 2810 psubsw m1, m2 2811 pmulhrsw m0, m7 ;out6 2812 pmulhrsw m1, m7 ;-out9 2813 mova [rsp+gprsize*2+16* 9], m0 2814 psubsw m2, m3, m4 2815 paddsw m3, m4 2816 pmulhrsw m2, m7 ;out10 2817 pmulhrsw m3, m7 ;-out5 2818 mova [rsp+gprsize*2+16* 8], m3 2819 mova m3, [rsp+gprsize*2+16* 7] 2820 mova m4, [rsp+gprsize*2+16*12] 2821 paddsw m0, m3, m4 2822 psubsw m3, m4 2823 pmulhrsw m0, m7 ;out4 2824 pmulhrsw m3, m7 ;-out11 2825 mova [rsp+gprsize*2+16* 7], m0 2826 mova m0, [rsp+gprsize*2+16*10] 2827 paddsw m4, m0, [rsp+gprsize*2+16*11] 2828 psubsw m0, [rsp+gprsize*2+16*11] 2829 pmulhrsw m4, m7 ;-out7 2830 pmulhrsw m0, m7 ;out8 2831 mova [rsp+gprsize*2+16*10], m4 2832 mova m4, [rsp+gprsize*2+16*2 ] ;out12 2833 ret 2834 2835 INV_TXFM_16X8_FN flipadst, dct 2836 INV_TXFM_16X8_FN flipadst, adst 2837 INV_TXFM_16X8_FN flipadst, flipadst 2838 INV_TXFM_16X8_FN flipadst, identity 2839 2840 cglobal iflipadst_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2841 mova m7, [o(pw_2896x8)] 2842 pmulhrsw m0, m7, [coeffq+16*0 ] 2843 pmulhrsw m1, m7, [coeffq+16*1 ] 2844 pmulhrsw m2, m7, [coeffq+16*14] 2845 pmulhrsw m3, m7, [coeffq+16*15] 2846 mova [rsp+gprsize+16*7], m0 2847 mova [rsp+gprsize+16*8], m1 2848 mova [rsp+gprsize+16*9], m2 2849 mova [rsp+gprsize+32*5], m3 2850 pmulhrsw m0, m7, [coeffq+16*6 ] 2851 pmulhrsw m1, m7, [coeffq+16*7 ] 2852 pmulhrsw m2, m7, [coeffq+16*8 ] 2853 pmulhrsw m3, m7, [coeffq+16*9 ] 2854 mova [rsp+gprsize+16*3], m2 2855 mova [rsp+gprsize+16*4], m3 2856 mova [rsp+gprsize+16*5], m0 2857 mova [rsp+gprsize+16*6], m1 2858 pmulhrsw m0, m7, [coeffq+16*2 ] 2859 pmulhrsw m1, m7, [coeffq+16*3 ] 2860 pmulhrsw m2, m7, [coeffq+16*4 ] 2861 pmulhrsw m3, m7, [coeffq+16*5 ] 2862 pmulhrsw m4, m7, [coeffq+16*10] 2863 pmulhrsw m5, m7, [coeffq+16*11] 2864 pmulhrsw m6, m7, [coeffq+16*12] 2865 pmulhrsw m7, [coeffq+16*13] 2866 2867 call m(iadst_16x8_internal_8bpc).main 2868 call m(iadst_16x8_internal_8bpc).main_pass1_end 2869 2870 mova m7, [rsp+gprsize+16*0] 2871 SAVE_8ROWS coeffq+16*0, 32 2872 LOAD_8ROWS rsp+gprsize+16*3, 16 2873 mova [rsp+gprsize+16*0], m7 2874 mov r3, tx2q 2875 lea tx2q, [o(.pass1_end)] 2876 jmp m(iflipadst_8x8_internal_8bpc).pass1_end 2877 2878 .pass1_end: 2879 SAVE_8ROWS coeffq+16*1, 32 2880 LOAD_8ROWS coeffq+16*0, 32 2881 mova [rsp+gprsize+16*0], m7 2882 mov tx2q, r3 2883 jmp m(iflipadst_8x8_internal_8bpc).pass1_end 2884 2885 .pass2: 2886 lea tx2q, [o(.end)] 2887 lea r3, [dstq+8] 2888 jmp m(iflipadst_8x8_internal_8bpc).pass2_main 2889 2890 .end: 2891 LOAD_8ROWS coeffq+16*1, 32 2892 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2893 mov dstq, r3 2894 jmp m(iflipadst_8x8_internal_8bpc).pass2_main 2895 2896 2897 INV_TXFM_16X8_FN identity, dct 2898 INV_TXFM_16X8_FN identity, adst 2899 INV_TXFM_16X8_FN identity, flipadst 2900 INV_TXFM_16X8_FN identity, identity 2901 2902 cglobal iidentity_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2903 add coeffq, 16*16 2904 mova m4, [coeffq-16*7] 2905 mova m5, [coeffq-16*5] 2906 mova m6, [coeffq-16*3] 2907 mova m7, [coeffq-16*1] 2908 mov r3, tx2q 2909 lea tx2q, [o(.pass1_end)] 2910 2911 .pass1: 2912 mova m0, [o(pw_2896x8)] 2913 mova m2, [o(pw_1697x16)] 2914 mova m3, [o(pw_16384)] 2915 sub coeffq, 8*16 2916 REPX {pmulhrsw x, m0}, m4, m5, m6, m7 2917 pmulhrsw m1, m2, m4 2918 pmulhrsw m1, m3 2919 paddsw m1, m4 ; 1 2920 pmulhrsw m4, m2, m5 2921 pmulhrsw m4, m3 2922 paddsw m4, m5 ; 3 2923 pmulhrsw m5, m2, m6 2924 pmulhrsw m5, m3 2925 paddsw m5, m6 ; 5 2926 pmulhrsw m6, m2, m7 2927 pmulhrsw m6, m3 2928 paddsw m7, m6 ; 7 2929 pmulhrsw m6, m0, [coeffq+16*6] 2930 mova [rsp+gprsize+16*0], m4 2931 pmulhrsw m4, m2, m6 2932 pmulhrsw m4, m3 2933 paddsw m6, m4 ; 6 2934 pmulhrsw m4, m0, [coeffq+16*4] 2935 mova [rsp+gprsize+16*1], m6 2936 pmulhrsw m6, m2, m4 2937 pmulhrsw m6, m3 2938 paddsw m4, m6 ; 4 2939 pmulhrsw m6, m0, [coeffq+16*2] 2940 pmulhrsw m0, [coeffq+16*0] 2941 pmulhrsw m2, m6 2942 pmulhrsw m2, m3 2943 paddsw m2, m6 ; 2 2944 pmulhrsw m6, m0, [o(pw_1697x16)] 2945 pmulhrsw m6, m3 2946 mova m3, [rsp+gprsize+16*0] 2947 paddsw m0, m6 2948 jmp m(idct_8x8_internal_8bpc).pass1_end3 2949 2950 .pass1_end: 2951 mova [coeffq+16*1], m4 2952 mova [coeffq+16*3], m5 2953 mova [coeffq+16*5], m6 2954 mova [coeffq+16*7], m7 2955 mova m4, [coeffq-16*7] 2956 mova m5, [coeffq-16*5] 2957 mova m6, [coeffq-16*3] 2958 mova m7, [coeffq-16*1] 2959 mova [coeffq-16*7], m0 2960 mova [coeffq-16*5], m1 2961 mova [coeffq-16*3], m2 2962 mova [coeffq-16*1], m3 2963 mov tx2q, r3 2964 jmp .pass1 2965 2966 .pass2: 2967 lea tx2q, [o(.end)] 2968 lea r3, [dstq+8] 2969 jmp m(iidentity_8x8_internal_8bpc).end 2970 2971 .end: 2972 LOAD_8ROWS coeffq+16*1, 32 2973 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 2974 mov dstq, r3 2975 jmp m(iidentity_8x8_internal_8bpc).end 2976 2977 2978 %macro INV_TXFM_16X16_FN 2 ; type1, type2 2979 INV_TXFM_FN %1, %2, 16x16, 8, 16*16 2980 %ifidn %1_%2, dct_dct 2981 movd m1, [o(pw_2896x8)] 2982 pmulhrsw m0, m1, [coeffq] 2983 movd m2, [o(pw_8192)] 2984 mov [coeffq], eobd 2985 mov r2d, 8 2986 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x16_8bpc).end)] 2987 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly 2988 .end: 2989 RET 2990 %endif 2991 %endmacro 2992 2993 INV_TXFM_16X16_FN dct, dct 2994 INV_TXFM_16X16_FN dct, adst 2995 INV_TXFM_16X16_FN dct, flipadst 2996 INV_TXFM_16X16_FN dct, identity 2997 2998 cglobal idct_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 2999 LOAD_8ROWS coeffq+16*1, 64 3000 call m(idct_8x8_internal_8bpc).main 3001 SAVE_7ROWS rsp+gprsize+16*3, 16 3002 LOAD_8ROWS coeffq+16*3, 64 3003 call m(idct_16x8_internal_8bpc).main 3004 mov r3, tx2q 3005 lea tx2q, [o(.pass1_end)] 3006 mova m7, [o(pw_8192)] 3007 jmp m(idct_8x8_internal_8bpc).pass1_end1 3008 3009 .pass1_end: 3010 SAVE_8ROWS coeffq+16*17, 32 3011 LOAD_8ROWS rsp+gprsize+16*3, 16 3012 mova [rsp+gprsize+16*0], m7 3013 lea tx2q, [o(.pass1_end1)] 3014 mova m7, [o(pw_8192)] 3015 jmp m(idct_8x8_internal_8bpc).pass1_end1 3016 3017 .pass1_end1: 3018 SAVE_8ROWS coeffq+16*1, 32 3019 LOAD_8ROWS coeffq+16*0, 64 3020 call m(idct_8x8_internal_8bpc).main 3021 SAVE_7ROWS rsp+gprsize+16*3, 16 3022 LOAD_8ROWS coeffq+16*2, 64 3023 call m(idct_16x8_internal_8bpc).main 3024 lea tx2q, [o(.pass1_end2)] 3025 mova m7, [o(pw_8192)] 3026 jmp m(idct_8x8_internal_8bpc).pass1_end1 3027 3028 .pass1_end2: 3029 SAVE_8ROWS coeffq+16*16, 32 3030 LOAD_8ROWS rsp+gprsize+16*3, 16 3031 mova [rsp+gprsize+16*0], m7 3032 mov tx2q, r3 3033 mova m7, [o(pw_8192)] 3034 jmp m(idct_8x8_internal_8bpc).pass1_end1 3035 3036 .pass2: 3037 lea tx2q, [o(.end)] 3038 jmp m(idct_8x16_internal_8bpc).pass2_pre 3039 3040 .end: 3041 LOAD_8ROWS rsp+gprsize+16*3, 16 3042 mova [rsp+gprsize+16*0], m7 3043 lea tx2q, [o(.end1)] 3044 mov dstq, r3 3045 lea r3, [dstq+8] 3046 jmp m(idct_8x8_internal_8bpc).end 3047 3048 .end1: 3049 pxor m7, m7 3050 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 3051 3052 add coeffq, 32*8 3053 mov dstq, r3 3054 3055 mova m0, [coeffq+16*0 ] 3056 mova m1, [coeffq+16*4 ] 3057 mova m2, [coeffq+16*8 ] 3058 mova m3, [coeffq+16*12] 3059 mova m4, [coeffq+16*1 ] 3060 mova m5, [coeffq+16*5 ] 3061 mova m6, [coeffq+16*9 ] 3062 mova m7, [coeffq+16*13] 3063 lea tx2q, [o(m(idct_8x16_internal_8bpc).end)] 3064 jmp m(idct_8x16_internal_8bpc).pass2_main 3065 3066 3067 %macro ITX_16X16_ADST_LOAD_ODD_COEFS 0 3068 mova m0, [coeffq+16*1 ] 3069 mova m1, [coeffq+16*3 ] 3070 mova m2, [coeffq+16*29] 3071 mova m3, [coeffq+16*31] 3072 mova [rsp+gprsize+16*7], m0 3073 mova [rsp+gprsize+16*8], m1 3074 mova [rsp+gprsize+16*9], m2 3075 mova [rsp+gprsize+32*5], m3 3076 mova m0, [coeffq+16*13] 3077 mova m1, [coeffq+16*15] 3078 mova m2, [coeffq+16*17] 3079 mova m3, [coeffq+16*19] 3080 mova [rsp+gprsize+16*3], m2 3081 mova [rsp+gprsize+16*4], m3 3082 mova [rsp+gprsize+16*5], m0 3083 mova [rsp+gprsize+16*6], m1 3084 mova m0, [coeffq+16*5 ] 3085 mova m1, [coeffq+16*7 ] 3086 mova m2, [coeffq+16*9 ] 3087 mova m3, [coeffq+16*11] 3088 mova m4, [coeffq+16*21] 3089 mova m5, [coeffq+16*23] 3090 mova m6, [coeffq+16*25] 3091 mova m7, [coeffq+16*27] 3092 %endmacro 3093 3094 %macro ITX_16X16_ADST_LOAD_EVEN_COEFS 0 3095 mova m0, [coeffq+16*0 ] 3096 mova m1, [coeffq+16*2 ] 3097 mova m2, [coeffq+16*28] 3098 mova m3, [coeffq+16*30] 3099 mova [rsp+gprsize+16*7], m0 3100 mova [rsp+gprsize+16*8], m1 3101 mova [rsp+gprsize+16*9], m2 3102 mova [rsp+gprsize+32*5], m3 3103 mova m0, [coeffq+16*12] 3104 mova m1, [coeffq+16*14] 3105 mova m2, [coeffq+16*16] 3106 mova m3, [coeffq+16*18] 3107 mova [rsp+gprsize+16*3], m2 3108 mova [rsp+gprsize+16*4], m3 3109 mova [rsp+gprsize+16*5], m0 3110 mova [rsp+gprsize+16*6], m1 3111 mova m0, [coeffq+16*4 ] 3112 mova m1, [coeffq+16*6 ] 3113 mova m2, [coeffq+16*8 ] 3114 mova m3, [coeffq+16*10] 3115 mova m4, [coeffq+16*20] 3116 mova m5, [coeffq+16*22] 3117 mova m6, [coeffq+16*24] 3118 mova m7, [coeffq+16*26] 3119 %endmacro 3120 3121 INV_TXFM_16X16_FN adst, dct 3122 INV_TXFM_16X16_FN adst, adst 3123 INV_TXFM_16X16_FN adst, flipadst 3124 3125 cglobal iadst_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 3126 ITX_16X16_ADST_LOAD_ODD_COEFS 3127 call m(iadst_16x8_internal_8bpc).main 3128 call m(iadst_16x8_internal_8bpc).main_pass1_end 3129 3130 mov r3, tx2q 3131 lea tx2q, [o(.pass1_end)] 3132 mova m7, [o(pw_8192)] 3133 jmp m(iadst_8x8_internal_8bpc).pass1_end1 3134 3135 .pass1_end: 3136 SAVE_8ROWS coeffq+16*17, 32 3137 LOAD_8ROWS rsp+gprsize+16*3, 16 3138 mova [rsp+gprsize+16*0], m7 3139 lea tx2q, [o(.pass1_end1)] 3140 mova m7, [o(pw_8192)] 3141 jmp m(iadst_8x8_internal_8bpc).pass1_end1 3142 3143 .pass1_end1: 3144 SAVE_8ROWS coeffq+16*1, 32 3145 ITX_16X16_ADST_LOAD_EVEN_COEFS 3146 call m(iadst_16x8_internal_8bpc).main 3147 call m(iadst_16x8_internal_8bpc).main_pass1_end 3148 3149 lea tx2q, [o(.pass1_end2)] 3150 mova m7, [o(pw_8192)] 3151 jmp m(iadst_8x8_internal_8bpc).pass1_end1 3152 3153 .pass1_end2: 3154 SAVE_8ROWS coeffq+16*16, 32 3155 LOAD_8ROWS rsp+gprsize+16*3, 16 3156 mova [rsp+gprsize+16*0], m7 3157 mov tx2q, r3 3158 mova m7, [o(pw_8192)] 3159 jmp m(iadst_8x8_internal_8bpc).pass1_end1 3160 3161 .pass2: 3162 lea tx2q, [o(.end)] 3163 jmp m(iadst_8x16_internal_8bpc).pass2_pre 3164 3165 .end: 3166 LOAD_8ROWS rsp+gprsize+16*3, 16 3167 mova [rsp+gprsize+16*0], m7 3168 lea tx2q, [o(.end1)] 3169 mov dstq, r3 3170 lea r3, [dstq+8] 3171 jmp m(iadst_8x8_internal_8bpc).end 3172 3173 .end1: 3174 pxor m7, m7 3175 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 3176 3177 add coeffq, 32*8 3178 mov dstq, r3 3179 3180 mova m4, [coeffq+16*0 ] 3181 mova m5, [coeffq+16*2 ] 3182 mova m0, [coeffq+16*4 ] 3183 mova m1, [coeffq+16*6 ] 3184 mova m2, [coeffq+16*8 ] 3185 mova m3, [coeffq+16*10] 3186 mova m6, [coeffq+16*12] 3187 mova m7, [coeffq+16*14] 3188 mova [rsp+gprsize+16*7], m4 3189 mova [rsp+gprsize+16*8], m5 3190 mova [rsp+gprsize+16*5], m6 3191 mova [rsp+gprsize+16*6], m7 3192 lea tx2q, [o(m(iadst_8x16_internal_8bpc).end)] 3193 jmp m(iadst_8x16_internal_8bpc).pass2_main 3194 3195 3196 INV_TXFM_16X16_FN flipadst, dct 3197 INV_TXFM_16X16_FN flipadst, adst 3198 INV_TXFM_16X16_FN flipadst, flipadst 3199 3200 cglobal iflipadst_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 3201 ITX_16X16_ADST_LOAD_ODD_COEFS 3202 call m(iadst_16x8_internal_8bpc).main 3203 call m(iadst_16x8_internal_8bpc).main_pass1_end 3204 3205 mov r3, tx2q 3206 lea tx2q, [o(.pass1_end)] 3207 mova m7, [o(pw_m8192)] 3208 jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 3209 3210 .pass1_end: 3211 SAVE_8ROWS coeffq+16*1, 32 3212 LOAD_8ROWS rsp+gprsize+16*3, 16 3213 mova [rsp+gprsize+16*0], m7 3214 lea tx2q, [o(.pass1_end1)] 3215 mova m7, [o(pw_m8192)] 3216 jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 3217 3218 .pass1_end1: 3219 SAVE_8ROWS coeffq+16*17, 32 3220 ITX_16X16_ADST_LOAD_EVEN_COEFS 3221 call m(iadst_16x8_internal_8bpc).main 3222 call m(iadst_16x8_internal_8bpc).main_pass1_end 3223 3224 mova m7, [rsp+gprsize+16*0] 3225 SAVE_8ROWS coeffq+16*0, 32 3226 LOAD_8ROWS rsp+gprsize+16*3, 16 3227 mova [rsp+gprsize+16*0], m7 3228 lea tx2q, [o(.pass1_end2)] 3229 mova m7, [o(pw_m8192)] 3230 jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 3231 3232 .pass1_end2: 3233 SAVE_8ROWS coeffq+16*16, 32 3234 LOAD_8ROWS coeffq+16* 0, 32 3235 mova [rsp+gprsize+16*0], m7 3236 mov tx2q, r3 3237 mova m7, [o(pw_m8192)] 3238 jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 3239 3240 .pass2: 3241 lea tx2q, [o(.end)] 3242 lea r3, [dstq+8] 3243 jmp m(iflipadst_8x16_internal_8bpc).pass2_pre 3244 3245 .end: 3246 LOAD_8ROWS rsp+gprsize+16*3, 16 3247 mova [rsp+gprsize+16*0], m7 3248 lea tx2q, [o(.end1)] 3249 lea dstq, [dstq+strideq*2] 3250 jmp m(iflipadst_8x8_internal_8bpc).end 3251 3252 .end1: 3253 pxor m7, m7 3254 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 3255 3256 add coeffq, 32*8 3257 3258 mova m4, [coeffq+16*0 ] 3259 mova m5, [coeffq+16*2 ] 3260 mova m0, [coeffq+16*4 ] 3261 mova m1, [coeffq+16*6 ] 3262 mova m2, [coeffq+16*8 ] 3263 mova m3, [coeffq+16*10] 3264 mova m6, [coeffq+16*12] 3265 mova m7, [coeffq+16*14] 3266 mova [rsp+gprsize+16*7], m4 3267 mova [rsp+gprsize+16*8], m5 3268 mova [rsp+gprsize+16*5], m6 3269 mova [rsp+gprsize+16*6], m7 3270 3271 lea tx2q, [o(.end2)] 3272 mov dstq, r3 3273 jmp m(iflipadst_8x16_internal_8bpc).pass2_main 3274 3275 .end2: 3276 LOAD_8ROWS rsp+gprsize+16*3, 16 3277 mova [rsp+gprsize+16*0], m7 3278 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 3279 lea dstq, [dstq+strideq*2] 3280 jmp m(iflipadst_8x8_internal_8bpc).end 3281 3282 3283 %macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 3284 pmulhrsw m%2, m%3, m%1 3285 psraw m%2, 1 3286 pavgw m%1, m%2 3287 %endmacro 3288 3289 INV_TXFM_16X16_FN identity, dct 3290 INV_TXFM_16X16_FN identity, identity 3291 3292 cglobal iidentity_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 3293 add coeffq, 16*17 3294 mov r3, tx2q 3295 lea tx2q, [o(.pass1_end)] 3296 3297 .pass1: 3298 mova m6, [o(pw_1697x16)] 3299 mova m7, [coeffq+32*6] 3300 mova m0, [coeffq+32*0] 3301 mova m1, [coeffq+32*1] 3302 mova m2, [coeffq+32*2] 3303 mova m3, [coeffq+32*3] 3304 mova m4, [coeffq+32*4] 3305 REPX {IDTX16B x, 5, 6}, 7, 0, 1, 2, 3, 4 3306 mova m5, [coeffq+32*5] 3307 mova [rsp+gprsize+16*1], m7 3308 IDTX16B 5, 7, 6 3309 mova m7, [coeffq+32*7] 3310 IDTX16B 7, 6, 6 3311 jmp m(idct_8x8_internal_8bpc).pass1_end3 3312 3313 .pass1_end: 3314 SAVE_8ROWS coeffq, 32 3315 sub coeffq, 16 3316 lea tx2q, [o(.pass1_end1)] 3317 jmp .pass1 3318 3319 .pass1_end1: 3320 SAVE_8ROWS coeffq, 32 3321 sub coeffq, 15*16 3322 lea tx2q, [o(.pass1_end2)] 3323 jmp .pass1 3324 3325 .pass1_end2: 3326 SAVE_8ROWS coeffq, 32 3327 sub coeffq, 16 3328 mov tx2q, r3 3329 jmp .pass1 3330 3331 .pass2: 3332 lea r3, [dstq+8] 3333 lea tx2q, [o(.end1)] 3334 3335 .end: 3336 mova [rsp+gprsize+16*0], m7 3337 mova [rsp+gprsize+16*1], m4 3338 mova m7, [o(pw_1697x16)] 3339 REPX {IDTX16 x, 4, 7}, 5, 6, 0, 1, 2, 3 3340 mova m4, [o(pw_2048)] 3341 pmulhrsw m5, m4 3342 pmulhrsw m6, m4 3343 mova [rsp+gprsize+16*2], m5 3344 mova m5, [rsp+gprsize+16*1] 3345 mova [rsp+gprsize+16*1], m6 3346 IDTX16 5, 6, 7 3347 mova m6, [rsp+gprsize+16*0] 3348 IDTX16 6, 7, 7 3349 REPX {pmulhrsw x, m4}, m0, m1, m2, m3, m6 3350 pmulhrsw m4, m5 3351 mova [rsp+gprsize+16*0], m6 3352 jmp m(idct_8x8_internal_8bpc).end3 3353 3354 .end1: 3355 LOAD_8ROWS coeffq+16*1, 32 3356 lea tx2q, [o(.end2)] 3357 lea dstq, [dstq+strideq*2] 3358 jmp .end 3359 3360 .end2: 3361 pxor m7, m7 3362 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 3363 3364 add coeffq, 32*8 3365 LOAD_8ROWS coeffq, 32 3366 lea tx2q, [o(.end3)] 3367 mov dstq, r3 3368 jmp .end 3369 3370 .end3: 3371 LOAD_8ROWS coeffq+16*1, 32 3372 lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] 3373 lea dstq, [dstq+strideq*2] 3374 jmp .end 3375 3376 3377 cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 3378 %if ARCH_X86_32 3379 LEA r5, $$ 3380 %endif 3381 test eobd, eobd 3382 jz .dconly 3383 call m(idct_8x32_internal_8bpc) 3384 RET 3385 3386 .dconly: 3387 movd m1, [o(pw_2896x8)] 3388 pmulhrsw m0, m1, [coeffq] 3389 movd m2, [o(pw_8192)] 3390 mov [coeffq], eobd 3391 pmulhrsw m0, m2 3392 psrlw m2, 2 ;pw_2048 3393 pmulhrsw m0, m1 3394 pmulhrsw m0, m2 3395 pshuflw m0, m0, q0000 3396 punpcklwd m0, m0 3397 mov r3d, 8 3398 lea tx2q, [o(.end)] 3399 jmp m(inv_txfm_add_dct_dct_8x8_8bpc).loop 3400 3401 .end: 3402 RET 3403 3404 3405 3406 cglobal idct_8x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 3407 cmp eobd, 106 3408 jle .fast 3409 3410 LOAD_8ROWS coeffq+16*3, 64 3411 call m(idct_8x8_internal_8bpc).main 3412 mova m7, [o(pw_8192)] 3413 lea tx2q, [o(.pass1)] 3414 jmp m(idct_8x8_internal_8bpc).pass1_end1 3415 3416 .pass1: 3417 mova [rsp+gprsize+16*9 ], m0 ;in24 3418 mova [rsp+gprsize+16*10], m4 ;in28 3419 mova [rsp+gprsize+16*17], m2 ;in26 3420 mova [rsp+gprsize+16*18], m6 ;in30 3421 mova [rsp+gprsize+16*31], m1 ;in25 3422 mova [rsp+gprsize+16*30], m3 ;in27 3423 mova [rsp+gprsize+16*27], m5 ;in29 3424 mova [rsp+gprsize+16*34], m7 ;in31 3425 LOAD_8ROWS coeffq+16*2, 64 3426 call m(idct_8x8_internal_8bpc).main 3427 mova m7, [o(pw_8192)] 3428 lea tx2q, [o(.pass1_1)] 3429 jmp m(idct_8x8_internal_8bpc).pass1_end1 3430 3431 .pass1_1: 3432 mova [rsp+gprsize+16*7 ], m0 ;in16 3433 mova [rsp+gprsize+16*8 ], m4 ;in20 3434 mova [rsp+gprsize+16*15], m2 ;in18 3435 mova [rsp+gprsize+16*16], m6 ;in22 3436 mova [rsp+gprsize+16*33], m1 ;in17 3437 mova [rsp+gprsize+16*28], m3 ;in19 3438 mova [rsp+gprsize+16*29], m5 ;in21 3439 mova [rsp+gprsize+16*32], m7 ;in23 3440 3441 .fast: 3442 LOAD_8ROWS coeffq+16*1, 64 3443 call m(idct_8x8_internal_8bpc).main 3444 mova m7, [o(pw_8192)] 3445 lea tx2q, [o(.pass1_end)] 3446 jmp m(idct_8x8_internal_8bpc).pass1_end1 3447 3448 .pass1_end: 3449 mova [rsp+gprsize+16*5 ], m0 ;in8 3450 mova [rsp+gprsize+16*6 ], m4 ;in12 3451 mova [rsp+gprsize+16*13], m2 ;in10 3452 mova [rsp+gprsize+16*14], m6 ;in14 3453 mova [rsp+gprsize+16*21], m1 ;in9 3454 mova [rsp+gprsize+16*24], m3 ;in11 3455 mova [rsp+gprsize+16*25], m5 ;in13 3456 mova [rsp+gprsize+16*20], m7 ;in15 3457 LOAD_8ROWS coeffq+16*0, 64 3458 call m(idct_8x8_internal_8bpc).main 3459 mova m7, [o(pw_8192)] 3460 lea tx2q, [o(.pass1_end1)] 3461 jmp m(idct_8x8_internal_8bpc).pass1_end1 3462 3463 .pass1_end1: 3464 mova [rsp+gprsize+16*11], m2 ;in2 3465 mova [rsp+gprsize+16*12], m6 ;in6 3466 mova [rsp+gprsize+16*19], m1 ;in1 3467 mova [rsp+gprsize+16*26], m3 ;in3 3468 mova [rsp+gprsize+16*23], m5 ;in5 3469 mova [rsp+gprsize+16*22], m7 ;in7 3470 mova m1, m4 ;in4 3471 mova m2, [rsp+gprsize+16*5 ] ;in8 3472 mova m3, [rsp+gprsize+16*6 ] ;in12 3473 3474 cmp eobd, 106 3475 jg .full 3476 3477 pxor m4, m4 3478 REPX {mova x, m4}, m5, m6, m7 3479 call m(idct_8x8_internal_8bpc).main 3480 SAVE_7ROWS rsp+gprsize+16*3 , 16 3481 mova m0, [rsp+gprsize+16*11] 3482 mova m1, [rsp+gprsize+16*12] 3483 mova m2, [rsp+gprsize+16*13] 3484 mova m3, [rsp+gprsize+16*14] 3485 pxor m4, m4 3486 REPX {mova x, m4}, m5, m6, m7 3487 call m(idct_16x8_internal_8bpc).main 3488 mova m7, [rsp+gprsize+16*0] 3489 SAVE_8ROWS rsp+gprsize+16*11, 16 3490 3491 call .main_fast 3492 jmp .pass2 3493 3494 .full: 3495 mova m4, [rsp+gprsize+16*7 ] ;in16 3496 mova m5, [rsp+gprsize+16*8 ] ;in20 3497 mova m6, [rsp+gprsize+16*9 ] ;in24 3498 mova m7, [rsp+gprsize+16*10] ;in28 3499 call m(idct_8x8_internal_8bpc).main 3500 SAVE_7ROWS rsp+gprsize+16*3 , 16 3501 LOAD_8ROWS rsp+gprsize+16*11, 16 3502 call m(idct_16x8_internal_8bpc).main 3503 mova m7, [rsp+gprsize+16*0] 3504 SAVE_8ROWS rsp+gprsize+16*11, 16 3505 call .main 3506 3507 .pass2: 3508 lea r3, [o(.end6)] 3509 3510 .end: 3511 mova [rsp+gprsize+16*0 ], m7 3512 lea tx2q, [o(.end2)] 3513 3514 .end1: 3515 pxor m7, m7 3516 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, \ 3517 8, 9, 10, 11, 12, 13, 14, 15, \ 3518 16, 17, 18, 19, 20, 21, 22, 23, \ 3519 24, 25, 26, 27, 28, 29, 30, 31 3520 3521 jmp tx2q 3522 3523 .end2: 3524 lea tx2q, [o(.end3)] 3525 jmp m(idct_8x8_internal_8bpc).end 3526 3527 .end3: 3528 LOAD_8ROWS rsp+gprsize+16*11, 16 3529 mova [rsp+gprsize+16*0 ], m7 3530 lea dstq, [dstq+strideq*2] 3531 lea tx2q, [o(.end4)] 3532 jmp m(idct_8x8_internal_8bpc).end 3533 3534 .end4: 3535 LOAD_8ROWS rsp+gprsize+16*19, 16 3536 mova [rsp+gprsize+16*0 ], m7 3537 lea dstq, [dstq+strideq*2] 3538 lea tx2q, [o(.end5)] 3539 jmp m(idct_8x8_internal_8bpc).end 3540 3541 .end5: 3542 LOAD_8ROWS rsp+gprsize+16*27, 16 3543 mova [rsp+gprsize+16*0 ], m7 3544 lea dstq, [dstq+strideq*2] 3545 mov tx2q, r3 3546 jmp m(idct_8x8_internal_8bpc).end 3547 3548 .end6: 3549 ret 3550 3551 ALIGN function_align 3552 cglobal_label .main_veryfast 3553 mova m0, [rsp+gprsize*2+16*19] ;in1 3554 pmulhrsw m3, m0, [o(pw_4091x8)] ;t30,t31 3555 pmulhrsw m0, [o(pw_201x8)] ;t16,t17 3556 mova m7, [o(pd_2048)] 3557 mova [rsp+gprsize*2+16*19], m0 ;t16 3558 mova [rsp+gprsize*2+16*34], m3 ;t31 3559 ITX_MULSUB_2W 3, 0, 1, 2, 7, 799, 4017 ;t17a, t30a 3560 mova [rsp+gprsize*2+16*20], m3 ;t17a 3561 mova [rsp+gprsize*2+16*33], m0 ;t30a 3562 mova m1, [rsp+gprsize*2+16*22] ;in7 3563 pmulhrsw m2, m1, [o(pw_3857x8)] ;t28,t29 3564 pmulhrsw m1, [o(pw_m1380x8)] ;t18,t19 3565 mova [rsp+gprsize*2+16*22], m1 ;t19 3566 mova [rsp+gprsize*2+16*31], m2 ;t28 3567 ITX_MULSUB_2W 2, 1, 0, 3, 7, m4017, 799 ;t18a, t29a 3568 mova [rsp+gprsize*2+16*21], m2 ;t18a 3569 mova [rsp+gprsize*2+16*32], m1 ;t29a 3570 mova m0, [rsp+gprsize*2+16*23] ;in5 3571 pmulhrsw m3, m0, [o(pw_3973x8)] ;t26, t27 3572 pmulhrsw m0, [o(pw_995x8)] ;t20, t21 3573 mova [rsp+gprsize*2+16*23], m0 ;t20 3574 mova [rsp+gprsize*2+16*30], m3 ;t27 3575 ITX_MULSUB_2W 3, 0, 1, 2, 7, 3406, 2276 ;t21a, t26a 3576 mova [rsp+gprsize*2+16*24], m3 ;t21a 3577 mova [rsp+gprsize*2+16*29], m0 ;t26a 3578 mova m2, [rsp+gprsize*2+16*26] ;in3 3579 pxor m0, m0 3580 mova m3, m0 3581 pmulhrsw m1, m2, [o(pw_4052x8)] 3582 pmulhrsw m2, [o(pw_m601x8)] 3583 jmp .main2 3584 3585 ALIGN function_align 3586 cglobal_label .main_fast ;bottom half is zero 3587 mova m0, [rsp+gprsize*2+16*19] ;in1 3588 mova m1, [rsp+gprsize*2+16*20] ;in15 3589 pmulhrsw m3, m0, [o(pw_4091x8)] ;t31a 3590 pmulhrsw m0, [o(pw_201x8)] ;t16a 3591 pmulhrsw m2, m1, [o(pw_3035x8)] ;t30a 3592 pmulhrsw m1, [o(pw_m2751x8)] ;t17a 3593 mova m7, [o(pd_2048)] 3594 psubsw m4, m0, m1 ;t17 3595 paddsw m0, m1 ;t16 3596 psubsw m5, m3, m2 ;t30 3597 paddsw m3, m2 ;t31 3598 ITX_MULSUB_2W 5, 4, 1, 2, 7, 799, 4017 ;t17a, t30a 3599 mova [rsp+gprsize*2+16*19], m0 ;t16 3600 mova [rsp+gprsize*2+16*20], m5 ;t17a 3601 mova [rsp+gprsize*2+16*33], m4 ;t30a 3602 mova [rsp+gprsize*2+16*34], m3 ;t31 3603 mova m0, [rsp+gprsize*2+16*21] ;in9 3604 mova m1, [rsp+gprsize*2+16*22] ;in7 3605 pmulhrsw m3, m0, [o(pw_3703x8)] 3606 pmulhrsw m0, [o(pw_1751x8)] 3607 pmulhrsw m2, m1, [o(pw_3857x8)] 3608 pmulhrsw m1, [o(pw_m1380x8)] 3609 psubsw m4, m1, m0 ;t18 3610 paddsw m0, m1 ;t19 3611 psubsw m5, m2, m3 ;t29 3612 paddsw m3, m2 ;t28 3613 ITX_MULSUB_2W 5, 4, 1, 2, 7, m4017, 799 ;t18a, t29a 3614 mova [rsp+gprsize*2+16*21], m5 ;t18a 3615 mova [rsp+gprsize*2+16*22], m0 ;t19 3616 mova [rsp+gprsize*2+16*31], m3 ;t28 3617 mova [rsp+gprsize*2+16*32], m4 ;t29a 3618 mova m0, [rsp+gprsize*2+16*23] ;in5 3619 mova m1, [rsp+gprsize*2+16*24] ;in11 3620 pmulhrsw m3, m0, [o(pw_3973x8)] 3621 pmulhrsw m0, [o(pw_995x8)] 3622 pmulhrsw m2, m1, [o(pw_3513x8)] 3623 pmulhrsw m1, [o(pw_m2106x8)] 3624 psubsw m4, m0, m1 ;t21 3625 paddsw m0, m1 ;t20 3626 psubsw m5, m3, m2 ;t26 3627 paddsw m3, m2 ;t27 3628 ITX_MULSUB_2W 5, 4, 1, 2, 7, 3406, 2276 ;t21a, t26a 3629 mova [rsp+gprsize*2+16*23], m0 ;t20 3630 mova [rsp+gprsize*2+16*24], m5 ;t21a 3631 mova [rsp+gprsize*2+16*29], m4 ;t26a 3632 mova [rsp+gprsize*2+16*30], m3 ;t27 3633 mova m0, [rsp+gprsize*2+16*25] ;in13 3634 mova m2, [rsp+gprsize*2+16*26] ;in3 3635 pmulhrsw m3, m0, [o(pw_3290x8)] 3636 pmulhrsw m0, [o(pw_2440x8)] 3637 pmulhrsw m1, m2, [o(pw_4052x8)] 3638 pmulhrsw m2, [o(pw_m601x8)] 3639 jmp .main2 3640 3641 ALIGN function_align 3642 cglobal_label .main 3643 mova m7, [o(pd_2048)] 3644 mova m0, [rsp+gprsize*2+16*19] ;in1 3645 mova m1, [rsp+gprsize*2+16*20] ;in15 3646 mova m2, [rsp+gprsize*2+16*33] ;in17 3647 mova m3, [rsp+gprsize*2+16*34] ;in31 3648 ITX_MULSUB_2W 0, 3, 4, 5, 7, 201, 4091 ;t16a, t31a 3649 ITX_MULSUB_2W 2, 1, 4, 5, 7, 3035, 2751 ;t17a, t30a 3650 psubsw m4, m0, m2 ;t17 3651 paddsw m0, m2 ;t16 3652 psubsw m5, m3, m1 ;t30 3653 paddsw m3, m1 ;t31 3654 ITX_MULSUB_2W 5, 4, 1, 2, 7, 799, 4017 ;t17a, t30a 3655 mova [rsp+gprsize*2+16*19], m0 ;t16 3656 mova [rsp+gprsize*2+16*20], m5 ;t17a 3657 mova [rsp+gprsize*2+16*33], m4 ;t30a 3658 mova [rsp+gprsize*2+16*34], m3 ;t31 3659 mova m0, [rsp+gprsize*2+16*21] ;in9 3660 mova m1, [rsp+gprsize*2+16*22] ;in7 3661 mova m2, [rsp+gprsize*2+16*31] ;in25 3662 mova m3, [rsp+gprsize*2+16*32] ;in23 3663 ITX_MULSUB_2W 0, 3, 4, 5, 7, 1751, 3703 ;t18a, t29a 3664 ITX_MULSUB_2W 2, 1, 4, 5, 7, 3857, 1380 ;t19a, t28a 3665 psubsw m4, m2, m0 ;t18 3666 paddsw m0, m2 ;t19 3667 psubsw m5, m1, m3 ;t29 3668 paddsw m3, m1 ;t28 3669 ITX_MULSUB_2W 5, 4, 1, 2, 7, m4017, 799 ;t18a, t29a 3670 mova [rsp+gprsize*2+16*21], m5 ;t18a 3671 mova [rsp+gprsize*2+16*22], m0 ;t19 3672 mova [rsp+gprsize*2+16*31], m3 ;t28 3673 mova [rsp+gprsize*2+16*32], m4 ;t29a 3674 mova m0, [rsp+gprsize*2+16*23] ;in5 3675 mova m1, [rsp+gprsize*2+16*24] ;in11 3676 mova m2, [rsp+gprsize*2+16*29] ;in21 3677 mova m3, [rsp+gprsize*2+16*30] ;in27 3678 ITX_MULSUB_2W 0, 3, 4, 5, 7, 995, 3973 ;t20a, t27a 3679 ITX_MULSUB_2W 2, 1, 4, 5, 7, 3513, 2106 ;t21a, t26a 3680 psubsw m4, m0, m2 ;t21 3681 paddsw m0, m2 ;t20 3682 psubsw m5, m3, m1 ;t26 3683 paddsw m3, m1 ;t27 3684 ITX_MULSUB_2W 5, 4, 1, 2, 7, 3406, 2276 ;t21a, t26a 3685 mova [rsp+gprsize*2+16*23], m0 ;t20 3686 mova [rsp+gprsize*2+16*24], m5 ;t21a 3687 mova [rsp+gprsize*2+16*29], m4 ;t26a 3688 mova [rsp+gprsize*2+16*30], m3 ;t27 3689 mova m0, [rsp+gprsize*2+16*25] ;in13 3690 mova m1, [rsp+gprsize*2+16*26] ;in3 3691 mova m2, [rsp+gprsize*2+16*27] ;in29 3692 mova m3, [rsp+gprsize*2+16*28] ;in19 3693 ITX_MULSUB_2W 0, 3, 4, 5, 7, 2440, 3290 ;t22a, t25a 3694 ITX_MULSUB_2W 2, 1, 4, 5, 7, 4052, 601 ;t23a, t24a 3695 3696 .main2: 3697 psubsw m4, m2, m0 ;t22 3698 paddsw m0, m2 ;t23 3699 psubsw m5, m1, m3 ;t25 3700 paddsw m3, m1 ;t24 3701 ITX_MULSUB_2W 5, 4, 1, 2, 7, m2276, 3406 ;t22a, t25a 3702 mova m2, [rsp+gprsize*2+16*24] ;t21a 3703 psubsw m1, m5, m2 ;t21 3704 paddsw m5, m2 ;t22 3705 mova [rsp+gprsize*2+16*25], m5 ;t22 3706 mova m2, [rsp+gprsize*2+16*29] ;t26a 3707 psubsw m5, m4, m2 ;t26 3708 paddsw m4, m2 ;t25 3709 mova [rsp+gprsize*2+16*28], m4 ;t25 3710 ITX_MULSUB_2W 5, 1, 2, 4, 7, m3784, 1567 ;t21a, t26a 3711 mova [rsp+gprsize*2+16*24], m5 ;t21a 3712 mova [rsp+gprsize*2+16*29], m1 ;t26a 3713 3714 mova m1, [rsp+gprsize*2+16*23] ;t20 3715 mova m5, [rsp+gprsize*2+16*30] ;t27 3716 psubsw m2, m0, m1 ;t20a 3717 paddsw m0, m1 ;t23a 3718 psubsw m6, m3, m5 ;t27a 3719 paddsw m3, m5 ;t24a 3720 ITX_MULSUB_2W 6, 2, 1, 5, 7, m3784, 1567 ;t20, t27 3721 mova [rsp+gprsize*2+16*26], m0 ;t23a 3722 mova [rsp+gprsize*2+16*27], m3 ;t24a 3723 mova [rsp+gprsize*2+16*30], m2 ;t27 3724 3725 mova m0, [rsp+gprsize*2+16*20] ;t17a 3726 mova m1, [rsp+gprsize*2+16*21] ;t18a 3727 mova m2, [rsp+gprsize*2+16*32] ;t29a 3728 mova m3, [rsp+gprsize*2+16*33] ;t30a 3729 psubsw m4, m0, m1 ;t18 3730 paddsw m0, m1 ;t17 3731 psubsw m5, m3, m2 ;t29 3732 paddsw m3, m2 ;t30 3733 ITX_MULSUB_2W 5, 4, 1, 2, 7, 1567, 3784 ;t18a, t29a 3734 mova [rsp+gprsize*2+16*20], m0 ;t17 3735 mova [rsp+gprsize*2+16*21], m5 ;t18a 3736 mova [rsp+gprsize*2+16*32], m4 ;t29a 3737 mova [rsp+gprsize*2+16*33], m3 ;t30 3738 mova m0, [rsp+gprsize*2+16*19] ;t16 3739 mova m1, [rsp+gprsize*2+16*22] ;t19 3740 mova m2, [rsp+gprsize*2+16*31] ;t28 3741 mova m3, [rsp+gprsize*2+16*34] ;t31 3742 psubsw m4, m0, m1 ;t19a 3743 paddsw m0, m1 ;t16a 3744 psubsw m5, m3, m2 ;t28a 3745 paddsw m3, m2 ;t31a 3746 ITX_MULSUB_2W 5, 4, 1, 2, 7, 1567, 3784 ;t19, t28 3747 mova m2, [rsp+gprsize*2+16*15] ;tmp12 3748 psubsw m1, m5, m6 ;t20a 3749 paddsw m5, m6 ;t19a 3750 psubsw m6, m2, m5 ;out19 3751 paddsw m2, m5 ;out12 3752 mova m5, [rsp+gprsize*2+16*30] ;t27 3753 mova [rsp+gprsize*2+16*22], m6 ;out19 3754 mova [rsp+gprsize*2+16*15], m2 ;out12 3755 psubsw m6, m4, m5 ;t27a 3756 paddsw m4, m5 ;t28a 3757 ITX_MULSUB_2W 6, 1, 2, 5, 7, 2896, 2896 ;t20, t27 3758 mova m2, [rsp+gprsize*2+16*6 ] ;tmp3 3759 psubsw m5, m2, m4 ;out28 3760 paddsw m2, m4 ;out3 3761 mova m4, [rsp+gprsize*2+16*14] ;tmp11 3762 mova [rsp+gprsize*2+16*31], m5 ;out28 3763 mova [rsp+gprsize*2+16*6 ], m2 ;out3 3764 psubsw m5, m4, m6 ;out20 3765 paddsw m4, m6 ;out11 3766 mova m2, [rsp+gprsize*2+16*7 ] ;tmp4 3767 mova [rsp+gprsize*2+16*23], m5 ;out20 3768 mova [rsp+gprsize*2+16*14], m4 ;out11 3769 psubsw m5, m2, m1 ;out27 3770 paddsw m2, m1 ;out4 3771 mova m1, [rsp+gprsize*2+16*26] ;t23a 3772 mova m4, [rsp+gprsize*2+16*27] ;t24a 3773 mova [rsp+gprsize*2+16*30], m5 ;out27 3774 mova [rsp+gprsize*2+16*7 ], m2 ;out4 3775 psubsw m5, m0, m1 ;t23 3776 paddsw m0, m1 ;t16 3777 psubsw m2, m3, m4 ;t24 3778 paddsw m3, m4 ;t31 3779 ITX_MULSUB_2W 2, 5, 4, 6, 7, 2896, 2896 ;t23a, t24a 3780 mova m6, [rsp+gprsize*2+16*18] ;tmp15 3781 psubsw m4, m6, m0 ;out16 3782 paddsw m6, m0 ;out15 3783 mova m0, [rsp+gprsize*2+16*3 ] ;tmp0 3784 mova m1, [rsp+gprsize*2+16*11] ;tmp8 3785 mova [rsp+gprsize*2+16*18], m6 ;out15 3786 mova [rsp+gprsize*2+16*19], m4 ;out16 3787 psubsw m6, m0, m3 ;out31 3788 paddsw m0, m3 ;out0 3789 psubsw m4, m1, m2 ;out23 3790 paddsw m1, m2 ;out8 3791 mova m3, [rsp+gprsize*2+16*10] ;tmp7 3792 mova [rsp+gprsize*2+16*34], m6 ;out31 3793 mova [rsp+gprsize*2+16*11], m1 ;out8 3794 mova [rsp+gprsize*2+16*26], m4 ;out23 3795 paddsw m6, m3, m5 ;out7 3796 psubsw m3, m5 ;out24 3797 mova m1, [rsp+gprsize*2+16*20] ;t17 3798 mova m5, [rsp+gprsize*2+16*25] ;t22 3799 mova m2, [rsp+gprsize*2+16*17] ;tmp14 3800 mova [rsp+gprsize*2+16*27], m3 ;out24 3801 psubsw m4, m1, m5 ;t22a 3802 paddsw m1, m5 ;t17a 3803 psubsw m3, m2, m1 ;out17 3804 paddsw m2, m1 ;out14 3805 mova m5, [rsp+gprsize*2+16*28] ;t25 3806 mova m1, [rsp+gprsize*2+16*33] ;t30 3807 mova [rsp+gprsize*2+16*17], m2 ;out14 3808 mova [rsp+gprsize*2+16*20], m3 ;out17 3809 psubsw m2, m1, m5 ;t25a 3810 paddsw m1, m5 ;t30a 3811 ITX_MULSUB_2W 2, 4, 3, 5, 7, 2896, 2896 ;t22, t25 3812 mova m5, [rsp+gprsize*2+16*4 ] ;tmp1 3813 psubsw m3, m5, m1 ;out30 3814 paddsw m5, m1 ;out1 3815 mova m1, [rsp+gprsize*2+16*12] ;tmp9 3816 mova [rsp+gprsize*2+16*33], m3 ;out30 3817 mova [rsp+gprsize*2+16*4 ], m5 ;out1 3818 psubsw m3, m1, m2 ;out22 3819 paddsw m1, m2 ;out9 3820 mova m5, [rsp+gprsize*2+16*9 ] ;tmp6 3821 mova [rsp+gprsize*2+16*25], m3 ;out22 3822 mova [rsp+gprsize*2+16*12], m1 ;out9 3823 psubsw m3, m5, m4 ;out25 3824 paddsw m5, m4 ;out6 3825 mova m4, [rsp+gprsize*2+16*21] ;t18a 3826 mova m1, [rsp+gprsize*2+16*24] ;t21a 3827 mova m2, [rsp+gprsize*2+16*16] ;tmp13 3828 mova [rsp+gprsize*2+16*28], m3 ;out25 3829 mova [rsp+gprsize*2+16*9 ], m5 ;out6 3830 paddsw m3, m4, m1 ;t18 3831 psubsw m4, m1 ;t21 3832 psubsw m5, m2, m3 ;out18 3833 paddsw m2, m3 ;out13 3834 mova m1, [rsp+gprsize*2+16*29] ;t26a 3835 mova m3, [rsp+gprsize*2+16*32] ;t29a 3836 mova [rsp+gprsize*2+16*21], m5 ;out18 3837 mova [rsp+gprsize*2+16*16], m2 ;out13 3838 psubsw m5, m3, m1 ;t26 3839 paddsw m3, m1 ;t29 3840 ITX_MULSUB_2W 5, 4, 1, 2, 7, 2896, 2896 ;t21a, t26a 3841 mova m2, [rsp+gprsize*2+16*5 ] ;tmp2 3842 psubsw m1, m2, m3 ;out29 3843 paddsw m2, m3 ;out2 3844 mova m3, [rsp+gprsize*2+16*13] ;tmp10 3845 mova [rsp+gprsize*2+16*32], m1 ;out29 3846 psubsw m7, m3, m5 ;out21 3847 paddsw m3, m5 ;out10 3848 mova m5, [rsp+gprsize*2+16*8 ] ;tmp5 3849 mova [rsp+gprsize*2+16*24], m7 ;out21 3850 mova [rsp+gprsize*2+16*13], m3 ;out10 3851 psubsw m1, m5, m4 ;out26 3852 paddsw m5, m4 ;out5 3853 mova m7, m6 ;out7 3854 mova m3, [rsp+gprsize*2+16*6 ] ;out3 3855 mova m4, [rsp+gprsize*2+16*7 ] ;out4 3856 mova [rsp+gprsize*2+16*29], m1 ;out26 3857 mova m6, [rsp+gprsize*2+16*9 ] ;out6 3858 mova m1, [rsp+gprsize*2+16*4 ] ;out1 3859 ret 3860 3861 3862 cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 3863 %if ARCH_X86_32 3864 LEA r5, $$ 3865 %endif 3866 test eobd, eobd 3867 jz .dconly 3868 call m(idct_32x8_internal_8bpc) 3869 RET 3870 3871 .dconly: 3872 movd m1, [o(pw_2896x8)] 3873 pmulhrsw m0, m1, [coeffq] 3874 movd m2, [o(pw_8192)] 3875 mov [coeffq], eobd 3876 mov r3d, 8 3877 lea tx2q, [o(.end)] 3878 3879 .body: 3880 pmulhrsw m0, m2 3881 movd m2, [o(pw_2048)] ;intentionally rip-relative 3882 pmulhrsw m0, m1 3883 pmulhrsw m0, m2 3884 pshuflw m0, m0, q0000 3885 punpcklwd m0, m0 3886 pxor m5, m5 3887 3888 .loop: 3889 mova m1, [dstq+16*0] 3890 mova m3, [dstq+16*1] 3891 punpckhbw m2, m1, m5 3892 punpcklbw m1, m5 3893 punpckhbw m4, m3, m5 3894 punpcklbw m3, m5 3895 paddw m2, m0 3896 paddw m1, m0 3897 paddw m4, m0 3898 paddw m3, m0 3899 packuswb m1, m2 3900 packuswb m3, m4 3901 mova [dstq+16*0], m1 3902 mova [dstq+16*1], m3 3903 add dstq, strideq 3904 dec r3d 3905 jg .loop 3906 jmp tx2q 3907 3908 .end: 3909 RET 3910 3911 3912 cglobal idct_32x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 3913 LOAD_8ROWS coeffq+16*0, 64 3914 call m(idct_8x8_internal_8bpc).main 3915 SAVE_7ROWS rsp+gprsize+16*3, 16 3916 3917 LOAD_8ROWS coeffq+16*2, 64 3918 call m(idct_16x8_internal_8bpc).main 3919 mova m7, [rsp+gprsize+16*0] 3920 SAVE_8ROWS rsp+gprsize+16*11, 16 3921 3922 LOAD_8ROWS coeffq+16*1, 32 3923 mova [rsp+gprsize+16*19], m0 ;in1 3924 mova [rsp+gprsize+16*26], m1 ;in3 3925 mova [rsp+gprsize+16*23], m2 ;in5 3926 mova [rsp+gprsize+16*22], m3 ;in7 3927 mova [rsp+gprsize+16*21], m4 ;in9 3928 mova [rsp+gprsize+16*24], m5 ;in11 3929 mova [rsp+gprsize+16*25], m6 ;in13 3930 mova [rsp+gprsize+16*20], m7 ;in15 3931 3932 cmp eobd, 106 3933 jg .full 3934 call m(idct_8x32_internal_8bpc).main_fast 3935 jmp .pass2 3936 3937 .full: 3938 LOAD_8ROWS coeffq+16*17, 32 3939 mova [rsp+gprsize+16*33], m0 ;in17 3940 mova [rsp+gprsize+16*28], m1 ;in19 3941 mova [rsp+gprsize+16*29], m2 ;in21 3942 mova [rsp+gprsize+16*32], m3 ;in23 3943 mova [rsp+gprsize+16*31], m4 ;in25 3944 mova [rsp+gprsize+16*30], m5 ;in27 3945 mova [rsp+gprsize+16*27], m6 ;in29 3946 mova [rsp+gprsize+16*34], m7 ;in31 3947 call m(idct_8x32_internal_8bpc).main 3948 3949 .pass2: 3950 mova [rsp+gprsize+16*0 ], m7 3951 lea tx2q, [o(.end)] 3952 jmp m(idct_8x32_internal_8bpc).end1 3953 3954 .end: 3955 mova m7, [o(pw_8192)] 3956 lea tx2q, [o(.end1)] 3957 jmp m(idct_8x8_internal_8bpc).pass1_end1 3958 3959 .end1: 3960 lea r3, [dstq+8] 3961 lea tx2q, [o(.end2)] 3962 jmp m(idct_8x8_internal_8bpc).pass2_main 3963 3964 .end2: 3965 LOAD_8ROWS rsp+gprsize+16*11, 16 3966 mova [rsp+gprsize+16*0 ], m7 3967 mova m7, [o(pw_8192)] 3968 lea tx2q, [o(.end3)] 3969 jmp m(idct_8x8_internal_8bpc).pass1_end1 3970 3971 .end3: 3972 mov dstq, r3 3973 add r3, 8 3974 lea tx2q, [o(.end4)] 3975 jmp m(idct_8x8_internal_8bpc).pass2_main 3976 3977 .end4: 3978 LOAD_8ROWS rsp+gprsize+16*19, 16 3979 mova [rsp+gprsize+16*0 ], m7 3980 mova m7, [o(pw_8192)] 3981 lea tx2q, [o(.end5)] 3982 jmp m(idct_8x8_internal_8bpc).pass1_end1 3983 3984 .end5: 3985 mov dstq, r3 3986 add r3, 8 3987 lea tx2q, [o(.end6)] 3988 jmp m(idct_8x8_internal_8bpc).pass2_main 3989 3990 .end6: 3991 LOAD_8ROWS rsp+gprsize+16*27, 16 3992 mova [rsp+gprsize+16*0 ], m7 3993 mova m7, [o(pw_8192)] 3994 lea tx2q, [o(.end7)] 3995 jmp m(idct_8x8_internal_8bpc).pass1_end1 3996 3997 .end7: 3998 mov dstq, r3 3999 lea tx2q, [o(.end8)] 4000 jmp m(idct_8x8_internal_8bpc).pass2_main 4001 4002 .end8: 4003 ret 4004 4005 4006 cglobal inv_txfm_add_identity_identity_8x32_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 4007 mov r5d, 4 4008 mov tx2d, 2 4009 cmp eobd, 107 4010 cmovns tx2d, r5d 4011 mov r3d, tx2d 4012 %if ARCH_X86_32 4013 LEA r5, $$ 4014 %endif 4015 lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] 4016 .loop: 4017 LOAD_8ROWS coeffq+16*0, 64 4018 paddsw m6, [o(pw_5)] 4019 mova [rsp+16*1], m6 4020 mova m6, [o(pw_5)] 4021 REPX {paddsw x, m6}, m0, m1, m2, m3, m4, m5, m7 4022 call m(idct_8x8_internal_8bpc).pass1_end3 4023 REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 4024 mova [rsp+16*2], m5 4025 mova [rsp+16*1], m6 4026 mova [rsp+16*0], m7 4027 call m(idct_8x8_internal_8bpc).end3 4028 lea dstq, [dstq+strideq*2] 4029 pxor m7, m7 4030 REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 4031 add coeffq, 16 4032 dec r3d 4033 jg .loop 4034 RET 4035 4036 cglobal inv_txfm_add_identity_identity_32x8_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 4037 mov r5d, 4 4038 mov tx2d, 2 4039 cmp eobd, 107 4040 cmovns tx2d, r5d 4041 mov r3d, tx2d 4042 %if ARCH_X86_32 4043 LEA r5, $$ 4044 %endif 4045 4046 .loop: 4047 LOAD_8ROWS coeffq+16*0, 16 4048 pmulhrsw m6, [o(pw_4096)] 4049 mova [rsp+16*1], m6 4050 mova m6, [o(pw_4096)] 4051 REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 4052 lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] 4053 call m(idct_8x8_internal_8bpc).pass1_end3 4054 4055 mov [rsp+16*3], dstq 4056 mova [rsp+16*2], m5 4057 mova [rsp+16*1], m6 4058 mova [rsp+16*0], m7 4059 lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] 4060 call m(idct_8x8_internal_8bpc).end3 4061 4062 add coeffq, 16*8 4063 mov dstq, [rsp+16*3] 4064 lea dstq, [dstq+8] 4065 dec r3d 4066 jg .loop 4067 jnc .loop 4068 RET 4069 4070 4071 cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 4072 %if ARCH_X86_32 4073 LEA r5, $$ 4074 %endif 4075 test eobd, eobd 4076 jz .dconly 4077 call m(idct_16x32_internal_8bpc) 4078 .end: 4079 RET 4080 4081 .dconly: 4082 movd m1, [o(pw_2896x8)] 4083 pmulhrsw m0, m1, [coeffq] 4084 movd m2, [o(pw_16384)] 4085 mov [coeffq], eobd 4086 pmulhrsw m0, m1 4087 mov r2d, 16 4088 lea tx2q, [o(.end)] 4089 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly 4090 4091 4092 cglobal idct_16x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 4093 LOAD_8ROWS coeffq+16*1, 128, 1 4094 call m(idct_8x8_internal_8bpc).main 4095 SAVE_7ROWS rsp+gprsize+16*3, 16 4096 LOAD_8ROWS coeffq+16*5, 128, 1 4097 call m(idct_16x8_internal_8bpc).main 4098 lea tx2q, [o(.pass1_end)] 4099 jmp m(idct_8x8_internal_8bpc).pass1_end 4100 4101 .pass1_end: 4102 SAVE_8ROWS coeffq+16*33, 64 ;in8~in15 4103 LOAD_8ROWS rsp+gprsize+16*3, 16 4104 mova [rsp+gprsize+16*0], m7 4105 lea tx2q, [o(.pass1_end1)] 4106 jmp m(idct_8x8_internal_8bpc).pass1_end 4107 4108 .pass1_end1: 4109 mova [coeffq+16*1 ], m0 ;in8 4110 mova [coeffq+16*5 ], m4 ;in12 4111 mova [rsp+gprsize+16*13], m2 ;in10 4112 mova [rsp+gprsize+16*14], m6 ;in14 4113 mova [rsp+gprsize+16*21], m1 ;in9 4114 mova [rsp+gprsize+16*24], m3 ;in11 4115 mova [rsp+gprsize+16*25], m5 ;in13 4116 mova [rsp+gprsize+16*20], m7 ;in15 4117 LOAD_8ROWS coeffq+16*0, 128, 1 4118 call m(idct_8x8_internal_8bpc).main 4119 SAVE_7ROWS rsp+gprsize+16*3, 16 4120 LOAD_8ROWS coeffq+16*4, 128, 1 4121 call m(idct_16x8_internal_8bpc).main 4122 lea tx2q, [o(.pass1_end2)] 4123 jmp m(idct_8x8_internal_8bpc).pass1_end 4124 4125 .pass1_end2: 4126 SAVE_8ROWS coeffq+16*32, 64 ;in0~in7 4127 LOAD_8ROWS rsp+gprsize+16*3, 16 4128 mova [rsp+gprsize+16*0], m7 4129 lea tx2q, [o(.pass1_end3)] 4130 jmp m(idct_8x8_internal_8bpc).pass1_end 4131 4132 .pass1_end3: 4133 mova [rsp+gprsize+16*11], m2 ;in2 4134 mova [rsp+gprsize+16*12], m6 ;in6 4135 mova [rsp+gprsize+16*19], m1 ;in1 4136 mova [rsp+gprsize+16*26], m3 ;in3 4137 mova [rsp+gprsize+16*23], m5 ;in5 4138 mova [rsp+gprsize+16*22], m7 ;in7 4139 4140 cmp eobd, 150 4141 jg .full 4142 4143 mova m1, m4 ;in4 4144 mova m2, [coeffq+16*1 ] ;in8 4145 mova m3, [coeffq+16*5 ] ;in12 4146 pxor m4, m4 4147 REPX {mova x, m4}, m5, m6, m7 4148 call m(idct_8x8_internal_8bpc).main 4149 SAVE_7ROWS rsp+gprsize+16*3, 16 4150 mova m0, [rsp+gprsize+16*11] ;in2 4151 mova m1, [rsp+gprsize+16*12] ;in6 4152 mova m2, [rsp+gprsize+16*13] ;in10 4153 mova m3, [rsp+gprsize+16*14] ;in14 4154 pxor m4, m4 4155 REPX {mova x, m4}, m5, m6, m7 4156 call m(idct_16x8_internal_8bpc).main 4157 mova m7, [rsp+gprsize+16*0] 4158 SAVE_8ROWS rsp+gprsize+16*11, 16 4159 4160 call m(idct_8x32_internal_8bpc).main_fast 4161 jmp .pass2 4162 4163 .full: 4164 mova [coeffq+16*0 ], m0 ;in0 4165 mova [coeffq+16*4 ], m4 ;in4 4166 4167 LOAD_8ROWS coeffq+16*2, 128, 1 4168 call m(idct_8x8_internal_8bpc).main 4169 SAVE_7ROWS rsp+gprsize+16*3, 16 4170 LOAD_8ROWS coeffq+16*6, 128, 1 4171 call m(idct_16x8_internal_8bpc).main 4172 lea tx2q, [o(.pass1_end4)] 4173 jmp m(idct_8x8_internal_8bpc).pass1_end 4174 4175 .pass1_end4: 4176 SAVE_8ROWS coeffq+16*34, 64 ;in16~in23 4177 LOAD_8ROWS rsp+gprsize+16*3, 16 4178 mova [rsp+gprsize+16*0], m7 4179 lea tx2q, [o(.pass1_end5)] 4180 jmp m(idct_8x8_internal_8bpc).pass1_end 4181 4182 .pass1_end5: 4183 mova [coeffq+16*2 ], m0 ;in16 4184 mova [coeffq+16*6 ], m4 ;in20 4185 mova [rsp+gprsize+16*15], m2 ;in18 4186 mova [rsp+gprsize+16*16], m6 ;in22 4187 mova [rsp+gprsize+16*33], m1 ;in17 4188 mova [rsp+gprsize+16*28], m3 ;in19 4189 mova [rsp+gprsize+16*29], m5 ;in21 4190 mova [rsp+gprsize+16*32], m7 ;in23 4191 4192 LOAD_8ROWS coeffq+16*3, 128, 1 4193 call m(idct_8x8_internal_8bpc).main 4194 SAVE_7ROWS rsp+gprsize+16*3, 16 4195 LOAD_8ROWS coeffq+16*7, 128, 1 4196 call m(idct_16x8_internal_8bpc).main 4197 lea tx2q, [o(.pass1_end6)] 4198 jmp m(idct_8x8_internal_8bpc).pass1_end 4199 4200 .pass1_end6: 4201 SAVE_8ROWS coeffq+16*35, 64 ;in24~in31 4202 LOAD_8ROWS rsp+gprsize+16*3, 16 4203 mova [rsp+gprsize+16*0], m7 4204 lea tx2q, [o(.pass1_end7)] 4205 jmp m(idct_8x8_internal_8bpc).pass1_end 4206 4207 .pass1_end7: 4208 mova [rsp+gprsize+16*17], m2 ;in26 4209 mova [rsp+gprsize+16*18], m6 ;in30 4210 mova [rsp+gprsize+16*31], m1 ;in25 4211 mova [rsp+gprsize+16*30], m3 ;in27 4212 mova [rsp+gprsize+16*27], m5 ;in29 4213 mova [rsp+gprsize+16*34], m7 ;in31 4214 4215 mova m6, m0 ;in24 4216 mova m7, m4 ;in28 4217 mova m0, [coeffq+16*0 ] ;in0 4218 mova m1, [coeffq+16*4 ] ;in4 4219 mova m2, [coeffq+16*1 ] ;in8 4220 mova m3, [coeffq+16*5 ] ;in12 4221 mova m4, [coeffq+16*2 ] ;in16 4222 mova m5, [coeffq+16*6 ] ;in20 4223 call m(idct_8x8_internal_8bpc).main 4224 SAVE_7ROWS rsp+gprsize+16*3 , 16 4225 LOAD_8ROWS rsp+gprsize+16*11, 16 4226 call m(idct_16x8_internal_8bpc).main 4227 mova m7, [rsp+gprsize+16*0] 4228 SAVE_8ROWS rsp+gprsize+16*11, 16 4229 4230 call m(idct_8x32_internal_8bpc).main 4231 4232 .pass2: 4233 mov [rsp+gprsize*1+16*35], eobd 4234 lea r3, [dstq+8] 4235 mov [rsp+gprsize*2+16*35], r3 4236 lea r3, [o(.end)] 4237 jmp m(idct_8x32_internal_8bpc).end 4238 4239 .end: 4240 mov dstq, [rsp+gprsize*2+16*35] 4241 mov eobd, [rsp+gprsize*1+16*35] 4242 add coeffq, 16*32 4243 4244 mova m0, [coeffq+16*4 ] ;in1 4245 mova m1, [coeffq+16*12] ;in3 4246 mova m2, [coeffq+16*20] ;in5 4247 mova m3, [coeffq+16*28] ;in7 4248 mova m4, [coeffq+16*5 ] ;in9 4249 mova m5, [coeffq+16*13] ;in11 4250 mova m6, [coeffq+16*21] ;in13 4251 mova m7, [coeffq+16*29] ;in15 4252 4253 mova [rsp+gprsize+16*19], m0 ;in1 4254 mova [rsp+gprsize+16*26], m1 ;in3 4255 mova [rsp+gprsize+16*23], m2 ;in5 4256 mova [rsp+gprsize+16*22], m3 ;in7 4257 mova [rsp+gprsize+16*21], m4 ;in9 4258 mova [rsp+gprsize+16*24], m5 ;in11 4259 mova [rsp+gprsize+16*25], m6 ;in13 4260 mova [rsp+gprsize+16*20], m7 ;in15 4261 4262 mova m0, [coeffq+16*0 ] ;in0 4263 mova m1, [coeffq+16*16] ;in4 4264 mova m2, [coeffq+16*1 ] ;in8 4265 mova m3, [coeffq+16*17] ;in12 4266 4267 cmp eobd, 150 4268 jg .full1 4269 4270 pxor m4, m4 4271 REPX {mova x, m4}, m5, m6, m7 4272 call m(idct_8x8_internal_8bpc).main 4273 SAVE_7ROWS rsp+gprsize+16*3, 16 4274 4275 mova m0, [coeffq+16*8 ] ;in2 4276 mova m1, [coeffq+16*24] ;in6 4277 mova m2, [coeffq+16*9 ] ;in10 4278 mova m3, [coeffq+16*25] ;in14 4279 pxor m4, m4 4280 REPX {mova x, m4}, m5, m6, m7 4281 call m(idct_16x8_internal_8bpc).main 4282 mova m7, [rsp+gprsize+16*0] 4283 SAVE_8ROWS rsp+gprsize+16*11, 16 4284 4285 call m(idct_8x32_internal_8bpc).main_fast 4286 jmp m(idct_8x32_internal_8bpc).pass2 4287 4288 .full1: 4289 mova m4, [coeffq+16*2 ] ;in16 4290 mova m5, [coeffq+16*18] ;in20 4291 mova m6, [coeffq+16*3 ] ;in24 4292 mova m7, [coeffq+16*19] ;in26 4293 call m(idct_8x8_internal_8bpc).main 4294 SAVE_7ROWS rsp+gprsize+16*3, 16 4295 4296 mova m0, [coeffq+16*8 ] ;in2 4297 mova m1, [coeffq+16*24] ;in6 4298 mova m2, [coeffq+16*9 ] ;in10 4299 mova m3, [coeffq+16*25] ;in14 4300 mova m4, [coeffq+16*10] ;in18 4301 mova m5, [coeffq+16*26] ;in22 4302 mova m6, [coeffq+16*11] ;in26 4303 mova m7, [coeffq+16*27] ;in30 4304 call m(idct_16x8_internal_8bpc).main 4305 mova m7, [rsp+gprsize+16*0] 4306 SAVE_8ROWS rsp+gprsize+16*11, 16 4307 4308 mova m0, [coeffq+16*6 ] ;in17 4309 mova m1, [coeffq+16*14] ;in19 4310 mova m2, [coeffq+16*22] ;in21 4311 mova m3, [coeffq+16*30] ;in23 4312 mova m4, [coeffq+16*7 ] ;in25 4313 mova m5, [coeffq+16*15] ;in27 4314 mova m6, [coeffq+16*23] ;in29 4315 mova m7, [coeffq+16*31] ;in31 4316 4317 mova [rsp+gprsize+16*33], m0 ;in17 4318 mova [rsp+gprsize+16*28], m1 ;in19 4319 mova [rsp+gprsize+16*29], m2 ;in21 4320 mova [rsp+gprsize+16*32], m3 ;in23 4321 mova [rsp+gprsize+16*31], m4 ;in25 4322 mova [rsp+gprsize+16*30], m5 ;in27 4323 mova [rsp+gprsize+16*27], m6 ;in29 4324 mova [rsp+gprsize+16*34], m7 ;in31 4325 4326 call m(idct_8x32_internal_8bpc).main 4327 jmp m(idct_8x32_internal_8bpc).pass2 4328 4329 4330 cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 4331 %if ARCH_X86_32 4332 LEA r5, $$ 4333 %endif 4334 test eobd, eobd 4335 jz .dconly 4336 4337 call m(idct_32x16_internal_8bpc) 4338 call m(idct_8x16_internal_8bpc).pass2 4339 4340 add coeffq, 16*16 4341 lea dstq, [r3+8] 4342 LOAD_8ROWS rsp+16*11, 16 4343 mova [rsp+16*0], m7 4344 lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] 4345 call m(idct_8x8_internal_8bpc).pass1_end 4346 call m(idct_8x16_internal_8bpc).pass2 4347 4348 add coeffq, 16*16 4349 lea dstq, [r3+8] 4350 LOAD_8ROWS rsp+16*19, 16 4351 mova [rsp+16*0], m7 4352 lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] 4353 call m(idct_8x8_internal_8bpc).pass1_end 4354 call m(idct_8x16_internal_8bpc).pass2 4355 4356 add coeffq, 16*16 4357 lea dstq, [r3+8] 4358 LOAD_8ROWS rsp+16*27, 16 4359 mova [rsp+16*0], m7 4360 lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] 4361 call m(idct_8x8_internal_8bpc).pass1_end 4362 call m(idct_8x16_internal_8bpc).pass2 4363 RET 4364 4365 .dconly: 4366 movd m1, [o(pw_2896x8)] 4367 pmulhrsw m0, m1, [coeffq] 4368 movd m2, [o(pw_16384)] 4369 mov [coeffq], eobd 4370 pmulhrsw m0, m1 4371 mov r3d, 16 4372 lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] 4373 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body 4374 4375 4376 cglobal idct_32x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 4377 add coeffq, 16 4378 lea r3, [o(.pass1_end1)] 4379 .pass1: 4380 LOAD_8ROWS coeffq+16*0, 128, 1 4381 call m(idct_8x8_internal_8bpc).main 4382 SAVE_7ROWS rsp+gprsize+16*3, 16 4383 4384 LOAD_8ROWS coeffq+16*4, 128, 1 4385 call m(idct_16x8_internal_8bpc).main 4386 mova m7, [rsp+gprsize+16*0] 4387 SAVE_8ROWS rsp+gprsize+16*11, 16 4388 4389 LOAD_8ROWS coeffq+16*2, 64, 1 4390 mova [rsp+gprsize+16*19], m0 ;in1 4391 mova [rsp+gprsize+16*26], m1 ;in3 4392 mova [rsp+gprsize+16*23], m2 ;in5 4393 mova [rsp+gprsize+16*22], m3 ;in7 4394 mova [rsp+gprsize+16*21], m4 ;in9 4395 mova [rsp+gprsize+16*24], m5 ;in11 4396 mova [rsp+gprsize+16*25], m6 ;in13 4397 mova [rsp+gprsize+16*20], m7 ;in15 4398 4399 LOAD_8ROWS coeffq+16*34, 64, 1 4400 mova [rsp+gprsize+16*33], m0 ;in17 4401 mova [rsp+gprsize+16*28], m1 ;in19 4402 mova [rsp+gprsize+16*29], m2 ;in21 4403 mova [rsp+gprsize+16*32], m3 ;in23 4404 mova [rsp+gprsize+16*31], m4 ;in25 4405 mova [rsp+gprsize+16*30], m5 ;in27 4406 mova [rsp+gprsize+16*27], m6 ;in29 4407 mova [rsp+gprsize+16*34], m7 ;in31 4408 call m(idct_8x32_internal_8bpc).main 4409 4410 .pass1_end: 4411 mova [rsp+gprsize+16*0 ], m7 4412 mov tx2q, r3 4413 jmp m(idct_8x8_internal_8bpc).pass1_end 4414 4415 .pass1_end1: 4416 SAVE_8ROWS coeffq+16*0, 32 4417 LOAD_8ROWS rsp+gprsize+16*11, 16 4418 mova [rsp+gprsize+16*0 ], m7 4419 lea tx2q, [o(.pass1_end2)] 4420 jmp m(idct_8x8_internal_8bpc).pass1_end 4421 4422 .pass1_end2: 4423 SAVE_8ROWS coeffq+16*16, 32 4424 LOAD_8ROWS rsp+gprsize+16*19, 16 4425 mova [rsp+gprsize+16*0 ], m7 4426 lea tx2q, [o(.pass1_end3)] 4427 jmp m(idct_8x8_internal_8bpc).pass1_end 4428 4429 .pass1_end3: 4430 SAVE_8ROWS coeffq+16*32, 32 4431 LOAD_8ROWS rsp+gprsize+16*27, 16 4432 mova [rsp+gprsize+16*0 ], m7 4433 lea tx2q, [o(.pass1_end4)] 4434 jmp m(idct_8x8_internal_8bpc).pass1_end 4435 4436 .pass1_end4: 4437 SAVE_8ROWS coeffq+16*48, 32 4438 4439 sub coeffq, 16 4440 lea r3, [o(.end)] 4441 jmp .pass1 4442 4443 .end: 4444 ret 4445 4446 4447 cglobal inv_txfm_add_identity_identity_16x32_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 4448 mov r4d, eobd 4449 cmp eobd, 43 ;if (eob > 43) 4450 sbb r3d, r3d ; iteration_count++ 4451 cmp r4d, 150 ;if (eob > 150) 4452 sbb r3d, 0 ; iteration_count++ 4453 cmp r4d, 278 ;if (eob > 278) 4454 sbb r3d, -4 ; iteration_count++ 4455 4456 %if ARCH_X86_32 4457 LEA r5, $$ 4458 %endif 4459 lea r4, [dstq+8] 4460 mov [rsp+16*3], r4 4461 mov [rsp+gprsize+16*3], r3d 4462 mov [rsp+gprsize*2+16*3], coeffq 4463 4464 .loop: 4465 LOAD_8ROWS coeffq, 64, 1 4466 mova [rsp+16*1], m6 4467 pxor m6, m6 4468 REPX {mova [coeffq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 4469 lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] 4470 call m(idct_8x8_internal_8bpc).pass1_end3 4471 mova [rsp+16*0], m2 4472 mova [rsp+16*1], m3 4473 mova [rsp+16*2], m4 4474 mova m3, [o(pw_1697x16)] 4475 mova m4, [o(pw_16384)] 4476 REPX {IDTX16 x, 2, 3, 4}, 5, 6, 7, 0, 1 4477 mova m2, [o(pw_8192)] 4478 REPX {pmulhrsw x, m2}, m5, m6, m7, m0, m1 4479 mova m2, [rsp+16*0] 4480 mova [rsp+16*0], m7 4481 IDTX16 2, 7, 3, 4 4482 mova m7, [rsp+16*2] 4483 mova [rsp+16*2], m5 4484 IDTX16 7, 5, 3, 4 4485 mova m5, [rsp+16*1] 4486 mova [rsp+16*1], m6 4487 pmulhrsw m3, m5 4488 pmulhrsw m3, m4 4489 psrlw m4, 1 ; pw_8192 4490 paddsw m3, m5 4491 pmulhrsw m2, m4 4492 pmulhrsw m3, m4 4493 pmulhrsw m4, m7 4494 call m(idct_8x8_internal_8bpc).end3 4495 lea dstq, [dstq+strideq*2] 4496 add coeffq, 16 4497 dec r3d 4498 jg .loop 4499 mov coeffq, [rsp+gprsize*2+16*3] 4500 add coeffq, 64*8 4501 mov r3d, [rsp+gprsize+16*3] 4502 xor dstq, dstq 4503 mov [rsp+gprsize+16*3], dstq 4504 mov dstq, [rsp+16*3] 4505 test r3d, r3d 4506 jnz .loop 4507 RET 4508 4509 4510 cglobal inv_txfm_add_identity_identity_32x16_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 4511 mov r4d, 12 ;0100b 4512 mov r5d, 136 ;1000 1000b 4513 cmp eobd, 44 ;if (eob > 43) 4514 cmovns r4d, r5d ; iteration_count+2 4515 cmp eobd, 151 ;if (eob > 150) 4516 mov r3d, 34952 ;1000 1000 1000 1000b 4517 cmovs r3d, r4d ; iteration_count += 4 4518 4519 %if ARCH_X86_32 4520 LEA r5, $$ 4521 %endif 4522 lea r4, [dstq+8] 4523 mov [rsp+16*3], r4 4524 4525 .loop: 4526 LOAD_8ROWS coeffq, 32, 1 4527 REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7 4528 mova [rsp+16*1], m6 4529 lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] 4530 call m(idct_8x8_internal_8bpc).pass1_end3 4531 mova [rsp+16*1], m5 4532 mova [rsp+16*2], m6 4533 mova m6, [o(pw_1697x16)] 4534 REPX {IDTX16 x, 5, 6}, 7, 0, 1, 2, 3, 4 4535 pmulhrsw m7, [o(pw_2048)] 4536 mova m5, [rsp+16*1] 4537 mova [rsp+16*0], m7 4538 IDTX16 5, 7, 6 4539 mova m7, [rsp+16*2] 4540 IDTX16 7, 6, 6 4541 mova m6, [o(pw_2048)] 4542 REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 4543 mova [rsp+16*2], m5 4544 mova [rsp+16*1], m7 4545 call m(idct_8x8_internal_8bpc).end3 4546 lea dstq, [dstq+strideq*2] 4547 pxor m7, m7 4548 REPX {mova [coeffq+32*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 4549 4550 .loop_end: 4551 add coeffq, 16 4552 shr r3d, 2 4553 jz .ret 4554 test r3d, 2 4555 jnz .loop 4556 mov r4d, r3d 4557 and r4d, 1 4558 lea coeffq, [coeffq+r4*8+32*7] 4559 mov dstq, [rsp+16*3] 4560 lea r4, [dstq+8] 4561 mov [rsp+16*3], r4 4562 jmp .loop 4563 4564 .ret: 4565 RET 4566 4567 4568 cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 4569 %if ARCH_X86_32 4570 LEA r5, $$ 4571 %endif 4572 test eobd, eobd 4573 jz .dconly 4574 4575 call m(idct_32x32_internal_8bpc) 4576 RET 4577 4578 .dconly: 4579 movd m1, [o(pw_2896x8)] 4580 pmulhrsw m0, m1, [coeffq] 4581 movd m2, [o(pw_8192)] 4582 mov [coeffq], eobd 4583 mov r3d, 32 4584 lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] 4585 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body 4586 4587 4588 cglobal idct_32x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 4589 mov r4d, 2 4590 sub eobd, 136 4591 mov [rsp+gprsize*1+16*35], eobd 4592 mov r3d, 4 4593 cmovs r3d, r4d 4594 4595 %if ARCH_X86_32 4596 LEA r5, $$ 4597 %endif 4598 4599 mov [rsp+gprsize*2+16*35], coeffq 4600 4601 .pass1_loop: 4602 LOAD_8ROWS coeffq+64*1, 64*2 4603 mova [rsp+gprsize+16*19], m0 ;in1 4604 mova [rsp+gprsize+16*26], m1 ;in3 4605 mova [rsp+gprsize+16*23], m2 ;in5 4606 mova [rsp+gprsize+16*22], m3 ;in7 4607 mova [rsp+gprsize+16*21], m4 ;in9 4608 mova [rsp+gprsize+16*24], m5 ;in11 4609 mova [rsp+gprsize+16*25], m6 ;in13 4610 mova [rsp+gprsize+16*20], m7 ;in15 4611 4612 mov tx2d, [rsp+gprsize*1+16*35] 4613 test tx2d, tx2d 4614 jl .fast 4615 4616 .full: 4617 LOAD_8ROWS coeffq+64*0, 64*4 4618 call m(idct_8x8_internal_8bpc).main 4619 SAVE_7ROWS rsp+gprsize+16*3, 16 4620 LOAD_8ROWS coeffq+64*2, 64*4 4621 call m(idct_16x8_internal_8bpc).main 4622 mova m7, [rsp+gprsize+16*0] 4623 SAVE_8ROWS rsp+gprsize+16*11, 16 4624 4625 LOAD_8ROWS coeffq+64*17, 64*2 4626 mova [rsp+gprsize+16*33], m0 ;in17 4627 mova [rsp+gprsize+16*28], m1 ;in19 4628 mova [rsp+gprsize+16*29], m2 ;in21 4629 mova [rsp+gprsize+16*32], m3 ;in23 4630 mova [rsp+gprsize+16*31], m4 ;in25 4631 mova [rsp+gprsize+16*30], m5 ;in27 4632 mova [rsp+gprsize+16*27], m6 ;in29 4633 mova [rsp+gprsize+16*34], m7 ;in31 4634 4635 call m(idct_8x32_internal_8bpc).main 4636 jmp .pass1_end 4637 4638 .fast: 4639 mova m0, [coeffq+256*0] 4640 mova m1, [coeffq+256*1] 4641 mova m2, [coeffq+256*2] 4642 mova m3, [coeffq+256*3] 4643 pxor m4, m4 4644 REPX {mova x, m4}, m5, m6, m7 4645 call m(idct_8x8_internal_8bpc).main 4646 4647 SAVE_7ROWS rsp+gprsize+16*3, 16 4648 mova m0, [coeffq+128*1] 4649 mova m1, [coeffq+128*3] 4650 mova m2, [coeffq+128*5] 4651 mova m3, [coeffq+128*7] 4652 pxor m4, m4 4653 REPX {mova x, m4}, m5, m6, m7 4654 call m(idct_16x8_internal_8bpc).main 4655 mova m7, [rsp+gprsize+16*0] 4656 SAVE_8ROWS rsp+gprsize+16*11, 16 4657 4658 call m(idct_8x32_internal_8bpc).main_fast 4659 4660 .pass1_end: 4661 mova [rsp+gprsize+16*0], m7 4662 mova m7, [o(pw_8192)] 4663 lea tx2q, [o(.pass1_end1)] 4664 jmp m(idct_8x8_internal_8bpc).pass1_end1 4665 4666 .pass1_end1: 4667 SAVE_8ROWS coeffq+64*0, 64 4668 LOAD_8ROWS rsp+gprsize+16*11, 16 4669 mova [rsp+gprsize+16*0], m7 4670 mova m7, [o(pw_8192)] 4671 lea tx2q, [o(.pass1_end2)] 4672 jmp m(idct_8x8_internal_8bpc).pass1_end1 4673 4674 .pass1_end2: 4675 SAVE_8ROWS coeffq+64*8, 64 4676 LOAD_8ROWS rsp+gprsize+16*19, 16 4677 mova [rsp+gprsize+16*0], m7 4678 mova m7, [o(pw_8192)] 4679 lea tx2q, [o(.pass1_end3)] 4680 jmp m(idct_8x8_internal_8bpc).pass1_end1 4681 4682 .pass1_end3: 4683 SAVE_8ROWS coeffq+64*16, 64 4684 LOAD_8ROWS rsp+gprsize+16*27, 16 4685 mova [rsp+gprsize+16*0], m7 4686 mova m7, [o(pw_8192)] 4687 lea tx2q, [o(.pass1_end4)] 4688 jmp m(idct_8x8_internal_8bpc).pass1_end1 4689 4690 .pass1_end4: 4691 SAVE_8ROWS coeffq+64*24, 64 4692 4693 add coeffq, 16 4694 dec r3d 4695 jg .pass1_loop 4696 4697 4698 .pass2: 4699 mov coeffq, [rsp+gprsize*2+16*35] 4700 mov r3d, 4 4701 lea tx2q, [o(.pass2_end)] 4702 4703 .pass2_loop: 4704 mov [rsp+gprsize*3+16*35], r3d 4705 lea r3, [dstq+8] 4706 mov [rsp+gprsize*2+16*35], r3 4707 4708 mova m0, [coeffq+16*4 ] 4709 mova m1, [coeffq+16*12] 4710 mova m2, [coeffq+16*20] 4711 mova m3, [coeffq+16*28] 4712 mova m4, [coeffq+16*5 ] 4713 mova m5, [coeffq+16*13] 4714 mova m6, [coeffq+16*21] 4715 mova m7, [coeffq+16*29] 4716 mova [rsp+gprsize+16*19], m0 ;in1 4717 mova [rsp+gprsize+16*26], m1 ;in3 4718 mova [rsp+gprsize+16*23], m2 ;in5 4719 mova [rsp+gprsize+16*22], m3 ;in7 4720 mova [rsp+gprsize+16*21], m4 ;in9 4721 mova [rsp+gprsize+16*24], m5 ;in11 4722 mova [rsp+gprsize+16*25], m6 ;in13 4723 mova [rsp+gprsize+16*20], m7 ;in15 4724 4725 mov eobd, [rsp+gprsize*1+16*35] 4726 test eobd, eobd 4727 jl .fast1 4728 4729 .full1: 4730 mova m0, [coeffq+16*0 ] 4731 mova m1, [coeffq+16*16] 4732 mova m2, [coeffq+16*1 ] 4733 mova m3, [coeffq+16*17] 4734 mova m4, [coeffq+16*2 ] 4735 mova m5, [coeffq+16*18] 4736 mova m6, [coeffq+16*3 ] 4737 mova m7, [coeffq+16*19] 4738 call m(idct_8x8_internal_8bpc).main 4739 SAVE_7ROWS rsp+gprsize+16*3, 16 4740 4741 mova m0, [coeffq+16*8 ] 4742 mova m1, [coeffq+16*24] 4743 mova m2, [coeffq+16*9 ] 4744 mova m3, [coeffq+16*25] 4745 mova m4, [coeffq+16*10] 4746 mova m5, [coeffq+16*26] 4747 mova m6, [coeffq+16*11] 4748 mova m7, [coeffq+16*27] 4749 call m(idct_16x8_internal_8bpc).main 4750 mova m7, [rsp+gprsize+16*0] 4751 SAVE_8ROWS rsp+gprsize+16*11, 16 4752 4753 mova m0, [coeffq+16*6 ] 4754 mova m1, [coeffq+16*14] 4755 mova m2, [coeffq+16*22] 4756 mova m3, [coeffq+16*30] 4757 mova m4, [coeffq+16*7 ] 4758 mova m5, [coeffq+16*15] 4759 mova m6, [coeffq+16*23] 4760 mova m7, [coeffq+16*31] 4761 mova [rsp+gprsize+16*33], m0 ;in17 4762 mova [rsp+gprsize+16*28], m1 ;in19 4763 mova [rsp+gprsize+16*29], m2 ;in21 4764 mova [rsp+gprsize+16*32], m3 ;in23 4765 mova [rsp+gprsize+16*31], m4 ;in25 4766 mova [rsp+gprsize+16*30], m5 ;in27 4767 mova [rsp+gprsize+16*27], m6 ;in29 4768 mova [rsp+gprsize+16*34], m7 ;in31 4769 4770 call m(idct_8x32_internal_8bpc).main 4771 jmp tx2q 4772 4773 .fast1: 4774 mova m0, [coeffq+16*0 ] 4775 mova m1, [coeffq+16*16] 4776 mova m2, [coeffq+16*1 ] 4777 mova m3, [coeffq+16*17] 4778 pxor m4, m4 4779 REPX {mova x, m4}, m5, m6, m7 4780 call m(idct_8x8_internal_8bpc).main 4781 SAVE_7ROWS rsp+gprsize+16*3, 16 4782 4783 mova m0, [coeffq+16*8 ] 4784 mova m1, [coeffq+16*24] 4785 mova m2, [coeffq+16*9 ] 4786 mova m3, [coeffq+16*25] 4787 pxor m4, m4 4788 REPX {mova x, m4}, m5, m6, m7 4789 call m(idct_16x8_internal_8bpc).main 4790 mova m7, [rsp+gprsize+16*0] 4791 SAVE_8ROWS rsp+gprsize+16*11, 16 4792 4793 call m(idct_8x32_internal_8bpc).main_fast 4794 jmp tx2q 4795 4796 .pass2_end: 4797 lea r3, [o(.pass2_end1)] 4798 jmp m(idct_8x32_internal_8bpc).end 4799 4800 .pass2_end1: 4801 lea tx2q, [o(.pass2_end)] 4802 add coeffq, 16*32 4803 mov dstq, [rsp+gprsize*2+16*35] 4804 mov r3d, [rsp+gprsize*3+16*35] 4805 dec r3d 4806 jg .pass2_loop 4807 4808 ret 4809 4810 4811 cglobal inv_txfm_add_identity_identity_32x32_8bpc, 4, 6, 8, 16*5, dst, stride, coeff, eob, tx2 4812 mov r4d, 2 4813 cmp eobd, 136 4814 mov r3d, 4 4815 cmovs r3d, r4d 4816 4817 %if ARCH_X86_32 4818 LEA r5, $$ 4819 %endif 4820 4821 lea r4, [dstq+8] 4822 mov [rsp+gprsize*0+16*3], r4 4823 mov [rsp+gprsize*1+16*3], r3d 4824 mov [rsp+gprsize*2+16*3], r3d 4825 mov [rsp+gprsize*3+16*3], coeffq 4826 4827 .loop: 4828 LOAD_8ROWS coeffq, 64 4829 mova [rsp+16*1], m6 4830 lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] 4831 call m(idct_8x8_internal_8bpc).pass1_end3 4832 pmulhrsw m7, [o(pw_8192)] 4833 mova [rsp+16*0], m7 4834 mova m7, [o(pw_8192)] 4835 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 4836 mova [rsp+16*1], m6 4837 mova [rsp+16*2], m5 4838 call m(idct_8x8_internal_8bpc).end3 4839 lea dstq, [dstq+strideq*2] 4840 4841 pxor m7, m7 4842 REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 4843 4844 add coeffq, 16 4845 dec r3d 4846 jg .loop 4847 4848 mov r4d, [rsp+gprsize*2+16*3] 4849 dec r4d 4850 jle .ret 4851 4852 mov dstq, [rsp+gprsize*0+16*3] 4853 mov coeffq, [rsp+gprsize*3+16*3] 4854 mov [rsp+gprsize*2+16*3], r4 4855 lea r3, [dstq+8] 4856 add coeffq, 64*8 4857 mov [rsp+gprsize*0+16*3], r3 4858 mov r3d, [rsp+gprsize*1+16*3] 4859 mov [rsp+gprsize*3+16*3], coeffq 4860 jmp .loop 4861 4862 .ret: 4863 RET 4864 4865 4866 cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 4867 %if ARCH_X86_32 4868 LEA r5, $$ 4869 %endif 4870 test eobd, eobd 4871 jz .dconly 4872 call m(idct_16x64_internal_8bpc) 4873 .end: 4874 RET 4875 4876 .dconly: 4877 movd m1, [o(pw_2896x8)] 4878 pmulhrsw m0, m1, [coeffq] 4879 movd m2, [o(pw_8192)] 4880 mov [coeffq], eobd 4881 mov r2d, 32 4882 lea tx2q, [o(.end)] 4883 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly 4884 4885 4886 cglobal idct_16x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 4887 mov r4d, 2 4888 sub eobd, 151 4889 mov [rsp+gprsize*1+16*67], eobd 4890 mov r3d, 4 4891 cmovs r3d, r4d 4892 4893 %if ARCH_X86_32 4894 LEA r5, $$ 4895 %endif 4896 4897 mov [rsp+gprsize*2+16*67], coeffq 4898 4899 .pass1_loop: 4900 LOAD_8ROWS coeffq+64*0, 64*2 4901 call m(idct_8x8_internal_8bpc).main 4902 SAVE_7ROWS rsp+gprsize+16*3, 16 4903 LOAD_8ROWS coeffq+64*1, 64*2 4904 call m(idct_16x8_internal_8bpc).main 4905 mova m7, [o(pw_8192)] 4906 lea tx2q, [o(.pass1_end)] 4907 jmp m(idct_8x8_internal_8bpc).pass1_end1 4908 4909 .pass1_end: 4910 SAVE_8ROWS coeffq+64*8, 64 4911 LOAD_8ROWS rsp+gprsize+16*3, 16 4912 mova [rsp+gprsize+16*0], m7 4913 mova m7, [o(pw_8192)] 4914 lea tx2q, [o(.pass1_end1)] 4915 jmp m(idct_8x8_internal_8bpc).pass1_end1 4916 4917 .pass1_end1: 4918 SAVE_8ROWS coeffq+64*0, 64 4919 4920 add coeffq, 16 4921 dec r3d 4922 jg .pass1_loop 4923 4924 mov coeffq, [rsp+gprsize*2+16*67] 4925 mov r3d, 2 4926 lea r4, [dstq+8] 4927 mov [rsp+gprsize*2+16*67], r4 4928 lea r4, [o(.end1)] 4929 4930 .pass2_loop: 4931 mov [rsp+gprsize*3+16*67], r3d 4932 mov eobd, [rsp+gprsize*1+16*67] 4933 4934 mova m0, [coeffq+16*4 ] ;in1 4935 mova m1, [coeffq+16*12] ;in3 4936 mova m2, [coeffq+16*20] ;in5 4937 mova m3, [coeffq+16*28] ;in7 4938 mova m4, [coeffq+16*5 ] ;in9 4939 mova m5, [coeffq+16*13] ;in11 4940 mova m6, [coeffq+16*21] ;in13 4941 mova m7, [coeffq+16*29] ;in15 4942 mova [rsp+gprsize+16*35], m0 ;in1 4943 mova [rsp+gprsize+16*49], m1 ;in3 4944 mova [rsp+gprsize+16*43], m2 ;in5 4945 mova [rsp+gprsize+16*41], m3 ;in7 4946 mova [rsp+gprsize+16*39], m4 ;in9 4947 mova [rsp+gprsize+16*45], m5 ;in11 4948 mova [rsp+gprsize+16*47], m6 ;in13 4949 mova [rsp+gprsize+16*37], m7 ;in15 4950 4951 pxor m4, m4 4952 mova m0, [coeffq+16*0] 4953 mova m1, [coeffq+16*1] 4954 4955 test eobd, eobd 4956 jl .fast 4957 4958 .full: 4959 mova m2, [coeffq+16*2] 4960 mova m3, [coeffq+16*3] 4961 4962 REPX {mova x, m4}, m5, m6, m7 4963 call m(idct_8x8_internal_8bpc).main 4964 SAVE_7ROWS rsp+gprsize+16*3, 16 4965 4966 pxor m4, m4 4967 mova m0, [coeffq+16*16] 4968 mova m1, [coeffq+16*17] 4969 mova m2, [coeffq+16*18] 4970 mova m3, [coeffq+16*19] 4971 4972 REPX {mova x, m4}, m5, m6, m7 4973 call m(idct_16x8_internal_8bpc).main 4974 mova m7, [rsp+gprsize+16*0] 4975 SAVE_8ROWS rsp+gprsize+16*11, 16 4976 4977 mova m0, [coeffq+16*8 ] 4978 mova m1, [coeffq+16*24] 4979 mova m2, [coeffq+16*9 ] 4980 mova m3, [coeffq+16*25] 4981 mova m4, [coeffq+16*10] 4982 mova m5, [coeffq+16*26] 4983 mova m6, [coeffq+16*11] 4984 mova m7, [coeffq+16*27] 4985 mova [rsp+gprsize+16*19], m0 4986 mova [rsp+gprsize+16*26], m1 4987 mova [rsp+gprsize+16*23], m2 4988 mova [rsp+gprsize+16*22], m3 4989 mova [rsp+gprsize+16*21], m4 4990 mova [rsp+gprsize+16*24], m5 4991 mova [rsp+gprsize+16*25], m6 4992 mova [rsp+gprsize+16*20], m7 4993 4994 call m(idct_8x32_internal_8bpc).main_fast 4995 SAVE_8ROWS rsp+gprsize+16*3, 16 4996 4997 mova m0, [coeffq+16*6 ] ;in17 4998 mova m1, [coeffq+16*14] ;in19 4999 mova m2, [coeffq+16*22] ;in21 5000 mova m3, [coeffq+16*30] ;in23 5001 mova m4, [coeffq+16*7 ] ;in25 5002 mova m5, [coeffq+16*15] ;in27 5003 mova m6, [coeffq+16*23] ;in29 5004 mova m7, [coeffq+16*31] ;in31 5005 mova [rsp+gprsize+16*63], m0 ;in17 5006 mova [rsp+gprsize+16*53], m1 ;in19 5007 mova [rsp+gprsize+16*55], m2 ;in21 5008 mova [rsp+gprsize+16*61], m3 ;in23 5009 mova [rsp+gprsize+16*59], m4 ;in25 5010 mova [rsp+gprsize+16*57], m5 ;in27 5011 mova [rsp+gprsize+16*51], m6 ;in29 5012 mova [rsp+gprsize+16*65], m7 ;in31 5013 5014 call .main 5015 jmp .end 5016 5017 .fast: 5018 REPX {mova x, m4}, m2, m3, m5, m6, m7 5019 call m(idct_8x8_internal_8bpc).main 5020 SAVE_7ROWS rsp+gprsize+16*3, 16 5021 5022 pxor m4, m4 5023 mova m0, [coeffq+16*16] 5024 mova m1, [coeffq+16*17] 5025 5026 REPX {mova x, m4}, m2, m3, m5, m6, m7 5027 call m(idct_16x8_internal_8bpc).main 5028 mova m7, [rsp+gprsize+16*0] 5029 SAVE_8ROWS rsp+gprsize+16*11, 16 5030 5031 mova m0, [coeffq+16*8 ] 5032 mova m1, [coeffq+16*24] 5033 mova m2, [coeffq+16*9 ] 5034 mova m3, [coeffq+16*25] 5035 mova [rsp+gprsize+16*19], m0 ;in1 5036 mova [rsp+gprsize+16*26], m1 ;in3 5037 mova [rsp+gprsize+16*23], m2 ;in5 5038 mova [rsp+gprsize+16*22], m3 ;in7 5039 5040 call m(idct_8x32_internal_8bpc).main_veryfast 5041 SAVE_8ROWS rsp+gprsize+16*3, 16 5042 5043 call .main_fast 5044 5045 .end: 5046 LOAD_8ROWS rsp+gprsize+16*3, 16 5047 mova [rsp+gprsize+16*0], m7 5048 mov r3, r4 5049 jmp m(idct_8x32_internal_8bpc).end2 5050 5051 .end1: 5052 LOAD_8ROWS rsp+gprsize+16*35, 16 5053 lea dstq, [dstq+strideq*2] 5054 lea r3, [rsp+16*32+gprsize] 5055 call .write 5056 mov dstq, [rsp+gprsize*2+16*67] 5057 mov r3d, [rsp+gprsize*3+16*67] 5058 lea r4, [dstq+8] 5059 mov [rsp+gprsize*2+16*67], r4 5060 lea r4, [o(.end1)] 5061 5062 dec r3d 5063 jg .pass2_loop 5064 ret 5065 .write: 5066 mova [r3+16*0], m7 5067 mov r4, -16*32 5068 pxor m7, m7 5069 sub coeffq, r4 5070 .zero_loop: 5071 mova [coeffq+r4+16*0], m7 5072 mova [coeffq+r4+16*1], m7 5073 add r4, 16*2 5074 jl .zero_loop 5075 call .write_main2 5076 LOAD_8ROWS r3+16*11, 16 5077 call .write_main 5078 LOAD_8ROWS r3+16*19, 16 5079 call .write_main 5080 LOAD_8ROWS r3+16*27, 16 5081 .write_main: 5082 mova [r3+16*0], m7 5083 .write_main2: 5084 mova m7, [o(pw_2048)] 5085 REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 5086 pmulhrsw m7, [r3+16*0] 5087 mova [r3+16*2], m5 5088 mova [r3+16*1], m6 5089 mova [r3+16*0], m7 5090 WRITE_8X4 0, 1, 2, 3, 5, 6, 7 5091 lea dstq, [dstq+strideq*2] 5092 WRITE_8X4 4, [r3+16*2], [r3+16*1], [r3+16*0], 5, 6, 7 5093 lea dstq, [dstq+strideq*2] 5094 ret 5095 5096 5097 ALIGN function_align 5098 cglobal_label .main_fast 5099 mova m0, [rsp+gprsize*2+16*35] ;in1 5100 pmulhrsw m3, m0, [o(pw_4095x8)] ;t62,t63 5101 pmulhrsw m0, [o(pw_101x8)] ;t32,t33 5102 mova m7, [o(pd_2048)] 5103 mova [rsp+gprsize*2+16*35], m0 ;t32 5104 mova [rsp+gprsize*2+16*66], m3 ;t63 5105 ITX_MULSUB_2W 3, 0, 1, 2, 7, 401, 4076 ;t33a, t62a 5106 mova [rsp+gprsize*2+16*36], m3 ;t33a 5107 mova [rsp+gprsize*2+16*65], m0 ;t62a 5108 5109 mova m1, [rsp+gprsize*2+16*37] ;in15 5110 pmulhrsw m2, m1, [o(pw_3822x8)] ;t60,t61 5111 pmulhrsw m1, [o(pw_m1474x8)] ;t34,t35 5112 mova [rsp+gprsize*2+16*38], m1 ;t35 5113 mova [rsp+gprsize*2+16*63], m2 ;t60 5114 ITX_MULSUB_2W 2, 1, 0, 3, 7, m4076, 401 ;t34a, t61a 5115 mova [rsp+gprsize*2+16*37], m2 ;t34a 5116 mova [rsp+gprsize*2+16*64], m1 ;t61a 5117 5118 mova m0, [rsp+gprsize*2+16*39] ;in9 5119 pmulhrsw m3, m0, [o(pw_3996x8)] ;t58,t59 5120 pmulhrsw m0, [o(pw_897x8)] ;t36,t37 5121 mova [rsp+gprsize*2+16*39], m0 ;t36 5122 mova [rsp+gprsize*2+16*62], m3 ;t59 5123 ITX_MULSUB_2W 3, 0, 1, 2, 7, 3166, 2598 ;t37a, t58a 5124 mova [rsp+gprsize*2+16*40], m3 ;t37a 5125 mova [rsp+gprsize*2+16*61], m0 ;t58a 5126 5127 mova m1, [rsp+gprsize*2+16*41] ;in7 5128 pmulhrsw m2, m1, [o(pw_4036x8)] ;t56,t57 5129 pmulhrsw m1, [o(pw_m700x8)] ;t38,t39 5130 mova [rsp+gprsize*2+16*42], m1 ;t39 5131 mova [rsp+gprsize*2+16*59], m2 ;t56 5132 ITX_MULSUB_2W 2, 1, 0, 3, 7, m2598, 3166 ;t38a, t57a 5133 mova [rsp+gprsize*2+16*41], m2 ;t38a 5134 mova [rsp+gprsize*2+16*60], m1 ;t57a 5135 5136 mova m0, [rsp+gprsize*2+16*43] ;in5 5137 pmulhrsw m3, m0, [o(pw_4065x8)] ;t54,t55 5138 pmulhrsw m0, [o(pw_501x8)] ;t40,t41 5139 mova [rsp+gprsize*2+16*43], m0 ;t40 5140 mova [rsp+gprsize*2+16*58], m3 ;t55 5141 ITX_MULSUB_2W 3, 0, 1, 2, 7, 1931, 3612 ;t41a, t54a 5142 mova [rsp+gprsize*2+16*44], m3 ;t41a 5143 mova [rsp+gprsize*2+16*57], m0 ;t54a 5144 5145 mova m1, [rsp+gprsize*2+16*45] ;in11 5146 pmulhrsw m2, m1, [o(pw_3948x8)] ;t52,t53 5147 pmulhrsw m1, [o(pw_m1092x8)] ;t42,t43 5148 mova [rsp+gprsize*2+16*46], m1 ;t43 5149 mova [rsp+gprsize*2+16*55], m2 ;t52 5150 ITX_MULSUB_2W 2, 1, 0, 3, 7, m3612, 1931 ;t42a, t53a 5151 mova [rsp+gprsize*2+16*45], m2 ;t42a 5152 mova [rsp+gprsize*2+16*56], m1 ;t53a 5153 5154 mova m0, [rsp+gprsize*2+16*47] ;in13 5155 pmulhrsw m3, m0, [o(pw_3889x8)] ;t50,t51 5156 pmulhrsw m0, [o(pw_1285x8)] ;t44,t45 5157 mova m6, m0 5158 mova [rsp+gprsize*2+16*54], m3 ;t51 5159 ITX_MULSUB_2W 3, 0, 1, 2, 7, 3920, 1189 ;t45a, t50a 5160 mova [rsp+gprsize*2+16*48], m3 ;t45a 5161 mova [rsp+gprsize*2+16*53], m0 ;t50a 5162 5163 mova m0, [rsp+gprsize*2+16*49] ;in3 5164 pmulhrsw m3, m0, [o(pw_4085x8)] ;t48,t49 5165 pmulhrsw m0, [o(pw_m301x8)] ;t46,t47 5166 mova m4, m3 5167 mova m5, m0 5168 5169 jmp .main2 5170 5171 ALIGN function_align 5172 cglobal_label .main 5173 mova m0, [rsp+gprsize*2+16*35] ;in1 5174 mova m1, [rsp+gprsize*2+16*65] ;in31 5175 pmulhrsw m3, m0, [o(pw_4095x8)] ;t63a 5176 pmulhrsw m0, [o(pw_101x8)] ;t32a 5177 pmulhrsw m2, m1, [o(pw_2967x8)] ;t62a 5178 pmulhrsw m1, [o(pw_m2824x8)] ;t33a 5179 mova m7, [o(pd_2048)] 5180 psubsw m4, m0, m1 ;t33 5181 paddsw m0, m1 ;t32 5182 psubsw m5, m3, m2 ;t62 5183 paddsw m3, m2 ;t63 5184 ITX_MULSUB_2W 5, 4, 1, 2, 7, 401, 4076 ;t33a, t62a 5185 mova [rsp+gprsize*2+16*35], m0 ;t32 5186 mova [rsp+gprsize*2+16*36], m5 ;t33a 5187 mova [rsp+gprsize*2+16*65], m4 ;t62a 5188 mova [rsp+gprsize*2+16*66], m3 ;t63 5189 5190 mova m0, [rsp+gprsize*2+16*63] ;in17 5191 mova m1, [rsp+gprsize*2+16*37] ;in15 5192 pmulhrsw m3, m0, [o(pw_3745x8)] ;t61a 5193 pmulhrsw m0, [o(pw_1660x8)] ;t34a 5194 pmulhrsw m2, m1, [o(pw_3822x8)] ;t60a 5195 pmulhrsw m1, [o(pw_m1474x8)] ;t35a 5196 psubsw m4, m1, m0 ;t34 5197 paddsw m0, m1 ;t35 5198 psubsw m5, m2, m3 ;t61 5199 paddsw m3, m2 ;t60 5200 ITX_MULSUB_2W 5, 4, 1, 2, 7, m4076, 401 ;t34a, t61a 5201 mova [rsp+gprsize*2+16*37], m5 ;t34a 5202 mova [rsp+gprsize*2+16*38], m0 ;t35 5203 mova [rsp+gprsize*2+16*63], m3 ;t60 5204 mova [rsp+gprsize*2+16*64], m4 ;t61a 5205 5206 mova m0, [rsp+gprsize*2+16*39] ;in9 5207 mova m1, [rsp+gprsize*2+16*61] ;in23 5208 pmulhrsw m3, m0, [o(pw_3996x8)] ;t59a 5209 pmulhrsw m0, [o(pw_897x8)] ;t36a 5210 pmulhrsw m2, m1, [o(pw_3461x8)] ;t58a 5211 pmulhrsw m1, [o(pw_m2191x8)] ;t37a 5212 psubsw m4, m0, m1 ;t37 5213 paddsw m0, m1 ;t36 5214 psubsw m5, m3, m2 ;t58 5215 paddsw m3, m2 ;t59 5216 ITX_MULSUB_2W 5, 4, 1, 2, 7, 3166, 2598 ;t37a, t58a 5217 mova [rsp+gprsize*2+16*39], m0 ;t36 5218 mova [rsp+gprsize*2+16*40], m5 ;t37a 5219 mova [rsp+gprsize*2+16*61], m4 ;t58a 5220 mova [rsp+gprsize*2+16*62], m3 ;t59 5221 5222 mova m0, [rsp+gprsize*2+16*59] ;in25 5223 mova m1, [rsp+gprsize*2+16*41] ;in7 5224 pmulhrsw m3, m0, [o(pw_3349x8)] ;t57a 5225 pmulhrsw m0, [o(pw_2359x8)] ;t38a 5226 pmulhrsw m2, m1, [o(pw_4036x8)] ;t56a 5227 pmulhrsw m1, [o(pw_m700x8)] ;t39a 5228 psubsw m4, m1, m0 ;t38 5229 paddsw m0, m1 ;t39 5230 psubsw m5, m2, m3 ;t57 5231 paddsw m3, m2 ;t56 5232 ITX_MULSUB_2W 5, 4, 1, 2, 7, m2598, 3166 ;t38a, t57a 5233 mova [rsp+gprsize*2+16*41], m5 ;t38a 5234 mova [rsp+gprsize*2+16*42], m0 ;t39 5235 mova [rsp+gprsize*2+16*59], m3 ;t56 5236 mova [rsp+gprsize*2+16*60], m4 ;t57a 5237 5238 mova m0, [rsp+gprsize*2+16*43] ;in5 5239 mova m1, [rsp+gprsize*2+16*57] ;in27 5240 pmulhrsw m3, m0, [o(pw_4065x8)] ;t55a 5241 pmulhrsw m0, [o(pw_501x8)] ;t40a 5242 pmulhrsw m2, m1, [o(pw_3229x8)] ;t54a 5243 pmulhrsw m1, [o(pw_m2520x8)] ;t41a 5244 psubsw m4, m0, m1 ;t41 5245 paddsw m0, m1 ;t40 5246 psubsw m5, m3, m2 ;t54 5247 paddsw m3, m2 ;t55 5248 ITX_MULSUB_2W 5, 4, 1, 2, 7, 1931, 3612 ;t41a, t54a 5249 mova [rsp+gprsize*2+16*43], m0 ;t40 5250 mova [rsp+gprsize*2+16*44], m5 ;t41a 5251 mova [rsp+gprsize*2+16*57], m4 ;t54a 5252 mova [rsp+gprsize*2+16*58], m3 ;t55 5253 5254 mova m0, [rsp+gprsize*2+16*55] ;in21 5255 mova m1, [rsp+gprsize*2+16*45] ;in11 5256 pmulhrsw m3, m0, [o(pw_3564x8)] ;t53a 5257 pmulhrsw m0, [o(pw_2019x8)] ;t42a 5258 pmulhrsw m2, m1, [o(pw_3948x8)] ;t52a 5259 pmulhrsw m1, [o(pw_m1092x8)] ;t43a 5260 psubsw m4, m1, m0 ;t42 5261 paddsw m0, m1 ;t43 5262 psubsw m5, m2, m3 ;t53 5263 paddsw m3, m2 ;t52 5264 ITX_MULSUB_2W 5, 4, 1, 2, 7, m3612, 1931 ;t42a, t53a 5265 mova [rsp+gprsize*2+16*45], m5 ;t42a 5266 mova [rsp+gprsize*2+16*46], m0 ;t43 5267 mova [rsp+gprsize*2+16*55], m3 ;t52 5268 mova [rsp+gprsize*2+16*56], m4 ;t53a 5269 5270 mova m0, [rsp+gprsize*2+16*47] ;in13 5271 mova m1, [rsp+gprsize*2+16*53] ;in19 5272 pmulhrsw m3, m0, [o(pw_3889x8)] ;t51a 5273 pmulhrsw m0, [o(pw_1285x8)] ;t44a 5274 pmulhrsw m2, m1, [o(pw_3659x8)] ;t50a 5275 pmulhrsw m1, [o(pw_m1842x8)] ;t45a 5276 psubsw m4, m0, m1 ;t45 5277 paddsw m0, m1 ;t44 5278 psubsw m5, m3, m2 ;t50 5279 paddsw m3, m2 ;t51 5280 ITX_MULSUB_2W 5, 4, 1, 2, 7, 3920, 1189 ;t45a, t50a 5281 mova m6, m0 5282 mova [rsp+gprsize*2+16*48], m5 ;t45a 5283 mova [rsp+gprsize*2+16*53], m4 ;t50a 5284 mova [rsp+gprsize*2+16*54], m3 ;t51 5285 5286 mova m0, [rsp+gprsize*2+16*51] ;in29 5287 mova m1, [rsp+gprsize*2+16*49] ;in3 5288 pmulhrsw m3, m0, [o(pw_3102x8)] ;t49a 5289 pmulhrsw m0, [o(pw_2675x8)] ;t46a 5290 pmulhrsw m2, m1, [o(pw_4085x8)] ;t48a 5291 pmulhrsw m1, [o(pw_m301x8)] ;t47a 5292 psubsw m5, m1, m0 ;t46 5293 paddsw m0, m1 ;t47 5294 psubsw m4, m2, m3 ;t49 5295 paddsw m3, m2 ;t48 5296 5297 ALIGN function_align 5298 .main2: 5299 ITX_MULSUB_2W 4, 5, 1, 2, 7, m1189, 3920 ;t46a, t49a 5300 mova m1, [rsp+gprsize*2+16*54] ;t51 5301 psubsw m2, m0, m6 ;t44a 5302 paddsw m0, m6 ;t47a 5303 psubsw m6, m3, m1 ;t51a 5304 paddsw m3, m1 ;t48a 5305 mova [rsp+gprsize*2+16*50], m0 ;t47a 5306 mova [rsp+gprsize*2+16*51], m3 ;t48a 5307 ITX_MULSUB_2W 6, 2, 0, 3, 7, m2276, 3406 ;t44, t51 5308 mova [rsp+gprsize*2+16*47], m6 ;t44 5309 mova [rsp+gprsize*2+16*54], m2 ;t51 5310 5311 mova m0, [rsp+gprsize*2+16*48] ;t45a 5312 mova m3, [rsp+gprsize*2+16*53] ;t50a 5313 psubsw m2, m4, m0 ;t45 5314 paddsw m4, m0 ;t46 5315 psubsw m6, m5, m3 ;t50 5316 paddsw m5, m3 ;t49 5317 ITX_MULSUB_2W 6, 2, 0, 3, 7, m2276, 3406 ;t45a, t50a 5318 mova [rsp+gprsize*2+16*48], m6 ;t45a 5319 mova [rsp+gprsize*2+16*49], m4 ;t46 5320 mova [rsp+gprsize*2+16*52], m5 ;t49 5321 mova [rsp+gprsize*2+16*53], m2 ;t50a 5322 5323 mova m0, [rsp+gprsize*2+16*43] ;t40 5324 mova m2, [rsp+gprsize*2+16*46] ;t43 5325 mova m3, [rsp+gprsize*2+16*55] ;t52 5326 mova m1, [rsp+gprsize*2+16*58] ;t55 5327 psubsw m4, m0, m2 ;t43a 5328 paddsw m0, m2 ;t40a 5329 psubsw m5, m1, m3 ;t52a 5330 paddsw m1, m3 ;t55a 5331 ITX_MULSUB_2W 5, 4, 2, 3, 7, 3406, 2276 ;t43, t52 5332 mova [rsp+gprsize*2+16*43], m0 ;t40a 5333 mova [rsp+gprsize*2+16*46], m5 ;t43 5334 mova [rsp+gprsize*2+16*55], m4 ;t52 5335 mova [rsp+gprsize*2+16*58], m1 ;t55a 5336 5337 mova m0, [rsp+gprsize*2+16*44] ;t41a 5338 mova m2, [rsp+gprsize*2+16*45] ;t42a 5339 mova m3, [rsp+gprsize*2+16*56] ;t53a 5340 mova m1, [rsp+gprsize*2+16*57] ;t54a 5341 psubsw m4, m0, m2 ;t42 5342 paddsw m0, m2 ;t41 5343 psubsw m5, m1, m3 ;t53 5344 paddsw m1, m3 ;t54 5345 ITX_MULSUB_2W 5, 4, 2, 3, 7, 3406, 2276 ;t42a, t53a 5346 mova [rsp+gprsize*2+16*44], m0 ;t41 5347 mova [rsp+gprsize*2+16*45], m5 ;t42a 5348 mova [rsp+gprsize*2+16*56], m4 ;t53a 5349 mova [rsp+gprsize*2+16*57], m1 ;t54 5350 5351 mova m0, [rsp+gprsize*2+16*41] ;t38a 5352 mova m2, [rsp+gprsize*2+16*40] ;t37a 5353 mova m3, [rsp+gprsize*2+16*61] ;t58a 5354 mova m1, [rsp+gprsize*2+16*60] ;t57a 5355 psubsw m4, m0, m2 ;t37 5356 paddsw m0, m2 ;t38 5357 psubsw m5, m1, m3 ;t58 5358 paddsw m1, m3 ;t57 5359 ITX_MULSUB_2W 5, 4, 2, 3, 7, m4017, 799 ;t37a, t58a 5360 mova [rsp+gprsize*2+16*41], m0 ;t38 5361 mova [rsp+gprsize*2+16*40], m5 ;t37a 5362 mova [rsp+gprsize*2+16*61], m4 ;t58a 5363 mova [rsp+gprsize*2+16*60], m1 ;t57 5364 5365 mova m0, [rsp+gprsize*2+16*42] ;t39 5366 mova m2, [rsp+gprsize*2+16*39] ;t36 5367 mova m3, [rsp+gprsize*2+16*62] ;t59 5368 mova m1, [rsp+gprsize*2+16*59] ;t56 5369 psubsw m4, m0, m2 ;t36a 5370 paddsw m0, m2 ;t39a 5371 psubsw m5, m1, m3 ;t59a 5372 paddsw m1, m3 ;t56a 5373 ITX_MULSUB_2W 5, 4, 2, 3, 7, m4017, 799 ;t36, t59 5374 mova [rsp+gprsize*2+16*42], m0 ;t39a 5375 mova [rsp+gprsize*2+16*39], m5 ;t36 5376 mova [rsp+gprsize*2+16*62], m4 ;t59 5377 mova [rsp+gprsize*2+16*59], m1 ;t56a 5378 5379 mova m0, [rsp+gprsize*2+16*35] ;t32 5380 mova m2, [rsp+gprsize*2+16*38] ;t35 5381 mova m3, [rsp+gprsize*2+16*63] ;t60 5382 mova m1, [rsp+gprsize*2+16*66] ;t63 5383 psubsw m4, m0, m2 ;t35a 5384 paddsw m0, m2 ;t32a 5385 psubsw m5, m1, m3 ;t60a 5386 paddsw m1, m3 ;t63a 5387 ITX_MULSUB_2W 5, 4, 2, 3, 7, 799, 4017 ;t35, t60 5388 mova [rsp+gprsize*2+16*35], m0 ;t32a 5389 mova [rsp+gprsize*2+16*38], m5 ;t35 5390 mova [rsp+gprsize*2+16*63], m4 ;t60 5391 mova [rsp+gprsize*2+16*66], m1 ;t63a 5392 5393 mova m0, [rsp+gprsize*2+16*36] ;t33a 5394 mova m2, [rsp+gprsize*2+16*37] ;t34a 5395 mova m3, [rsp+gprsize*2+16*64] ;t61a 5396 mova m1, [rsp+gprsize*2+16*65] ;t62a 5397 psubsw m4, m0, m2 ;t34 5398 paddsw m0, m2 ;t33 5399 psubsw m5, m1, m3 ;t61 5400 paddsw m1, m3 ;t62 5401 ITX_MULSUB_2W 5, 4, 2, 3, 7, 799, 4017 ;t34a, t61a 5402 5403 mova m2, [rsp+gprsize*2+16*41] ;t38 5404 mova m3, [rsp+gprsize*2+16*60] ;t57 5405 psubsw m6, m0, m2 ;t38a 5406 paddsw m0, m2 ;t33a 5407 psubsw m2, m1, m3 ;t57a 5408 paddsw m1, m3 ;t62a 5409 mova [rsp+gprsize*2+16*36], m0 ;t33a 5410 mova [rsp+gprsize*2+16*65], m1 ;t62a 5411 ITX_MULSUB_2W 2, 6, 0, 3, 7, 1567, 3784 ;t38, t57 5412 mova [rsp+gprsize*2+16*41], m2 ;t38 5413 mova [rsp+gprsize*2+16*60], m6 ;t57 5414 5415 mova m2, [rsp+gprsize*2+16*40] ;t37 5416 mova m3, [rsp+gprsize*2+16*61] ;t58 5417 psubsw m0, m5, m2 ;t37 5418 paddsw m5, m2 ;t34 5419 psubsw m1, m4, m3 ;t58 5420 paddsw m4, m3 ;t61 5421 ITX_MULSUB_2W 1, 0, 2, 3, 7, 1567, 3784 ;t37a, t58a 5422 mova [rsp+gprsize*2+16*37], m5 ;t34 5423 mova [rsp+gprsize*2+16*64], m4 ;t61 5424 mova [rsp+gprsize*2+16*40], m1 ;t37a 5425 mova [rsp+gprsize*2+16*61], m0 ;t58a 5426 5427 mova m0, [rsp+gprsize*2+16*38] ;t35 5428 mova m2, [rsp+gprsize*2+16*39] ;t36 5429 mova m3, [rsp+gprsize*2+16*62] ;t59 5430 mova m1, [rsp+gprsize*2+16*63] ;t60 5431 psubsw m4, m0, m2 ;t36a 5432 paddsw m0, m2 ;t35a 5433 psubsw m5, m1, m3 ;t59a 5434 paddsw m1, m3 ;t60a 5435 ITX_MULSUB_2W 5, 4, 2, 3, 7, 1567, 3784 ;t36, t59 5436 mova [rsp+gprsize*2+16*38], m0 ;t35a 5437 mova [rsp+gprsize*2+16*39], m5 ;t36 5438 mova [rsp+gprsize*2+16*62], m4 ;t59 5439 mova [rsp+gprsize*2+16*63], m1 ;t60a 5440 5441 mova m0, [rsp+gprsize*2+16*35] ;t32a 5442 mova m2, [rsp+gprsize*2+16*42] ;t39a 5443 mova m3, [rsp+gprsize*2+16*59] ;t56a 5444 mova m1, [rsp+gprsize*2+16*66] ;t63a 5445 psubsw m4, m0, m2 ;t39 5446 paddsw m0, m2 ;t32 5447 psubsw m5, m1, m3 ;t56 5448 paddsw m1, m3 ;t63 5449 ITX_MULSUB_2W 5, 4, 2, 3, 7, 1567, 3784 ;t39a, t56a 5450 mova [rsp+gprsize*2+16*35], m0 ;t32 5451 mova [rsp+gprsize*2+16*42], m5 ;t39a 5452 mova [rsp+gprsize*2+16*59], m4 ;t56a 5453 mova [rsp+gprsize*2+16*66], m1 ;t63 5454 5455 mova m0, [rsp+gprsize*2+16*50] ;t47a 5456 mova m2, [rsp+gprsize*2+16*43] ;t40a 5457 mova m3, [rsp+gprsize*2+16*58] ;t55a 5458 mova m1, [rsp+gprsize*2+16*51] ;t48a 5459 psubsw m4, m0, m2 ;t40 5460 paddsw m0, m2 ;t47 5461 psubsw m5, m1, m3 ;t55 5462 paddsw m1, m3 ;t48 5463 ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t40a, t55a 5464 mova [rsp+gprsize*2+16*50], m0 ;t47 5465 mova [rsp+gprsize*2+16*43], m5 ;t40a 5466 mova [rsp+gprsize*2+16*58], m4 ;t55a 5467 mova [rsp+gprsize*2+16*51], m1 ;t48 5468 5469 mova m0, [rsp+gprsize*2+16*49] ;t46 5470 mova m2, [rsp+gprsize*2+16*44] ;t41 5471 mova m3, [rsp+gprsize*2+16*57] ;t54 5472 mova m1, [rsp+gprsize*2+16*52] ;t49 5473 psubsw m4, m0, m2 ;t41a 5474 paddsw m0, m2 ;t46a 5475 psubsw m5, m1, m3 ;t54a 5476 paddsw m1, m3 ;t49a 5477 ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t41, t54 5478 mova [rsp+gprsize*2+16*49], m0 ;t46a 5479 mova [rsp+gprsize*2+16*44], m5 ;t41 5480 mova [rsp+gprsize*2+16*57], m4 ;t54 5481 mova [rsp+gprsize*2+16*52], m1 ;t49a 5482 5483 mova m0, [rsp+gprsize*2+16*48] ;t45a 5484 mova m2, [rsp+gprsize*2+16*45] ;t42a 5485 mova m3, [rsp+gprsize*2+16*56] ;t53a 5486 mova m1, [rsp+gprsize*2+16*53] ;t50a 5487 psubsw m4, m0, m2 ;t42 5488 paddsw m0, m2 ;t45 5489 psubsw m5, m1, m3 ;t53 5490 paddsw m1, m3 ;t50 5491 ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t42a, t53a 5492 mova [rsp+gprsize*2+16*48], m0 ;t45 5493 mova [rsp+gprsize*2+16*45], m5 ;t42a 5494 mova [rsp+gprsize*2+16*56], m4 ;t53a 5495 mova [rsp+gprsize*2+16*53], m1 ;t50 5496 5497 mova m0, [rsp+gprsize*2+16*47] ;t44 5498 mova m2, [rsp+gprsize*2+16*46] ;t43 5499 mova m3, [rsp+gprsize*2+16*55] ;t52 5500 mova m1, [rsp+gprsize*2+16*54] ;t51 5501 psubsw m4, m0, m2 ;t43a 5502 paddsw m0, m2 ;t44a 5503 psubsw m5, m1, m3 ;t52a 5504 paddsw m1, m3 ;t51a 5505 ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t43, t52 5506 5507 mova m2, [rsp+gprsize*2+16*38] ;t35a 5508 mova m3, [rsp+gprsize*2+16*31] ;tmp[28] 5509 psubsw m6, m2, m0 ;t44 5510 paddsw m2, m0 ;t35 5511 psubsw m0, m3, m2 ;out35 5512 paddsw m2, m3 ;out28 5513 mova m3, [rsp+gprsize*2+16*63] ;t60a 5514 mova [rsp+gprsize*2+16*38], m0 ;out35 5515 mova [rsp+gprsize*2+16*31], m2 ;out28 5516 psubsw m0, m3, m1 ;t51 5517 paddsw m3, m1 ;t60 5518 ITX_MULSUB_2W 0, 6, 1, 2, 7, 2896, 2896 ;t44a, t51a 5519 mova m2, [rsp+gprsize*2+16*6 ] ;tmp[3] 5520 psubsw m1, m2, m3 ;out60 5521 paddsw m2, m3 ;out3 5522 mova m3, [rsp+gprsize*2+16*22] ;tmp[19] 5523 mova [rsp+gprsize*2+16*63], m1 ;out60 5524 mova [rsp+gprsize*2+16*6 ], m2 ;out3 5525 psubsw m1, m3, m0 ;out44 5526 paddsw m3, m0 ;out19 5527 mova m2, [rsp+gprsize*2+16*15] ;tmp[12] 5528 5529 mova m0, [rsp+gprsize*2+16*39] ;t36 5530 mova [rsp+gprsize*2+16*47], m1 ;out44 5531 mova [rsp+gprsize*2+16*22], m3 ;out19 5532 mova m1, [rsp+gprsize*2+16*62] ;t59 5533 psubsw m3, m2, m6 ;out51 5534 paddsw m2, m6 ;out12 5535 mova [rsp+gprsize*2+16*54], m3 ;out51 5536 mova [rsp+gprsize*2+16*15], m2 ;out12 5537 psubsw m2, m0, m5 ;t43a 5538 paddsw m0, m5 ;t36a 5539 mova m5, [rsp+gprsize*2+16*30] ;tmp[27] 5540 psubsw m3, m1, m4 ;t52a 5541 paddsw m1, m4 ;t59a 5542 ITX_MULSUB_2W 3, 2, 4, 6, 7, 2896, 2896 ;t43, t52 5543 mova m4, [rsp+gprsize*2+16*7 ] ;tmp[4 ] 5544 psubsw m6, m5, m0 ;out36 5545 paddsw m5, m0 ;out27 5546 psubsw m0, m4, m1 ;out59 5547 paddsw m4, m1 ;out4 5548 mova [rsp+gprsize*2+16*39], m6 ;out36 5549 mova [rsp+gprsize*2+16*30], m5 ;out27 5550 mova [rsp+gprsize*2+16*62], m0 ;out59 5551 mova [rsp+gprsize*2+16*7 ], m4 ;out4 5552 mova m0, [rsp+gprsize*2+16*23] ;tmp[20] 5553 mova m5, [rsp+gprsize*2+16*14] ;tmp[11] 5554 psubsw m4, m0, m3 ;out43 5555 paddsw m0, m3 ;out20 5556 psubsw m6, m5, m2 ;out52 5557 paddsw m5, m2 ;out11 5558 mova [rsp+gprsize*2+16*46], m4 ;out43 5559 mova [rsp+gprsize*2+16*23], m0 ;out20 5560 mova [rsp+gprsize*2+16*55], m6 ;out52 5561 mova [rsp+gprsize*2+16*14], m5 ;out11 5562 5563 mova m0, [rsp+gprsize*2+16*40] ;t37a 5564 mova m5, [rsp+gprsize*2+16*45] ;t42a 5565 mova m3, [rsp+gprsize*2+16*56] ;t53a 5566 mova m1, [rsp+gprsize*2+16*61] ;t58a 5567 mova m2, [rsp+gprsize*2+16*29] ;tmp[26] 5568 psubsw m4, m0, m5 ;t42 5569 paddsw m0, m5 ;t37 5570 psubsw m5, m1, m3 ;t53 5571 paddsw m1, m3 ;t58 5572 ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t43, t52 5573 mova m3, [rsp+gprsize*2+16*8 ] ;tmp[5 ] 5574 psubsw m6, m2, m0 ;out37 5575 paddsw m2, m0 ;out26 5576 psubsw m0, m3, m1 ;out58 5577 paddsw m3, m1 ;out5 5578 mova [rsp+gprsize*2+16*40], m6 ;out37 5579 mova [rsp+gprsize*2+16*29], m2 ;out26 5580 mova [rsp+gprsize*2+16*61], m0 ;out58 5581 mova [rsp+gprsize*2+16*8 ], m3 ;out5 5582 mova m0, [rsp+gprsize*2+16*24] ;tmp[21] 5583 mova m1, [rsp+gprsize*2+16*13] ;tmp[10] 5584 psubsw m2, m0, m5 ;out42 5585 paddsw m0, m5 ;out21 5586 psubsw m3, m1, m4 ;out53 5587 paddsw m1, m4 ;out10 5588 mova [rsp+gprsize*2+16*45], m2 ;out42 5589 mova [rsp+gprsize*2+16*24], m0 ;out21 5590 mova [rsp+gprsize*2+16*56], m3 ;out53 5591 mova [rsp+gprsize*2+16*13], m1 ;out10 5592 5593 mova m0, [rsp+gprsize*2+16*41] ;t38 5594 mova m5, [rsp+gprsize*2+16*44] ;t41 5595 mova m3, [rsp+gprsize*2+16*57] ;t54 5596 mova m1, [rsp+gprsize*2+16*60] ;t57 5597 mova m2, [rsp+gprsize*2+16*28] ;tmp[25] 5598 psubsw m4, m0, m5 ;t41a 5599 paddsw m0, m5 ;t38a 5600 psubsw m5, m1, m3 ;t54a 5601 paddsw m1, m3 ;t57a 5602 ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t41a, t54a 5603 mova m3, [rsp+gprsize*2+16*9 ] ;tmp[6 ] 5604 psubsw m6, m2, m0 ;out38 5605 paddsw m2, m0 ;out25 5606 psubsw m0, m3, m1 ;out57 5607 paddsw m3, m1 ;out6 5608 mova [rsp+gprsize*2+16*41], m6 ;out38 5609 mova [rsp+gprsize*2+16*28], m2 ;out25 5610 mova [rsp+gprsize*2+16*60], m0 ;out57 5611 mova [rsp+gprsize*2+16*9 ], m3 ;out6 5612 mova m0, [rsp+gprsize*2+16*25] ;tmp[22] 5613 mova m1, [rsp+gprsize*2+16*12] ;tmp[9 ] 5614 psubsw m2, m0, m5 ;out41 5615 paddsw m0, m5 ;out22 5616 psubsw m3, m1, m4 ;out54 5617 paddsw m1, m4 ;out9 5618 mova [rsp+gprsize*2+16*44], m2 ;out41 5619 mova [rsp+gprsize*2+16*25], m0 ;out22 5620 mova [rsp+gprsize*2+16*57], m3 ;out54 5621 mova [rsp+gprsize*2+16*12], m1 ;out9 5622 5623 mova m0, [rsp+gprsize*2+16*42] ;t39a 5624 mova m5, [rsp+gprsize*2+16*43] ;t40a 5625 mova m3, [rsp+gprsize*2+16*58] ;t55a 5626 mova m1, [rsp+gprsize*2+16*59] ;t56a 5627 mova m2, [rsp+gprsize*2+16*27] ;tmp[24] 5628 psubsw m4, m0, m5 ;t40 5629 paddsw m0, m5 ;t39 5630 psubsw m5, m1, m3 ;t55 5631 paddsw m1, m3 ;t56 5632 ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t40a, t55a 5633 mova m3, [rsp+gprsize*2+16*10] ;tmp[7 ] 5634 psubsw m6, m2, m0 ;out39 5635 paddsw m2, m0 ;out24 5636 psubsw m0, m3, m1 ;out56 5637 paddsw m3, m1 ;out7 5638 mova [rsp+gprsize*2+16*42], m6 ;out39 5639 mova [rsp+gprsize*2+16*27], m2 ;out24 5640 mova [rsp+gprsize*2+16*59], m0 ;out56 5641 mova [rsp+gprsize*2+16*10], m3 ;out7 5642 mova m0, [rsp+gprsize*2+16*26] ;tmp[23] 5643 mova m1, [rsp+gprsize*2+16*11] ;tmp[8 ] 5644 psubsw m2, m0, m5 ;out40 5645 paddsw m0, m5 ;out23 5646 psubsw m3, m1, m4 ;out55 5647 paddsw m1, m4 ;out8 5648 mova [rsp+gprsize*2+16*43], m2 ;out40 5649 mova [rsp+gprsize*2+16*26], m0 ;out23 5650 mova [rsp+gprsize*2+16*58], m3 ;out55 5651 mova [rsp+gprsize*2+16*11], m1 ;out8 5652 5653 mova m0, [rsp+gprsize*2+16*37] ;t34 5654 mova m5, [rsp+gprsize*2+16*48] ;t45 5655 mova m3, [rsp+gprsize*2+16*53] ;t50 5656 mova m1, [rsp+gprsize*2+16*64] ;t61 5657 mova m2, [rsp+gprsize*2+16*32] ;tmp[29] 5658 psubsw m4, m0, m5 ;t45a 5659 paddsw m0, m5 ;t34a 5660 psubsw m5, m1, m3 ;t50a 5661 paddsw m1, m3 ;t61a 5662 ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t45, t50 5663 mova m3, [rsp+gprsize*2+16*5 ] ;tmp[2 ] 5664 psubsw m6, m2, m0 ;out34 5665 paddsw m2, m0 ;out29 5666 psubsw m0, m3, m1 ;out61 5667 paddsw m3, m1 ;out2 5668 mova [rsp+gprsize*2+16*37], m6 ;out34 5669 mova [rsp+gprsize*2+16*32], m2 ;out29 5670 mova [rsp+gprsize*2+16*64], m0 ;out61 5671 mova [rsp+gprsize*2+16*5 ], m3 ;out2 5672 mova m0, [rsp+gprsize*2+16*21] ;tmp[18] 5673 mova m1, [rsp+gprsize*2+16*16] ;tmp[13] 5674 psubsw m2, m0, m5 ;out45 5675 paddsw m0, m5 ;out18 5676 psubsw m3, m1, m4 ;out50 5677 paddsw m1, m4 ;out13 5678 mova [rsp+gprsize*2+16*48], m2 ;out45 5679 mova [rsp+gprsize*2+16*21], m0 ;out18 5680 mova [rsp+gprsize*2+16*53], m3 ;out50 5681 mova [rsp+gprsize*2+16*16], m1 ;out13 5682 5683 mova m0, [rsp+gprsize*2+16*36] ;t33a 5684 mova m5, [rsp+gprsize*2+16*49] ;t46a 5685 mova m3, [rsp+gprsize*2+16*52] ;t49a 5686 mova m1, [rsp+gprsize*2+16*65] ;t62a 5687 mova m2, [rsp+gprsize*2+16*33] ;tmp[30] 5688 psubsw m4, m0, m5 ;t46 5689 paddsw m0, m5 ;t33 5690 psubsw m5, m1, m3 ;t49 5691 paddsw m1, m3 ;t62 5692 ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t45, t50 5693 mova m3, [rsp+gprsize*2+16*4 ] ;tmp[1 ] 5694 psubsw m6, m2, m0 ;out33 5695 paddsw m2, m0 ;out30 5696 psubsw m0, m3, m1 ;out62 5697 paddsw m3, m1 ;out1 5698 mova [rsp+gprsize*2+16*36], m6 ;out33 5699 mova [rsp+gprsize*2+16*33], m2 ;out30 5700 mova [rsp+gprsize*2+16*65], m0 ;out62 5701 mova [rsp+gprsize*2+16*4 ], m3 ;out1 5702 mova m0, [rsp+gprsize*2+16*20] ;tmp[17] 5703 mova m1, [rsp+gprsize*2+16*17] ;tmp[14] 5704 psubsw m2, m0, m5 ;out46 5705 paddsw m0, m5 ;out17 5706 psubsw m3, m1, m4 ;out49 5707 paddsw m1, m4 ;out14 5708 mova [rsp+gprsize*2+16*49], m2 ;out46 5709 mova [rsp+gprsize*2+16*20], m0 ;out17 5710 mova [rsp+gprsize*2+16*52], m3 ;out49 5711 mova [rsp+gprsize*2+16*17], m1 ;out14 5712 5713 mova m0, [rsp+gprsize*2+16*35] ;t32 5714 mova m5, [rsp+gprsize*2+16*50] ;t47 5715 mova m3, [rsp+gprsize*2+16*51] ;t48 5716 mova m1, [rsp+gprsize*2+16*66] ;t63 5717 mova m2, [rsp+gprsize*2+16*34] ;tmp[31] 5718 psubsw m4, m0, m5 ;t47a 5719 paddsw m0, m5 ;t32a 5720 psubsw m5, m1, m3 ;t48a 5721 paddsw m1, m3 ;t63a 5722 ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t47, t48 5723 mova m3, [rsp+gprsize*2+16*3 ] ;tmp[0 ] 5724 psubsw m6, m2, m0 ;out32 5725 paddsw m2, m0 ;out31 5726 psubsw m0, m3, m1 ;out63 5727 paddsw m3, m1 ;out0 5728 mova [rsp+gprsize*2+16*35], m6 ;out32 5729 mova [rsp+gprsize*2+16*34], m2 ;out31 5730 mova [rsp+gprsize*2+16*66], m0 ;out63 5731 mova [rsp+gprsize*2+16*3 ], m3 ;out0 5732 mova m0, [rsp+gprsize*2+16*19] ;tmp[16] 5733 mova m1, [rsp+gprsize*2+16*18] ;tmp[15] 5734 psubsw m2, m0, m5 ;out47 5735 paddsw m0, m5 ;out16 5736 psubsw m3, m1, m4 ;out48 5737 paddsw m1, m4 ;out15 5738 mova [rsp+gprsize*2+16*50], m2 ;out47 5739 mova [rsp+gprsize*2+16*19], m0 ;out16 5740 mova [rsp+gprsize*2+16*51], m3 ;out48 5741 mova [rsp+gprsize*2+16*18], m1 ;out15 5742 ret 5743 5744 5745 cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 6, 8, 16*132, dst, stride, coeff, eob, tx2 5746 %if ARCH_X86_32 5747 LEA r5, $$ 5748 %endif 5749 test eobd, eobd 5750 jz .dconly 5751 5752 call m(idct_64x16_internal_8bpc) 5753 RET 5754 5755 .dconly: 5756 movd m1, [o(pw_2896x8)] 5757 pmulhrsw m0, m1, [coeffq] 5758 movd m2, [o(pw_8192)] 5759 mov [coeffq], eobd 5760 mov r3d, 16 5761 lea tx2q, [o(.end)] 5762 5763 .body: 5764 pmulhrsw m0, m2 5765 movd m2, [o(pw_2048)] ;intentionally rip-relative 5766 pmulhrsw m0, m1 5767 pmulhrsw m0, m2 5768 pshuflw m0, m0, q0000 5769 punpcklwd m0, m0 5770 pxor m7, m7 5771 5772 .loop: 5773 mova m1, [dstq+16*0] 5774 mova m3, [dstq+16*1] 5775 mova m5, [dstq+16*2] 5776 mova m6, [dstq+16*3] 5777 punpckhbw m2, m1, m7 5778 punpcklbw m1, m7 5779 punpckhbw m4, m3, m7 5780 punpcklbw m3, m7 5781 paddw m2, m0 5782 paddw m1, m0 5783 paddw m4, m0 5784 paddw m3, m0 5785 packuswb m1, m2 5786 packuswb m3, m4 5787 punpckhbw m2, m5, m7 5788 punpcklbw m5, m7 5789 punpckhbw m4, m6, m7 5790 punpcklbw m6, m7 5791 paddw m2, m0 5792 paddw m5, m0 5793 paddw m4, m0 5794 paddw m6, m0 5795 packuswb m5, m2 5796 packuswb m6, m4 5797 mova [dstq+16*0], m1 5798 mova [dstq+16*1], m3 5799 mova [dstq+16*2], m5 5800 mova [dstq+16*3], m6 5801 add dstq, strideq 5802 dec r3d 5803 jg .loop 5804 jmp tx2q 5805 5806 .end: 5807 RET 5808 5809 5810 %macro LOAD_4ROWS 2-3 0 ;src, stride, is_rect2 5811 5812 %if %3 5813 mova m3, [o(pw_2896x8)] 5814 pmulhrsw m0, m3, [%1+%2*0] 5815 pmulhrsw m1, m3, [%1+%2*1] 5816 pmulhrsw m2, m3, [%1+%2*2] 5817 pmulhrsw m3, [%1+%2*3] 5818 %else 5819 mova m0, [%1+%2*0] 5820 mova m1, [%1+%2*1] 5821 mova m2, [%1+%2*2] 5822 mova m3, [%1+%2*3] 5823 %endif 5824 %endmacro 5825 5826 %macro LOAD_4ROWS_H 2 ;src, stride 5827 mova m4, [%1+%2*0] 5828 mova m5, [%1+%2*1] 5829 mova m6, [%1+%2*2] 5830 mova m7, [%1+%2*3] 5831 %endmacro 5832 5833 cglobal idct_64x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 5834 mov r3d, 2 5835 mov [rsp+gprsize*2+16*67], dstq 5836 lea dstq, [rsp+gprsize+16*68] 5837 5838 .pass1_loop: 5839 LOAD_4ROWS coeffq+32*0, 32*8 5840 pxor m4, m4 5841 REPX {mova x, m4}, m5, m6, m7 5842 call m(idct_8x8_internal_8bpc).main 5843 SAVE_7ROWS rsp+gprsize+16*3, 16 5844 5845 pxor m4, m4 5846 LOAD_4ROWS coeffq+32*4, 32*8 5847 5848 REPX {mova x, m4}, m5, m6, m7 5849 call m(idct_16x8_internal_8bpc).main 5850 mova m7, [rsp+gprsize+16*0] 5851 SAVE_8ROWS rsp+gprsize+16*11, 16 5852 5853 LOAD_8ROWS coeffq+32*2, 32*4 5854 mova [rsp+gprsize+16*19], m0 5855 mova [rsp+gprsize+16*26], m1 5856 mova [rsp+gprsize+16*23], m2 5857 mova [rsp+gprsize+16*22], m3 5858 mova [rsp+gprsize+16*21], m4 5859 mova [rsp+gprsize+16*24], m5 5860 mova [rsp+gprsize+16*25], m6 5861 mova [rsp+gprsize+16*20], m7 5862 5863 call m(idct_8x32_internal_8bpc).main_fast 5864 SAVE_8ROWS rsp+gprsize+16*3, 16 5865 5866 LOAD_8ROWS coeffq+32*1, 32*2 5867 mova [rsp+gprsize+16*35], m0 ;in1 5868 mova [rsp+gprsize+16*49], m1 ;in3 5869 mova [rsp+gprsize+16*43], m2 ;in5 5870 mova [rsp+gprsize+16*41], m3 ;in7 5871 mova [rsp+gprsize+16*39], m4 ;in9 5872 mova [rsp+gprsize+16*45], m5 ;in11 5873 mova [rsp+gprsize+16*47], m6 ;in13 5874 mova [rsp+gprsize+16*37], m7 ;in15 5875 5876 LOAD_8ROWS coeffq+32*17, 32*2 5877 mova [rsp+gprsize+16*63], m0 ;in17 5878 mova [rsp+gprsize+16*53], m1 ;in19 5879 mova [rsp+gprsize+16*55], m2 ;in21 5880 mova [rsp+gprsize+16*61], m3 ;in23 5881 mova [rsp+gprsize+16*59], m4 ;in25 5882 mova [rsp+gprsize+16*57], m5 ;in27 5883 mova [rsp+gprsize+16*51], m6 ;in29 5884 mova [rsp+gprsize+16*65], m7 ;in31 5885 5886 call m(idct_16x64_internal_8bpc).main 5887 5888 LOAD_8ROWS rsp+gprsize+16*3, 16 5889 mova [rsp+gprsize+16*0], m7 5890 mova m7, [o(pw_8192)] 5891 lea tx2q, [o(.pass1_end)] 5892 jmp m(idct_8x8_internal_8bpc).pass1_end1 5893 5894 .pass1_end: 5895 SAVE_8ROWS coeffq+32*0, 32 5896 LOAD_8ROWS rsp+gprsize+16*11, 16 5897 mova [rsp+gprsize+16*0], m7 5898 mova m7, [o(pw_8192)] 5899 lea tx2q, [o(.pass1_end1)] 5900 jmp m(idct_8x8_internal_8bpc).pass1_end1 5901 5902 .pass1_end1: 5903 SAVE_8ROWS coeffq+32*8, 32 5904 LOAD_8ROWS rsp+gprsize+16*19, 16 5905 mova [rsp+gprsize+16*0], m7 5906 mova m7, [o(pw_8192)] 5907 lea tx2q, [o(.pass1_end2)] 5908 jmp m(idct_8x8_internal_8bpc).pass1_end1 5909 5910 .pass1_end2: 5911 SAVE_8ROWS coeffq+32*16, 32 5912 LOAD_8ROWS rsp+gprsize+16*27, 16 5913 mova [rsp+gprsize+16*0], m7 5914 mova m7, [o(pw_8192)] 5915 lea tx2q, [o(.pass1_end3)] 5916 jmp m(idct_8x8_internal_8bpc).pass1_end1 5917 5918 .pass1_end3: 5919 SAVE_8ROWS coeffq+32*24, 32 5920 LOAD_8ROWS rsp+gprsize+16*35, 16 5921 mova [rsp+gprsize+16*0], m7 5922 mova m7, [o(pw_8192)] 5923 lea tx2q, [o(.pass1_end4)] 5924 jmp m(idct_8x8_internal_8bpc).pass1_end1 5925 5926 .pass1_end4: 5927 SAVE_8ROWS dstq+32*0, 32 5928 LOAD_8ROWS rsp+gprsize+16*43, 16 5929 mova [rsp+gprsize+16*0], m7 5930 mova m7, [o(pw_8192)] 5931 lea tx2q, [o(.pass1_end5)] 5932 jmp m(idct_8x8_internal_8bpc).pass1_end1 5933 5934 .pass1_end5: 5935 SAVE_8ROWS dstq+32*8, 32 5936 LOAD_8ROWS rsp+gprsize+16*51, 16 5937 mova [rsp+gprsize+16*0], m7 5938 mova m7, [o(pw_8192)] 5939 lea tx2q, [o(.pass1_end6)] 5940 jmp m(idct_8x8_internal_8bpc).pass1_end1 5941 5942 .pass1_end6: 5943 SAVE_8ROWS dstq+32*16, 32 5944 LOAD_8ROWS rsp+gprsize+16*59, 16 5945 mova [rsp+gprsize+16*0], m7 5946 mova m7, [o(pw_8192)] 5947 lea tx2q, [o(.pass1_end7)] 5948 jmp m(idct_8x8_internal_8bpc).pass1_end1 5949 5950 .pass1_end7: 5951 SAVE_8ROWS dstq+32*24, 32 5952 5953 add coeffq, 16 5954 add dstq, 16 5955 dec r3d 5956 jg .pass1_loop 5957 5958 .pass2: 5959 mov dstq, [rsp+gprsize*2+16*67] 5960 sub coeffq, 32 5961 mov r3d, 4 5962 5963 .pass2_loop: 5964 mov [rsp+gprsize*1+16*67], r3d 5965 5966 LOAD_4ROWS coeffq+16*0, 32*2 5967 LOAD_4ROWS_H coeffq+16*1, 32*2 5968 call m(idct_8x8_internal_8bpc).main 5969 SAVE_7ROWS rsp+gprsize+16*3, 16 5970 LOAD_4ROWS coeffq+16*2, 32*2 5971 LOAD_4ROWS_H coeffq+16*3, 32*2 5972 call m(idct_16x8_internal_8bpc).main 5973 5974 mov r3, dstq 5975 lea tx2q, [o(.end)] 5976 lea dstq, [dstq+strideq*8] 5977 jmp m(idct_8x8_internal_8bpc).end 5978 5979 .end: 5980 LOAD_8ROWS rsp+gprsize+16*3, 16 5981 mova [rsp+gprsize+16*0], m7 5982 lea tx2q, [o(.end1)] 5983 mov dstq, r3 5984 jmp m(idct_8x8_internal_8bpc).end 5985 5986 .end1: 5987 pxor m7, m7 5988 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 5989 5990 add coeffq, 16*16 5991 mov r3d, [rsp+gprsize*1+16*67] 5992 mov dstq, [rsp+gprsize*2+16*67] 5993 add dstq, 8 5994 mov [rsp+gprsize*2+16*67], dstq 5995 dec r3d 5996 jg .pass2_loop 5997 5998 mov r3d, 4 5999 lea coeffq, [rsp+gprsize+16*68] 6000 .pass2_loop2: 6001 mov [rsp+gprsize*1+16*67], r3d 6002 6003 LOAD_4ROWS coeffq+16*0, 32*2 6004 LOAD_4ROWS_H coeffq+16*1, 32*2 6005 call m(idct_8x8_internal_8bpc).main 6006 SAVE_7ROWS rsp+gprsize+16*3, 16 6007 LOAD_4ROWS coeffq+16*2, 32*2 6008 LOAD_4ROWS_H coeffq+16*3, 32*2 6009 call m(idct_16x8_internal_8bpc).main 6010 6011 mov r3, dstq 6012 lea tx2q, [o(.end2)] 6013 lea dstq, [dstq+strideq*8] 6014 jmp m(idct_8x8_internal_8bpc).end 6015 6016 .end2: 6017 LOAD_8ROWS rsp+gprsize+16*3, 16 6018 mova [rsp+gprsize+16*0], m7 6019 lea tx2q, [o(.end3)] 6020 mov dstq, r3 6021 jmp m(idct_8x8_internal_8bpc).end 6022 6023 .end3: 6024 6025 add coeffq, 16*16 6026 mov r3d, [rsp+gprsize*1+16*67] 6027 mov dstq, [rsp+gprsize*2+16*67] 6028 add dstq, 8 6029 mov [rsp+gprsize*2+16*67], dstq 6030 dec r3d 6031 jg .pass2_loop2 6032 ret 6033 6034 6035 cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 6036 %if ARCH_X86_32 6037 LEA r5, $$ 6038 %endif 6039 test eobd, eobd 6040 jz .dconly 6041 call m(idct_32x64_internal_8bpc) 6042 .end: 6043 RET 6044 6045 .dconly: 6046 movd m1, [o(pw_2896x8)] 6047 pmulhrsw m0, m1, [coeffq] 6048 movd m2, [o(pw_16384)] 6049 mov [coeffq], eobd 6050 pmulhrsw m0, m1 6051 mov r3d, 64 6052 lea tx2q, [o(.end)] 6053 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body 6054 6055 6056 cglobal idct_32x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 6057 mov r4d, 2 6058 sub eobd, 136 6059 mov [rsp+gprsize*1+16*67], eobd 6060 mov r3d, 4 6061 cmovs r3d, r4d 6062 6063 %if ARCH_X86_32 6064 LEA r5, $$ 6065 %endif 6066 6067 mov [rsp+gprsize*2+16*67], coeffq 6068 6069 .pass1_loop: 6070 LOAD_8ROWS coeffq+64*1, 64*2, 1 6071 mova [rsp+gprsize+16*19], m0 ;in1 6072 mova [rsp+gprsize+16*26], m1 ;in3 6073 mova [rsp+gprsize+16*23], m2 ;in5 6074 mova [rsp+gprsize+16*22], m3 ;in7 6075 mova [rsp+gprsize+16*21], m4 ;in9 6076 mova [rsp+gprsize+16*24], m5 ;in11 6077 mova [rsp+gprsize+16*25], m6 ;in13 6078 mova [rsp+gprsize+16*20], m7 ;in15 6079 6080 mov tx2d, [rsp+gprsize*1+16*67] 6081 test tx2d, tx2d 6082 jl .fast 6083 6084 .full: 6085 LOAD_8ROWS coeffq+64*0, 64*4, 1 6086 call m(idct_8x8_internal_8bpc).main 6087 SAVE_7ROWS rsp+gprsize+16*3, 16 6088 LOAD_8ROWS coeffq+64*2, 64*4, 1 6089 call m(idct_16x8_internal_8bpc).main 6090 mova m7, [rsp+gprsize+16*0] 6091 SAVE_8ROWS rsp+gprsize+16*11, 16 6092 6093 LOAD_8ROWS coeffq+64*17, 64*2, 1 6094 mova [rsp+gprsize+16*33], m0 ;in17 6095 mova [rsp+gprsize+16*28], m1 ;in19 6096 mova [rsp+gprsize+16*29], m2 ;in21 6097 mova [rsp+gprsize+16*32], m3 ;in23 6098 mova [rsp+gprsize+16*31], m4 ;in25 6099 mova [rsp+gprsize+16*30], m5 ;in27 6100 mova [rsp+gprsize+16*27], m6 ;in29 6101 mova [rsp+gprsize+16*34], m7 ;in31 6102 6103 call m(idct_8x32_internal_8bpc).main 6104 jmp .pass1_end 6105 6106 .fast: 6107 LOAD_4ROWS coeffq, 256, 1 6108 pxor m4, m4 6109 REPX {mova x, m4}, m5, m6, m7 6110 call m(idct_8x8_internal_8bpc).main 6111 6112 SAVE_7ROWS rsp+gprsize+16*3, 16 6113 LOAD_4ROWS coeffq+128*1, 256, 1 6114 pxor m4, m4 6115 REPX {mova x, m4}, m5, m6, m7 6116 call m(idct_16x8_internal_8bpc).main 6117 mova m7, [rsp+gprsize+16*0] 6118 SAVE_8ROWS rsp+gprsize+16*11, 16 6119 6120 call m(idct_8x32_internal_8bpc).main_fast 6121 6122 .pass1_end: 6123 mova [rsp+gprsize+16*0], m7 6124 lea tx2q, [o(.pass1_end1)] 6125 jmp m(idct_8x8_internal_8bpc).pass1_end 6126 6127 .pass1_end1: 6128 SAVE_8ROWS coeffq+64*0, 64 6129 LOAD_8ROWS rsp+gprsize+16*11, 16 6130 mova [rsp+gprsize+16*0], m7 6131 lea tx2q, [o(.pass1_end2)] 6132 jmp m(idct_8x8_internal_8bpc).pass1_end 6133 6134 .pass1_end2: 6135 SAVE_8ROWS coeffq+64*8, 64 6136 LOAD_8ROWS rsp+gprsize+16*19, 16 6137 mova [rsp+gprsize+16*0], m7 6138 lea tx2q, [o(.pass1_end3)] 6139 jmp m(idct_8x8_internal_8bpc).pass1_end 6140 6141 .pass1_end3: 6142 SAVE_8ROWS coeffq+64*16, 64 6143 LOAD_8ROWS rsp+gprsize+16*27, 16 6144 mova [rsp+gprsize+16*0], m7 6145 lea tx2q, [o(.pass1_end4)] 6146 jmp m(idct_8x8_internal_8bpc).pass1_end 6147 6148 .pass1_end4: 6149 SAVE_8ROWS coeffq+64*24, 64 6150 6151 add coeffq, 16 6152 dec r3d 6153 jg .pass1_loop 6154 6155 .pass2: 6156 mov coeffq, [rsp+gprsize*2+16*67] 6157 mov r3d, 4 6158 lea r4, [dstq+8] 6159 mov [rsp+gprsize*2+16*67], r4 6160 lea r4, [o(m(idct_16x64_internal_8bpc).end1)] 6161 jmp m(idct_16x64_internal_8bpc).pass2_loop 6162 6163 6164 cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 6165 %if ARCH_X86_32 6166 LEA r5, $$ 6167 %endif 6168 test eobd, eobd 6169 jz .dconly 6170 call m(idct_64x32_internal_8bpc) 6171 .end: 6172 RET 6173 6174 .dconly: 6175 movd m1, [o(pw_2896x8)] 6176 pmulhrsw m0, m1, [coeffq] 6177 movd m2, [o(pw_16384)] 6178 pmulhrsw m0, m1 6179 mov [coeffq], eobd 6180 mov r3d, 32 6181 lea tx2q, [o(.end)] 6182 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).body 6183 6184 6185 cglobal idct_64x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 6186 mov r4d, 2 6187 sub eobd, 136 6188 mov [rsp+gprsize*1+16*67], eobd 6189 mov r3d, 4 6190 cmovs r3d, r4d 6191 6192 %if ARCH_X86_32 6193 LEA r5, $$ 6194 %endif 6195 6196 mov [rsp+gprsize*2+16*67], coeffq 6197 mov [rsp+gprsize*3+16*67], dstq 6198 lea dstq, [rsp+gprsize+16*69] 6199 mov [rsp+gprsize*4+16*67], dstq 6200 6201 .pass1_loop: 6202 LOAD_4ROWS coeffq+64*0, 64*8, 1 6203 pxor m4, m4 6204 REPX {mova x, m4}, m5, m6, m7 6205 call m(idct_8x8_internal_8bpc).main 6206 SAVE_7ROWS rsp+gprsize+16*3, 16 6207 6208 pxor m4, m4 6209 LOAD_4ROWS coeffq+64*4, 64*8, 1 6210 6211 REPX {mova x, m4}, m5, m6, m7 6212 call m(idct_16x8_internal_8bpc).main 6213 mova m7, [rsp+gprsize+16*0] 6214 SAVE_8ROWS rsp+gprsize+16*11, 16 6215 6216 LOAD_8ROWS coeffq+64*2, 64*4, 1 6217 mova [rsp+gprsize+16*19], m0 6218 mova [rsp+gprsize+16*26], m1 6219 mova [rsp+gprsize+16*23], m2 6220 mova [rsp+gprsize+16*22], m3 6221 mova [rsp+gprsize+16*21], m4 6222 mova [rsp+gprsize+16*24], m5 6223 mova [rsp+gprsize+16*25], m6 6224 mova [rsp+gprsize+16*20], m7 6225 6226 call m(idct_8x32_internal_8bpc).main_fast 6227 SAVE_8ROWS rsp+gprsize+16*3, 16 6228 6229 LOAD_8ROWS coeffq+64*1, 64*2, 1 6230 mova [rsp+gprsize+16*35], m0 ;in1 6231 mova [rsp+gprsize+16*49], m1 ;in3 6232 mova [rsp+gprsize+16*43], m2 ;in5 6233 mova [rsp+gprsize+16*41], m3 ;in7 6234 mova [rsp+gprsize+16*39], m4 ;in9 6235 mova [rsp+gprsize+16*45], m5 ;in11 6236 mova [rsp+gprsize+16*47], m6 ;in13 6237 mova [rsp+gprsize+16*37], m7 ;in15 6238 6239 LOAD_8ROWS coeffq+64*17, 64*2, 1 6240 mova [rsp+gprsize+16*63], m0 ;in17 6241 mova [rsp+gprsize+16*53], m1 ;in19 6242 mova [rsp+gprsize+16*55], m2 ;in21 6243 mova [rsp+gprsize+16*61], m3 ;in23 6244 mova [rsp+gprsize+16*59], m4 ;in25 6245 mova [rsp+gprsize+16*57], m5 ;in27 6246 mova [rsp+gprsize+16*51], m6 ;in29 6247 mova [rsp+gprsize+16*65], m7 ;in31 6248 6249 call m(idct_16x64_internal_8bpc).main 6250 6251 LOAD_8ROWS rsp+gprsize+16*3, 16 6252 mova [rsp+gprsize+16*0], m7 6253 lea tx2q, [o(.pass1_end)] 6254 jmp m(idct_8x8_internal_8bpc).pass1_end 6255 6256 .pass1_end: 6257 SAVE_8ROWS coeffq+64*0, 64 6258 LOAD_8ROWS rsp+gprsize+16*11, 16 6259 mova [rsp+gprsize+16*0], m7 6260 lea tx2q, [o(.pass1_end1)] 6261 jmp m(idct_8x8_internal_8bpc).pass1_end 6262 6263 .pass1_end1: 6264 SAVE_8ROWS coeffq+64*8, 64 6265 LOAD_8ROWS rsp+gprsize+16*19, 16 6266 mova [rsp+gprsize+16*0], m7 6267 lea tx2q, [o(.pass1_end2)] 6268 jmp m(idct_8x8_internal_8bpc).pass1_end 6269 6270 .pass1_end2: 6271 SAVE_8ROWS coeffq+64*16, 64 6272 LOAD_8ROWS rsp+gprsize+16*27, 16 6273 mova [rsp+gprsize+16*0], m7 6274 lea tx2q, [o(.pass1_end3)] 6275 jmp m(idct_8x8_internal_8bpc).pass1_end 6276 6277 .pass1_end3: 6278 SAVE_8ROWS coeffq+64*24, 64 6279 LOAD_8ROWS rsp+gprsize+16*35, 16 6280 mova [rsp+gprsize+16*0], m7 6281 lea tx2q, [o(.pass1_end4)] 6282 jmp m(idct_8x8_internal_8bpc).pass1_end 6283 6284 .pass1_end4: 6285 SAVE_8ROWS dstq+64*0, 64 6286 LOAD_8ROWS rsp+gprsize+16*43, 16 6287 mova [rsp+gprsize+16*0], m7 6288 lea tx2q, [o(.pass1_end5)] 6289 jmp m(idct_8x8_internal_8bpc).pass1_end 6290 6291 .pass1_end5: 6292 SAVE_8ROWS dstq+64*8, 64 6293 LOAD_8ROWS rsp+gprsize+16*51, 16 6294 mova [rsp+gprsize+16*0], m7 6295 lea tx2q, [o(.pass1_end6)] 6296 jmp m(idct_8x8_internal_8bpc).pass1_end 6297 6298 .pass1_end6: 6299 SAVE_8ROWS dstq+64*16, 64 6300 LOAD_8ROWS rsp+gprsize+16*59, 16 6301 mova [rsp+gprsize+16*0], m7 6302 lea tx2q, [o(.pass1_end7)] 6303 jmp m(idct_8x8_internal_8bpc).pass1_end 6304 6305 .pass1_end7: 6306 SAVE_8ROWS dstq+64*24, 64 6307 6308 add coeffq, 16 6309 add dstq, 16 6310 dec r3d 6311 jg .pass1_loop 6312 6313 .pass2: 6314 mov coeffq, [rsp+gprsize*4+16*67] 6315 mov dstq, [rsp+gprsize*3+16*67] 6316 mov eobd, [rsp+gprsize*1+16*67] 6317 lea dstq, [dstq+32] 6318 mov [rsp+gprsize*1+16*35], eobd 6319 lea tx2q, [o(.pass2_end)] 6320 mov r3d, 4 6321 jmp m(idct_32x32_internal_8bpc).pass2_loop 6322 6323 .pass2_end: 6324 mova [rsp+gprsize+16*0], m7 6325 lea r3, [o(.pass2_end1)] 6326 jmp m(idct_8x32_internal_8bpc).end2 6327 6328 .pass2_end1: 6329 lea tx2q, [o(.pass2_end)] 6330 add coeffq, 16*32 6331 mov dstq, [rsp+gprsize*2+16*35] 6332 mov r3d, [rsp+gprsize*3+16*35] 6333 dec r3d 6334 jg m(idct_32x32_internal_8bpc).pass2_loop 6335 6336 .pass2_end2: 6337 mov dstq, [rsp+gprsize*3+16*67] 6338 mov coeffq, [rsp+gprsize*2+16*67] 6339 lea tx2q, [o(m(idct_32x32_internal_8bpc).pass2_end)] 6340 mov r3d, 4 6341 jmp m(idct_32x32_internal_8bpc).pass2_loop 6342 6343 6344 cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 6345 %if ARCH_X86_32 6346 LEA r5, $$ 6347 %endif 6348 test eobd, eobd 6349 jz .dconly 6350 6351 call m(idct_64x64_internal_8bpc) 6352 RET 6353 6354 .dconly: 6355 movd m1, [o(pw_2896x8)] 6356 pmulhrsw m0, m1, [coeffq] 6357 movd m2, [o(pw_8192)] 6358 mov [coeffq], eobd 6359 mov r3d, 64 6360 lea tx2q, [o(m(inv_txfm_add_dct_dct_64x32_8bpc).end)] 6361 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).body 6362 6363 cglobal idct_64x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 6364 mov r5d, 4 6365 mov r4d, 2 6366 sub eobd, 136 6367 cmovns r4d, r5d 6368 6369 %if ARCH_X86_32 6370 LEA r5, $$ 6371 %endif 6372 6373 mov [rsp+gprsize*1+16*67], eobd 6374 mov r3d, r4d 6375 mov [rsp+gprsize*4+16*67], coeffq 6376 mov [rsp+gprsize*3+16*67], dstq 6377 lea dstq, [rsp+gprsize+16*69] 6378 mov [rsp+gprsize*2+16*67], dstq 6379 6380 .pass1_loop: 6381 LOAD_4ROWS coeffq+64*0, 64*8 6382 pxor m4, m4 6383 REPX {mova x, m4}, m5, m6, m7 6384 call m(idct_8x8_internal_8bpc).main 6385 SAVE_7ROWS rsp+gprsize+16*3, 16 6386 6387 pxor m4, m4 6388 LOAD_4ROWS coeffq+64*4, 64*8 6389 6390 REPX {mova x, m4}, m5, m6, m7 6391 call m(idct_16x8_internal_8bpc).main 6392 mova m7, [rsp+gprsize+16*0] 6393 SAVE_8ROWS rsp+gprsize+16*11, 16 6394 6395 LOAD_8ROWS coeffq+64*2, 64*4 6396 mova [rsp+gprsize+16*19], m0 6397 mova [rsp+gprsize+16*26], m1 6398 mova [rsp+gprsize+16*23], m2 6399 mova [rsp+gprsize+16*22], m3 6400 mova [rsp+gprsize+16*21], m4 6401 mova [rsp+gprsize+16*24], m5 6402 mova [rsp+gprsize+16*25], m6 6403 mova [rsp+gprsize+16*20], m7 6404 6405 call m(idct_8x32_internal_8bpc).main_fast 6406 SAVE_8ROWS rsp+gprsize+16*3, 16 6407 6408 LOAD_8ROWS coeffq+64*1, 64*2 6409 mova [rsp+gprsize+16*35], m0 ;in1 6410 mova [rsp+gprsize+16*49], m1 ;in3 6411 mova [rsp+gprsize+16*43], m2 ;in5 6412 mova [rsp+gprsize+16*41], m3 ;in7 6413 mova [rsp+gprsize+16*39], m4 ;in9 6414 mova [rsp+gprsize+16*45], m5 ;in11 6415 mova [rsp+gprsize+16*47], m6 ;in13 6416 mova [rsp+gprsize+16*37], m7 ;in15 6417 6418 LOAD_8ROWS coeffq+64*17, 64*2 6419 mova [rsp+gprsize+16*63], m0 ;in17 6420 mova [rsp+gprsize+16*53], m1 ;in19 6421 mova [rsp+gprsize+16*55], m2 ;in21 6422 mova [rsp+gprsize+16*61], m3 ;in23 6423 mova [rsp+gprsize+16*59], m4 ;in25 6424 mova [rsp+gprsize+16*57], m5 ;in27 6425 mova [rsp+gprsize+16*51], m6 ;in29 6426 mova [rsp+gprsize+16*65], m7 ;in31 6427 6428 call m(idct_16x64_internal_8bpc).main 6429 6430 LOAD_8ROWS rsp+gprsize+16*3, 16 6431 mova [rsp+gprsize+16*0], m7 6432 mova m7, [o(pw_8192)] 6433 lea tx2q, [o(.pass1_end)] 6434 jmp m(idct_8x8_internal_8bpc).pass1_end1 6435 6436 .pass1_end: 6437 SAVE_8ROWS coeffq+64*0, 64 6438 LOAD_8ROWS rsp+gprsize+16*11, 16 6439 mova [rsp+gprsize+16*0], m7 6440 mova m7, [o(pw_8192)] 6441 lea tx2q, [o(.pass1_end1)] 6442 jmp m(idct_8x8_internal_8bpc).pass1_end1 6443 6444 .pass1_end1: 6445 SAVE_8ROWS coeffq+64*8, 64 6446 LOAD_8ROWS rsp+gprsize+16*19, 16 6447 mova [rsp+gprsize+16*0], m7 6448 mova m7, [o(pw_8192)] 6449 lea tx2q, [o(.pass1_end2)] 6450 jmp m(idct_8x8_internal_8bpc).pass1_end1 6451 6452 .pass1_end2: 6453 SAVE_8ROWS coeffq+64*16, 64 6454 LOAD_8ROWS rsp+gprsize+16*27, 16 6455 mova [rsp+gprsize+16*0], m7 6456 mova m7, [o(pw_8192)] 6457 lea tx2q, [o(.pass1_end3)] 6458 jmp m(idct_8x8_internal_8bpc).pass1_end1 6459 6460 .pass1_end3: 6461 SAVE_8ROWS coeffq+64*24, 64 6462 LOAD_8ROWS rsp+gprsize+16*35, 16 6463 mova [rsp+gprsize+16*0], m7 6464 mova m7, [o(pw_8192)] 6465 lea tx2q, [o(.pass1_end4)] 6466 jmp m(idct_8x8_internal_8bpc).pass1_end1 6467 6468 .pass1_end4: 6469 SAVE_8ROWS dstq+64*0, 64 6470 LOAD_8ROWS rsp+gprsize+16*43, 16 6471 mova [rsp+gprsize+16*0], m7 6472 mova m7, [o(pw_8192)] 6473 lea tx2q, [o(.pass1_end5)] 6474 jmp m(idct_8x8_internal_8bpc).pass1_end1 6475 6476 .pass1_end5: 6477 SAVE_8ROWS dstq+64*8, 64 6478 LOAD_8ROWS rsp+gprsize+16*51, 16 6479 mova [rsp+gprsize+16*0], m7 6480 mova m7, [o(pw_8192)] 6481 lea tx2q, [o(.pass1_end6)] 6482 jmp m(idct_8x8_internal_8bpc).pass1_end1 6483 6484 .pass1_end6: 6485 SAVE_8ROWS dstq+64*16, 64 6486 LOAD_8ROWS rsp+gprsize+16*59, 16 6487 mova [rsp+gprsize+16*0], m7 6488 mova m7, [o(pw_8192)] 6489 lea tx2q, [o(.pass1_end7)] 6490 jmp m(idct_8x8_internal_8bpc).pass1_end1 6491 6492 .pass1_end7: 6493 SAVE_8ROWS dstq+64*24, 64 6494 6495 add coeffq, 16 6496 add dstq, 16 6497 dec r3d 6498 jg .pass1_loop 6499 6500 .pass2: 6501 mov dstq, [rsp+gprsize*3+16*67] 6502 mov coeffq, [rsp+gprsize*2+16*67] 6503 lea dstq, [dstq+32] 6504 mov r3d, 4 6505 lea r4, [dstq+8] 6506 mov [rsp+gprsize*2+16*67], r4 6507 lea r4, [o(.pass2_end)] 6508 jmp m(idct_16x64_internal_8bpc).pass2_loop 6509 6510 .pass2_end: 6511 LOAD_8ROWS rsp+gprsize+16*35, 16 6512 lea dstq, [dstq+strideq*2] 6513 lea r3, [rsp+16*32+gprsize] 6514 mova [rsp+gprsize+16*0], m7 6515 call m(idct_16x64_internal_8bpc).write 6516 mov dstq, [rsp+gprsize*2+16*67] 6517 mov r3d, [rsp+gprsize*3+16*67] 6518 lea r4, [dstq+8] 6519 mov [rsp+gprsize*2+16*67], r4 6520 lea r4, [o(.pass2_end)] 6521 6522 dec r3d 6523 jg m(idct_16x64_internal_8bpc).pass2_loop 6524 6525 .pass2_end2: 6526 mov coeffq, [rsp+gprsize*4+16*67] 6527 mov dstq, [rsp+gprsize*2+16*67] 6528 mov r3d, 4 6529 sub dstq, 72 6530 lea r4, [dstq+8] 6531 mov [rsp+gprsize*2+16*67], r4 6532 lea r4, [o(m(idct_16x64_internal_8bpc).end1)] 6533 jmp m(idct_16x64_internal_8bpc).pass2_loop