itx_avx512.asm (304805B)
1 ; Copyright © 2020-2023, VideoLAN and dav1d authors 2 ; Copyright © 2020-2023, Two Orioles, LLC 3 ; All rights reserved. 4 ; 5 ; Redistribution and use in source and binary forms, with or without 6 ; modification, are permitted provided that the following conditions are met: 7 ; 8 ; 1. Redistributions of source code must retain the above copyright notice, this 9 ; list of conditions and the following disclaimer. 10 ; 11 ; 2. Redistributions in binary form must reproduce the above copyright notice, 12 ; this list of conditions and the following disclaimer in the documentation 13 ; and/or other materials provided with the distribution. 14 ; 15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 26 %include "config.asm" 27 %include "ext/x86/x86inc.asm" 28 29 %if ARCH_X86_64 30 31 SECTION_RODATA 64 32 const \ 33 dup16_perm, db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7 34 db 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15 35 db 16, 17, 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23 36 db 24, 25, 24, 25, 26, 27, 26, 27, 28, 29, 28, 29, 30, 31, 30, 31 37 const \ 38 int8_permA, db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51 39 db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55 40 db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59 41 db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63 42 int8_permB: db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51 43 db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59 44 db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55 45 db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63 46 int16_perm: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 47 db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 48 db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 49 db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 50 idtx_16x4p: db 0, 1, 4, 5, 16, 17, 20, 21, 2, 3, 6, 7, 18, 19, 22, 23 51 db 32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55 52 db 8, 9, 12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31 53 db 40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63 54 idct_8x32p: db 60, 61, 4, 5, 32, 33, 0, 1, 28, 29, 36, 37, 56, 57, 8, 9 55 db 12, 13, 52, 53, 24, 25, 40, 41, 44, 45, 20, 21, 48, 49, 16, 17 56 db 62, 63, 2, 3, 6, 7, 58, 59, 54, 55, 10, 11, 14, 15, 50, 51 57 db 46, 47, 18, 19, 22, 23, 42, 43, 38, 39, 26, 27, 30, 31, 34, 35 58 idct_16x32p: db 6, 7, 58, 59, 38, 39, 26, 27, 32, 33, 0, 1, 30, 31, 34, 35 59 db 46, 47, 18, 19, 22, 23, 42, 43, 24, 25, 40, 41, 44, 45, 20, 21 60 db 62, 63, 2, 3, 48, 49, 16, 17, 56, 57, 8, 9, 14, 15, 50, 51 61 db 54, 55, 10, 11, 60, 61, 4, 5, 12, 13, 52, 53, 28, 29, 36, 37 62 end_16x32p: db 0, 32, 1, 48, 2, 36, 3, 52, 16, 40, 17, 56, 18, 44, 19, 60 63 db 4, 33, 5, 49, 6, 37, 7, 53, 20, 41, 21, 57, 22, 45, 23, 61 64 db 8, 35, 9, 51, 10, 39, 11, 55, 24, 43, 25, 59, 26, 47, 27, 63 65 db 12, 34, 13, 50, 14, 38, 15, 54, 28, 42, 29, 58, 30, 46, 31, 62 66 67 ; packed 4-bit qword shuffle indices 68 permA: dq 0x1c0d0d1ce0d94040, 0x5849495868fb6262 69 dq 0x3e2f2f3ef1c85151, 0x7a6b6b7a79ea7373 70 dq 0x94858594a451c8d9, 0xd0c1c1d02c73eafb 71 dq 0xb6a7a7b6b540d9c8, 0xf2e3e3f23d62fbea 72 permB: dq 0x40acbd0fcadb0f40, 0x518e9f3ce8f99604 73 dq 0xc824352d56128751, 0xd906171e74301e15 74 dq 0x6271604b03472d62, 0x735342782165b426 75 dq 0xeaf9e8699f8ea573, 0xfbdbca5abdac3c37 76 permC: dq 0x9d409d041551c2e0, 0xbf62bf263773a486 77 dq 0xc88c8c15409dd3f1, 0xeaaeae3762bfb597 78 dq 0x04d9158c8cc84a68, 0x26fb37aeaeea2c0e 79 dq 0x5115049dd9045b79, 0x733726bffb263d1f 80 permD: dq 0x0cda098800041504, 0x0edb09b2028c3726 81 dq 0x0f11fa9c01150415, 0x0988f326039d2637 82 dq 0x05640f1108269d8c, 0x05290edb0aaebfae 83 dq 0x0005000509378c9d, 0xffffffff0bbfaebf 84 85 pd_0to15: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 86 gather8a: dd 0, 2, 1, 3, 8, 10, 9, 11 87 gather8b: dd 0, 1, 4, 5, 8, 9, 12, 13 88 gather8c: dd 0, 4, 2, 6, 12, 8, 14, 10 89 gather8d: dd 0, 19, 1, 18, 2, 17, 3, 16 90 91 int_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 92 int_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 93 int_shuf3: db 0, 1, 8, 9, 4, 5, 12, 13, 2, 3, 10, 11, 6, 7, 14, 15 94 int_shuf4: db 8, 9, 0, 1, 12, 13, 4, 5, 10, 11, 2, 3, 14, 15, 6, 7 95 deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 96 int_mshift: db 12, 20, 0, 0, 44, 52, 0, 0 97 98 pb_32: times 4 db 32 99 pw_2048: times 2 dw 2048 100 pw_4096: times 2 dw 4096 101 pw_8192: times 2 dw 8192 102 pw_16384: times 2 dw 16384 103 pw_1697x16: times 2 dw 1697*16 104 pw_1697x8: times 2 dw 1697*8 105 pw_2896x8: times 2 dw 2896*8 106 pd_2048: dd 2048 107 108 %define pw_5 (permD+52) 109 %define pd_m1 (permD+60) 110 %define pw_3803_1321 (permD+44) 111 %define pw_2482_3803 (permD+12) 112 %define pw_2440_3290 (permD+ 4) 113 %define pw_m3290_2440 (permD+28) 114 %define pw_3857_1380 (permD+36) 115 %define pw_m1380_3857 (permD+20) 116 117 pw_8192_m8192: dw 8192, -8192 118 pw_m8192_8192: dw -8192, 8192 119 pw_16384_m16384: dw 16384, -16384 120 pw_m16384_16384: dw -16384, 16384 121 122 pw_m1321_2482: dw -1321, 2482 123 pw_m3344_3344: dw -3344, 3344 124 pw_2482_3344: dw 2482, 3344 125 pw_m3803_3344: dw -3803, 3344 126 pd_3344: dd 3344 127 pw_m1321_m3344: dw -1321, -3344 128 pw_2896_m2896: dw 2896, -2896 129 130 pw_1567_m3784: dw 1567, -3784 131 pw_3784_m1567: dw 3784, -1567 132 pw_4017_m799: dw 4017, -799 133 pw_2276_m3406: dw 2276, -3406 134 pw_m799_m4017: dw -799, -4017 135 pw_m3406_m2276: dw -3406, -2276 136 137 %macro COEF_PAIR 2-3 0 138 pw_%1_%2: dw %1, %2 139 pw_m%2_%1: dw -%2, %1 140 %if %3 141 pw_m%1_m%2: dw -%1, -%2 142 %endif 143 %endmacro 144 145 COEF_PAIR 2896, 2896 146 COEF_PAIR 1567, 3784, 1 147 COEF_PAIR 3784, 1567 148 COEF_PAIR 201, 4091 149 COEF_PAIR 995, 3973 150 COEF_PAIR 1751, 3703 151 COEF_PAIR 3035, 2751 152 COEF_PAIR 3513, 2106 153 COEF_PAIR 4052, 601 154 COEF_PAIR 3166, 2598, 1 155 COEF_PAIR 3920, 1189, 1 156 COEF_PAIR 2276, 3406 157 COEF_PAIR 4017, 799 158 159 %macro COEF_X8 1-* 160 %rep %0 161 dw %1*8, %1*8 162 %rotate 1 163 %endrep 164 %endmacro 165 166 pw_m2276x8: COEF_X8 -2276 167 pw_3406x8: COEF_X8 3406 168 pw_4017x8: COEF_X8 4017 169 pw_799x8: COEF_X8 799 170 pw_3784x8: COEF_X8 3784 171 pw_1567x8: COEF_X8 1567 172 173 pw_4076x8: COEF_X8 4076 174 pw_401x8: COEF_X8 401 175 pw_m2598x8: COEF_X8 -2598 176 pw_3166x8: COEF_X8 3166 177 pw_3612x8: COEF_X8 3612 178 pw_1931x8: COEF_X8 1931 179 pw_m1189x8: COEF_X8 -1189 180 pw_3920x8: COEF_X8 3920 181 182 pw_4091x8: COEF_X8 4091 183 pw_201x8: COEF_X8 201 184 pw_m2751x8: COEF_X8 -2751 185 pw_3035x8: COEF_X8 3035 186 pw_3703x8: COEF_X8 3703 187 pw_1751x8: COEF_X8 1751 188 pw_m1380x8: COEF_X8 -1380 189 pw_3857x8: COEF_X8 3857 190 pw_3973x8: COEF_X8 3973 191 pw_995x8: COEF_X8 995 192 pw_m2106x8: COEF_X8 -2106 193 pw_3513x8: COEF_X8 3513 194 pw_3290x8: COEF_X8 3290 195 pw_2440x8: COEF_X8 2440 196 pw_m601x8: COEF_X8 -601 197 pw_4052x8: COEF_X8 4052 198 199 pw_401_4076x8: dw 401*8, 4076*8 200 pw_m2598_3166x8: dw -2598*8, 3166*8 201 pw_1931_3612x8: dw 1931*8, 3612*8 202 pw_m1189_3920x8: dw -1189*8, 3920*8 203 pw_799_4017x8: dw 799*8, 4017*8 204 pw_m2276_3406x8: dw -2276*8, 3406*8 205 206 pw_201_4091x8: dw 201*8, 4091*8 207 pw_m601_4052x8: dw -601*8, 4052*8 208 pw_995_3973x8: dw 995*8, 3973*8 209 pw_m1380_3857x8: dw -1380*8, 3857*8 210 pw_1751_3703x8: dw 1751*8, 3703*8 211 pw_m2106_3513x8: dw -2106*8, 3513*8 212 pw_2440_3290x8: dw 2440*8, 3290*8 213 pw_m2751_3035x8: dw -2751*8, 3035*8 214 215 pw_101_4095x8: dw 101*8, 4095*8 216 pw_m2824_2967x8: dw -2824*8, 2967*8 217 pw_1660_3745x8: dw 1660*8, 3745*8 218 pw_m1474_3822x8: dw -1474*8, 3822*8 219 pw_897_3996x8: dw 897*8, 3996*8 220 pw_m2191_3461x8: dw -2191*8, 3461*8 221 pw_2359_3349x8: dw 2359*8, 3349*8 222 pw_m700_4036x8: dw -700*8, 4036*8 223 pw_501_4065x8: dw 501*8, 4065*8 224 pw_m2520_3229x8: dw -2520*8, 3229*8 225 pw_2019_3564x8: dw 2019*8, 3564*8 226 pw_m1092_3948x8: dw -1092*8, 3948*8 227 pw_1285_3889x8: dw 1285*8, 3889*8 228 pw_m1842_3659x8: dw -1842*8, 3659*8 229 pw_2675_3102x8: dw 2675*8, 3102*8 230 pw_m301_4085x8: dw -301*8, 4085*8 231 232 idct64_mul: COEF_X8 4095, 101, 2967, -2824, 3745, 1660, 3822, -1474 233 COEF_PAIR 401, 4076, 1 234 COEF_PAIR 799, 4017 235 COEF_X8 -700, 4036, 2359, 3349, -2191, 3461, 897, 3996 236 dw -2598, -3166, 3166, -2598, 2598, 3166, -4017, -799, 799, -4017 237 COEF_X8 4065, 501, 3229, -2520, 3564, 2019, 3948, -1092 238 COEF_PAIR 1931, 3612, 1 239 COEF_PAIR 3406, 2276 240 COEF_X8 -301, 4085, 2675, 3102, -1842, 3659, 1285, 3889 241 dw -1189, -3920, 3920, -1189, 1189, 3920, -2276, -3406, 3406, -2276 242 243 SECTION .text 244 245 %define o_base int8_permA+64*18 246 %define o(x) (r5 - (o_base) + (x)) 247 %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) 248 249 ; flags: 1 = swap, 2 = interleave (l), 4 = interleave (t), 8 = no_pack, 250 ; 16 = special_mul1, 32 = special_mul2 251 %macro ITX_MUL2X_PACK 6-7 0 ; dst/src, tmp[1-2], rnd, coef[1-2], flags 252 mova m%2, m%4 253 %if %7 & 16 254 vpdpwssd m%2, m%1, [o(pw_%5)] {bcstd} 255 mova m%3, m%4 256 %if %7 & 32 257 vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd} 258 %else 259 vpdpwssd m%3, m%1, m%6 260 %endif 261 %elif %7 & 32 262 vpdpwssd m%2, m%1, m%5 263 mova m%3, m%4 264 vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd} 265 %elif %6 < 32 266 vpdpwssd m%2, m%1, m%5 267 mova m%3, m%4 268 vpdpwssd m%3, m%1, m%6 269 %elif %7 & 1 270 vpdpwssd m%2, m%1, [o(pw_%5_%6)] {bcstd} 271 mova m%3, m%4 272 vpdpwssd m%3, m%1, [o(pw_m%6_%5)] {bcstd} 273 %else 274 vpdpwssd m%2, m%1, [o(pw_m%6_%5)] {bcstd} 275 mova m%3, m%4 276 vpdpwssd m%3, m%1, [o(pw_%5_%6)] {bcstd} 277 %endif 278 %if %7 & 2 279 psrld m%2, 12 280 pslld m%3, 4 281 vpshrdd m%1, m%3, m%2, 16 282 %elif %7 & 4 283 ; compared to using shifts (as above) this has better throughput, 284 ; but worse latency and requires setting up the opmask/index 285 ; registers, so only use this method for the larger transforms 286 pslld m%1, m%2, 4 287 vpmultishiftqb m%1{k7}, m13, m%3 288 %else 289 psrad m%2, 12 290 psrad m%3, 12 291 %if %7 & 8 == 0 292 packssdw m%1, m%3, m%2 293 %endif 294 %endif 295 %endmacro 296 297 ; flags: same as ITX_MUL2X_PACK 298 %macro ITX_MUL4X_PACK 10-11 0 ; dst/src, tmp[1-2], coef_tmp[1-2], rnd, coef[1-4], flags 299 %if %11 & 1 300 vpbroadcastd m%4, [o(pw_%9_%10)] 301 vpbroadcastd m%4{k1}, [o(pw_%7_%8)] 302 vpbroadcastd m%5, [o(pw_m%10_%9)] 303 vpbroadcastd m%5{k1}, [o(pw_m%8_%7)] 304 %else 305 vpbroadcastd m%4, [o(pw_m%10_%9)] 306 vpbroadcastd m%4{k1}, [o(pw_m%8_%7)] 307 vpbroadcastd m%5, [o(pw_%9_%10)] 308 vpbroadcastd m%5{k1}, [o(pw_%7_%8)] 309 %endif 310 ITX_MUL2X_PACK %1, %2, %3, %6, %4, %5, %11 311 %endmacro 312 313 ; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 314 ; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 315 %macro ITX_MULSUB_2W 7-8 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2], dst2 316 punpcklwd m%3, m%2, m%1 317 punpckhwd m%2, m%1 318 %if %7 < 32 319 mova m%1, m%5 320 vpdpwssd m%1, m%3, m%7 321 mova m%4, m%5 322 vpdpwssd m%4, m%2, m%7 323 %else 324 mova m%1, m%5 325 vpdpwssd m%1, m%3, [o(pw_m%7_%6)] {bcstd} 326 mova m%4, m%5 327 vpdpwssd m%4, m%2, [o(pw_m%7_%6)] {bcstd} 328 %endif 329 psrad m%1, 12 330 psrad m%4, 12 331 packssdw m%1, m%4 332 mova m%4, m%5 333 %if %7 < 32 334 vpdpwssd m%4, m%2, m%6 335 mova m%2, m%5 336 vpdpwssd m%2, m%3, m%6 337 %else 338 vpdpwssd m%4, m%2, [o(pw_%6_%7)] {bcstd} 339 mova m%2, m%5 340 vpdpwssd m%2, m%3, [o(pw_%6_%7)] {bcstd} 341 %endif 342 psrad m%4, 12 343 psrad m%2, 12 344 %if %0 == 8 345 packssdw m%8, m%2, m%4 346 %else 347 packssdw m%2, m%4 348 %endif 349 %endmacro 350 351 %macro WRAP_XMM 1+ 352 %xdefine %%reset RESET_MM_PERMUTATION 353 INIT_XMM cpuname 354 DEFINE_MMREGS xmm 355 AVX512_MM_PERMUTATION 356 %1 357 %%reset 358 %endmacro 359 360 %macro WRAP_YMM 1+ 361 INIT_YMM cpuname 362 %1 363 INIT_ZMM cpuname 364 %endmacro 365 366 %macro ITX4_END 4-5 2048 ; row[1-4], rnd 367 %if %5 368 vpbroadcastd m2, [o(pw_%5)] 369 pmulhrsw m0, m2 370 pmulhrsw m1, m2 371 %endif 372 lea r2, [dstq+strideq*2] 373 %assign %%i 1 374 %rep 4 375 %if %1 & 2 376 CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1) 377 %else 378 CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1) 379 %endif 380 %assign %%i %%i + 1 381 %rotate 1 382 %endrep 383 movd m2, [%%row_adr1] 384 pinsrd m2, [%%row_adr2], 1 385 movd m3, [%%row_adr3] 386 pinsrd m3, [%%row_adr4], 1 387 pmovzxbw m2, m2 388 pmovzxbw m3, m3 389 paddw m0, m2 390 paddw m1, m3 391 packuswb m0, m1 392 movd [%%row_adr1], m0 393 pextrd [%%row_adr2], m0, 1 394 pextrd [%%row_adr3], m0, 2 395 pextrd [%%row_adr4], m0, 3 396 ret 397 %endmacro 398 399 %macro INV_TXFM_FN 3 ; type1, type2, size 400 cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 6, 0, dst, stride, c, eob, tx2, base 401 %define %%p1 m(i%1_%3_internal_8bpc) 402 lea baseq, [o_base] 403 ; Jump to the 1st txfm function if we're not taking the fast path, which 404 ; in turn performs an indirect jump to the 2nd txfm function. 405 lea tx2q, [m(i%2_%3_internal_8bpc).pass2] 406 %ifidn %1_%2, dct_dct 407 test eobd, eobd 408 jnz %%p1 409 %else 410 ; jump to the 1st txfm function unless it's located directly after this 411 times ((%%end - %%p1) >> 31) & 1 jmp %%p1 412 ALIGN function_align 413 %%end: 414 %endif 415 %endmacro 416 417 %macro INV_TXFM_4X4_FN 2 ; type1, type2 418 INV_TXFM_FN %1, %2, 4x4 419 %ifidn %1_%2, dct_dct 420 vpbroadcastw m0, [cq] 421 vpbroadcastd m1, [o(pw_2896x8)] 422 pmulhrsw m0, m1 423 mov [cq], eobd 424 pmulhrsw m0, m1 425 mova m1, m0 426 jmp m(iadst_4x4_internal_8bpc).end2 427 %endif 428 %endmacro 429 430 %macro IDCT4_1D_PACKED 0 431 vpbroadcastd m4, [o(pd_2048)] 432 punpckhwd m2, m1, m0 433 punpcklwd m1, m0 434 ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784 435 ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896 436 paddsw m0, m1, m2 ; out0 out1 437 psubsw m1, m2 ; out3 out2 438 %endmacro 439 440 %macro IADST4_1D_PACKED 0 441 punpcklwd m4, m1, m0 ; in2 in0 442 punpckhwd m5, m1, m0 ; in3 in1 443 .main2: 444 vpbroadcastd m3, [o(pd_2048)] 445 mova m0, m3 446 vpdpwssd m0, m4, [o(pw_3803_1321)] {bcstd} 447 mova m2, m3 448 vpdpwssd m2, m4, [o(pw_m1321_2482)] {bcstd} 449 mova m1, m3 450 vpdpwssd m1, m4, [o(pw_m3344_3344)] {bcstd} 451 vpdpwssd m3, m4, [o(pw_2482_3803)] {bcstd} 452 vpdpwssd m0, m5, [o(pw_2482_3344)] {bcstd} 453 vpdpwssd m2, m5, [o(pw_m3803_3344)] {bcstd} 454 vpdpwssd m1, m5, [o(pd_3344)] {bcstd} 455 vpdpwssd m3, m5, [o(pw_m1321_m3344)] {bcstd} 456 REPX {psrad x, 12}, m0, m2, m1, m3 457 packssdw m0, m2 ; out0 out1 458 packssdw m1, m3 ; out2 out3 459 %endmacro 460 461 INIT_XMM avx512icl 462 INV_TXFM_4X4_FN dct, dct 463 INV_TXFM_4X4_FN dct, adst 464 INV_TXFM_4X4_FN dct, flipadst 465 INV_TXFM_4X4_FN dct, identity 466 467 cglobal idct_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 468 mova m0, [cq+16*0] 469 mova m1, [cq+16*1] 470 IDCT4_1D_PACKED 471 mova m2, [o(deint_shuf)] 472 shufps m3, m0, m1, q1331 473 shufps m0, m0, m1, q0220 474 pshufb m0, m2 475 pshufb m1, m3, m2 476 jmp tx2q 477 .pass2: 478 IDCT4_1D_PACKED 479 pxor ymm16, ymm16 480 mova [cq], ymm16 481 ITX4_END 0, 1, 3, 2 482 483 INV_TXFM_4X4_FN adst, dct 484 INV_TXFM_4X4_FN adst, adst 485 INV_TXFM_4X4_FN adst, flipadst 486 INV_TXFM_4X4_FN adst, identity 487 488 cglobal iadst_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 489 mova m0, [cq+16*0] 490 mova m1, [cq+16*1] 491 call .main 492 punpckhwd m3, m0, m1 493 punpcklwd m0, m1 494 punpckhwd m1, m0, m3 495 punpcklwd m0, m3 496 jmp tx2q 497 .pass2: 498 call .main 499 .end: 500 pxor ymm16, ymm16 501 mova [cq], ymm16 502 .end2: 503 ITX4_END 0, 1, 2, 3 504 ALIGN function_align 505 .main: 506 IADST4_1D_PACKED 507 ret 508 509 INV_TXFM_4X4_FN flipadst, dct 510 INV_TXFM_4X4_FN flipadst, adst 511 INV_TXFM_4X4_FN flipadst, flipadst 512 INV_TXFM_4X4_FN flipadst, identity 513 514 cglobal iflipadst_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 515 mova m0, [cq+16*0] 516 mova m1, [cq+16*1] 517 call m(iadst_4x4_internal_8bpc).main 518 punpcklwd m2, m1, m0 519 punpckhwd m1, m0 520 punpcklwd m0, m1, m2 521 punpckhwd m1, m2 522 jmp tx2q 523 .pass2: 524 call m(iadst_4x4_internal_8bpc).main 525 .end: 526 pxor ymm16, ymm16 527 mova [cq], ymm16 528 .end2: 529 ITX4_END 3, 2, 1, 0 530 531 INV_TXFM_4X4_FN identity, dct 532 INV_TXFM_4X4_FN identity, adst 533 INV_TXFM_4X4_FN identity, flipadst 534 INV_TXFM_4X4_FN identity, identity 535 536 cglobal iidentity_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 537 mova m0, [cq+16*0] 538 mova m1, [cq+16*1] 539 vpbroadcastd m3, [o(pw_1697x8)] 540 pmulhrsw m2, m3, m0 541 pmulhrsw m3, m1 542 paddsw m0, m2 543 paddsw m1, m3 544 punpckhwd m2, m0, m1 545 punpcklwd m0, m1 546 punpckhwd m1, m0, m2 547 punpcklwd m0, m2 548 jmp tx2q 549 .pass2: 550 vpbroadcastd m3, [o(pw_1697x8)] 551 pmulhrsw m2, m3, m0 552 pmulhrsw m3, m1 553 paddsw m0, m2 554 paddsw m1, m3 555 jmp m(iadst_4x4_internal_8bpc).end 556 557 %macro INV_TXFM_4X8_FN 2 ; type1, type2 558 INV_TXFM_FN %1, %2, 4x8 559 %ifidn %1_%2, dct_dct 560 movd xmm1, [o(pw_2896x8)] 561 pmulhrsw xmm0, xmm1, [cq] 562 movd xmm2, [o(pw_2048)] 563 pmulhrsw xmm0, xmm1 564 pmulhrsw xmm0, xmm1 565 pmulhrsw xmm0, xmm2 566 vpbroadcastw ym0, xmm0 567 mova ym1, ym0 568 jmp m(iadst_4x8_internal_8bpc).end3 569 %endif 570 %endmacro 571 572 %macro IDCT8_1D_PACKED 0 573 punpckhwd m5, m3, m0 ; in7 in1 574 punpckhwd m4, m1, m2 ; in3 in5 575 punpcklwd m3, m1 ; in6 in2 576 punpcklwd m2, m0 ; in4 in0 577 .main2: 578 vpbroadcastd m6, [o(pd_2048)] 579 ITX_MUL2X_PACK 5, 0, 1, 6, 799, 4017, 3 ; t4a t7a 580 ITX_MUL2X_PACK 4, 0, 1, 6, 3406, 2276, 3 ; t5a t6a 581 ITX_MUL2X_PACK 3, 0, 1, 6, 1567, 3784 ; t3 t2 582 psubsw m0, m5, m4 ; t5a t6a (interleaved) 583 paddsw m4, m5 ; t4 t7 (interleaved) 584 ITX_MUL2X_PACK 2, 1, 5, 6, 2896, 2896 ; t0 t1 585 ITX_MUL2X_PACK 0, 1, 5, 6, 2896, 2896, 1 ; t6 t5 586 %if mmsize > 16 587 vbroadcasti32x4 m1, [o(deint_shuf)] 588 pshufb m4, m1 589 %else 590 pshufb m4, [o(deint_shuf)] 591 %endif 592 psubsw m1, m2, m3 ; tmp3 tmp2 593 paddsw m3, m2 ; tmp0 tmp1 594 punpckhqdq m2, m4, m0 ; t7 t6 595 punpcklqdq m4, m0 ; t4 t5 596 paddsw m0, m3, m2 ; out0 out1 597 psubsw m3, m2 ; out7 out6 598 psubsw m2, m1, m4 ; out4 out5 599 paddsw m1, m4 ; out3 out2 600 %endmacro 601 602 %macro IADST8_1D_PACKED 1 ; pass 603 vpbroadcastd m6, [o(pd_2048)] 604 %if %1 == 1 605 ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076, 3 ; t1a t0a 606 ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612, 2 ; t2a t3a 607 ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598, 3 ; t5a t4a 608 ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189, 2 ; t6a t7a 609 psubsw m4, m0, m2 ; t5 t4 610 paddsw m0, m2 ; t1 t0 611 psubsw m5, m1, m3 ; t6 t7 612 paddsw m1, m3 ; t2 t3 613 ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784, 3 ; t5a t4a 614 ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 2 ; t7a t6a 615 %if mmsize > 16 616 vbroadcasti32x4 m2, [o(deint_shuf)] 617 %else 618 mova m2, [o(deint_shuf)] 619 %endif 620 vprord m1, 16 621 psubsw m3, m0, m1 ; t3 t2 622 paddsw m0, m1 ; -out7 out0 623 psubsw m1, m4, m5 ; t7 t6 624 paddsw m4, m5 ; out6 -out1 625 pshufb m0, m2 626 pshufb m4, m2 627 mova m2, m6 628 vpdpwssd m2, m3, [o(pw_m2896_2896)] {bcstd} 629 mova m5, m6 630 vpdpwssd m5, m1, [o(pw_m2896_2896)] {bcstd} 631 psrad m2, 12 632 psrad m5, 12 633 packssdw m2, m5 ; out4 -out5 634 mova m5, m6 635 vpdpwssd m5, m3, [o(pw_2896_2896)] {bcstd} 636 mova m3, m6 637 vpdpwssd m3, m1, [o(pw_2896_2896)] {bcstd} 638 psrad m5, 12 639 psrad m3, 12 640 packssdw m1, m3, m5 ; out2 -out3 641 %else 642 punpckhwd m0, m4, m3 ; 0 7 643 punpckhwd m1, m5, m2 ; 2 5 644 punpcklwd m2, m5 ; 4 3 645 punpcklwd m3, m4 ; 6 1 646 ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076 ; t0a t1a 647 ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612 ; t2a t3a 648 ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598 ; t4a t5a 649 ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189 ; t6a t7a 650 psubsw m4, m0, m2 ; t4 t5 651 paddsw m0, m2 ; t0 t1 652 psubsw m5, m1, m3 ; t6 t7 653 paddsw m1, m3 ; t2 t3 654 shufps m2, m5, m4, q1032 655 punpckhwd m4, m2 656 punpcklwd m5, m2 657 ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784 ; t4a t5a 658 ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 1 ; t6a t7a 659 psubsw m2, m0, m1 ; t2 t3 660 paddsw m0, m1 ; out0 -out7 661 psubsw m1, m4, m5 ; t6 t7 662 paddsw m4, m5 ; -out1 out6 663 vpbroadcastd m5, [o(pw_2896x8)] 664 punpckhqdq m3, m2, m1 ; t3 t7 665 punpcklqdq m2, m1 ; t2 t6 666 paddsw m1, m2, m3 ; t2+t3 t6+t7 667 psubsw m2, m3 ; t2-t3 t6-t7 668 punpckhqdq m3, m4, m0 ; out6 -out7 669 punpcklqdq m0, m4 ; out0 -out1 670 pmulhrsw m2, m5 ; out4 -out5 671 pshufd m1, m1, q1032 672 pmulhrsw m1, m5 ; out2 -out3 673 %endif 674 %endmacro 675 676 INIT_YMM avx512icl 677 INV_TXFM_4X8_FN dct, dct 678 INV_TXFM_4X8_FN dct, identity 679 INV_TXFM_4X8_FN dct, adst 680 INV_TXFM_4X8_FN dct, flipadst 681 682 cglobal idct_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 683 vpermq m0, [cq+32*0], q3120 684 vpermq m1, [cq+32*1], q3120 685 vpbroadcastd m2, [o(pw_2896x8)] 686 pmulhrsw m0, m2 687 pmulhrsw m1, m2 688 IDCT4_1D_PACKED 689 vbroadcasti32x4 m2, [o(deint_shuf)] 690 shufps m3, m0, m1, q1331 691 shufps m0, m0, m1, q0220 692 pshufb m0, m2 693 pshufb m1, m3, m2 694 jmp tx2q 695 .pass2: 696 vextracti32x4 xm2, m0, 1 697 vextracti32x4 xm3, m1, 1 698 call .main 699 vpbroadcastd m4, [o(pw_2048)] 700 vinserti32x4 m0, m0, xm2, 1 701 vinserti32x4 m1, m1, xm3, 1 702 pshufd m1, m1, q1032 703 jmp m(iadst_4x8_internal_8bpc).end2 704 ALIGN function_align 705 .main: 706 WRAP_XMM IDCT8_1D_PACKED 707 ret 708 709 INV_TXFM_4X8_FN adst, dct 710 INV_TXFM_4X8_FN adst, adst 711 INV_TXFM_4X8_FN adst, flipadst 712 INV_TXFM_4X8_FN adst, identity 713 714 cglobal iadst_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 715 vpermq m0, [cq+32*0], q3120 716 vpermq m1, [cq+32*1], q3120 717 vpbroadcastd m2, [o(pw_2896x8)] 718 pmulhrsw m0, m2 719 pmulhrsw m1, m2 720 call m(iadst_8x4_internal_8bpc).main 721 punpckhwd m3, m0, m1 722 punpcklwd m0, m1 723 punpckhwd m1, m0, m3 724 punpcklwd m0, m3 725 jmp tx2q 726 .pass2: 727 vextracti32x4 xm2, m0, 1 728 vextracti32x4 xm3, m1, 1 729 pshufd xm4, xm0, q1032 730 pshufd xm5, xm1, q1032 731 call .main_pass2 732 vpbroadcastd m4, [o(pw_2048)] 733 vinserti32x4 m0, xm2, 1 734 vinserti32x4 m1, xm3, 1 735 pxor m5, m5 736 psubw m5, m4 737 .end: 738 punpcklqdq m4, m5 739 .end2: 740 pmulhrsw m0, m4 741 pmulhrsw m1, m4 742 .end3: 743 vpbroadcastd m3, strided 744 pmulld m5, m3, [o(pd_0to15)] 745 kxnorb k1, k1, k1 746 kmovb k2, k1 747 vpgatherdd m3{k1}, [dstq+m5] 748 pxor m4, m4 749 mova [cq], zmm20 750 punpcklbw m2, m3, m4 751 punpckhbw m3, m4 752 paddw m0, m2 753 paddw m1, m3 754 packuswb m0, m1 755 vpscatterdd [dstq+m5]{k2}, m0 756 RET 757 ALIGN function_align 758 .main_pass1: 759 punpckhwd xm0, xm4, xm3 ; 0 7 760 punpckhwd xm1, xm5, xm2 ; 2 5 761 punpcklwd xm2, xm5 ; 4 3 762 punpcklwd xm3, xm4 ; 6 1 763 WRAP_XMM IADST8_1D_PACKED 1 764 punpcklqdq xm3, xm4, xm0 ; out6 -out7 765 punpckhqdq xm0, xm4 ; out0 -out1 766 ret 767 ALIGN function_align 768 .main_pass2: 769 WRAP_XMM IADST8_1D_PACKED 2 770 ret 771 772 INV_TXFM_4X8_FN flipadst, dct 773 INV_TXFM_4X8_FN flipadst, adst 774 INV_TXFM_4X8_FN flipadst, flipadst 775 INV_TXFM_4X8_FN flipadst, identity 776 777 cglobal iflipadst_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 778 vpermq m0, [cq+32*0], q3120 779 vpermq m1, [cq+32*1], q3120 780 vpbroadcastd m2, [o(pw_2896x8)] 781 pmulhrsw m0, m2 782 pmulhrsw m1, m2 783 call m(iadst_8x4_internal_8bpc).main 784 punpcklwd m3, m1, m0 785 punpckhwd m1, m0 786 punpcklwd m0, m1, m3 787 punpckhwd m1, m3 788 jmp tx2q 789 .pass2: 790 vextracti32x4 xm2, m0, 1 791 vextracti32x4 xm3, m1, 1 792 pshufd xm4, xm0, q1032 793 pshufd xm5, xm1, q1032 794 call m(iadst_4x8_internal_8bpc).main_pass2 795 vpbroadcastd m5, [o(pw_2048)] 796 vinserti32x4 m3, xm1, 1 797 vinserti32x4 m2, xm0, 1 798 pxor m4, m4 799 psubw m4, m5 800 pshufd m0, m3, q1032 801 pshufd m1, m2, q1032 802 jmp m(iadst_4x8_internal_8bpc).end 803 804 INIT_ZMM avx512icl 805 INV_TXFM_4X8_FN identity, dct 806 INV_TXFM_4X8_FN identity, adst 807 INV_TXFM_4X8_FN identity, flipadst 808 INV_TXFM_4X8_FN identity, identity 809 810 cglobal iidentity_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 811 vpbroadcastd m0, [o(pw_2896x8)] 812 pmulhrsw m0, [cq] 813 mova m1, [o(int8_permB)] 814 vpbroadcastd m2, [o(pw_1697x8)] 815 vpermb m0, m1, m0 816 pmulhrsw m2, m0 817 paddsw m0, m2 818 vextracti32x8 ym1, m0, 1 819 jmp tx2q 820 .pass2: 821 vpbroadcastd ym4, [o(pw_4096)] 822 jmp m(iadst_4x8_internal_8bpc).end2 823 824 %macro INV_TXFM_4X16_FN 2 ; type1, type2 825 INV_TXFM_FN %1, %2, 4x16 826 %ifidn %1_%2, dct_dct 827 movsx r6d, word [cq] 828 mov [cq], eobd 829 imul r6d, 181 830 add r6d, 128+256 831 sar r6d, 8+1 832 imul r6d, 181 833 add r6d, 128+2048 834 sar r6d, 8+4 835 vpbroadcastw m0, r6d 836 mova m1, m0 837 jmp m(iadst_4x16_internal_8bpc).end3 838 %endif 839 %endmacro 840 841 %macro IDCT16_1D_PACKED 0 842 punpckhwd m8, m7, m0 ; dct16 in15 in1 843 punpcklwd m9, m4, m0 ; dct4 in2 in0 844 punpckhwd m0, m3, m4 ; dct16 in7 in9 845 punpcklwd m7, m1 ; dct8 in7 in1 846 punpckhwd m1, m6 ; dct16 in3 in13 847 punpcklwd m3, m5 ; dct8 in3 in5 848 punpckhwd m5, m2 ; dct16 in11 in5 849 punpcklwd m6, m2 ; dct4 in3 in1 850 cglobal_label .main2 851 vpbroadcastd m10, [o(pd_2048)] 852 .main3: 853 vpbroadcastq m13, [o(int_mshift)] 854 vpcmpub k7, m13, m10, 6 ; 0x33... 855 ITX_MUL2X_PACK 8, 2, 4, 10, 401, 4076, 5 ; t8a t15a 856 ITX_MUL2X_PACK 0, 2, 4, 10, 3166, 2598, 5 ; t9a t14a 857 ITX_MUL2X_PACK 5, 2, 4, 10, 1931, 3612, 5 ; t10a t13a 858 ITX_MUL2X_PACK 1, 2, 4, 10, 3920, 1189, 5 ; t11a t12a 859 ITX_MUL2X_PACK 7, 2, 4, 10, 799, 4017, 5 ; t4a t7a 860 ITX_MUL2X_PACK 3, 2, 4, 10, 3406, 2276, 5 ; t5a t6a 861 .main4: 862 psubsw m2, m8, m0 ; t9 t14 863 paddsw m8, m0 ; t8 t15 864 psubsw m4, m1, m5 ; t10 t13 865 paddsw m1, m5 ; t11 t12 866 ITX_MUL2X_PACK 6, 0, 5, 10, 1567, 3784 ; t3 t2 867 psubsw m0, m8, m1 ; t11a t12a 868 paddsw m8, m1 ; t8a t15a 869 psubsw m1, m7, m3 ; t5a t6a 870 paddsw m7, m3 ; t4 t7 871 .main5: 872 ITX_MUL2X_PACK 2, 3, 5, 10, 1567, 3784, 5 ; t9a t14a 873 ITX_MUL2X_PACK 4, 3, 5, 10, m3784, 1567, 5 ; t10a t13a 874 %if mmsize > 16 875 vbroadcasti32x4 m5, [o(deint_shuf)] 876 %else 877 mova m5, [o(deint_shuf)] 878 %endif 879 vpbroadcastd m11, [o(pw_m2896_2896)] 880 vpbroadcastd m12, [o(pw_2896_2896)] 881 paddsw m3, m2, m4 ; t9 t14 882 psubsw m2, m4 ; t10 t13 883 pshufb m8, m5 884 pshufb m7, m5 885 pshufb m3, m5 886 ITX_MUL2X_PACK 9, 4, 5, 10, 11, 12 ; t0 t1 887 ITX_MUL2X_PACK 1, 4, 5, 10, 12, 11 ; t5 t6 888 ITX_MUL2X_PACK 0, 4, 5, 10, 11, 12, 8 ; t11 t12 889 ITX_MUL2X_PACK 2, 0, 11, 10, 11, 12, 8 ; t10a t13a 890 punpckhqdq m2, m7, m1 ; t7 t6 891 punpcklqdq m7, m1 ; t4 t5 892 psubsw m1, m9, m6 ; dct4 out3 out2 893 paddsw m9, m6 ; dct4 out0 out1 894 packssdw m5, m11 ; t12 t13a 895 packssdw m4, m0 ; t11 t10a 896 punpckhqdq m0, m8, m3 ; t15a t14 897 punpcklqdq m8, m3 ; t8a t9 898 psubsw m3, m9, m2 ; dct8 out7 out6 899 paddsw m9, m2 ; dct8 out0 out1 900 psubsw m2, m1, m7 ; dct8 out4 out5 901 paddsw m1, m7 ; dct8 out3 out2 902 psubsw m7, m9, m0 ; out15 out14 903 paddsw m0, m9 ; out0 out1 904 psubsw m6, m1, m5 ; out12 out13 905 paddsw m1, m5 ; out3 out2 906 psubsw m5, m2, m4 ; out11 out10 907 paddsw m2, m4 ; out4 out5 908 psubsw m4, m3, m8 ; out8 out9 909 paddsw m3, m8 ; out7 out6 910 %endmacro 911 912 INV_TXFM_4X16_FN dct, dct 913 INV_TXFM_4X16_FN dct, identity 914 INV_TXFM_4X16_FN dct, adst 915 INV_TXFM_4X16_FN dct, flipadst 916 917 cglobal idct_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 918 mova ym1, [cq+32*2] 919 vinserti32x8 m1, [cq+32*0], 1 920 mova m0, [o(int16_perm)] 921 mova ym2, [cq+32*3] 922 vinserti32x8 m2, [cq+32*1], 1 923 vpbroadcastd m4, [o(pd_2048)] 924 vpermb m1, m0, m1 ; c0 a0 c1 a1 c2 a2 c3 a3 925 vpermb m2, m0, m2 ; d0 b0 d1 b1 d2 b2 d3 b3 926 ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896, 2 927 ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784, 2 928 vpbroadcastd m4, [o(pw_16384)] 929 psubsw m3, m1, m2 930 paddsw m1, m2 ; out0 out1 931 vprord m3, 16 ; out2 out3 932 punpckldq m0, m1, m3 933 punpckhdq m1, m3 934 pmulhrsw m0, m4 935 pmulhrsw m1, m4 936 jmp tx2q 937 .pass2: 938 vextracti32x4 xm2, ym0, 1 939 vextracti32x4 xm3, ym1, 1 940 vextracti32x4 xm4, m0, 2 941 vextracti32x4 xm5, m1, 2 942 vextracti32x4 xm6, m0, 3 943 vextracti32x4 xm7, m1, 3 944 call .main 945 vinserti32x4 ym0, xm2, 1 946 vinserti32x4 ym1, xm3, 1 947 vinserti32x4 ym4, xm6, 1 948 vinserti32x4 ym5, xm7, 1 949 vinserti32x8 m0, ym4, 1 950 vinserti32x8 m1, ym5, 1 951 vpbroadcastd m5, [o(pw_2048)] 952 pshufd m1, m1, q1032 953 jmp m(iadst_4x16_internal_8bpc).end2 954 ALIGN function_align 955 .main: 956 WRAP_XMM IDCT16_1D_PACKED 957 ret 958 959 INV_TXFM_4X16_FN adst, dct 960 INV_TXFM_4X16_FN adst, adst 961 INV_TXFM_4X16_FN adst, flipadst 962 INV_TXFM_4X16_FN adst, identity 963 964 cglobal iadst_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 965 mova m1, [o(permB)] 966 vpermq m0, m1, [cq+64*0] 967 vpermq m1, m1, [cq+64*1] 968 call m(iadst_16x4_internal_8bpc).main 969 vpbroadcastd m3, [o(pw_16384)] 970 punpckhwd m2, m0, m1 971 punpcklwd m0, m1 972 pmulhrsw m2, m3 973 pmulhrsw m0, m3 974 punpckhwd m1, m0, m2 975 punpcklwd m0, m2 976 jmp tx2q 977 .pass2: 978 call .main 979 vpbroadcastd m5, [o(pw_2048)] 980 psrlq m10, 4 981 psubw m6, m8, m5 982 .end: 983 vpbroadcastd m7, [o(pw_2896x8)] 984 paddsw ym1, ym2, ym4 985 psubsw ym2, ym4 986 vinserti32x8 m1, ym2, 1 987 pmulhrsw m1, m7 ; -out7 out4 out6 -out5 out8 -out11 -out9 out10 988 psrlq m0, m10, 4 989 vpermi2q m0, m1, m3 ; 0 1 4 5 8 9 c d 990 vpermt2q m1, m10, m3 ; 2 3 6 7 a b e f 991 punpcklqdq m5, m6 992 .end2: 993 pmulhrsw m0, m5 994 pmulhrsw m1, m5 995 .end3: 996 vpbroadcastd m3, strided 997 pmulld m5, m3, [o(pd_0to15)] 998 kxnorw k1, k1, k1 999 kmovw k2, k1 1000 vpgatherdd m3{k1}, [dstq+m5] 1001 pxor m4, m4 1002 mova [cq+64*0], m4 1003 mova [cq+64*1], m4 1004 punpcklbw m2, m3, m4 1005 punpckhbw m3, m4 1006 paddw m0, m2 1007 paddw m1, m3 1008 packuswb m0, m1 1009 vpscatterdd [dstq+m5]{k2}, m0 1010 RET 1011 ALIGN function_align 1012 .main: 1013 movu m3, [o(permB+1)] 1014 psrlq m10, m3, 4 1015 .main2: 1016 vpermi2q m3, m0, m1 ; in15 in12 in13 in14 in11 in8 in9 in10 1017 vpermt2q m0, m10, m1 ; in0 in3 in2 in1 in4 in7 in6 in5 1018 vpbroadcastd m9, [o(pd_2048)] 1019 vpbroadcastq ym13, [o(int_mshift)] 1020 kxnorb k1, k1, k1 1021 punpckhwd m4, m3, m0 ; in12 in3 in14 in1 1022 punpcklwd m0, m3 ; in0 in15 in2 in13 1023 kshiftrb k1, k1, 4 1024 vextracti32x8 ym3, m4, 1 ; in8 in7 in10 in5 1025 vextracti32x8 ym1, m0, 1 ; in4 in11 in6 in9 1026 INIT_YMM avx512icl 1027 vpcmpub k7, m13, m9, 6 ; 0x33... 1028 pxor m8, m8 1029 ITX_MUL4X_PACK 0, 2, 5, 6, 7, 9, 201, 4091, 995, 3973, 5 1030 ITX_MUL4X_PACK 1, 2, 5, 6, 7, 9, 1751, 3703, 2440, 3290, 5 1031 ITX_MUL4X_PACK 3, 2, 5, 6, 7, 9, 3035, 2751, 3513, 2106, 5 1032 ITX_MUL4X_PACK 4, 2, 5, 6, 7, 9, 3857, 1380, 4052, 601, 5 1033 psubsw m2, m0, m3 ; t9a t8a t11a t10a 1034 paddsw m0, m3 ; t1a t0a t3a t2a 1035 psubsw m3, m1, m4 ; t13a t12a t15a t14a 1036 paddsw m4, m1 ; t5a t4a t7a t6a 1037 ITX_MUL4X_PACK 2, 1, 5, 6, 7, 9, 799, 4017, 3406, 2276, 5 1038 psubw m7, m8, m7 1039 ITX_MUL2X_PACK 3, 1, 5, 9, 7, 6, 4 1040 vpbroadcastd m6, [o(pw_3784_m1567)] 1041 vpbroadcastd m6{k1}, [o(pw_m3784_1567)] 1042 psubsw m1, m0, m4 ; t5 t4 t7 t6 1043 paddsw m0, m4 ; t1 t0 t3 t2 1044 psubsw m4, m2, m3 ; t13a t12a t15a t14a 1045 paddsw m2, m3 ; t9a t8a t11a t10a 1046 ITX_MUL2X_PACK 1, 3, 5, 9, 1567_3784, 6, 16 ; t4a t5a t7a t6a 1047 ITX_MUL2X_PACK 4, 3, 5, 9, 1567_3784, 6, 16 ; t12 t13 t15 t14 1048 vbroadcasti32x4 m5, [o(deint_shuf)] 1049 pshufb m0, m5 1050 pshufb m2, m5 1051 vshufi32x4 m3, m0, m2, 0x03 ; t3 t2 t11a t10a 1052 vinserti32x4 m0, xm2, 1 ; t1 t0 t9a t8a 1053 vshufi32x4 m2, m1, m4, 0x03 ; t7a t6a t15 t14 1054 vinserti32x4 m1, xm4, 1 ; t4a t5a t12 t13 1055 pshufd m2, m2, q1032 ; t6a t7a t14 t15 1056 psubsw m4, m0, m3 ; t3a t2a t11 t10 1057 paddsw m0, m3 ; -out15 out0 out14 -out1 1058 paddsw m3, m1, m2 ; out12 -out3 -out13 out2 1059 psubsw m1, m2 ; t7 t6 t15a t14a 1060 punpckhqdq m2, m4, m1 ; t2a t6 t10 t14a 1061 punpcklqdq m4, m1 ; t3a t7 t11 t15a 1062 INIT_ZMM avx512icl 1063 vinserti32x8 m3, ym0, 1 ; out12 -out3 -out13 out2 -out15 out0 out14 -out1 1064 ret 1065 1066 INV_TXFM_4X16_FN flipadst, dct 1067 INV_TXFM_4X16_FN flipadst, adst 1068 INV_TXFM_4X16_FN flipadst, flipadst 1069 INV_TXFM_4X16_FN flipadst, identity 1070 1071 cglobal iflipadst_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1072 mova m1, [o(permB)] 1073 vpermq m0, m1, [cq+64*0] 1074 vpermq m1, m1, [cq+64*1] 1075 call m(iadst_16x4_internal_8bpc).main 1076 vpbroadcastd m3, [o(pw_16384)] 1077 punpcklwd m2, m1, m0 1078 punpckhwd m1, m0 1079 pmulhrsw m2, m3 1080 pmulhrsw m1, m3 1081 punpcklwd m0, m1, m2 1082 punpckhwd m1, m2 1083 jmp tx2q 1084 .pass2: 1085 call m(iadst_4x16_internal_8bpc).main 1086 vpbroadcastd m6, [o(pw_2048)] 1087 psrlq m10, 12 1088 psubw m5, m8, m6 1089 jmp m(iadst_4x16_internal_8bpc).end 1090 1091 INV_TXFM_4X16_FN identity, dct 1092 INV_TXFM_4X16_FN identity, adst 1093 INV_TXFM_4X16_FN identity, flipadst 1094 INV_TXFM_4X16_FN identity, identity 1095 1096 cglobal iidentity_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1097 mova m2, [o(int16_perm)] 1098 vpermb m1, m2, [cq+64*0] 1099 vpermb m2, m2, [cq+64*1] 1100 vpbroadcastd m4, [o(pw_1697x8)] 1101 vpbroadcastd m0, [o(pd_m1)] 1102 pmulhrsw m3, m4, m1 ; we want to do a signed avg, but pavgw is 1103 vpcmpw k1, m1, m0, 4 ; unsigned. as long as both signs are equal 1104 pmulhrsw m4, m2 ; it still works, but if the input is -1 the 1105 vpcmpw k2, m2, m0, 4 ; pmulhrsw result will become 0 which causes 1106 vpavgw m1{k1}{z}, m3 ; pavgw to output -32768 instead of 0 unless 1107 vpavgw m2{k2}{z}, m4 ; we explicitly deal with that case here. 1108 punpckldq m0, m1, m2 1109 punpckhdq m1, m2 1110 jmp tx2q 1111 .pass2: 1112 vpbroadcastd m3, [o(pw_1697x16)] 1113 vpbroadcastd m5, [o(pw_2048)] 1114 pmulhrsw m2, m3, m0 1115 pmulhrsw m3, m1 1116 paddsw m0, m0 1117 paddsw m1, m1 1118 paddsw m0, m2 1119 paddsw m1, m3 1120 jmp m(iadst_4x16_internal_8bpc).end2 1121 1122 %macro WRITE_8X4 4-7 strideq*1, strideq*2, r6 ; coefs[1-2], tmp[1-2], off[1-3] 1123 movq xm%3, [dstq ] 1124 movhps xm%3, [dstq+%5] 1125 movq xm%4, [dstq+%6] 1126 movhps xm%4, [dstq+%7] 1127 pmovzxbw m%3, xm%3 1128 pmovzxbw m%4, xm%4 1129 %ifnum %1 1130 paddw m%3, m%1 1131 %else 1132 paddw m%3, %1 1133 %endif 1134 %ifnum %2 1135 paddw m%4, m%2 1136 %else 1137 paddw m%4, %2 1138 %endif 1139 packuswb m%3, m%4 1140 vextracti32x4 xm%4, m%3, 1 1141 movq [dstq ], xm%3 1142 movhps [dstq+%6], xm%3 1143 movq [dstq+%5], xm%4 1144 movhps [dstq+%7], xm%4 1145 %endmacro 1146 1147 %macro INV_TXFM_8X4_FN 2 ; type1, type2 1148 INV_TXFM_FN %1, %2, 8x4 1149 %ifidn %1_%2, dct_dct 1150 movd xm1, [o(pw_2896x8)] 1151 pmulhrsw xm0, xm1, [cq] 1152 movd xm2, [o(pw_2048)] 1153 pmulhrsw xm0, xm1 1154 pmulhrsw xm0, xm1 1155 pmulhrsw xm0, xm2 1156 vpbroadcastw m0, xm0 1157 mova m1, m0 1158 jmp m(iadst_8x4_internal_8bpc).end3 1159 %endif 1160 %endmacro 1161 1162 INIT_YMM avx512icl 1163 INV_TXFM_8X4_FN dct, dct 1164 INV_TXFM_8X4_FN dct, adst 1165 INV_TXFM_8X4_FN dct, flipadst 1166 INV_TXFM_8X4_FN dct, identity 1167 1168 cglobal idct_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1169 vpbroadcastd xm3, [o(pw_2896x8)] 1170 pmulhrsw xm0, xm3, [cq+16*0] 1171 pmulhrsw xm1, xm3, [cq+16*1] 1172 pmulhrsw xm2, xm3, [cq+16*2] 1173 pmulhrsw xm3, [cq+16*3] 1174 call m(idct_4x8_internal_8bpc).main 1175 vbroadcasti32x4 m4, [o(deint_shuf)] 1176 vinserti32x4 m3, m1, xm3, 1 1177 vinserti32x4 m1, m0, xm2, 1 1178 shufps m0, m1, m3, q0220 1179 shufps m1, m3, q1331 1180 pshufb m0, m4 1181 pshufb m1, m4 1182 jmp tx2q 1183 .pass2: 1184 IDCT4_1D_PACKED 1185 vpermq m0, m0, q3120 1186 vpermq m1, m1, q2031 1187 jmp m(iadst_8x4_internal_8bpc).end2 1188 1189 INV_TXFM_8X4_FN adst, dct 1190 INV_TXFM_8X4_FN adst, adst 1191 INV_TXFM_8X4_FN adst, flipadst 1192 INV_TXFM_8X4_FN adst, identity 1193 1194 cglobal iadst_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1195 vpbroadcastd xm0, [o(pw_2896x8)] 1196 pshufd xm4, [cq+16*0], q1032 1197 pmulhrsw xm3, xm0, [cq+16*3] 1198 pshufd xm5, [cq+16*1], q1032 1199 pmulhrsw xm2, xm0, [cq+16*2] 1200 pmulhrsw xm4, xm0 1201 pmulhrsw xm5, xm0 1202 call m(iadst_4x8_internal_8bpc).main_pass1 1203 vinserti32x4 m0, xm2, 1 1204 vinserti32x4 m1, xm3, 1 1205 pxor m3, m3 1206 punpckhwd m2, m0, m1 1207 punpcklwd m0, m1 1208 psubsw m3, m2 1209 punpckhwd m1, m0, m3 1210 punpcklwd m0, m3 1211 jmp tx2q 1212 .pass2: 1213 call .main 1214 .end: 1215 vpermq m0, m0, q3120 1216 vpermq m1, m1, q3120 1217 .end2: 1218 vpbroadcastd m2, [o(pw_2048)] 1219 pmulhrsw m0, m2 1220 pmulhrsw m1, m2 1221 .end3: 1222 pxor m2, m2 1223 mova [cq], zmm18 1224 lea r6, [strideq*3] 1225 WRITE_8X4 0, 1, 4, 5 1226 RET 1227 ALIGN function_align 1228 .main: 1229 IADST4_1D_PACKED 1230 ret 1231 1232 INV_TXFM_8X4_FN flipadst, dct 1233 INV_TXFM_8X4_FN flipadst, adst 1234 INV_TXFM_8X4_FN flipadst, flipadst 1235 INV_TXFM_8X4_FN flipadst, identity 1236 1237 cglobal iflipadst_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1238 vpbroadcastd xm0, [o(pw_2896x8)] 1239 pshufd xm4, [cq+16*0], q1032 1240 pmulhrsw xm3, xm0, [cq+16*3] 1241 pshufd xm5, [cq+16*1], q1032 1242 pmulhrsw xm2, xm0, [cq+16*2] 1243 pmulhrsw xm4, xm0 1244 pmulhrsw xm5, xm0 1245 call m(iadst_4x8_internal_8bpc).main_pass1 1246 vinserti32x4 m3, m3, xm1, 1 1247 vinserti32x4 m2, m2, xm0, 1 1248 punpckhwd m1, m3, m2 1249 punpcklwd m3, m2 1250 pxor m0, m0 1251 psubsw m0, m1 1252 punpckhwd m1, m0, m3 1253 punpcklwd m0, m3 1254 jmp tx2q 1255 .pass2: 1256 call m(iadst_8x4_internal_8bpc).main 1257 mova m2, m1 1258 vpermq m1, m0, q2031 1259 vpermq m0, m2, q2031 1260 jmp m(iadst_8x4_internal_8bpc).end2 1261 1262 INV_TXFM_8X4_FN identity, dct 1263 INV_TXFM_8X4_FN identity, adst 1264 INV_TXFM_8X4_FN identity, flipadst 1265 INV_TXFM_8X4_FN identity, identity 1266 1267 cglobal iidentity_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1268 mova xm2, [cq+16*0] 1269 mova xm0, [cq+16*1] 1270 vinserti32x4 m2, [cq+16*2], 1 1271 vinserti32x4 m0, [cq+16*3], 1 1272 vpbroadcastd m3, [o(pw_2896x8)] 1273 punpcklwd m1, m2, m0 1274 punpckhwd m2, m0 1275 pmulhrsw m1, m3 1276 pmulhrsw m2, m3 1277 punpcklwd m0, m1, m2 1278 punpckhwd m1, m2 1279 paddsw m0, m0 1280 paddsw m1, m1 1281 jmp tx2q 1282 .pass2: 1283 vpbroadcastd m3, [o(pw_1697x8)] 1284 pmulhrsw m2, m3, m0 1285 pmulhrsw m3, m1 1286 paddsw m0, m2 1287 paddsw m1, m3 1288 jmp m(iadst_8x4_internal_8bpc).end 1289 1290 %macro INV_TXFM_8X8_FN 2 ; type1, type2 1291 INV_TXFM_FN %1, %2, 8x8 1292 %ifidn %1_%2, dct_dct 1293 INIT_ZMM avx512icl 1294 movsx r6d, word [cq] 1295 mov [cq], eobd 1296 .dconly: 1297 imul r6d, 181 1298 add r6d, 128+256 1299 sar r6d, 8+1 1300 .dconly2: 1301 vpbroadcastd ym2, strided 1302 imul r6d, 181 1303 pmulld ym5, ym2, [o(pd_0to15)] 1304 kxnorb k1, k1, k1 1305 add r6d, 128+2048 1306 sar r6d, 8+4 1307 pxor m3, m3 1308 vpbroadcastw m4, r6d 1309 .dconly_loop: 1310 kmovb k2, k1 1311 vpgatherdq m2{k1}, [dstq+ym5] 1312 punpcklbw m0, m2, m3 1313 punpckhbw m1, m2, m3 1314 paddw m0, m4 1315 paddw m1, m4 1316 packuswb m0, m1 1317 kmovb k1, k2 1318 vpscatterdq [dstq+ym5]{k2}, m0 1319 lea dstq, [dstq+strideq*8] 1320 sub r3d, 8 1321 jg .dconly_loop 1322 RET 1323 INIT_YMM avx512icl 1324 %endif 1325 %endmacro 1326 1327 INV_TXFM_8X8_FN dct, dct 1328 INV_TXFM_8X8_FN dct, identity 1329 INV_TXFM_8X8_FN dct, adst 1330 INV_TXFM_8X8_FN dct, flipadst 1331 1332 cglobal idct_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1333 vpermq m0, [cq+32*0], q3120 ; 0 1 1334 vpermq m3, [cq+32*3], q3120 ; 6 7 1335 vpermq m2, [cq+32*2], q3120 ; 4 5 1336 vpermq m1, [cq+32*1], q3120 ; 2 3 1337 call .main 1338 shufps m4, m0, m1, q0220 1339 shufps m5, m0, m1, q1331 1340 shufps m1, m2, m3, q0220 1341 shufps m3, m2, m3, q1331 1342 vbroadcasti32x4 m0, [o(deint_shuf)] 1343 vpbroadcastd m2, [o(pw_16384)] 1344 REPX {pshufb x, m0}, m4, m5, m1, m3 1345 REPX {pmulhrsw x, m2}, m4, m5, m1, m3 1346 vinserti32x4 m0, m4, xm1, 1 1347 vshufi32x4 m2, m4, m1, 0x03 1348 vinserti32x4 m1, m5, xm3, 1 1349 vshufi32x4 m3, m5, m3, 0x03 1350 jmp tx2q 1351 .pass2: 1352 call .main 1353 vpbroadcastd m4, [o(pw_2048)] 1354 vpermq m0, m0, q3120 1355 vpermq m1, m1, q2031 1356 vpermq m2, m2, q3120 1357 vpermq m3, m3, q2031 1358 jmp m(iadst_8x8_internal_8bpc).end2 1359 ALIGN function_align 1360 cglobal_label .main 1361 IDCT8_1D_PACKED 1362 ret 1363 1364 INV_TXFM_8X8_FN adst, dct 1365 INV_TXFM_8X8_FN adst, adst 1366 INV_TXFM_8X8_FN adst, flipadst 1367 INV_TXFM_8X8_FN adst, identity 1368 1369 cglobal iadst_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1370 vpermq m4, [cq+32*0], q1302 ; 1 0 1371 vpermq m3, [cq+32*3], q3120 ; 6 7 1372 vpermq m5, [cq+32*1], q1302 ; 3 2 1373 vpermq m2, [cq+32*2], q3120 ; 4 5 1374 call .main_pass1 1375 vpbroadcastd m5, [o(pw_16384_m16384)] 1376 punpcklwd m4, m0, m1 1377 punpckhwd m0, m1 1378 punpcklwd m1, m2, m3 1379 punpckhwd m2, m3 1380 punpcklwd m3, m4, m0 1381 punpckhwd m4, m0 1382 punpcklwd m0, m1, m2 1383 punpckhwd m1, m2 1384 REPX {pmulhrsw x, m5}, m3, m4, m0, m1 1385 vshufi32x4 m2, m3, m0, 0x03 1386 vinserti32x4 m0, m3, xm0, 1 1387 vshufi32x4 m3, m4, m1, 0x03 1388 vinserti32x4 m1, m4, xm1, 1 1389 jmp tx2q 1390 .pass2: 1391 pshufd m4, m0, q1032 1392 pshufd m5, m1, q1032 1393 call .main_pass2 1394 vpbroadcastd m5, [o(pw_2048)] 1395 vpbroadcastd xm4, [o(pw_4096)] 1396 psubw m4, m5 ; lower half = 2048, upper half = -2048 1397 .end: 1398 REPX {vpermq x, x, q3120}, m0, m1, m2, m3 1399 .end2: 1400 pmulhrsw m0, m4 1401 pmulhrsw m1, m4 1402 .end3: 1403 pmulhrsw m2, m4 1404 pmulhrsw m3, m4 1405 .end4: 1406 pxor m4, m4 1407 mova [cq+32*0], m4 1408 mova [cq+32*1], m4 1409 mova [cq+32*2], m4 1410 mova [cq+32*3], m4 1411 lea r6, [strideq*3] 1412 WRITE_8X4 0, 1, 4, 5 1413 lea dstq, [dstq+strideq*4] 1414 WRITE_8X4 2, 3, 4, 5 1415 RET 1416 ALIGN function_align 1417 .main_pass1: 1418 punpckhwd m0, m4, m3 ; 0 7 1419 punpckhwd m1, m5, m2 ; 2 5 1420 punpcklwd m2, m5 ; 4 3 1421 punpcklwd m3, m4 ; 6 1 1422 IADST8_1D_PACKED 1 1423 punpcklqdq m3, m4, m0 ; out6 -out7 1424 punpckhqdq m0, m4 ; out0 -out1 1425 ret 1426 ALIGN function_align 1427 cglobal_label .main_pass2 1428 IADST8_1D_PACKED 2 1429 ret 1430 1431 INV_TXFM_8X8_FN flipadst, dct 1432 INV_TXFM_8X8_FN flipadst, adst 1433 INV_TXFM_8X8_FN flipadst, flipadst 1434 INV_TXFM_8X8_FN flipadst, identity 1435 1436 cglobal iflipadst_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1437 vpermq m4, [cq+32*0], q1302 ; 1 0 1438 vpermq m3, [cq+32*3], q3120 ; 6 7 1439 vpermq m5, [cq+32*1], q1302 ; 3 2 1440 vpermq m2, [cq+32*2], q3120 ; 4 5 1441 call m(iadst_8x8_internal_8bpc).main_pass1 1442 vpbroadcastd m5, [o(pw_m16384_16384)] 1443 punpckhwd m4, m3, m2 1444 punpcklwd m3, m2 1445 punpckhwd m2, m1, m0 1446 punpcklwd m1, m0 1447 punpckhwd m0, m4, m3 1448 punpcklwd m4, m3 1449 punpckhwd m3, m2, m1 1450 punpcklwd m2, m1 1451 REPX {pmulhrsw x, m5}, m0, m4, m3, m2 1452 vinserti32x4 m1, m0, xm3, 1 1453 vshufi32x4 m3, m0, m3, 0x03 1454 vinserti32x4 m0, m4, xm2, 1 1455 vshufi32x4 m2, m4, m2, 0x03 1456 jmp tx2q 1457 .pass2: 1458 pshufd m4, m0, q1032 1459 pshufd m5, m1, q1032 1460 call m(iadst_8x8_internal_8bpc).main_pass2 1461 vpbroadcastd m4, [o(pw_2048)] 1462 vpbroadcastd xm5, [o(pw_4096)] 1463 psubw m4, m5 ; lower half = -2048, upper half = 2048 1464 vpermq m5, m3, q2031 1465 vpermq m3, m0, q2031 1466 vpermq m0, m2, q2031 1467 vpermq m2, m1, q2031 1468 pmulhrsw m1, m0, m4 1469 pmulhrsw m0, m5, m4 1470 jmp m(iadst_8x8_internal_8bpc).end3 1471 1472 INV_TXFM_8X8_FN identity, dct 1473 INV_TXFM_8X8_FN identity, adst 1474 INV_TXFM_8X8_FN identity, flipadst 1475 INV_TXFM_8X8_FN identity, identity 1476 1477 cglobal iidentity_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1478 mova xm3, [cq+16*0] 1479 mova xm2, [cq+16*1] 1480 vinserti32x4 m3, [cq+16*4], 1 1481 vinserti32x4 m2, [cq+16*5], 1 1482 mova xm4, [cq+16*2] 1483 mova xm0, [cq+16*3] 1484 vinserti32x4 m4, [cq+16*6], 1 1485 vinserti32x4 m0, [cq+16*7], 1 1486 punpcklwd m1, m3, m2 1487 punpckhwd m3, m2 1488 punpcklwd m2, m4, m0 1489 punpckhwd m4, m0 1490 punpckldq m0, m1, m2 1491 punpckhdq m1, m2 1492 punpckldq m2, m3, m4 1493 punpckhdq m3, m4 1494 jmp tx2q 1495 .pass2: 1496 vpbroadcastd m4, [o(pw_4096)] 1497 jmp m(iadst_8x8_internal_8bpc).end 1498 1499 %macro INV_TXFM_8X16_FN 2 ; type1, type2 1500 INV_TXFM_FN %1, %2, 8x16 1501 %ifidn %1_%2, dct_dct 1502 movsx r6d, word [cq] 1503 mov [cq], eobd 1504 or r3d, 16 1505 imul r6d, 181 1506 add r6d, 128 1507 sar r6d, 8 1508 jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly 1509 %endif 1510 %endmacro 1511 1512 %macro ITX_8X16_LOAD_COEFS 0 1513 vpbroadcastd m4, [o(pw_2896x8)] 1514 pmulhrsw m0, m4, [cq+32*0] 1515 add cq, 32*4 1516 pmulhrsw m7, m4, [cq+32*3] 1517 pmulhrsw m1, m4, [cq-32*3] 1518 pmulhrsw m6, m4, [cq+32*2] 1519 pmulhrsw m2, m4, [cq-32*2] 1520 pmulhrsw m5, m4, [cq+32*1] 1521 pmulhrsw m3, m4, [cq-32*1] 1522 pmulhrsw m4, [cq+32*0] 1523 %endmacro 1524 1525 INIT_ZMM avx512icl 1526 INV_TXFM_8X16_FN dct, dct 1527 INV_TXFM_8X16_FN dct, identity 1528 INV_TXFM_8X16_FN dct, adst 1529 INV_TXFM_8X16_FN dct, flipadst 1530 1531 cglobal idct_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1532 mova m3, [o(permB)] 1533 vpermq m0, m3, [cq+64*0] 1534 vpbroadcastd m4, [o(pw_2896x8)] 1535 vpermq m1, m3, [cq+64*1] 1536 vpermq m2, m3, [cq+64*2] 1537 vpermq m3, m3, [cq+64*3] 1538 REPX {pmulhrsw x, m4}, m0, m1, m2, m3 1539 call m(idct_16x8_internal_8bpc).main 1540 vpbroadcastd m5, [o(pw_16384)] 1541 punpckhwd m4, m0, m2 ; b0 f0 b1 f1 b2 f2 b3 f3 1542 punpcklwd m0, m2 ; a0 e0 a1 e1 a2 e2 a3 e3 1543 punpckhwd m2, m1, m3 ; c0 g0 c1 g1 c2 g2 c3 g3 1544 punpcklwd m1, m3 ; d0 h0 d1 h1 d2 h2 d3 h3 1545 REPX {pmulhrsw x, m5}, m4, m0, m2, m1 1546 punpckhwd m3, m0, m4 ; a2 b2 e2 f2 a3 b3 e3 f3 1547 punpcklwd m0, m4 ; a0 b0 e0 f0 a1 b1 e1 f1 1548 punpckhwd m4, m2, m1 ; c2 d2 g2 h2 c3 d3 g3 h3 1549 punpcklwd m2, m1 ; c0 d0 g0 h0 c1 d1 g1 h1 1550 punpckhdq m1, m0, m2 ; 1 5 9 13 1551 punpckldq m0, m2 ; 0 4 8 12 1552 punpckldq m2, m3, m4 ; 2 6 10 14 1553 punpckhdq m3, m4 ; 3 7 11 15 1554 jmp tx2q 1555 .pass2: 1556 vprord m5, [o(int16_perm)], 16 1557 vshufi32x4 m2, m2, q1320 ; 2 10 14 6 1558 vshufi32x4 m4, m1, m3, q2310 ; 1 5 15 11 1559 vshufi32x4 m1, m3, q0132 ; 9 13 7 3 1560 vpermb m9, m5, m0 1561 vpermb m7, m5, m2 1562 vpermb m8, m5, m4 1563 vpermb m0, m5, m1 1564 vextracti32x8 ym6, m9, 1 1565 vextracti32x8 ym3, m7, 1 1566 vextracti32x8 ym5, m8, 1 1567 vextracti32x8 ym1, m0, 1 1568 call .main2 1569 mova ym8, [o(gather8a)] 1570 lea r3, [dstq+strideq*4] 1571 pmovzxdq m9, ym8 1572 pshufd ym8, ym8, q1230 1573 vpermt2q m0, m9, m4 1574 vpermt2q m1, m9, m5 1575 vpermt2q m2, m9, m6 1576 vpermt2q m3, m9, m7 1577 .end: 1578 vpbroadcastd m7, [o(pw_2048)] 1579 .end2: 1580 pmulhrsw m0, m7 1581 pmulhrsw m1, m7 1582 .end3: 1583 pmulhrsw m2, m7 1584 pmulhrsw m3, m7 1585 .end4: 1586 vpbroadcastd ym6, strided 1587 kxnorb k1, k1, k1 1588 pxor m4, m4 1589 pmulld ym8, ym6 1590 kmovb k2, k1 1591 vpgatherdq m6{k1}, [dstq+ym8] 1592 kmovb k1, k2 1593 vpgatherdq m7{k2}, [r3+ym8] 1594 mova [cq+64*0], m4 1595 mova [cq+64*1], m4 1596 kmovb k2, k1 1597 mova [cq+64*2], m4 1598 mova [cq+64*3], m4 1599 punpcklbw m5, m6, m4 1600 punpckhbw m6, m4 1601 paddw m0, m5 1602 paddw m1, m6 1603 packuswb m0, m1 1604 vpscatterdq [dstq+ym8]{k1}, m0 1605 punpcklbw m6, m7, m4 1606 punpckhbw m7, m4 1607 paddw m2, m6 1608 paddw m3, m7 1609 packuswb m2, m3 1610 vpscatterdq [r3+ym8]{k2}, m2 1611 RET 1612 ALIGN function_align 1613 cglobal_label .main_fast2 ; bottom three-quarters are zero 1614 vpbroadcastd ym10, [o(pd_2048)] 1615 vpbroadcastq ym13, [o(int_mshift)] 1616 vpbroadcastd ym3, [o(pw_401_4076x8)] 1617 vpbroadcastd ym5, [o(pw_799_4017x8)] 1618 vpbroadcastd ym4, [o(pw_m1189_3920x8)] 1619 pxor ym6, ym6 1620 punpckhwd ym2, ym0, ym0 1621 pmulhrsw ym2, ym3 ; t8a t15a 1622 punpcklwd ym7, ym1, ym1 1623 pmulhrsw ym7, ym5 ; t4a t7a 1624 punpckhwd ym1, ym1 1625 pmulhrsw ym4, ym1 ; t11a t12a 1626 vpcmpub k7, ym13, ym10, 6 1627 punpcklwd ym9, ym6, ym0 1628 psubsw ym0, ym2, ym4 ; t11a t12a 1629 paddsw ym8, ym2, ym4 ; t8a t15a 1630 mova ym1, ym7 1631 jmp .main5 1632 ALIGN function_align 1633 cglobal_label .main_fast ; bottom half is zero 1634 vpbroadcastd ym10, [o(pd_2048)] 1635 vpbroadcastq ym13, [o(int_mshift)] 1636 pxor ym6, ym6 1637 punpckhwd ym8, ym0, ym0 1638 punpckhwd ym4, ym3, ym3 1639 punpckhwd ym5, ym2, ym2 1640 punpcklwd ym7, ym1, ym1 1641 punpckhwd ym1, ym1 1642 punpcklwd ym3, ym3 1643 punpcklwd ym9, ym6, ym0 1644 punpcklwd ym6, ym2 1645 vpbroadcastd ym2, [o(pw_401_4076x8)] 1646 vpbroadcastd ym0, [o(pw_m2598_3166x8)] 1647 vpbroadcastd ym11, [o(pw_1931_3612x8)] 1648 vpbroadcastd ym12, [o(pw_m1189_3920x8)] 1649 pmulhrsw ym8, ym2 ; t8a t15a 1650 vpbroadcastd ym2, [o(pw_799_4017x8)] 1651 pmulhrsw ym0, ym4 ; t9a t14a 1652 vpbroadcastd ym4, [o(pw_m2276_3406x8)] 1653 pmulhrsw ym5, ym11 ; t10a t13a 1654 pmulhrsw ym1, ym12 ; t11a t12a 1655 pmulhrsw ym7, ym2 ; t4a t7a 1656 pmulhrsw ym3, ym4 ; t5a t6a 1657 vpcmpub k7, ym13, ym10, 6 1658 jmp .main4 1659 ALIGN function_align 1660 cglobal_label .main 1661 WRAP_YMM IDCT16_1D_PACKED 1662 ret 1663 1664 INV_TXFM_8X16_FN adst, dct 1665 INV_TXFM_8X16_FN adst, adst 1666 INV_TXFM_8X16_FN adst, flipadst 1667 INV_TXFM_8X16_FN adst, identity 1668 1669 cglobal iadst_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1670 call m(iadst_16x8_internal_8bpc).main_pass1 1671 vbroadcasti32x4 m6, [o(int_shuf1)] 1672 vpbroadcastd m7, [o(pw_16384_m16384)] 1673 punpckhwd m3, m0, m4 ; a0 b0 a1 b1 a2 b2 a3 b3 1674 punpcklwd m4, m0 ; g0 h0 g1 h1 g2 h2 g3 h3 1675 pshufb m5, m1, m6 ; c0 d0 c1 d1 c2 d2 c3 d3 1676 pshufb m2, m6 ; e0 f0 e1 f1 e2 f2 e3 f3 1677 .pass1_end: 1678 REPX {pmulhrsw x, m7}, m3, m5, m4, m2 1679 punpckldq m0, m3, m5 ; a0 b0 c0 d0 a1 b1 c1 d1 1680 punpckhdq m3, m5 ; a2 b2 c2 d2 a3 b3 c3 d3 1681 punpckhdq m5, m2, m4 ; e2 f2 g2 h2 e3 f3 g3 h3 1682 punpckldq m2, m4 ; e0 f0 g0 h0 e1 f1 g1 h1 1683 punpckhqdq m1, m0, m2 1684 punpcklqdq m0, m2 1685 punpcklqdq m2, m3, m5 1686 punpckhqdq m3, m5 1687 jmp tx2q 1688 .pass2: 1689 call .main_pass2 1690 vpbroadcastd m6, [o(pw_2048)] 1691 psrlq m10, 4 1692 psubw m7, m8, m6 1693 .pass2_end: 1694 vpbroadcastd m5, [o(pw_2896x8)] 1695 paddsw m1, m2, m4 1696 psubsw m2, m4 1697 pmulhrsw m1, m5 ; -out7 out4 out6 -out5 1698 pmulhrsw m5, m2 ; out8 -out11 -out9 out10 1699 mova ym8, [o(gather8c)] 1700 lea r3, [dstq+strideq] 1701 psrlq m2, m10, 4 1702 vpermi2q m2, m0, m3 ; 1 3 13 15 1703 vpermt2q m0, m10, m3 ; 0 2 12 14 1704 psrlq m3, m10, 8 1705 vpermi2q m3, m1, m5 ; 5 7 9 11 1706 psrlq m10, 12 1707 vpermt2q m1, m10, m5 ; 4 6 8 10 1708 pmulhrsw m0, m6 1709 pmulhrsw m1, m6 1710 jmp m(idct_8x16_internal_8bpc).end3 1711 ALIGN function_align 1712 .main_pass1: 1713 vpbroadcastd m2, [o(pw_2896x8)] 1714 pmulhrsw m5, m2, [cq+64*0] 1715 pmulhrsw m3, m2, [cq+64*3] 1716 pmulhrsw m1, m2, [cq+64*1] 1717 pmulhrsw m2, [cq+64*2] 1718 movu m4, [o(permA+3)] 1719 psrlq m10, m4, 4 1720 mova m6, m4 1721 vpermi2q m4, m5, m3 ; in0 in12 in2 in14 1722 vpermt2q m5, m10, m3 ; in15 in3 in13 in1 1723 vpermi2q m6, m1, m2 ; in4 in8 in6 in10 1724 vpermt2q m1, m10, m2 ; in11 in7 in9 in5 1725 jmp .main 1726 ALIGN function_align 1727 .main_pass2: 1728 mova m4, [o(permC)] 1729 psrlq m5, m4, 4 1730 vpermi2q m4, m0, m2 ; in0 in12 in2 in14 1731 psrlq m6, m5, 4 1732 vpermi2q m5, m1, m3 ; in15 in3 in13 in1 1733 psrlq m10, m6, 4 1734 vpermi2q m6, m0, m2 ; in4 in8 in6 in10 1735 vpermt2q m1, m10, m3 ; in11 in7 in9 in5 1736 .main: 1737 punpcklwd m0, m4, m5 ; in0 in15 in2 in13 1738 punpckhwd m4, m5 ; in12 in3 in14 in1 1739 punpcklwd m5, m6, m1 ; in4 in11 in6 in9 1740 punpckhwd m6, m1 ; in8 in7 in10 in5 1741 cglobal_label .main2 1742 vpbroadcastd m9, [o(pd_2048)] 1743 vpbroadcastq m13, [o(int_mshift)] 1744 kxnorb k1, k1, k1 1745 vpcmpub k7, m13, m9, 6 ; 0x33... 1746 pxor m8, m8 1747 ITX_MUL4X_PACK 0, 1, 2, 3, 7, 9, 201, 4091, 995, 3973, 5 1748 ITX_MUL4X_PACK 6, 1, 2, 3, 7, 9, 3035, 2751, 3513, 2106, 5 1749 ITX_MUL4X_PACK 4, 1, 2, 3, 7, 9, 3857, 1380, 4052, 601, 5 1750 ITX_MUL4X_PACK 5, 1, 2, 3, 7, 9, 1751, 3703, 2440, 3290, 5 1751 psubsw m2, m0, m6 ; t9a t8a t11a t10a 1752 paddsw m0, m6 ; t1a t0a t3a t2a 1753 psubsw m3, m5, m4 ; t13a t12a t15a t14a 1754 paddsw m5, m4 ; t5a t4a t7a t6a 1755 ITX_MUL4X_PACK 2, 4, 1, 6, 7, 9, 799, 4017, 3406, 2276, 5 1756 psubw m7, m8, m7 1757 ITX_MUL2X_PACK 3, 4, 1, 9, 7, 6, 4 1758 vpbroadcastd m6, [o(pw_3784_m1567)] 1759 vpbroadcastd m6{k1}, [o(pw_m3784_1567)] 1760 psubsw m1, m0, m5 ; t5 t4 t7 t6 1761 paddsw m0, m5 ; t1 t0 t3 t2 1762 psubsw m4, m2, m3 ; t13a t12a t15a t14a 1763 paddsw m2, m3 ; t9a t8a t11a t10a 1764 ITX_MUL2X_PACK 1, 3, 5, 9, 1567_3784, 6, 16 ; t5a t4a t6a t7a 1765 ITX_MUL2X_PACK 4, 3, 5, 9, 1567_3784, 6, 16 ; t13 t12 t14 t15 1766 vbroadcasti32x4 m5, [o(deint_shuf)] 1767 pshufb m0, m5 1768 pshufb m2, m5 1769 vshufi32x4 m3, m0, m2, q3232 ; t3 t2 t11a t10a 1770 vinserti32x8 m0, ym2, 1 ; t1 t0 t9a t8a 1771 vshufi32x4 m2, m1, m4, q3232 ; t6a t7a t14 t15 1772 vinserti32x8 m1, ym4, 1 ; t5a t4a t13 t12 1773 pshufd m2, m2, q1032 ; t7a t6a t15 t14 1774 psubsw m4, m0, m3 ; t3a t2a t11 t10 1775 paddsw m0, m3 ; -out15 out0 out14 -out1 1776 paddsw m3, m1, m2 ; out12 -out3 -out13 out2 1777 psubsw m1, m2 ; t7 t6 t15a t14a 1778 punpckhqdq m2, m4, m1 ; t2a t6 t10 t14a 1779 punpcklqdq m4, m1 ; t3a t7 t11 t15a 1780 ret 1781 1782 INV_TXFM_8X16_FN flipadst, dct 1783 INV_TXFM_8X16_FN flipadst, adst 1784 INV_TXFM_8X16_FN flipadst, flipadst 1785 INV_TXFM_8X16_FN flipadst, identity 1786 1787 cglobal iflipadst_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1788 call m(iadst_16x8_internal_8bpc).main_pass1 1789 vbroadcasti32x4 m6, [o(int_shuf2)] 1790 vpbroadcastd m7, [o(pw_m16384_16384)] 1791 punpcklwd m3, m0, m4 ; a0 b0 a1 b1 a2 b2 a3 b3 1792 punpckhwd m4, m0 ; g0 h0 g1 h1 g2 h2 g3 h3 1793 pshufb m5, m2, m6 ; c0 d0 c1 d1 c2 d2 c3 d3 1794 pshufb m2, m1, m6 ; e0 f0 e1 f1 e2 f2 e3 f3 1795 jmp m(iadst_8x16_internal_8bpc).pass1_end 1796 .pass2: 1797 call m(iadst_8x16_internal_8bpc).main_pass2 1798 vpbroadcastd m7, [o(pw_2048)] 1799 psrlq m10, 36 1800 psubw m6, m8, m7 1801 jmp m(iadst_8x16_internal_8bpc).pass2_end 1802 1803 INV_TXFM_8X16_FN identity, dct 1804 INV_TXFM_8X16_FN identity, adst 1805 INV_TXFM_8X16_FN identity, flipadst 1806 INV_TXFM_8X16_FN identity, identity 1807 1808 cglobal iidentity_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1809 mova m0, [o(int16_perm)] 1810 vpermb m3, m0, [cq+64*0] ; a0 b0 a1 b1 a2 b2 a3 b3 1811 vpermb m2, m0, [cq+64*1] ; c0 d0 c1 d1 c2 d2 c3 d3 1812 vpermb m4, m0, [cq+64*2] ; e0 f0 e1 f1 e2 f2 e3 f3 1813 vpermb m0, m0, [cq+64*3] ; g0 h0 g1 h1 g2 h2 g3 h3 1814 vpbroadcastd m5, [o(pw_2896x8)] 1815 punpckldq m1, m3, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 1816 punpckhdq m3, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 1817 punpckldq m2, m4, m0 ; e0 f0 g0 h0 a1 f1 g1 h1 1818 punpckhdq m4, m0 ; e2 f2 g2 h2 e3 f3 g3 h3 1819 REPX {pmulhrsw x, m5}, m1, m2, m3, m4 1820 punpcklqdq m0, m1, m2 ; a0 b0 c0 d0 e0 f0 g0 h0 1821 punpckhqdq m1, m2 ; a1 b1 c1 d1 e1 f1 g1 h1 1822 punpcklqdq m2, m3, m4 ; a2 b2 c2 d2 e2 f2 g2 h2 1823 punpckhqdq m3, m4 ; a3 b3 c3 d3 e3 f3 g3 h3 1824 jmp tx2q 1825 .pass2: 1826 vpbroadcastd m7, [o(pw_1697x16)] 1827 mova ym8, [o(gather8b)] 1828 lea r3, [dstq+strideq*2] 1829 pmulhrsw m4, m7, m0 1830 pmulhrsw m5, m7, m1 1831 pmulhrsw m6, m7, m2 1832 pmulhrsw m7, m3 1833 REPX {paddsw x, x}, m0, m1, m2, m3 1834 paddsw m0, m4 1835 paddsw m1, m5 1836 paddsw m2, m6 1837 paddsw m3, m7 1838 jmp m(idct_8x16_internal_8bpc).end 1839 1840 %macro WRITE_16X2 6 ; coefs[1-2], tmp[1-2], offset[1-2] 1841 pmovzxbw m%3, [dstq+%5] 1842 %ifnum %1 1843 paddw m%3, m%1 1844 %else 1845 paddw m%3, %1 1846 %endif 1847 pmovzxbw m%4, [dstq+%6] 1848 %ifnum %2 1849 paddw m%4, m%2 1850 %else 1851 paddw m%4, %2 1852 %endif 1853 packuswb m%3, m%4 1854 vpermq m%3, m%3, q3120 1855 mova [dstq+%5], xm%3 1856 vextracti32x4 [dstq+%6], m%3, 1 1857 %endmacro 1858 1859 %macro INV_TXFM_16X4_FN 2 ; type1, type2 1860 INV_TXFM_FN %1, %2, 16x4 1861 %ifidn %1_%2, dct_dct 1862 movsx r6d, word [cq] 1863 mov [cq], eobd 1864 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly2 1865 %endif 1866 %endmacro 1867 1868 INIT_ZMM avx512icl 1869 INV_TXFM_16X4_FN dct, dct 1870 INV_TXFM_16X4_FN dct, adst 1871 INV_TXFM_16X4_FN dct, flipadst 1872 INV_TXFM_16X4_FN dct, identity 1873 1874 cglobal idct_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1875 mova xm0, [cq+16*0] 1876 mova xm1, [cq+16*1] 1877 mova xm2, [cq+16*2] 1878 mova xm3, [cq+16*3] 1879 mova xm4, [cq+16*4] 1880 mova xm5, [cq+16*5] 1881 mova xm6, [cq+16*6] 1882 mova xm7, [cq+16*7] 1883 call m(idct_4x16_internal_8bpc).main 1884 vpbroadcastd m8, [o(pw_16384)] 1885 vinserti32x4 ym1, xm3, 1 ; 3 2 7 6 1886 vinserti32x4 ym5, xm7, 1 ; b a f e 1887 vinserti32x4 ym0, xm2, 1 ; 0 1 4 5 1888 vinserti32x4 ym4, xm6, 1 ; 8 9 c d 1889 vinserti32x8 m1, ym5, 1 ; 3 2 7 6 b a f e 1890 vinserti32x8 m0, ym4, 1 ; 0 1 4 5 8 9 c d 1891 pmulhrsw m1, m8 1892 pmulhrsw m0, m8 1893 pshufd m1, m1, q1032 1894 punpckhwd m2, m0, m1 1895 punpcklwd m0, m1 1896 punpckhwd m1, m0, m2 1897 punpcklwd m0, m2 1898 jmp tx2q 1899 .pass2: 1900 IDCT4_1D_PACKED 1901 mova m2, [o(permA)] 1902 jmp m(iadst_16x4_internal_8bpc).end 1903 1904 INV_TXFM_16X4_FN adst, dct 1905 INV_TXFM_16X4_FN adst, adst 1906 INV_TXFM_16X4_FN adst, flipadst 1907 INV_TXFM_16X4_FN adst, identity 1908 1909 cglobal iadst_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1910 mova m0, [cq+64*0] 1911 mova m1, [cq+64*1] 1912 movshdup m3, [o(permB)] 1913 psrlq m10, m3, 4 1914 call m(iadst_4x16_internal_8bpc).main2 1915 vpbroadcastd m6, [o(pw_16384_m16384)] 1916 psrlq m0, m10, 4 1917 psrlq m10, 8 1918 .pass1_end: 1919 punpcklwd ym5, ym4, ym2 1920 punpckhwd ym4, ym2 1921 vinserti32x8 m5, ym4, 1 1922 mova m1, m9 1923 vpdpwssd m1, m5, [o(pw_m2896_2896)] {1to16} 1924 mova m4, m9 1925 vpdpwssd m4, m5, [o(pw_2896_2896)] {1to16} 1926 psrad m1, 12 1927 psrad m4, 12 1928 packssdw m1, m4 ; out8 -out7 -out9 out6 -out11 out4 out10 -out5 1929 vpermi2q m0, m1, m3 ; 0 1 4 5 8 9 c d 1930 vpermt2q m1, m10, m3 ; 2 3 6 7 a b e f 1931 punpckhwd m2, m0, m1 1932 punpcklwd m0, m1 1933 punpckhwd m1, m0, m2 1934 punpcklwd m0, m2 1935 pmulhrsw m0, m6 1936 pmulhrsw m1, m6 1937 jmp tx2q 1938 .pass2: 1939 call .main 1940 movu m2, [o(permA+1)] 1941 .end: 1942 vpbroadcastd m3, [o(pw_2048)] 1943 pmulhrsw m0, m3 1944 pmulhrsw m1, m3 1945 .end2: 1946 psrlq m3, m2, 4 1947 vpermi2q m2, m0, m1 1948 vpermi2q m3, m0, m1 1949 .end3: 1950 lea r3, [dstq+strideq*2] 1951 mova xm1, [dstq+strideq*0] 1952 vinserti32x4 ym1, [dstq+strideq*1], 1 1953 vinserti32x4 m1, [r3 +strideq*0], 2 1954 vinserti32x4 m1, [r3 +strideq*1], 3 1955 pxor m4, m4 1956 mova [cq+64*0], m4 1957 mova [cq+64*1], m4 1958 punpcklbw m0, m1, m4 1959 punpckhbw m1, m4 1960 paddw m0, m2 1961 paddw m1, m3 1962 packuswb m0, m1 1963 mova [dstq+strideq*0], xm0 1964 vextracti32x4 [dstq+strideq*1], ym0, 1 1965 vextracti32x4 [r3 +strideq*0], m0, 2 1966 vextracti32x4 [r3 +strideq*1], m0, 3 1967 RET 1968 ALIGN function_align 1969 .main: 1970 IADST4_1D_PACKED 1971 ret 1972 1973 INV_TXFM_16X4_FN flipadst, dct 1974 INV_TXFM_16X4_FN flipadst, adst 1975 INV_TXFM_16X4_FN flipadst, flipadst 1976 INV_TXFM_16X4_FN flipadst, identity 1977 1978 cglobal iflipadst_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1979 mova m0, [cq+64*0] 1980 mova m1, [cq+64*1] 1981 movshdup m3, [o(permB)] 1982 psrlq m10, m3, 4 1983 call m(iadst_4x16_internal_8bpc).main2 1984 vpbroadcastd m6, [o(pw_m16384_16384)] 1985 psrlq m0, m10, 12 1986 psrlq m10, 16 1987 jmp m(iadst_16x4_internal_8bpc).pass1_end 1988 .pass2: 1989 call m(iadst_16x4_internal_8bpc).main 1990 movu m2, [o(permA+2)] 1991 jmp m(iadst_16x4_internal_8bpc).end 1992 1993 INV_TXFM_16X4_FN identity, dct 1994 INV_TXFM_16X4_FN identity, adst 1995 INV_TXFM_16X4_FN identity, flipadst 1996 INV_TXFM_16X4_FN identity, identity 1997 1998 cglobal iidentity_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 1999 mova m1, [cq+64*0] 2000 mova m2, [cq+64*1] 2001 vpbroadcastd m3, [o(pw_1697x16)] 2002 vpbroadcastd m4, [o(pw_16384)] 2003 mova m5, [o(idtx_16x4p)] 2004 shufps m0, m1, m2, q2020 2005 shufps m1, m2, q3131 2006 pmulhrsw m2, m3, m0 2007 pmulhrsw m3, m1 2008 pmulhrsw m2, m4 2009 pmulhrsw m3, m4 2010 paddsw m0, m2 2011 paddsw m1, m3 2012 vpermb m0, m5, m0 2013 vpermb m1, m5, m1 2014 jmp tx2q 2015 .pass2: 2016 vpbroadcastd m3, [o(pw_1697x8)] 2017 pmulhrsw m2, m3, m0 2018 pmulhrsw m3, m1 2019 paddsw m0, m2 2020 paddsw m1, m3 2021 movu m2, [o(permA+1)] 2022 jmp m(iadst_16x4_internal_8bpc).end 2023 2024 %macro INV_TXFM_16X8_FN 2 ; type1, type2 2025 INV_TXFM_FN %1, %2, 16x8 2026 %ifidn %1_%2, dct_dct 2027 movsx r6d, word [cq] 2028 mov [cq], eobd 2029 or r3d, 8 2030 .dconly: 2031 imul r6d, 181 2032 add r6d, 128 2033 sar r6d, 8 2034 .dconly2: 2035 imul r6d, 181 2036 add r6d, 128+256 2037 sar r6d, 8+1 2038 .dconly3: 2039 imul r6d, 181 2040 lea r2, [strideq*3] 2041 add r6d, 128+2048 2042 sar r6d, 8+4 2043 pxor m2, m2 2044 vpbroadcastw m3, r6d 2045 .dconly_loop: 2046 mova xm1, [dstq+strideq*0] 2047 vinserti32x4 ym1, [dstq+strideq*1], 1 2048 vinserti32x4 m1, [dstq+strideq*2], 2 2049 vinserti32x4 m1, [dstq+r2 ], 3 2050 punpcklbw m0, m1, m2 2051 punpckhbw m1, m2 2052 paddw m0, m3 2053 paddw m1, m3 2054 packuswb m0, m1 2055 mova [dstq+strideq*0], xm0 2056 vextracti32x4 [dstq+strideq*1], ym0, 1 2057 vextracti32x4 [dstq+strideq*2], m0, 2 2058 vextracti32x4 [dstq+r2 ], m0, 3 2059 lea dstq, [dstq+strideq*4] 2060 sub r3d, 4 2061 jg .dconly_loop 2062 RET 2063 %endif 2064 %endmacro 2065 2066 %macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd 2067 vpbroadcastd m8, [o(pw_2896x8)] 2068 vpermq m0, [cq+32*0], q3120 2069 add cq, 32*4 2070 vpermq m7, [cq+32*3], q%1 2071 vpermq m1, [cq-32*3], q%1 2072 vpermq m6, [cq+32*2], q3120 2073 vpermq m2, [cq-32*2], q3120 2074 vpermq m5, [cq+32*1], q%1 2075 vpermq m3, [cq-32*1], q%1 2076 vpermq m4, [cq+32*0], q3120 2077 REPX {pmulhrsw x, m8}, m0, m7, m1, m6, m2, m5, m3, m4 2078 %endmacro 2079 2080 INV_TXFM_16X8_FN dct, dct 2081 INV_TXFM_16X8_FN dct, identity 2082 INV_TXFM_16X8_FN dct, adst 2083 INV_TXFM_16X8_FN dct, flipadst 2084 2085 cglobal idct_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2086 vpbroadcastd m1, [o(pw_2896x8)] 2087 vpermq m0, [cq+64*0], q3120 2088 vpermq m2, [cq+64*1], q3120 2089 vpermq m4, [cq+64*2], q3120 2090 vpermq m6, [cq+64*3], q3120 2091 REPX {pmulhrsw x, m1}, m0, m2, m4, m6 2092 vextracti32x8 ym1, m0, 1 2093 vextracti32x8 ym3, m2, 1 2094 vextracti32x8 ym5, m4, 1 2095 vextracti32x8 ym7, m6, 1 2096 call m(idct_8x16_internal_8bpc).main 2097 vbroadcasti32x4 m8, [o(int_shuf1)] 2098 vbroadcasti32x4 m9, [o(int_shuf2)] 2099 vinserti32x8 m0, ym2, 1 ; a0 a1 a2 a3 b0 b1 b2 b3 2100 vinserti32x8 m1, ym3, 1 ; d0 d1 d2 d3 c0 c1 c2 c3 2101 vinserti32x8 m4, ym6, 1 ; i0 i1 i2 i3 j0 j1 j2 j3 2102 vinserti32x8 m5, ym7, 1 ; l0 l1 l2 l3 k0 k1 k2 k3 2103 vpbroadcastd m2, [o(pw_16384)] 2104 pshufb m0, m8 ; a0 b0 a1 b1 a2 b2 a3 b3 2105 pshufb m1, m9 ; c0 d0 c1 d1 c2 d2 c3 d3 2106 pshufb m6, m4, m8 ; i0 j0 i1 j1 i2 j2 i3 j3 2107 pshufb m7, m5, m9 ; m0 n0 m1 n1 m2 n2 m3 n3 2108 REPX {pmulhrsw x, m2}, m0, m1, m6, m7 2109 punpckldq m2, m0, m1 ; a0 b0 c0 d0 a1 b1 c1 d1 2110 punpckhdq m3, m0, m1 ; a2 b2 c2 d2 a3 b3 c3 d3 2111 punpckldq m4, m6, m7 ; i0 j0 k0 l0 i1 j1 k1 l1 2112 punpckhdq m5, m6, m7 ; i2 j2 k2 l2 i3 j3 k3 l3 2113 jmp tx2q 2114 .pass2: 2115 vshufi32x4 m0, m2, m4, q2020 ; 0 1 2116 vshufi32x4 m2, m4, q3131 ; 4 5 2117 vshufi32x4 m1, m3, m5, q2020 ; 2 3 2118 vshufi32x4 m3, m5, q3131 ; 6 7 2119 call .main 2120 movshdup m4, [o(permC)] 2121 psrlq m6, m4, 4 2122 vpermq m5, m4, q1032 2123 vpermi2q m4, m0, m2 ; a2 a3 b2 b3 e2 e3 f2 f3 2124 vpermt2q m0, m6, m2 ; a0 a1 b0 b1 e0 e1 f0 f1 2125 psrlq m6, m5, 4 2126 vpermi2q m5, m1, m3 ; c2 c3 d2 d3 g2 g3 h2 h3 2127 vpermt2q m1, m6, m3 ; c0 c1 d0 d1 g0 g1 h0 h1 2128 vpbroadcastd m6, [o(pw_2048)] 2129 .end: 2130 REPX {pmulhrsw x, m6}, m0, m4, m1, m5 2131 .end2: 2132 lea r3, [dstq+strideq*4] 2133 lea r4, [strideq*3] 2134 mova xm3, [dstq+strideq*0] 2135 mova xm6, [dstq+strideq*2] 2136 vinserti32x4 ym3, [dstq+strideq*1], 1 2137 vinserti32x4 ym6, [dstq+r4 ], 1 2138 vinserti32x4 m3, [r3 +strideq*0], 2 2139 vinserti32x4 m6, [r3 +strideq*2], 2 2140 vinserti32x4 m3, [r3 +strideq*1], 3 2141 vinserti32x4 m6, [r3 +r4 ], 3 2142 pxor m7, m7 2143 mova [cq+64*0], m7 2144 mova [cq+64*1], m7 2145 mova [cq+64*2], m7 2146 mova [cq+64*3], m7 2147 punpcklbw m2, m3, m7 2148 punpckhbw m3, m7 2149 paddw m0, m2 2150 paddw m4, m3 2151 packuswb m0, m4 2152 mova [dstq+strideq*0], xm0 2153 vextracti32x4 [dstq+strideq*1], ym0, 1 2154 vextracti32x4 [r3 +strideq*0], m0, 2 2155 vextracti32x4 [r3 +strideq*1], m0, 3 2156 punpcklbw m3, m6, m7 2157 punpckhbw m6, m7 2158 paddw m1, m3 2159 paddw m5, m6 2160 packuswb m1, m5 2161 mova [dstq+strideq*2], xm1 2162 vextracti32x4 [dstq+r4 ], ym1, 1 2163 vextracti32x4 [r3 +strideq*2], m1, 2 2164 vextracti32x4 [r3 +r4 ], m1, 3 2165 RET 2166 ALIGN function_align 2167 cglobal_label .main 2168 IDCT8_1D_PACKED 2169 ret 2170 2171 INV_TXFM_16X8_FN adst, dct 2172 INV_TXFM_16X8_FN adst, adst 2173 INV_TXFM_16X8_FN adst, flipadst 2174 INV_TXFM_16X8_FN adst, identity 2175 2176 cglobal iadst_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2177 call m(iadst_8x16_internal_8bpc).main_pass1 2178 vpbroadcastd m7, [o(pw_16384_m16384)] 2179 psrlq m10, 4 2180 .pass1_end: 2181 punpcklwd m5, m4, m2 2182 punpckhwd m4, m2 2183 mova m1, m9 2184 vpdpwssd m1, m5, [o(pw_m2896_2896)] {1to16} 2185 mova m6, m9 2186 vpdpwssd m6, m5, [o(pw_2896_2896)] {1to16} 2187 mova m2, m9 2188 vpdpwssd m2, m4, [o(pw_m2896_2896)] {1to16} 2189 vpdpwssd m9, m4, [o(pw_2896_2896)] {1to16} 2190 psrad m1, 12 2191 psrad m6, 12 2192 packssdw m1, m6 ; out8 -out7 -out9 out6 2193 psrad m2, 12 2194 psrad m9, 12 2195 packssdw m2, m9 ; -out11 out4 out10 -out5 2196 psrlq m4, m10, 4 2197 vpermi2q m4, m0, m2 2198 vpermt2q m0, m10, m2 2199 psrlq m5, m10, 8 2200 vpermi2q m5, m1, m3 2201 psrlq m10, 12 2202 vpermt2q m1, m10, m3 2203 punpcklwd m3, m4, m5 ; a0 c0 a1 c1 a2 c2 a3 c3 2204 punpckhwd m4, m5 ; b0 d0 b1 d1 b2 d2 b3 d3 2205 punpcklwd m5, m1, m0 ; i0 k0 i1 k1 2i k2 i3 k3 2206 punpckhwd m1, m0 ; j0 l0 j1 l1 j2 l2 j3 l3 2207 punpcklwd m2, m3, m4 ; a0 b0 c0 d0 a1 b1 c1 d1 2208 punpckhwd m3, m4 ; a2 b2 c2 d2 a3 b3 c3 d3 2209 punpcklwd m4, m5, m1 ; i0 j0 k0 l0 i1 j1 k1 l1 2210 punpckhwd m5, m1 ; i2 j2 k2 l2 i3 j3 k3 l3 2211 REPX {pmulhrsw x, m7}, m2, m3, m4, m5 2212 jmp tx2q 2213 .pass2: 2214 vshufi32x4 m0, m2, m4, q2020 2215 vshufi32x4 m2, m4, q3131 ; 4 5 2216 vshufi32x4 m1, m3, m5, q2020 2217 vshufi32x4 m3, m5, q3131 ; 6 7 2218 pshufd m4, m0, q1032 ; 1 0 2219 pshufd m5, m1, q1032 ; 3 2 2220 call .main_pass2 2221 movshdup m4, [o(permC)] 2222 pmulhrsw m0, m6 2223 pmulhrsw m1, m6 2224 psrlq m6, m4, 4 2225 mova m5, m4 2226 vpermi2q m4, m0, m2 2227 vpermt2q m0, m6, m2 2228 vpermi2q m5, m1, m3 2229 vpermt2q m1, m6, m3 2230 jmp m(idct_16x8_internal_8bpc).end2 2231 ALIGN function_align 2232 .main_pass1: 2233 vpbroadcastd m4, [o(pw_2896x8)] 2234 pmulhrsw m3, m4, [cq+64*0] 2235 pmulhrsw m1, m4, [cq+64*3] 2236 pmulhrsw m2, m4, [cq+64*1] 2237 pmulhrsw m4, [cq+64*2] 2238 mova m5, [o(int16_perm)] 2239 kxnorb k1, k1, k1 2240 vpblendmd m0{k1}, m1, m3 ; 0 7 2241 vmovdqa32 m3{k1}, m1 ; 6 1 2242 vpblendmd m1{k1}, m4, m2 ; 2 5 2243 vmovdqa32 m2{k1}, m4 ; 4 3 2244 REPX {vpermb x, m5, x}, m0, m1, m2, m3 2245 IADST8_1D_PACKED 1 2246 ret 2247 ALIGN function_align 2248 cglobal_label .main_pass2 2249 IADST8_1D_PACKED 2 2250 pxor m5, m5 2251 psubd m5, m6 2252 packssdw m6, m5 2253 pmulhrsw m2, m6 2254 pmulhrsw m3, m6 2255 ret 2256 2257 INV_TXFM_16X8_FN flipadst, dct 2258 INV_TXFM_16X8_FN flipadst, adst 2259 INV_TXFM_16X8_FN flipadst, flipadst 2260 INV_TXFM_16X8_FN flipadst, identity 2261 2262 cglobal iflipadst_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2263 call m(iadst_8x16_internal_8bpc).main_pass1 2264 vpbroadcastd m7, [o(pw_m16384_16384)] 2265 psrlq m10, 20 2266 jmp m(iadst_16x8_internal_8bpc).pass1_end 2267 .pass2: 2268 vshufi32x4 m0, m2, m4, q2020 2269 vshufi32x4 m2, m4, q3131 ; 4 5 2270 vshufi32x4 m1, m3, m5, q2020 2271 vshufi32x4 m3, m5, q3131 ; 6 7 2272 pshufd m4, m0, q1032 ; 1 0 2273 pshufd m5, m1, q1032 ; 3 2 2274 call m(iadst_16x8_internal_8bpc).main_pass2 2275 movshdup m4, [o(permC)] 2276 pmulhrsw m5, m6, m0 2277 pmulhrsw m0, m6, m1 2278 psrlq m1, m4, 12 2279 psrlq m4, 8 2280 mova m7, m4 2281 vpermi2q m4, m0, m3 2282 vpermt2q m0, m1, m3 2283 vpermi2q m1, m5, m2 2284 vpermt2q m5, m7, m2 2285 jmp m(idct_16x8_internal_8bpc).end2 2286 2287 INV_TXFM_16X8_FN identity, dct 2288 INV_TXFM_16X8_FN identity, adst 2289 INV_TXFM_16X8_FN identity, flipadst 2290 INV_TXFM_16X8_FN identity, identity 2291 2292 cglobal iidentity_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2293 vpbroadcastd m0, [o(pw_2896x8)] 2294 pmulhrsw m3, m0, [cq+64*0] 2295 pmulhrsw m4, m0, [cq+64*1] 2296 pmulhrsw m5, m0, [cq+64*2] 2297 pmulhrsw m0, [cq+64*3] 2298 vpbroadcastd m7, [o(pw_1697x16)] 2299 vpbroadcastd m8, [o(pw_16384)] 2300 shufps m2, m3, m4, q2020 ; a0 a1 a4 a5 e0 e1 e4 e5 2301 shufps m3, m4, q3131 ; a2 a3 a6 a7 e2 e3 e6 e7 2302 shufps m4, m5, m0, q2020 ; i0 i1 i4 i5 m0 m1 m4 m5 2303 shufps m5, m0, q3131 ; i2 i3 i6 i7 m2 m3 m6 m7 2304 mova m9, [o(int8_permA)] 2305 pmulhrsw m0, m7, m2 2306 pmulhrsw m1, m7, m3 2307 pmulhrsw m6, m7, m4 2308 pmulhrsw m7, m5 2309 REPX {pmulhrsw x, m8}, m0, m1, m6, m7 2310 paddsw m2, m0 2311 paddsw m3, m1 2312 paddsw m4, m6 2313 paddsw m5, m7 2314 REPX {vpermb x, m9, x}, m2, m3, m4, m5 2315 jmp tx2q 2316 .pass2: 2317 mova m7, [o(permB)] 2318 vpbroadcastd m6, [o(pw_4096)] 2319 vpermq m0, m7, m2 2320 vpermq m4, m7, m4 2321 vpermq m1, m7, m3 2322 vpermq m5, m7, m5 2323 jmp m(idct_16x8_internal_8bpc).end 2324 2325 %macro INV_TXFM_16X16_FN 2 ; type1, type2 2326 INV_TXFM_FN %1, %2, 16x16 2327 %ifidn %1_%2, dct_dct 2328 movsx r6d, word [cq] 2329 mov [cq], eobd 2330 or r3d, 16 2331 imul r6d, 181 2332 add r6d, 128+512 2333 sar r6d, 8+2 2334 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3 2335 %endif 2336 %endmacro 2337 2338 INV_TXFM_16X16_FN dct, dct 2339 INV_TXFM_16X16_FN dct, identity 2340 INV_TXFM_16X16_FN dct, adst 2341 INV_TXFM_16X16_FN dct, flipadst 2342 2343 cglobal idct_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2344 mova m7, [o(permB)] 2345 vpermq m0, m7, [cq+64*0] 2346 vpermq m1, m7, [cq+64*1] 2347 vpermq m2, m7, [cq+64*2] 2348 vpermq m3, m7, [cq+64*3] 2349 vpermq m4, m7, [cq+64*4] 2350 vpermq m5, m7, [cq+64*5] 2351 vpermq m6, m7, [cq+64*6] 2352 vpermq m7, m7, [cq+64*7] 2353 call .main 2354 vbroadcasti32x4 m12, [o(int_shuf1)] 2355 vbroadcasti32x4 m11, [o(int_shuf2)] 2356 vpbroadcastd m13, [o(pw_8192)] 2357 pshufb m0, m12 2358 pshufb m8, m1, m11 2359 pshufb m2, m12 2360 pshufb m9, m3, m11 2361 pshufb m4, m12 2362 pshufb m10, m5, m11 2363 pshufb m6, m12 2364 pshufb m11, m7, m11 2365 REPX {pmulhrsw x, m13}, m0, m8, m2, m9, m4, m10, m6, m11 2366 punpckhdq m1, m0, m8 2367 punpckldq m0, m8 2368 punpckhdq m3, m2, m9 2369 punpckldq m2, m9 2370 punpckhdq m5, m4, m10 2371 punpckldq m4, m10 2372 punpckhdq m7, m6, m11 2373 punpckldq m6, m11 2374 jmp tx2q 2375 .pass2: 2376 vshufi32x4 m8, m4, m6, q3232 ; i8 ic m8 mc 2377 vinserti32x8 m4, ym6, 1 ; i0 i4 m0 m4 2378 vshufi32x4 m6, m0, m2, q3232 ; a8 ac e8 ec 2379 vinserti32x8 m0, ym2, 1 ; a0 a4 e0 e4 2380 vshufi32x4 m9, m5, m7, q3232 ; ia ie ma me 2381 vinserti32x8 m5, ym7, 1 ; i2 i6 m2 m6 2382 vshufi32x4 m7, m1, m3, q3232 ; aa ae ea ee 2383 vinserti32x8 m1, ym3, 1 ; a2 a6 e2 e6 2384 vshufi32x4 m2, m0, m4, q3131 ; 4 5 2385 vshufi32x4 m0, m4, q2020 ; 0 1 2386 vshufi32x4 m4, m6, m8, q2020 ; 8 9 2387 vshufi32x4 m6, m8, q3131 ; 12 13 2388 vshufi32x4 m3, m1, m5, q3131 ; 6 7 2389 vshufi32x4 m1, m5, q2020 ; 2 3 2390 vshufi32x4 m5, m7, m9, q2020 ; 10 11 2391 vshufi32x4 m7, m9, q3131 ; 14 15 2392 call .main 2393 mova m8, [o(permD)] 2394 psrlq m12, m8, 4 2395 psrlq m9, m8, 8 2396 psrlq m13, m8, 12 2397 mova m10, m8 2398 vpermi2q m8, m0, m2 ; 0 1 4 5 2399 vpermt2q m0, m12, m2 2400 mova m11, m9 2401 vpermi2q m9, m1, m3 ; 2 3 6 7 2402 vpermt2q m1, m13, m3 2403 vpermi2q m10, m4, m6 ; 8 9 12 13 2404 vpermt2q m4, m12, m6 2405 vpermi2q m11, m5, m7 ; 10 11 14 15 2406 vpermt2q m5, m13, m7 2407 .end: 2408 vpbroadcastd m12, [o(pw_2048)] 2409 .end2: 2410 REPX {pmulhrsw x, m12}, m0, m1, m4, m5 2411 .end3: 2412 REPX {pmulhrsw x, m12}, m8, m9, m10, m11 2413 lea r3, [strideq*3] 2414 lea r4, [dstq+strideq*4] 2415 lea r5, [dstq+strideq*8] 2416 lea r6, [r4 +strideq*8] 2417 mova xm3, [dstq+strideq*0] 2418 mova xm6, [dstq+strideq*2] 2419 vinserti32x4 ym3, [dstq+strideq*1], 1 2420 vinserti32x4 ym6, [dstq+r3 ], 1 2421 vinserti32x4 m3, [r4+strideq*0], 2 2422 vinserti32x4 m6, [r4+strideq*2], 2 2423 vinserti32x4 m3, [r4+strideq*1], 3 2424 vinserti32x4 m6, [r4+r3 ], 3 2425 mova xm12, [r5+strideq*0] 2426 mova xm13, [r5+strideq*2] 2427 vinserti32x4 ym12, [r5+strideq*1], 1 2428 vinserti32x4 ym13, [r5+r3 ], 1 2429 vinserti32x4 m12, [r6+strideq*0], 2 2430 vinserti32x4 m13, [r6+strideq*2], 2 2431 vinserti32x4 m12, [r6+strideq*1], 3 2432 vinserti32x4 m13, [r6+r3 ], 3 2433 pxor m7, m7 2434 REPX {mova [cq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 2435 punpcklbw m2, m3, m7 2436 punpckhbw m3, m7 2437 paddw m0, m2 2438 paddw m8, m3 2439 packuswb m0, m8 2440 punpcklbw m2, m6, m7 2441 punpckhbw m6, m7 2442 paddw m1, m2 2443 paddw m9, m6 2444 packuswb m1, m9 2445 punpcklbw m2, m12, m7 2446 punpckhbw m12, m7 2447 paddw m2, m4 2448 paddw m10, m12 2449 packuswb m2, m10 2450 punpcklbw m3, m13, m7 2451 punpckhbw m13, m7 2452 paddw m3, m5 2453 paddw m11, m13 2454 packuswb m3, m11 2455 mova [dstq+strideq*0], xm0 2456 vextracti32x4 [dstq+strideq*1], ym0, 1 2457 mova [dstq+strideq*2], xm1 2458 vextracti32x4 [dstq+r3 ], ym1, 1 2459 vextracti32x4 [r4+strideq*0], m0, 2 2460 vextracti32x4 [r4+strideq*1], m0, 3 2461 vextracti32x4 [r4+strideq*2], m1, 2 2462 vextracti32x4 [r4+r3 ], m1, 3 2463 mova [r5+strideq*0], xm2 2464 vextracti32x4 [r5+strideq*1], ym2, 1 2465 mova [r5+strideq*2], xm3 2466 vextracti32x4 [r5+r3 ], ym3, 1 2467 vextracti32x4 [r6+strideq*0], m2, 2 2468 vextracti32x4 [r6+strideq*1], m2, 3 2469 vextracti32x4 [r6+strideq*2], m3, 2 2470 vextracti32x4 [r6+r3 ], m3, 3 2471 RET 2472 ALIGN function_align 2473 cglobal_label .main_fast2 ; bottom three-quarters are zero 2474 vpbroadcastd m10, [o(pd_2048)] 2475 vpbroadcastq m13, [o(int_mshift)] 2476 vpcmpub k7, m13, m10, 6 2477 .main_fast4: 2478 vpbroadcastd m2, [o(pw_401_4076x8)] 2479 vpbroadcastd m4, [o(pw_m1189_3920x8)] 2480 vpbroadcastd m3, [o(pw_799_4017x8)] 2481 pmulhrsw m2, m8 ; t8a t15a 2482 pmulhrsw m4, m1 ; t11a t12a 2483 pmulhrsw m7, m3 ; t4a t7a 2484 pxor m6, m6 2485 psubsw m0, m2, m4 ; t11a t12a 2486 paddsw m8, m2, m4 ; t8a t15a 2487 mova m1, m7 2488 jmp .main5 2489 ALIGN function_align 2490 cglobal_label .main_fast ; bottom half is zero 2491 vpbroadcastd m10, [o(pd_2048)] 2492 .main_fast3: 2493 vpbroadcastq m13, [o(int_mshift)] 2494 vpcmpub k7, m13, m10, 6 2495 .main_fast5: 2496 vpbroadcastd m2, [o(pw_401_4076x8)] 2497 vpbroadcastd m4, [o(pw_m2598_3166x8)] 2498 vpbroadcastd m11, [o(pw_1931_3612x8)] 2499 vpbroadcastd m12, [o(pw_m1189_3920x8)] 2500 pmulhrsw m8, m2 ; t8a t15a 2501 vpbroadcastd m2, [o(pw_799_4017x8)] 2502 pmulhrsw m0, m4 ; t9a t14a 2503 vpbroadcastd m4, [o(pw_m2276_3406x8)] 2504 pmulhrsw m5, m11 ; t10a t13a 2505 pmulhrsw m1, m12 ; t11a t12a 2506 pmulhrsw m7, m2 ; t4a t7a 2507 pmulhrsw m3, m4 ; t5a t6a 2508 jmp .main4 2509 ALIGN function_align 2510 cglobal_label .main 2511 IDCT16_1D_PACKED 2512 ret 2513 2514 INV_TXFM_16X16_FN adst, dct 2515 INV_TXFM_16X16_FN adst, adst 2516 INV_TXFM_16X16_FN adst, flipadst 2517 2518 cglobal iadst_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2519 call .main_pass1 2520 vpbroadcastd m10, [o(pw_8192_m8192)] 2521 punpcklwd m8, m0, m1 ; b0 d0 b1 d1 b2 d2 b3 d3 2522 punpckhwd m0, m1 ; a0 c0 a1 c1 a2 c2 a3 c3 2523 punpckhwd m1, m0, m8 ; a2 b2 c2 d2 a3 b3 c3 d3 2524 punpcklwd m0, m8 ; a0 b0 c0 d0 a1 b1 c1 d1 2525 punpcklwd m8, m2, m3 ; f0 h0 f1 h1 f2 h2 f3 h3 2526 punpckhwd m2, m3 ; e0 g0 e1 g1 e2 g2 e3 g3 2527 punpckhwd m3, m2, m8 ; e2 f2 g2 h2 e3 f3 g3 h3 2528 punpcklwd m2, m8 ; e0 f0 g0 h0 e1 f1 g1 h1 2529 punpckhwd m8, m4, m5 ; i0 k0 i1 k1 i2 k2 i3 k3 2530 punpcklwd m4, m5 ; j0 l0 j1 l1 j2 l2 j3 l3 2531 punpckhwd m5, m4, m8 ; i2 j2 k2 l2 i3 j3 k3 l3 2532 punpcklwd m4, m8 ; i0 j0 k0 l0 i1 j1 k1 l1 2533 punpckhwd m8, m6, m7 ; m0 o0 m1 o1 m2 o2 m3 o3 2534 punpcklwd m6, m7 ; n0 p0 n1 p1 n2 p2 n3 p3 2535 punpckhwd m7, m6, m8 ; m2 n2 o2 p2 m3 n3 o3 p3 2536 punpcklwd m6, m8 ; m0 n0 o0 p0 m1 n1 o1 p1 2537 .pass1_end: 2538 REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 2539 jmp tx2q 2540 .pass2: 2541 call .main_pass2 2542 mova m10, [o(permD)] 2543 psrlq m8, m10, 8 2544 psrlq m12, m10, 12 2545 psrlq m13, m10, 4 2546 mova m9, m8 2547 vpermi2q m8, m0, m2 ; 0 1 4 5 2548 vpermt2q m0, m12, m2 2549 vpermi2q m9, m1, m3 ; 2 3 6 7 2550 vpermt2q m1, m12, m3 2551 vpbroadcastd m12, [o(pw_2048)] 2552 mov r3d, 0xff00ff00 2553 mova m11, m10 2554 vpermi2q m10, m4, m6 ; 8 9 12 13 2555 vpermt2q m4, m13, m6 2556 kmovd k1, r3d 2557 vpermi2q m11, m5, m7 ; 10 11 14 15 2558 vpermt2q m5, m13, m7 2559 pxor m7, m7 2560 vpsubw m12{k1}, m7, m12 2561 jmp m(idct_16x16_internal_8bpc).end2 2562 ALIGN function_align 2563 .main_pass1: 2564 mova m4, [o(permB)] 2565 psrlq m3, m4, 4 2566 vpermq m0, m4, [cq+64*0] 2567 vpermq m7, m3, [cq+64*7] 2568 vpermq m6, m4, [cq+64*6] 2569 vpermq m1, m3, [cq+64*1] 2570 vpermq m2, m4, [cq+64*2] 2571 vpermq m5, m3, [cq+64*5] 2572 vpermq m4, m4, [cq+64*4] 2573 vpermq m3, m3, [cq+64*3] 2574 call .main 2575 vpbroadcastd m13, [o(pw_2896_2896)] 2576 vpbroadcastd m12, [o(pw_m2896_2896)] 2577 mova m2, m10 2578 vpdpwssd m2, m5, m13 ; -out5 2579 mova m8, m10 2580 vpdpwssd m8, m11, m13 ; out4 2581 mova m9, m10 2582 vpdpwssd m9, m5, m12 ; out10 2583 mova m5, m10 2584 vpdpwssd m5, m11, m12 ; -out11 2585 mova m11, m10 2586 vpdpwssd m11, m3, m13 ; -out7 2587 mova m14, m10 2588 vpdpwssd m14, m4, m13 ; out6 2589 mova m13, m10 2590 vpdpwssd m13, m3, m12 ; out8 2591 vpdpwssd m10, m4, [o(pw_2896_m2896)] {1to16} ; -out9 2592 REPX {psrad x, 12}, m2, m8, m9, m5, m11, m14, m13, m10 2593 packssdw m2, m8 ; -out5 out4 2594 packssdw m5, m9, m5 ; out10 -out11 2595 packssdw m3, m11, m14 ; -out7 out6 2596 packssdw m4, m13, m10 ; out8 -out9 2597 ret 2598 ALIGN function_align 2599 .main_pass2: 2600 vshufi32x4 m8, m4, m6, q3232 ; i8 ic m8 mc 2601 vinserti32x8 m4, ym6, 1 ; i0 i4 m0 m4 2602 vshufi32x4 m6, m0, m2, q3232 ; a8 ac e8 ec 2603 vinserti32x8 m0, ym2, 1 ; a0 a4 e0 e4 2604 vshufi32x4 m9, m5, m7, q3232 ; ia ie ma me 2605 vinserti32x8 m5, ym7, 1 ; i2 i6 m2 m6 2606 vshufi32x4 m7, m1, m3, q3232 ; aa ae ea ee 2607 vinserti32x8 m1, ym3, 1 ; a2 a6 e2 e6 2608 vshufi32x4 m2, m0, m4, q3131 ; 4 5 2609 vshufi32x4 m0, m4, q2020 ; 0 1 2610 vshufi32x4 m4, m6, m8, q2020 ; 8 9 2611 vshufi32x4 m6, m8, q3131 ; 12 13 2612 vshufi32x4 m3, m1, m5, q3131 ; 6 7 2613 vshufi32x4 m1, m5, q2020 ; 2 3 2614 vshufi32x4 m5, m7, m9, q2020 ; 10 11 2615 vshufi32x4 m7, m9, q3131 ; 14 15 2616 cglobal_label .main_pass2b 2617 REPX {pshufd x, x, q1032}, m1, m3, m5, m7 2618 call .main 2619 vpbroadcastd m8, [o(pw_2896x8)] 2620 pshufb m2, m11, m12 2621 pshufb m5, m12 2622 pshufb m3, m12 2623 pshufb m4, m12 2624 punpcklqdq m9, m5, m2 ; t15a t7 2625 punpckhqdq m5, m2 ; t14a t6 2626 shufps m2, m3, m4, q1032 ; t2a t10 2627 shufps m3, m4, q3210 ; t3a t11 2628 psubsw m4, m2, m3 ; out8 -out9 2629 paddsw m3, m2 ; -out7 out6 2630 paddsw m2, m5, m9 ; -out5 out4 2631 psubsw m5, m9 ; out10 -out11 2632 REPX {pmulhrsw x, m8}, m2, m3, m4, m5 2633 ret 2634 ALIGN function_align 2635 .main: 2636 vpbroadcastd m10, [o(pd_2048)] 2637 vpbroadcastq m13, [o(int_mshift)] 2638 punpckhwd m8, m7, m0 ; in14 in1 2639 punpcklwd m0, m7 ; in0 in15 2640 punpcklwd m7, m6, m1 ; in12 in3 2641 punpckhwd m1, m6 ; in2 in13 2642 punpckhwd m6, m5, m2 ; in10 in5 2643 punpcklwd m2, m5 ; in4 in11 2644 punpcklwd m5, m4, m3 ; in8 in7 2645 punpckhwd m3, m4 ; in6 in9 2646 vpcmpub k7, m13, m10, 6 ; 0x33... 2647 ITX_MUL2X_PACK 0, 4, 9, 10, 201, 4091, 5 ; t0 t1 2648 ITX_MUL2X_PACK 1, 4, 9, 10, 995, 3973, 5 ; t2 t3 2649 ITX_MUL2X_PACK 2, 4, 9, 10, 1751, 3703, 5 ; t4 t5 2650 ITX_MUL2X_PACK 3, 4, 9, 10, 2440, 3290, 5 ; t6 t7 2651 ITX_MUL2X_PACK 5, 4, 9, 10, 3035, 2751, 5 ; t8 t9 2652 ITX_MUL2X_PACK 6, 4, 9, 10, 3513, 2106, 5 ; t10 t11 2653 ITX_MUL2X_PACK 7, 4, 9, 10, 3857, 1380, 5 ; t12 t13 2654 ITX_MUL2X_PACK 8, 4, 9, 10, 4052, 601, 5 ; t14 t15 2655 psubsw m4, m0, m5 ; t9a t8a 2656 paddsw m0, m5 ; t1a t0a 2657 psubsw m5, m1, m6 ; t11a t10a 2658 paddsw m1, m6 ; t3a t2a 2659 psubsw m6, m2, m7 ; t13a t12a 2660 paddsw m2, m7 ; t5a t4a 2661 psubsw m7, m3, m8 ; t15a t14a 2662 paddsw m3, m8 ; t7a t6a 2663 ITX_MUL2X_PACK 4, 8, 9, 10, 799, 4017, 4 ; t8 t9 2664 ITX_MUL2X_PACK 6, 8, 9, 10, 799_4017, 4017_m799, 52 ; t12 t13 2665 ITX_MUL2X_PACK 5, 8, 9, 10, 3406, 2276, 4 ; t10 t11 2666 ITX_MUL2X_PACK 7, 8, 9, 10, 3406_2276, 2276_m3406, 52 ; t14 t15 2667 psubsw m8, m1, m3 ; t7 t6 2668 paddsw m1, m3 ; t3 t2 2669 psubsw m3, m0, m2 ; t5 t4 2670 paddsw m0, m2 ; t1 t0 2671 psubsw m2, m5, m7 ; t14a t15a 2672 paddsw m7, m5 ; t10a t11a 2673 psubsw m5, m4, m6 ; t12a t13a 2674 paddsw m4, m6 ; t8a t9a 2675 ITX_MUL2X_PACK 3, 6, 9, 10, 1567, 3784, 5 ; t5a t4a 2676 ITX_MUL2X_PACK 8, 6, 9, 10, 3784_m1567, 1567_3784, 52 ; t7a t6a 2677 ITX_MUL2X_PACK 2, 6, 9, 10, 3784, 1567, 4 ; t15 t14 2678 ITX_MUL2X_PACK 5, 6, 9, 10, 3784_1567, 1567_m3784, 52 ; t13 t12 2679 vbroadcasti32x4 m12, [o(deint_shuf)] 2680 paddsw m6, m4, m7 ; -out1 out14 2681 psubsw m4, m7 ; t10 t11 2682 psubsw m11, m3, m8 ; t7 t6 2683 paddsw m8, m3 ; out12 -out3 2684 psubsw m3, m0, m1 ; t3a t2a 2685 paddsw m0, m1 ; -out15 out0 2686 paddsw m1, m2, m5 ; -out13 out2 2687 psubsw m5, m2 ; t15a t14a 2688 pshufb m0, m12 2689 pshufb m6, m12 2690 pshufb m8, m12 2691 pshufb m1, m12 2692 shufps m7, m6, m0, q1032 ; out14 -out15 2693 shufps m0, m6, m0, q3210 ; -out1 out0 2694 punpcklqdq m6, m8, m1 ; out12 -out13 2695 punpckhqdq m1, m8, m1 ; -out3 out2 2696 ret 2697 2698 INV_TXFM_16X16_FN flipadst, dct 2699 INV_TXFM_16X16_FN flipadst, adst 2700 INV_TXFM_16X16_FN flipadst, flipadst 2701 2702 cglobal iflipadst_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2703 call m(iadst_16x16_internal_8bpc).main_pass1 2704 vpbroadcastd m10, [o(pw_m8192_8192)] 2705 punpcklwd m8, m1, m0 ; m0 o0 m1 o1 m2 o2 m3 o3 2706 punpckhwd m9, m1, m0 ; n0 p0 n1 p1 n2 p2 n3 p3 2707 punpckhwd m1, m7, m6 ; a0 c0 a1 c1 a2 c2 a3 c3 2708 punpcklwd m7, m6 ; b0 d0 b1 d1 b2 d2 b3 d3 2709 punpcklwd m0, m1, m7 ; a0 b0 c0 d0 a1 b1 c1 d1 2710 punpckhwd m1, m7 ; a2 b2 c2 d2 a3 b3 c3 d3 2711 punpcklwd m6, m8, m9 ; m0 n0 o0 p0 m1 n1 o1 p1 2712 punpckhwd m7, m8, m9 ; m2 n2 o2 p2 m3 n3 o3 p3 2713 punpcklwd m8, m3, m2 ; i0 k0 i1 k1 i2 k2 i3 k3 2714 punpckhwd m9, m3, m2 ; j0 l0 j1 l1 j2 l2 j3 l3 2715 punpckhwd m3, m5, m4 ; e0 g0 e1 g1 e2 g2 e3 g3 2716 punpcklwd m5, m4 ; f0 h0 f1 h1 f2 h2 f3 h3 2717 punpcklwd m2, m3, m5 ; e0 f0 g0 h0 e1 f1 g1 h1 2718 punpckhwd m3, m5 ; e2 f2 g2 h2 e3 f3 g3 h3 2719 punpcklwd m4, m8, m9 ; i0 j0 k0 l0 i1 j1 k1 l1 2720 punpckhwd m5, m8, m9 ; i2 j2 k2 l2 i3 j3 k3 l3 2721 jmp m(iadst_16x16_internal_8bpc).pass1_end 2722 .pass2: 2723 call m(iadst_16x16_internal_8bpc).main_pass2 2724 mova m10, [o(permD)] 2725 psrlq m8, m10, 8 2726 psrlq m12, m10, 12 2727 psrlq m13, m10, 4 2728 mova m9, m8 2729 vpermi2q m8, m7, m5 ; 0 1 4 5 2730 vpermt2q m7, m12, m5 2731 vpermi2q m9, m6, m4 ; 2 3 6 7 2732 vpermt2q m6, m12, m4 2733 vpbroadcastd m12, [o(pw_2048)] 2734 mov r3d, 0x00ff00ff 2735 mova m11, m10 2736 vpermi2q m10, m3, m1 ; 8 9 12 13 2737 vpermt2q m3, m13, m1 2738 kmovd k1, r3d 2739 vpermi2q m11, m2, m0 ; 10 11 14 15 2740 vpermt2q m2, m13, m0 2741 pxor m0, m0 2742 vpsubw m12{k1}, m0, m12 2743 pmulhrsw m0, m7, m12 2744 pmulhrsw m1, m6, m12 2745 pmulhrsw m4, m3, m12 2746 pmulhrsw m5, m2, m12 2747 jmp m(idct_16x16_internal_8bpc).end3 2748 2749 INV_TXFM_16X16_FN identity, dct 2750 INV_TXFM_16X16_FN identity, identity 2751 2752 cglobal iidentity_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 2753 mova m8, [o(int16_perm)] 2754 vpermb m1, m8, [cq+64*0] ; a0 b0 a1 b1 a2 b2 a3 b3 2755 vpermb m2, m8, [cq+64*1] ; c0 d0 c1 d1 c2 d2 c3 d3 2756 vpbroadcastd m0, [o(pw_1697x16)] 2757 vpermb m3, m8, [cq+64*2] ; e0 f0 e1 f1 e2 f2 e3 f3 2758 vpermb m4, m8, [cq+64*3] ; g0 h0 g1 h1 g2 h2 g3 h3 2759 vpermb m5, m8, [cq+64*4] ; i0 j0 i1 j1 i2 j2 i3 j3 2760 vpermb m6, m8, [cq+64*5] ; k0 l0 k1 l1 k2 l2 k3 l3 2761 vpermb m7, m8, [cq+64*6] ; m0 n0 m1 n1 m2 n2 m3 n3 2762 vpermb m8, m8, [cq+64*7] ; o0 p0 o1 p1 o2 p2 o3 p3 2763 pmulhrsw m9, m0, m1 2764 pmulhrsw m10, m0, m2 2765 pmulhrsw m11, m0, m3 2766 pmulhrsw m12, m0, m4 2767 pmulhrsw m13, m0, m5 2768 pmulhrsw m14, m0, m6 2769 pmulhrsw m15, m0, m7 2770 pmulhrsw m0, m8 2771 REPX {psraw x, 1}, m9, m10, m11, m12 2772 pavgw m1, m9 2773 pavgw m2, m10 2774 pavgw m3, m11 2775 pavgw m4, m12 2776 REPX {psraw x, 1}, m13, m14, m15, m0 2777 pavgw m5, m13 2778 pavgw m6, m14 2779 pavgw m7, m15 2780 pavgw m8, m0 2781 punpckldq m0, m1, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 2782 punpckhdq m1, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 2783 punpckldq m2, m3, m4 ; e0 f0 g0 h0 e1 f1 g1 h1 2784 punpckhdq m3, m4 ; e2 f2 g2 h2 e3 f3 g3 h3 2785 punpckldq m4, m5, m6 ; i0 j0 k0 l0 i1 j1 k1 l1 2786 punpckhdq m5, m6 ; i2 j2 k2 l2 i3 j3 k3 l3 2787 punpckldq m6, m7, m8 ; m0 n0 o0 p0 m1 n1 o1 p1 2788 punpckhdq m7, m8 ; m2 n2 o2 p2 m3 n3 o3 p3 2789 jmp tx2q 2790 ALIGN function_align 2791 .pass2: 2792 vpbroadcastd m11, [o(pw_1697x16)] 2793 pmulhrsw m12, m11, m0 2794 pmulhrsw m13, m11, m1 2795 pmulhrsw m14, m11, m2 2796 pmulhrsw m15, m11, m3 2797 pmulhrsw m8, m11, m4 2798 pmulhrsw m9, m11, m5 2799 pmulhrsw m10, m11, m6 2800 pmulhrsw m11, m7 2801 REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7 2802 paddsw m0, m12 2803 paddsw m1, m13 2804 paddsw m2, m14 2805 paddsw m3, m15 2806 paddsw m8, m4 2807 movu m4, [o(permD+2)] 2808 paddsw m9, m5 2809 paddsw m6, m10 2810 paddsw m7, m11 2811 psrlq m12, m4, 4 2812 mova m5, m4 2813 mova m10, m4 2814 mova m11, m4 2815 vpermi2q m4, m0, m2 ; 8 9 12 13 2816 vpermt2q m0, m12, m2 ; 0 1 4 5 2817 vpermi2q m5, m1, m3 ; 10 11 14 15 2818 vpermt2q m1, m12, m3 ; 2 3 6 7 2819 vpermi2q m10, m8, m6 2820 vpermt2q m8, m12, m6 2821 vpermi2q m11, m9, m7 2822 vpermt2q m9, m12, m7 2823 jmp m(idct_16x16_internal_8bpc).end 2824 2825 %macro ITX_UNPACK_MULHRSW 8 ; dst[1-2], src, tmp, coef[1-4] 2826 vpbroadcastd m%4, [o(pw_%5_%6x8)] 2827 punpcklwd m%1, m%3, m%3 2828 pmulhrsw m%1, m%4 2829 vpbroadcastd m%4, [o(pw_%7_%8x8)] 2830 punpckhwd m%2, m%3, m%3 2831 pmulhrsw m%2, m%4 2832 %endmacro 2833 2834 cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 4, 0, dst, stride, c, eob 2835 %undef cmp 2836 lea r5, [o_base] 2837 test eobd, eobd 2838 jz .dconly 2839 cmp eobd, 107 2840 jb .fast 2841 mova m5, [cq+64*5] 2842 mova m3, [cq+64*3] 2843 mova m1, [cq+64*1] 2844 mova m7, [cq+64*7] 2845 mova m2, [cq+64*2] 2846 mova m6, [cq+64*6] 2847 mova m0, [cq+64*0] 2848 mova m4, [cq+64*4] 2849 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 2850 mova m8, [o(idct_8x32p)] 2851 vpbroadcastd m9, [o(pw_8192)] 2852 REPX {vpermb x, m8, x}, m0, m1, m2, m3, m4, m5, m6, m7 2853 punpckldq m8, m0, m1 ; ab 2854 punpckhdq m0, m1 2855 punpckldq m1, m2, m3 ; cd 2856 punpckhdq m2, m3 2857 punpckldq m3, m4, m5 ; ef 2858 punpckhdq m4, m5 2859 punpckldq m5, m6, m7 ; gh 2860 punpckhdq m6, m7 2861 REPX {pmulhrsw x, m9}, m8, m0, m1, m2, m3, m4, m5, m6 2862 punpcklqdq m18, m8, m1 ; 30 2 6 26 31 1 23 9 2863 punpckhqdq m14, m8, m1 ; 16 0 12 20 3 29 11 21 2864 punpcklqdq m21, m0, m2 ; 14 18 22 10 27 5 19 13 2865 punpckhqdq m15, m0, m2 ; 18 4 24 8 7 25 15 17 2866 punpcklqdq m20, m3, m5 2867 punpckhqdq m16, m3, m5 2868 punpcklqdq m19, m4, m6 2869 punpckhqdq m17, m4, m6 2870 vinserti32x4 ym8, ym18, xm20, 1 2871 vshufi32x4 ym1, ym18, ym20, 0x03 2872 vinserti32x4 ym9, ym14, xm16, 1 2873 vshufi32x4 ym3, ym14, ym16, 0x03 2874 vinserti32x4 ym0, ym21, xm19, 1 2875 vshufi32x4 ym5, ym21, ym19, 0x03 2876 vinserti32x4 ym7, ym15, xm17, 1 2877 vshufi32x4 ym6, ym15, ym17, 0x03 2878 call m(idct_8x16_internal_8bpc).main2 2879 psrlq m12, [o(permB)], 60 2880 vpermt2q m14, m12, m16 2881 vpermt2q m21, m12, m19 2882 vpermt2q m15, m12, m17 2883 vpermi2q m12, m18, m20 2884 vextracti32x8 ym16, m14, 1 2885 vextracti32x8 ym19, m21, 1 2886 vextracti32x8 ym17, m15, 1 2887 vextracti32x8 ym20, m12, 1 2888 call .main2 2889 jmp .end 2890 .fast: ; right half is zero 2891 mova m0, [o(int16_perm)] 2892 mova ym2, [cq+64*4] 2893 vinserti32x8 m2, [cq+64*0], 1 2894 mova ym3, [cq+64*6] 2895 vinserti32x8 m3, [cq+64*2], 1 2896 mova ym4, [cq+64*3] 2897 vinserti32x8 m4, [cq+64*5], 1 2898 mova ym5, [cq+64*7] 2899 vinserti32x8 m5, [cq+64*1], 1 2900 REPX {vpermb x, m0, x}, m2, m3, m4, m5 2901 call m(idct_16x8_internal_8bpc).main2 2902 vbroadcasti32x4 m4, [o(int_shuf3)] 2903 vbroadcasti32x4 m5, [o(int_shuf4)] 2904 pshufb m2, m4 ; e0 f0 e2 f2 e1 f1 e3 f3 2905 pshufb m3, m5 ; g0 h0 g2 h2 g1 h1 g3 h3 2906 pshufb m0, m4 ; a0 b0 a2 b2 a1 b1 a3 b3 2907 pshufb m1, m5 ; c0 d0 c2 d2 c1 d1 c3 d3 2908 vpbroadcastd m4, [o(pw_8192)] 2909 psrlq m5, [o(permB)], 60 2910 punpckldq m6, m2, m3 ; e0 f0 g0 h0 e2 f2 g2 h2 2911 punpckhdq m17, m2, m3 ; e1 f1 g1 h1 e3 f3 g3 h3 2912 punpckldq m2, m0, m1 ; a0 b0 c0 d0 a2 b2 c2 d2 2913 punpckhdq m16, m0, m1 ; a1 b1 c1 d1 a3 b3 c3 d3 2914 REPX {pmulhrsw x, m4}, m6, m17, m2, m16 2915 vinserti32x4 ym0, ym2, xm6, 1 ; 0 2 2916 vshufi32x4 ym1, ym2, ym6, 0x03 ; 4 6 2917 vinserti32x4 ym14, ym16, xm17, 1 ; 1 3 2918 vshufi32x4 ym15, ym16, ym17, 0x03 ; 5 7 2919 vpermt2q m2, m5, m6 ; 8 10 2920 vpermt2q m16, m5, m17 ; 9 11 2921 vextracti32x8 ym3, m2, 1 ; 12 14 2922 vextracti32x8 ym17, m16, 1 ; 13 15 2923 call m(idct_8x16_internal_8bpc).main_fast 2924 call .main_fast 2925 .end: 2926 vpbroadcastd ym8, strided 2927 pmulld ym8, [o(gather8d)] 2928 call .main_end 2929 lea r3, [dstq+strideq*4] 2930 kxnorb k1, k1, k1 2931 lea r4, [dstq+strideq*8] 2932 pxor m9, m9 2933 lea r1, [r3+strideq*8] 2934 kmovb k2, k1 2935 vpgatherdq m12{k1}, [r0+ym8] 2936 kmovb k1, k2 2937 vpgatherdq m13{k2}, [r3+ym8] 2938 kmovb k2, k1 2939 vpgatherdq m14{k1}, [r4+ym8] 2940 kmovb k1, k2 2941 vpgatherdq m15{k2}, [r1+ym8] 2942 REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 2943 REPX {mova [cq+64*x], m9}, 0, 1, 2, 3, 4, 5, 6, 7 2944 punpcklbw m11, m12, m9 2945 punpckhbw m12, m9 2946 paddw m0, m11 2947 paddw m1, m12 2948 packuswb m0, m1 2949 kmovb k2, k1 2950 vpscatterdq [r0+ym8]{k1}, m0 2951 punpcklbw m12, m13, m9 2952 punpckhbw m13, m9 2953 paddw m2, m12 2954 paddw m3, m13 2955 packuswb m2, m3 2956 kmovb k1, k2 2957 vpscatterdq [r3+ym8]{k2}, m2 2958 punpcklbw m13, m14, m9 2959 punpckhbw m14, m9 2960 paddw m4, m13 2961 paddw m5, m14 2962 packuswb m4, m5 2963 kmovb k2, k1 2964 vpscatterdq [r4+ym8]{k1}, m4 2965 punpcklbw m14, m15, m9 2966 punpckhbw m15, m9 2967 paddw m6, m14 2968 paddw m7, m15 2969 packuswb m6, m7 2970 vpscatterdq [r1+ym8]{k2}, m6 2971 RET 2972 .dconly: 2973 movsx r6d, word [cq] 2974 mov [cq], eobd 2975 or r3d, 32 2976 imul r6d, 181 2977 add r6d, 128+512 2978 sar r6d, 8+2 2979 jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly2 2980 INIT_YMM avx512icl 2981 ALIGN function_align 2982 cglobal_label .main_fast2 ; bottom three-quarters are zero 2983 ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a 2984 ITX_UNPACK_MULHRSW 21, 20, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a 2985 mova m11, m12 2986 mova m17, m20 2987 mova m15, m21 2988 mova m16, m14 2989 jmp .main4 2990 ALIGN function_align 2991 cglobal_label .main_fast ; bottom half is zero 2992 ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a 2993 ITX_UNPACK_MULHRSW 21, 15, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a 2994 ITX_UNPACK_MULHRSW 20, 16, 16, 8, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a 2995 ITX_UNPACK_MULHRSW 19, 17, 17, 8, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a 2996 jmp .main3 2997 ALIGN function_align 2998 cglobal_label .main 2999 punpcklwd m12, m21, m14 ; in31 in1 3000 punpckhwd m14, m21 ; in3 in29 3001 punpcklwd m21, m20, m15 ; in27 in5 3002 punpckhwd m15, m20 ; in7 in25 3003 punpcklwd m20, m19, m16 ; in23 in9 3004 punpckhwd m16, m19 ; in11 in21 3005 punpcklwd m19, m18, m17 ; in19 in13 3006 punpckhwd m17, m18 ; in15 in17 3007 .main2: 3008 ITX_MUL2X_PACK 12, 8, 9, 10, 201, 4091, 5 ; t16a, t31a 3009 ITX_MUL2X_PACK 14, 8, 9, 10, 4052, 601, 5 ; t23a, t24a 3010 ITX_MUL2X_PACK 21, 8, 9, 10, 995, 3973, 5 ; t20a, t27a 3011 ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a 3012 ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a 3013 ITX_MUL2X_PACK 16, 8, 9, 10, 3513, 2106, 5 ; t21a, t26a 3014 ITX_MUL2X_PACK 19, 8, 9, 10, 2440, 3290, 5 ; t22a, t25a 3015 ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a 3016 .main3: 3017 psubsw m11, m12, m17 ; t17 t30 3018 paddsw m12, m17 ; t16 t31 3019 psubsw m17, m15, m20 ; t18 t29 3020 paddsw m20, m15 ; t19 t28 3021 psubsw m15, m21, m16 ; t21 t26 3022 paddsw m21, m16 ; t20 t27 3023 psubsw m16, m14, m19 ; t22 t25 3024 paddsw m14, m19 ; t23 t24 3025 .main4: 3026 ITX_MUL2X_PACK 11, 18, 19, 10, 799, 4017, 5 ; t17a t30a 3027 ITX_MUL2X_PACK 17, 18, 19, 10, m4017, 799, 5 ; t18a t29a 3028 ITX_MUL2X_PACK 15, 18, 19, 10, 3406, 2276, 5 ; t21a t26a 3029 ITX_MUL2X_PACK 16, 18, 19, 10, m2276, 3406, 5 ; t22a t25a 3030 vpbroadcastd m8, [o(pw_m3784_1567)] 3031 psubsw m19, m12, m20 ; t19a t28a 3032 paddsw m20, m12 ; t16a t31a 3033 psubsw m12, m14, m21 ; t20a t27a 3034 paddsw m14, m21 ; t23a t24a 3035 psubsw m21, m11, m17 ; t18 t29 3036 paddsw m11, m17 ; t17 t30 3037 psubsw m17, m16, m15 ; t21 t26 3038 paddsw m16, m15 ; t22 t25 3039 ITX_MUL2X_PACK 21, 18, 15, 10, 1567_3784, 8, 20 ; t18a t29a 3040 ITX_MUL2X_PACK 19, 18, 15, 10, 1567_3784, 8, 20 ; t19 t28 3041 ITX_MUL2X_PACK 12, 18, 15, 10, 8, m1567_m3784, 36 ; t20 t27 3042 ITX_MUL2X_PACK 17, 18, 15, 10, 8, m1567_m3784, 36 ; t21a t26a 3043 vbroadcasti32x4 m18, [o(deint_shuf)] 3044 vpbroadcastd m8, [o(pw_m2896_2896)] 3045 vpbroadcastd m9, [o(pw_2896_2896)] 3046 psubsw m15, m20, m14 ; t23 t24 3047 paddsw m20, m14 ; t16 t31 3048 psubsw m14, m11, m16 ; t22a t25a 3049 paddsw m11, m16 ; t17a t30a 3050 psubsw m16, m21, m17 ; t21 t26 3051 paddsw m21, m17 ; t18 t29 3052 psubsw m17, m19, m12 ; t20a t27a 3053 paddsw m19, m12 ; t19a t28a 3054 REPX {pshufb x, m18}, m20, m11, m21, m19 3055 ITX_MUL2X_PACK 15, 18, 12, 10, 8, 9, 8 ; t23a t22a 3056 ITX_MUL2X_PACK 14, 13, 15, 10, 8, 9, 8 ; t22 t25 3057 packssdw m18, m13 ; t23a t22 3058 packssdw m12, m15 ; t24a t25 3059 ITX_MUL2X_PACK 16, 13, 15, 10, 8, 9, 8 ; t21a t26a 3060 ITX_MUL2X_PACK 17, 16, 14, 10, 8, 9, 8 ; t20 t27 3061 packssdw m16, m13 ; t20 t21a 3062 packssdw m14, m15 ; t27 t26a 3063 punpcklqdq m13, m19, m21 ; t19a t18 3064 punpckhqdq m19, m21 ; t28a t29 3065 punpcklqdq m21, m20, m11 ; t16 t17a 3066 punpckhqdq m20, m11 ; t31 t30a 3067 INIT_ZMM avx512icl 3068 mova m15, [o(permA)] 3069 ret 3070 cglobal_label .main_end 3071 vpbroadcastd m10, [o(pw_2048)] 3072 vpermt2q m0, m15, m1 ; t0 t1 t2 t3 3073 vpermt2q m20, m15, m19 ; t31 t30a t29 t28a 3074 vpermt2q m2, m15, m3 ; t4 t5 t6 t7 3075 vpermt2q m14, m15, m12 ; t27 t26a t25 t24a 3076 vpermt2q m4, m15, m5 ; t8 t9 t10 t11 3077 vpermt2q m18, m15, m16 ; t23a t22 t21a t20 3078 vpermt2q m6, m15, m7 ; t12 t13 t14 t15 3079 vpermt2q m13, m15, m21 ; t19a t18 t17a t16 3080 psubsw m7, m0, m20 ; out31 out30 out29 out28 3081 paddsw m0, m20 ; out0 out1 out2 out3 3082 psubsw m5, m2, m14 ; out27 out26 out25 out24 3083 paddsw m2, m14 ; out4 out5 out6 out7 3084 psubsw m3, m4, m18 ; out23 out22 out21 out20 3085 paddsw m4, m18 ; out8 out9 out10 out11 3086 psubsw m1, m6, m13 ; out19 out18 out17 out16 3087 paddsw m6, m13 ; out12 out13 out14 out15 3088 vzeroupper 3089 ret 3090 3091 %macro LOAD_PACKED_16X2 3 ; dst, row[1-2] 3092 vbroadcasti32x4 ym%1, [cq+16*%2] 3093 vbroadcasti32x4 ym8, [cq+16*%3] 3094 shufpd ym%1, ym8, 0x0c 3095 %endmacro 3096 3097 cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 4, 0, dst, stride, c, eob 3098 %undef cmp 3099 test eobd, eobd 3100 jz .dconly 3101 lea r5, [o_base] 3102 LOAD_PACKED_16X2 0, 0, 2 ; in0 in2 3103 LOAD_PACKED_16X2 1, 4, 6 ; in4 in6 3104 LOAD_PACKED_16X2 2, 8, 10 ; in8 in10 3105 LOAD_PACKED_16X2 3, 12, 14 ; in12 in14 3106 LOAD_PACKED_16X2 14, 1, 3 ; in1 in3 3107 LOAD_PACKED_16X2 15, 5, 7 ; in5 in7 3108 LOAD_PACKED_16X2 16, 9, 11 ; in9 in11 3109 LOAD_PACKED_16X2 17, 13, 15 ; in13 in15 3110 pxor m4, m4 3111 REPX {mova [cq+64*x], m4}, 0, 1, 2, 3 3112 cmp eobd, 107 3113 jb .fast 3114 LOAD_PACKED_16X2 4, 16, 18 ; in16 in18 3115 LOAD_PACKED_16X2 5, 20, 22 ; in20 in22 3116 LOAD_PACKED_16X2 6, 24, 26 ; in24 in26 3117 LOAD_PACKED_16X2 7, 28, 30 ; in28 in30 3118 call m(idct_8x16_internal_8bpc).main 3119 LOAD_PACKED_16X2 18, 19, 17 ; in19 in17 3120 LOAD_PACKED_16X2 19, 23, 21 ; in23 in21 3121 LOAD_PACKED_16X2 20, 27, 25 ; in27 in25 3122 LOAD_PACKED_16X2 21, 31, 29 ; in31 in29 3123 pxor m8, m8 3124 REPX {mova [cq+64*x], m8}, 4, 5, 6, 7 3125 call m(inv_txfm_add_dct_dct_8x32_8bpc).main 3126 jmp .pass2 3127 .fast: ; bottom half is zero 3128 mova ym5, ym4 3129 mova ym6, ym4 3130 mova ym7, ym4 3131 call m(idct_8x16_internal_8bpc).main 3132 call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast 3133 .pass2: 3134 vpbroadcastd m10, [o(pw_8192)] 3135 vpermt2q m0, m15, m4 ; t0 t1 t9 t8 3136 vpermt2q m20, m15, m18 ; t31 t30a t23a t22 3137 vpermt2q m3, m15, m7 ; t7 t6 t14 t15 3138 vpermt2q m12, m15, m21 ; t25 t24a t17a t16 3139 vpermt2q m2, m15, m6 ; t4 t5 t13 t12 3140 vpermt2q m14, m15, m13 ; t23a t22 t21a t20 3141 vpermt2q m1, m15, m5 ; t3 t2 t10 t11 3142 vpermt2q m19, m15, m16 ; t27 t26a t19a t18 3143 psubsw m8, m0, m20 ; out31 out30 out22 out23 3144 paddsw m0, m20 ; out0 out1 out9 out8 3145 paddsw m6, m3, m12 ; out7 out6 out14 out15 3146 psubsw m3, m12 ; out24 out25 out17 out16 3147 psubsw m5, m2, m14 ; out27 out26 out18 out19 3148 paddsw m4, m2, m14 ; out4 out5 out13 out12 3149 psubsw m7, m1, m19 ; out28 out29 out21 out20 3150 paddsw m2, m1, m19 ; out3 out2 out10 out11 3151 vzeroupper 3152 vshufi32x4 m1, m0, m3, q1221 ; out1 out9 out17 out25 3153 vshufi32x4 m0, m3, q0330 ; out0 out8 out16 out24 3154 vshufi32x4 m3, m2, m5, q0330 ; out3 out11 out19 out27 3155 vshufi32x4 m2, m5, q1221 ; out2 out10 out18 out26 3156 vshufi32x4 m5, m4, m7, q1221 ; out5 out13 out21 out29 3157 vshufi32x4 m4, m7, q0330 ; out4 out12 out20 out28 3158 vshufi32x4 m7, m6, m8, q0330 ; out7 out15 out23 out31 3159 vshufi32x4 m6, m8, q1221 ; out6 out14 out22 out30 3160 REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 3161 call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8 3162 call .main 3163 vpbroadcastd m8, [o(pw_2048)] 3164 REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 3165 lea r2, [strideq*3] 3166 lea r3, [dstq+strideq*4] 3167 movshdup m12, [o(permD)] 3168 pmovzxbw m8, [dstq+strideq*0] 3169 pmovzxbw m9, [dstq+strideq*1] 3170 pmovzxbw m10, [dstq+strideq*2] 3171 pmovzxbw m11, [dstq+r2 ] 3172 paddw m0, m8 3173 paddw m1, m9 3174 paddw m2, m10 3175 paddw m3, m11 3176 pmovzxbw m8, [r3+strideq*0] 3177 pmovzxbw m9, [r3+strideq*1] 3178 pmovzxbw m10, [r3+strideq*2] 3179 pmovzxbw m11, [r3+r2 ] 3180 paddw m4, m8 3181 paddw m5, m9 3182 paddw m6, m10 3183 paddw m7, m11 3184 packuswb m0, m1 3185 packuswb m2, m3 3186 vpermq m0, m12, m0 3187 vpermq m2, m12, m2 3188 mova [dstq+strideq*0], ym0 3189 vextracti32x8 [dstq+strideq*1], m0, 1 3190 mova [dstq+strideq*2], ym2 3191 vextracti32x8 [dstq+r2 ], m2, 1 3192 packuswb m4, m5 3193 packuswb m6, m7 3194 vpermq m4, m12, m4 3195 vpermq m6, m12, m6 3196 mova [r3+strideq*0], ym4 3197 vextracti32x8 [r3+strideq*1], m4, 1 3198 mova [r3+strideq*2], ym6 3199 vextracti32x8 [r3+r2 ], m6, 1 3200 RET 3201 .dconly: 3202 movsx r6d, word [cq] 3203 mov [cq], eobd 3204 or r3d, 8 3205 .dconly2: 3206 imul r6d, 181 3207 add r6d, 128+512 3208 sar r6d, 8+2 3209 .dconly3: 3210 imul r6d, 181 3211 add r6d, 128+2048 3212 sar r6d, 8+4 3213 pxor m2, m2 3214 vpbroadcastw m3, r6d 3215 .dconly_loop: 3216 mova ym1, [dstq+strideq*0] 3217 vinserti32x8 m1, [dstq+strideq*1], 1 3218 punpcklbw m0, m1, m2 3219 punpckhbw m1, m2 3220 paddw m0, m3 3221 paddw m1, m3 3222 packuswb m0, m1 3223 mova [dstq+strideq*0], ym0 3224 vextracti32x8 [dstq+strideq*1], m0, 1 3225 lea dstq, [dstq+strideq*2] 3226 sub r3d, 2 3227 jg .dconly_loop 3228 RET 3229 ALIGN function_align 3230 cglobal_label .main 3231 vpbroadcastd m10, [o(pd_2048)] 3232 .main2: 3233 ITX_MULSUB_2W 5, 3, 8, 9, 10, 3406, 2276 ; t5a, t6a 3234 ITX_MULSUB_2W 1, 7, 8, 9, 10, 799, 4017 ; t4a, t7a 3235 ITX_MULSUB_2W 2, 6, 8, 9, 10, 1567, 3784 ; t2, t3 3236 vpbroadcastd m11, [o(pw_2896_2896)] 3237 vpbroadcastd m12, [o(pw_m2896_2896)] 3238 ITX_MULSUB_2W 0, 4, 8, 9, 10, 11, 12 ; t1, t0 3239 .main3: 3240 paddsw m8, m1, m5 ; t4 3241 psubsw m1, m5 ; t5a 3242 paddsw m9, m7, m3 ; t7 3243 psubsw m7, m3 ; t6a 3244 ITX_MULSUB_2W 7, 1, 3, 5, 10, 11, 12 ; t5, t6 3245 psubsw m5, m0, m2 ; dct4 out2 3246 paddsw m2, m0 ; dct4 out1 3247 paddsw m0, m4, m6 ; dct4 out0 3248 psubsw m4, m6 ; dct4 out3 3249 psubsw m6, m2, m1 ; out6 3250 paddsw m1, m2 ; out1 3251 paddsw m2, m5, m7 ; out2 3252 psubsw m5, m7 ; out5 3253 psubsw m7, m0, m9 ; out7 3254 paddsw m0, m9 ; out0 3255 paddsw m3, m4, m8 ; out3 3256 psubsw m4, m8 ; out4 3257 ret 3258 3259 cglobal inv_txfm_add_identity_identity_8x32_8bpc, 3, 5, 0, dst, stride, c 3260 vpbroadcastd m7, [pw_5] 3261 paddsw m0, m7, [cq+64*0] 3262 paddsw m1, m7, [cq+64*1] 3263 vpbroadcastd ym9, strided 3264 paddsw m2, m7, [cq+64*2] 3265 paddsw m3, m7, [cq+64*3] 3266 paddsw m4, m7, [cq+64*4] 3267 paddsw m5, m7, [cq+64*5] 3268 paddsw m6, m7, [cq+64*6] 3269 paddsw m7, [cq+64*7] 3270 pmulld ym14, ym9, [pd_0to15] 3271 lea r3, [dstq+strideq*1] 3272 lea r4, [dstq+strideq*2] 3273 kxnorb k1, k1, k1 3274 pxor m13, m13 3275 add r1, r4 ; dstq+strideq*3 3276 kmovb k2, k1 3277 vpgatherdq m9{k1}, [r0+ym14*4] 3278 kmovb k1, k2 3279 vpgatherdq m10{k2}, [r3+ym14*4] 3280 kmovb k2, k1 3281 call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8 3282 REPX {psraw x, 3}, m0, m1, m2, m3, m4, m5, m6, m7 3283 vpgatherdq m11{k1}, [r4+ym14*4] 3284 kmovb k1, k2 3285 vpgatherdq m12{k2}, [r1+ym14*4] 3286 REPX {mova [cq+64*x], m13}, 0, 1, 2, 3, 4, 5, 6, 7 3287 punpcklbw m8, m9, m13 ; 0 8 16 24 3288 punpckhbw m9, m13 ; 4 12 20 28 3289 paddw m0, m8 3290 paddw m4, m9 3291 packuswb m0, m4 3292 kmovb k2, k1 3293 vpscatterdq [r0+ym14*4]{k1}, m0 3294 punpcklbw m8, m10, m13 ; 1 9 17 25 3295 punpckhbw m10, m13 ; 5 13 21 29 3296 paddw m1, m8 3297 paddw m5, m10 3298 packuswb m1, m5 3299 kmovb k1, k2 3300 vpscatterdq [r3+ym14*4]{k2}, m1 3301 punpcklbw m8, m11, m13 ; 2 10 18 26 3302 punpckhbw m11, m13 ; 6 14 22 30 3303 paddw m2, m8 3304 paddw m6, m11 3305 packuswb m2, m6 3306 kmovb k2, k1 3307 vpscatterdq [r4+ym14*4]{k1}, m2 3308 punpcklbw m8, m12, m13 ; 3 11 19 27 3309 punpckhbw m12, m13 ; 7 15 23 31 3310 paddw m3, m8 3311 paddw m7, m12 3312 packuswb m3, m7 3313 vpscatterdq [r1+ym14*4]{k2}, m3 3314 RET 3315 3316 cglobal inv_txfm_add_identity_identity_32x8_8bpc, 3, 5, 0, dst, stride, c 3317 vpbroadcastd m0, [pw_4096] 3318 pmulhrsw m3, m0, [cq+64*0] 3319 pmulhrsw m4, m0, [cq+64*4] 3320 pmulhrsw m6, m0, [cq+64*1] 3321 pmulhrsw m5, m0, [cq+64*5] 3322 pmulhrsw m7, m0, [cq+64*2] 3323 pmulhrsw m2, m0, [cq+64*6] 3324 pmulhrsw m8, m0, [cq+64*3] 3325 pmulhrsw m0, [cq+64*7] 3326 mova m13, [int8_permA] 3327 lea r3, [strideq*3] 3328 lea r4, [dstq+strideq*4] 3329 punpckldq m1, m3, m4 3330 punpckhdq m3, m4 3331 punpckldq m4, m6, m5 3332 punpckhdq m6, m5 3333 punpckldq m5, m7, m2 3334 punpckhdq m7, m2 3335 punpckldq m2, m8, m0 3336 punpckhdq m8, m0 3337 mova ym9, [dstq+strideq*0] 3338 vinserti32x8 m9, [dstq+strideq*2], 1 3339 mova ym10, [dstq+strideq*1] 3340 vinserti32x8 m10, [dstq+r3 ], 1 3341 mova ym11, [r4+strideq*0] 3342 vinserti32x8 m11, [r4+strideq*2], 1 3343 mova ym12, [r4+strideq*1] 3344 vinserti32x8 m12, [r4+r3 ], 1 3345 REPX {vpermb x, m13, x}, m1, m4, m5, m2, m3, m6, m7, m8 3346 pxor m13, m13 3347 REPX {mova [cq+64*x], m13}, 0, 1, 2, 3, 4, 5, 6, 7 3348 punpcklqdq m0, m1, m4 ; a0 a2 c0 c2 3349 punpckhqdq m1, m4 ; b0 b2 d0 d2 3350 punpcklqdq m4, m5, m2 ; a1 a3 c1 c3 3351 punpckhqdq m5, m2 ; b1 b3 d1 d3 3352 punpcklqdq m2, m3, m6 ; e0 e2 g0 g2 3353 punpckhqdq m3, m6 ; f0 f2 h0 h2 3354 punpcklqdq m6, m7, m8 ; e1 e3 g1 g3 3355 punpckhqdq m7, m8 ; f1 f3 h1 h3 3356 punpcklbw m8, m9, m13 3357 punpckhbw m9, m13 3358 paddw m0, m8 3359 paddw m4, m9 3360 packuswb m0, m4 3361 mova [dstq+strideq*0], ym0 3362 vextracti32x8 [dstq+strideq*2], m0, 1 3363 punpcklbw m8, m10, m13 3364 punpckhbw m10, m13 3365 paddw m1, m8 3366 paddw m5, m10 3367 packuswb m1, m5 3368 mova [dstq+strideq*1], ym1 3369 vextracti32x8 [dstq+r3 ], m1, 1 3370 punpcklbw m8, m11, m13 3371 punpckhbw m11, m13 3372 paddw m2, m8 3373 paddw m6, m11 3374 packuswb m2, m6 3375 mova [r4+strideq*0], ym2 3376 vextracti32x8 [r4+strideq*2], m2, 1 3377 punpcklbw m8, m12, m13 3378 punpckhbw m12, m13 3379 paddw m3, m8 3380 paddw m7, m12 3381 packuswb m3, m7 3382 mova [r4+strideq*1], ym3 3383 vextracti32x8 [r4+r3 ], m3, 1 3384 RET 3385 3386 %macro IDCT_16x32_END 3 ; src[1-2], row 3387 mova xm8, [dstq+strideq*0] 3388 vinserti32x4 ym8, [dstq+strideq*1], 1 3389 mova xm9, [dstq+r3 ] 3390 vinserti32x4 ym9, [dstq+strideq*2], 1 3391 pmulhrsw m%1, m10 3392 pmulhrsw m%2, m10 3393 vpermb m8, m11, m8 3394 vpermb m9, m11, m9 3395 mova [cq+64*(%3*2+0)], m13 3396 mova [cq+64*(%3*2+1)], m13 3397 paddw m8, m%1 3398 paddw m9, m%2 3399 packuswb m8, m9 3400 vpermd m8, m12, m8 3401 mova [dstq+strideq*0], xm8 3402 vextracti32x4 [dstq+strideq*1], ym8, 1 3403 vextracti32x4 [dstq+strideq*2], m8, 2 3404 vextracti32x4 [dstq+r3 ], m8, 3 3405 %if %1 != 20 3406 lea dstq, [dstq+strideq*4] 3407 %endif 3408 %endmacro 3409 3410 cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 4, 22, dst, stride, c, eob 3411 %undef cmp 3412 lea r5, [o_base] 3413 test eobd, eobd 3414 jz .dconly 3415 vpbroadcastd m15, [o(pw_2896x8)] 3416 cmp eobd, 151 3417 jb .fast 3418 pmulhrsw m5, m15, [cq+64*10] 3419 pmulhrsw m3, m15, [cq+64* 6] 3420 pmulhrsw m1, m15, [cq+64* 2] 3421 pmulhrsw m7, m15, [cq+64*14] 3422 pmulhrsw m2, m15, [cq+64* 4] 3423 pmulhrsw m6, m15, [cq+64*12] 3424 pmulhrsw m0, m15, [cq+64* 0] 3425 pmulhrsw m4, m15, [cq+64* 8] 3426 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 3427 pmulhrsw m14, m15, [cq+64* 1] 3428 pmulhrsw m21, m15, [cq+64*15] 3429 pmulhrsw m18, m15, [cq+64* 9] 3430 pmulhrsw m17, m15, [cq+64* 7] 3431 pmulhrsw m16, m15, [cq+64* 5] 3432 pmulhrsw m19, m15, [cq+64*11] 3433 pmulhrsw m20, m15, [cq+64*13] 3434 pmulhrsw m15, [cq+64* 3] 3435 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 3436 mova m8, [o(idct_16x32p)] 3437 vpbroadcastd m9, [o(pw_16384)] 3438 REPX {vpermb x, m8, x}, m0, m1, m2, m3, m4, m5, m6, m7, \ 3439 m14, m15, m16, m17, m18, m19, m20, m21 3440 punpckldq m8, m0, m1 3441 punpckhdq m0, m1 3442 punpckldq m1, m2, m3 3443 punpckhdq m2, m3 3444 REPX {pmulhrsw x, m9}, m8, m0, m1, m2 3445 punpckldq m3, m4, m5 3446 punpckhdq m4, m5 3447 punpckldq m5, m6, m7 3448 punpckhdq m6, m7 3449 REPX {pmulhrsw x, m9}, m3, m4, m5, m6 3450 punpckldq m7, m14, m15 3451 punpckhdq m14, m15 3452 punpckldq m15, m16, m17 3453 punpckhdq m16, m17 3454 REPX {pmulhrsw x, m9}, m7, m14, m15, m16 3455 punpckldq m17, m18, m19 3456 punpckhdq m18, m19 3457 punpckldq m19, m20, m21 3458 punpckhdq m20, m21 3459 REPX {pmulhrsw x, m9}, m17, m18, m19, m20 3460 punpcklqdq m21, m8, m1 3461 punpckhqdq m8, m1 3462 punpcklqdq m1, m0, m2 3463 punpckhqdq m0, m2 3464 punpcklqdq m2, m3, m5 3465 punpckhqdq m3, m5 3466 punpcklqdq m5, m4, m6 3467 punpckhqdq m4, m6 3468 punpcklqdq m6, m7, m15 3469 punpckhqdq m7, m15 3470 punpcklqdq m15, m14, m16 3471 punpckhqdq m14, m16 3472 punpcklqdq m16, m17, m19 3473 punpckhqdq m17, m19 3474 punpcklqdq m19, m18, m20 3475 punpckhqdq m18, m20 3476 vinserti32x8 m20, m21, ym2, 1 3477 vshufi32x4 m21, m2, q3232 3478 vinserti32x8 m2, m8, ym3, 1 3479 vshufi32x4 m8, m3, q3232 3480 vinserti32x8 m3, m1, ym5, 1 3481 vshufi32x4 m1, m5, q3232 3482 vinserti32x8 m5, m0, ym4, 1 3483 vshufi32x4 m0, m4, q3232 3484 vinserti32x8 m4, m6, ym16, 1 3485 vshufi32x4 m6, m16, q3232 3486 vinserti32x8 m16, m7, ym17, 1 3487 vshufi32x4 m7, m17, q3232 3488 vinserti32x8 m17, m15, ym19, 1 3489 vshufi32x4 m15, m19, q3232 3490 vinserti32x8 m19, m14, ym18, 1 3491 vshufi32x4 m14, m18, q3232 3492 vshufi32x4 m18, m21, m6, q3131 ; 27 5 3493 vshufi32x4 m21, m6, q2020 ; 31 1 3494 vshufi32x4 m6, m8, m7, q2020 ; 24 8 3495 vshufi32x4 m8, m7, q3131 ; 30 2 3496 vshufi32x4 m7, m1, m15, q2020 ; 28 4 3497 vshufi32x4 m1, m15, q3131 ; 6 26 3498 vshufi32x4 m15, m0, m14, q2020 ; 7 25 3499 vshufi32x4 m0, m14, q3131 ; 14 18 3500 vshufi32x4 m14, m20, m4, q2020 ; 3 29 3501 vshufi32x4 m20, m4, q3131 ; 23 9 3502 vshufi32x4 m9, m3, m17, q2020 ; 16 0 3503 vshufi32x4 m3, m17, q3131 ; 12 20 3504 vshufi32x4 m17, m5, m19, q2020 ; 15 17 3505 vshufi32x4 m5, m19, q3131 ; 22 10 3506 vshufi32x4 m19, m2, m16, q2020 ; 19 13 3507 vshufi32x4 m16, m2, m16, q3131 ; 11 21 3508 call m(idct_16x16_internal_8bpc).main3 3509 call .main_oddhalf 3510 jmp .pass2 3511 .fast: ; right half is zero 3512 mova ym8, [cq+64*15] 3513 vinserti32x8 m8, [cq+64* 1], 1 3514 mova m2, [o(int16_perm)] 3515 mova ym9, [cq+64* 8] 3516 vinserti32x8 m9, [cq+64* 0], 1 3517 mova ym0, [cq+64* 7] 3518 vinserti32x8 m0, [cq+64* 9], 1 3519 mova ym7, [cq+64*14] 3520 vinserti32x8 m7, [cq+64* 2], 1 3521 mova ym1, [cq+64* 3] 3522 vinserti32x8 m1, [cq+64*13], 1 3523 mova ym3, [cq+64* 6] 3524 vinserti32x8 m3, [cq+64*10], 1 3525 mova ym5, [cq+64*11] 3526 vinserti32x8 m5, [cq+64* 5], 1 3527 mova ym6, [cq+64*12] 3528 vinserti32x8 m6, [cq+64* 4], 1 3529 REPX {pmulhrsw x, m15}, m8, m9, m0, m7, m1, m3, m5, m6 3530 REPX {vpermb x, m2, x}, m8, m9, m0, m7, m1, m3, m5, m6 3531 call m(idct_16x16_internal_8bpc).main2 3532 vbroadcasti32x4 m8, [o(int_shuf3)] 3533 vbroadcasti32x4 m9, [o(int_shuf4)] 3534 vpbroadcastd m11, [o(pw_16384)] 3535 pshufb m0, m8 3536 pshufb m1, m9 3537 pshufb m2, m8 3538 pshufb m3, m9 3539 REPX {pmulhrsw x, m11}, m0, m1, m2, m3 3540 pshufb m4, m8 3541 pshufb m5, m9 3542 pshufb m6, m8 3543 pshufb m7, m9 3544 REPX {pmulhrsw x, m11}, m4, m5, m6, m7 3545 punpckhdq m17, m0, m1 3546 punpckldq m0, m1 3547 punpckhdq m16, m2, m3 3548 punpckldq m2, m3 3549 punpckhdq m18, m4, m5 3550 punpckldq m4, m5 3551 punpckhdq m5, m6, m7 3552 punpckldq m6, m7 3553 vinserti32x8 m1, m0, ym2, 1 3554 vshufi32x4 m3, m0, m2, q3232 3555 vinserti32x8 m2, m4, ym6, 1 3556 vshufi32x4 m4, m6, q3232 3557 vinserti32x8 m15, m17, ym16, 1 3558 vshufi32x4 m17, m16, q3232 3559 vinserti32x8 m16, m18, ym5, 1 3560 vshufi32x4 m18, m5, q3232 3561 vshufi32x4 m0, m1, m2, q2020 ; 0 2 3562 vshufi32x4 m1, m2, q3131 ; 4 6 3563 vshufi32x4 m2, m3, m4, q2020 ; 8 10 3564 vshufi32x4 m3, m4, q3131 ; 12 14 3565 vshufi32x4 m14, m15, m16, q2020 ; 1 3 3566 vshufi32x4 m15, m16, q3131 ; 5 7 3567 vshufi32x4 m16, m17, m18, q2020 ; 9 11 3568 vshufi32x4 m17, m18, q3131 ; 13 15 3569 pxor m6, m6 3570 punpckhwd m8, m0, m0 3571 punpcklwd m9, m6, m0 3572 punpckhwd m0, m3, m3 3573 punpckhwd m5, m2, m2 3574 punpcklwd m7, m1, m1 3575 punpckhwd m1, m1 3576 punpcklwd m3, m3 3577 punpcklwd m6, m2 3578 call m(idct_16x16_internal_8bpc).main_fast5 3579 punpcklwd m21, m14, m14 3580 punpckhwd m14, m14 3581 punpcklwd m18, m15, m15 3582 punpckhwd m15, m15 3583 punpcklwd m20, m16, m16 3584 punpckhwd m16, m16 3585 punpcklwd m19, m17, m17 3586 punpckhwd m17, m17 3587 call .main_oddhalf_fast 3588 .pass2: 3589 vpbroadcastd m10, [o(pw_2048)] 3590 mova m11, [o(end_16x32p)] 3591 lea r3, [strideq*3] 3592 pxor m13, m13 3593 psrld m12, m11, 8 3594 IDCT_16x32_END 0, 1, 0 3595 IDCT_16x32_END 2, 3, 1 3596 IDCT_16x32_END 4, 5, 2 3597 IDCT_16x32_END 6, 7, 3 3598 IDCT_16x32_END 14, 15, 4 3599 IDCT_16x32_END 16, 17, 5 3600 IDCT_16x32_END 18, 19, 6 3601 IDCT_16x32_END 20, 21, 7 3602 RET 3603 ALIGN function_align 3604 .dconly: 3605 movsx r6d, word [cq] 3606 mov [cq], eobd 3607 or r3d, 32 3608 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly 3609 ALIGN function_align 3610 cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero 3611 vpbroadcastd m8, [o(pw_201_4091x8)] 3612 vpbroadcastd m20, [o(pw_m1380_3857x8)] 3613 vpbroadcastd m9, [o(pw_995_3973x8)] 3614 vpbroadcastd m16, [o(pw_m601_4052x8)] 3615 pmulhrsw m21, m8 ; t16a, t31a 3616 pmulhrsw m20, m15 ; t19a, t28a 3617 pmulhrsw m18, m9 ; t20a, t27a 3618 pmulhrsw m14, m16 ; t23a, t24a 3619 mova m8, m21 3620 mova m17, m20 3621 mova m15, m18 3622 mova m16, m14 3623 jmp .main3 3624 ALIGN function_align 3625 cglobal_label .main_oddhalf_fast ; bottom half is zero 3626 vpbroadcastd m8, [o(pw_201_4091x8)] 3627 vpbroadcastd m9, [o(pw_m2751_3035x8)] 3628 vpbroadcastd m11, [o(pw_1751_3703x8)] 3629 vpbroadcastd m12, [o(pw_m1380_3857x8)] 3630 pmulhrsw m21, m8 ; t16a, t31a 3631 vpbroadcastd m8, [o(pw_995_3973x8)] 3632 pmulhrsw m17, m9 ; t17a, t30a 3633 vpbroadcastd m9, [o(pw_m2106_3513x8)] 3634 pmulhrsw m20, m11 ; t18a, t29a 3635 vpbroadcastd m11, [o(pw_2440_3290x8)] 3636 pmulhrsw m15, m12 ; t19a, t28a 3637 vpbroadcastd m12, [o(pw_m601_4052x8)] 3638 pmulhrsw m18, m8 ; t20a, t27a 3639 pmulhrsw m16, m9 ; t21a, t26a 3640 pmulhrsw m19, m11 ; t22a, t25a 3641 pmulhrsw m14, m12 ; t23a, t24a 3642 jmp .main2 3643 ALIGN function_align 3644 cglobal_label .main_oddhalf 3645 ITX_MUL2X_PACK 21, 8, 9, 10, 201, 4091, 5 ; t16a, t31a 3646 ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a 3647 ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a 3648 ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a 3649 ITX_MUL2X_PACK 18, 8, 9, 10, 995, 3973, 5 ; t20a, t27a 3650 ITX_MUL2X_PACK 16, 8, 9, 10, 3513, 2106, 5 ; t21a, t26a 3651 ITX_MUL2X_PACK 19, 8, 9, 10, 2440, 3290, 5 ; t22a, t25a 3652 ITX_MUL2X_PACK 14, 8, 9, 10, 4052, 601, 5 ; t23a, t24a 3653 .main2: 3654 psubsw m8, m21, m17 ; t17 t30 3655 paddsw m21, m17 ; t16 t31 3656 psubsw m17, m15, m20 ; t18 t29 3657 paddsw m20, m15 ; t19 t28 3658 psubsw m15, m18, m16 ; t21 t26 3659 paddsw m18, m16 ; t20 t27 3660 psubsw m16, m14, m19 ; t22 t25 3661 paddsw m14, m19 ; t23 t24 3662 .main3: 3663 ITX_MUL2X_PACK 8, 9, 19, 10, 799, 4017, 5 ; t17a t30a 3664 ITX_MUL2X_PACK 17, 9, 19, 10, m4017, 799, 5 ; t18a t29a 3665 ITX_MUL2X_PACK 15, 9, 19, 10, 3406, 2276, 5 ; t21a t26a 3666 ITX_MUL2X_PACK 16, 9, 19, 10, m2276, 3406, 5 ; t22a t25a 3667 vpbroadcastd m11, [o(pw_m3784_1567)] 3668 psubsw m19, m21, m20 ; t19a t28a 3669 paddsw m21, m20 ; t16a t31a 3670 psubsw m20, m14, m18 ; t20a t27a 3671 paddsw m14, m18 ; t23a t24a 3672 psubsw m18, m8, m17 ; t18 t29 3673 paddsw m8, m17 ; t17 t30 3674 psubsw m17, m16, m15 ; t21 t26 3675 paddsw m15, m16 ; t22 t25 3676 ITX_MUL2X_PACK 18, 9, 16, 10, 1567_3784, 11, 20 ; t18a t29a 3677 ITX_MUL2X_PACK 19, 9, 16, 10, 1567_3784, 11, 20 ; t19 t28 3678 ITX_MUL2X_PACK 20, 9, 16, 10, 11, m1567_m3784, 36 ; t20 t27 3679 ITX_MUL2X_PACK 17, 9, 16, 10, 11, m1567_m3784, 36 ; t21a t26a 3680 vbroadcasti32x4 m9, [o(deint_shuf)] 3681 psubsw m16, m21, m14 ; t23 t24 3682 paddsw m14, m21 ; t16 t31 3683 psubsw m21, m8, m15 ; t22a t25a 3684 paddsw m15, m8 ; t17a t30a 3685 psubsw m8, m18, m17 ; t21 t26 3686 paddsw m18, m17 ; t18 t29 3687 paddsw m17, m19, m20 ; t19a t28a 3688 psubsw m19, m20 ; t20a t27a 3689 vpbroadcastd m11, [o(pw_m2896_2896)] 3690 vpbroadcastd m12, [o(pw_2896_2896)] 3691 REPX {pshufb x, m9}, m14, m15, m18, m17 3692 mova m9, m10 3693 vpdpwssd m9, m16, m11 3694 mova m20, m10 3695 vpdpwssd m20, m21, m11 3696 psrad m9, 12 3697 psrad m20, 12 3698 packssdw m9, m20 ; t23a t22 3699 mova m20, m10 3700 vpdpwssd m20, m16, m12 3701 mova m16, m10 3702 vpdpwssd m16, m21, m12 3703 psrad m20, 12 3704 psrad m16, 12 3705 packssdw m16, m20, m16 ; t24a t25 3706 ITX_MUL2X_PACK 8, 21, 20, 10, 11, 12, 8 ; t21a t26a 3707 ITX_MUL2X_PACK 19, 8, 11, 10, 11, 12, 8 ; t20 t27 3708 packssdw m11, m20 ; t27 t26a 3709 packssdw m8, m21 ; t20 t21a 3710 punpcklqdq m20, m14, m15 ; t16 t17a 3711 punpckhqdq m14, m15 ; t31 t30a 3712 punpckhqdq m15, m17, m18 ; t28a t29 3713 punpcklqdq m17, m18 ; t19a t18 3714 psubsw m21, m0, m14 ; out31 out30 3715 paddsw m0, m14 ; out0 out1 3716 psubsw m14, m7, m20 ; out16 out17 3717 paddsw m7, m20 ; out15 out14 3718 psubsw m20, m1, m15 ; out28 out29 3719 paddsw m1, m15 ; out3 out2 3720 psubsw m15, m6, m17 ; out19 out18 3721 paddsw m6, m17 ; out12 out13 3722 psubsw m17, m4, m9 ; out23 out22 3723 paddsw m4, m9 ; out8 out9 3724 psubsw m18, m3, m16 ; out24 out25 3725 paddsw m3, m16 ; out7 out6 3726 psubsw m16, m5, m8 ; out20 out21 3727 paddsw m5, m8 ; out11 out10 3728 psubsw m19, m2, m11 ; out27 out26 3729 paddsw m2, m11 ; out4 out5 3730 ret 3731 3732 cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 6, 22, dst, stride, c, eob 3733 %undef cmp 3734 lea r5, [o_base] 3735 test eobd, eobd 3736 jz .dconly 3737 mova m21, [o(permB)] 3738 vpermq m1, m21, [cq+64* 0] ; 0 1 3739 vpermq m14, m21, [cq+64* 1] ; 2 3 3740 vpermq m20, m21, [cq+64* 2] ; 4 5 3741 vpermq m15, m21, [cq+64* 3] ; 6 7 3742 vpbroadcastd m8, [o(pw_2896x8)] 3743 vpermq m2, m21, [cq+64* 4] ; 8 9 3744 vpermq m16, m21, [cq+64* 5] ; 10 11 3745 vpermq m3, m21, [cq+64* 6] ; 12 13 3746 vpermq m17, m21, [cq+64* 7] ; 14 15 3747 REPX {pmulhrsw x, m8}, m1, m14, m20, m15, m2, m16, m3, m17 3748 pxor m12, m12 3749 REPX {mova [cq+64*x], m12}, 0, 1, 2, 3, 4, 5, 6, 7 3750 cmp eobd, 151 3751 jb .fast 3752 vpermq m9, m21, [cq+64* 8] ; 16 17 3753 vpermq m19, m21, [cq+64* 9] ; 18 19 3754 vpermq m4, m21, [cq+64*10] ; 20 21 3755 vpermq m5, m21, [cq+64*11] ; 22 23 3756 vpermq m6, m21, [cq+64*12] ; 24 25 3757 vpermq m18, m21, [cq+64*13] ; 26 27 3758 vpermq m7, m21, [cq+64*14] ; 28 29 3759 vpermq m21, m21, [cq+64*15] ; 30 31 3760 REPX {pmulhrsw x, m8}, m9, m19, m4, m5, m6, m18, m7, m21 3761 REPX {mova [cq+64*x], m12}, 8, 9, 10, 11, 12, 13, 14, 15 3762 punpcklwd m8, m21, m14 ; 30 2 3763 punpckhwd m21, m1 ; 31 1 3764 punpcklwd m0, m17, m19 ; 14 18 3765 punpckhwd m17, m9 ; 15 17 3766 punpcklwd m9, m1 ; 16 0 3767 punpckhwd m14, m7 ; 3 29 3768 punpcklwd m1, m15, m18 ; 6 26 3769 punpckhwd m15, m6 ; 7 25 3770 punpcklwd m6, m2 ; 24 8 3771 punpckhwd m19, m3 ; 19 13 3772 punpcklwd m3, m4 ; 12 20 3773 punpckhwd m18, m20 ; 27 5 3774 punpcklwd m7, m20 ; 28 4 3775 punpckhwd m20, m5, m2 ; 23 9 3776 punpcklwd m5, m16 ; 22 10 3777 punpckhwd m16, m4 ; 11 21 3778 call m(idct_16x16_internal_8bpc).main2 3779 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf 3780 jmp .pass2 3781 .fast: ; bottom half zero 3782 punpcklwd m8, m14, m14 ; 2 3783 punpcklwd m0, m17, m17 ; 14 3784 punpcklwd m5, m16, m16 ; 10 3785 punpcklwd m9, m12, m1 ; __ 0 3786 punpckhwd m21, m1, m1 ; 1 3787 punpcklwd m1, m15, m15 ; 6 3788 punpcklwd m7, m20, m20 ; 4 3789 punpckhwd m19, m3, m3 ; 13 3790 punpcklwd m3, m3 ; 12 3791 punpcklwd m6, m12, m2 ; __ 8 3792 punpckhwd m18, m20, m20 ; 5 3793 punpckhwd m20, m2, m2 ; 9 3794 call m(idct_16x16_internal_8bpc).main_fast 3795 punpckhwd m15, m15 ; 7 3796 punpckhwd m14, m14 ; 3 3797 punpckhwd m16, m16 ; 11 3798 punpckhwd m17, m17 ; 15 3799 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast 3800 .pass2: 3801 vpbroadcastd m9, [o(pw_16384)] 3802 call .transpose_round 3803 vshufi32x4 m16, m14, m2, q3131 ; 5 3804 vshufi32x4 m14, m2, q2020 ; 1 3805 vshufi32x4 m2, m0, m3, q3131 ; 4 3806 vshufi32x4 m0, m3, q2020 ; 0 3807 vshufi32x4 m3, m1, m18, q3131 ; 6 3808 vshufi32x4 m1, m18, q2020 ; 2 3809 vshufi32x4 m18, m20, m6, q2020 ; 9 3810 vshufi32x4 m20, m6, q3131 ; 13 3811 vshufi32x4 m6, m21, m4, q3131 ; 12 3812 vshufi32x4 m4, m21, m4, q2020 ; 8 3813 vshufi32x4 m21, m19, m7, q3131 ; 15 3814 vshufi32x4 m19, m7, q2020 ; 11 3815 vshufi32x4 m7, m5, m15, q3131 ; 14 3816 vshufi32x4 m5, m15, q2020 ; 10 3817 vshufi32x4 m15, m17, m9, q2020 ; 3 3818 vshufi32x4 m17, m9, q3131 ; 7 3819 call m(inv_txfm_add_dct_dct_32x8_8bpc).main2 3820 call .main_oddhalf 3821 vpbroadcastd m12, [o(pw_2048)] 3822 movshdup m13, [o(permD)] 3823 lea r2, [strideq*3] 3824 pmovzxbw m8, [dstq+strideq*0] 3825 pmovzxbw m9, [dstq+strideq*1] 3826 pmovzxbw m10, [dstq+strideq*2] 3827 pmovzxbw m11, [dstq+r2 ] 3828 REPX {pmulhrsw x, m12}, m0, m1, m2, m3 3829 lea r3, [dstq+strideq*4] 3830 paddw m0, m8 3831 paddw m1, m9 3832 paddw m2, m10 3833 paddw m3, m11 3834 pmovzxbw m8, [r3+strideq*0] 3835 pmovzxbw m9, [r3+strideq*1] 3836 pmovzxbw m10, [r3+strideq*2] 3837 pmovzxbw m11, [r3+r2 ] 3838 REPX {pmulhrsw x, m12}, m4, m5, m6, m7 3839 lea r4, [dstq+strideq*8] 3840 packuswb m0, m1 3841 paddw m4, m8 3842 paddw m5, m9 3843 packuswb m2, m3 3844 paddw m6, m10 3845 paddw m7, m11 3846 pmovzxbw m8, [r4+strideq*0] 3847 pmovzxbw m9, [r4+strideq*1] 3848 pmovzxbw m10, [r4+strideq*2] 3849 pmovzxbw m11, [r4+r2 ] 3850 REPX {pmulhrsw x, m12}, m14, m15, m16, m17 3851 lea r5, [r3+strideq*8] 3852 packuswb m4, m5 3853 paddw m14, m8 3854 paddw m15, m9 3855 packuswb m6, m7 3856 paddw m16, m10 3857 paddw m17, m11 3858 pmovzxbw m8, [r5+strideq*0] 3859 pmovzxbw m9, [r5+strideq*1] 3860 pmovzxbw m10, [r5+strideq*2] 3861 pmovzxbw m11, [r5+r2 ] 3862 REPX {pmulhrsw x, m12}, m18, m19, m20, m21 3863 packuswb m14, m15 3864 paddw m18, m8 3865 paddw m19, m9 3866 packuswb m16, m17 3867 paddw m20, m10 3868 paddw m21, m11 3869 packuswb m18, m19 3870 packuswb m20, m21 3871 REPX {vpermq x, m13, x}, m0, m2, m4, m6, m14, m16, m18, m20 3872 mova [dstq+strideq*0], ym0 3873 vextracti32x8 [dstq+strideq*1], m0, 1 3874 mova [dstq+strideq*2], ym2 3875 vextracti32x8 [dstq+r2 ], m2, 1 3876 mova [r3+strideq*0], ym4 3877 vextracti32x8 [r3+strideq*1], m4, 1 3878 mova [r3+strideq*2], ym6 3879 vextracti32x8 [r3+r2 ], m6, 1 3880 mova [r4+strideq*0], ym14 3881 vextracti32x8 [r4+strideq*1], m14, 1 3882 mova [r4+strideq*2], ym16 3883 vextracti32x8 [r4+r2 ], m16, 1 3884 mova [r5+strideq*0], ym18 3885 vextracti32x8 [r5+strideq*1], m18, 1 3886 mova [r5+strideq*2], ym20 3887 vextracti32x8 [r5+r2 ], m20, 1 3888 RET 3889 ALIGN function_align 3890 .dconly: 3891 movsx r6d, word [cq] 3892 mov [cq], eobd 3893 or r3d, 16 3894 imul r6d, 181 3895 add r6d, 128 3896 sar r6d, 8 3897 imul r6d, 181 3898 add r6d, 128+256 3899 sar r6d, 8+1 3900 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3 3901 ALIGN function_align 3902 cglobal_label .main_oddhalf_fast3 ; bottom seven-eights are zero 3903 vpbroadcastd m8, [o(pw_2896x8)] 3904 vpbroadcastd m4, [o(pw_4076x8)] 3905 vpbroadcastd m3, [o(pw_401x8)] 3906 pmulhrsw m8, m0 ; t0 3907 pmulhrsw m4, m14 ; t15a 3908 pmulhrsw m3, m14 ; t8a 3909 punpcklwd m9, m3, m4 3910 punpckhwd m5, m3, m4 3911 mova m2, m10 3912 vpdpwssd m2, m9, [o(pw_m3784_1567)] {bcstd} 3913 mova m1, m10 3914 vpdpwssd m1, m5, [o(pw_m3784_1567)] {bcstd} 3915 mova m6, m10 3916 vpdpwssd m6, m5, [o(pw_1567_3784)] {bcstd} 3917 mova m5, m10 3918 vpdpwssd m5, m9, [o(pw_1567_3784)] {bcstd} 3919 vpbroadcastd m11, [o(pw_2896_2896)] 3920 vpbroadcastd m12, [o(pw_m2896_2896)] 3921 psubsw m21, m8, m4 ; out15 3922 paddsw m0, m8, m4 ; out0 3923 psubsw m14, m8, m3 ; out8 3924 paddsw m7, m8, m3 ; out7 3925 REPX {psrad x, 12}, m2, m1, m6, m5 3926 packssdw m2, m1 ; t9a 3927 packssdw m5, m6 ; t14a 3928 ITX_MULSUB_2W 4, 3, 16, 17, 10, 11, 12 ; t11, t12 3929 psubsw m20, m8, m5 ; out14 3930 paddsw m1, m8, m5 ; out1 3931 psubsw m15, m8, m2 ; out9 3932 paddsw m6, m8, m2 ; out6 3933 ITX_MULSUB_2W 5, 2, 16, 17, 10, 11, 12 ; t10a, t13a 3934 psubsw m18, m8, m3 ; out12 3935 paddsw m3, m8 ; out3 3936 psubsw m17, m8, m4 ; out11 3937 paddsw m4, m8 ; out4 3938 psubsw m19, m8, m2 ; out13 3939 paddsw m2, m8 ; out2 3940 psubsw m16, m8, m5 ; out10 3941 paddsw m5, m8 ; out5 3942 ret 3943 cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero 3944 vpbroadcastd m9, [o(pw_2896x8)] 3945 vpbroadcastd m2, [o(pw_4017x8)] 3946 vpbroadcastd m3, [o(pw_799x8)] 3947 vpbroadcastd m18, [o(pw_4076x8)] 3948 vpbroadcastd m19, [o(pw_401x8)] 3949 vpbroadcastd m20, [o(pw_m1189x8)] 3950 vpbroadcastd m16, [o(pw_3920x8)] 3951 pmulhrsw m9, m0 ; t0 3952 pmulhrsw m2, m1 ; t7a 3953 pmulhrsw m1, m3 ; t4a 3954 pmulhrsw m18, m14 ; t15a 3955 pmulhrsw m14, m19 ; t8a 3956 pmulhrsw m20, m15 ; t11a 3957 pmulhrsw m15, m16 ; t12a 3958 psubsw m7, m9, m2 ; idct8 out7 3959 paddsw m0, m9, m2 ; idct8 out0 3960 psubsw m4, m9, m1 ; idct8 out4 3961 paddsw m3, m9, m1 ; idct8 out3 3962 ITX_MULSUB_2W 2, 1, 5, 6, 10, 2896, 2896 ; t5, t6 3963 mova m21, m18 3964 mova m19, m14 3965 mova m16, m15 3966 mova m8, m20 3967 psubsw m6, m9, m1 ; idct8 out6 3968 paddsw m1, m9 ; idct8 out1 3969 psubsw m5, m9, m2 ; idct8 out5 3970 paddsw m2, m9 ; idct8 out2 3971 jmp .main3 3972 ALIGN function_align 3973 cglobal_label .main_oddhalf_fast ; bottom half is zero 3974 vpbroadcastd m5, [o(pw_m2276x8)] 3975 vpbroadcastd m11, [o(pw_3406x8)] 3976 vpbroadcastd m7, [o(pw_4017x8)] 3977 vpbroadcastd m12, [o(pw_799x8)] 3978 vpbroadcastd m6, [o(pw_3784x8)] 3979 vpbroadcastd m10, [o(pw_1567x8)] 3980 vpbroadcastd m4, [o(pw_2896x8)] 3981 pmulhrsw m5, m3 ; t5a 3982 pmulhrsw m3, m11 ; t6a 3983 pmulhrsw m7, m1 ; t7a 3984 pmulhrsw m1, m12 ; t4a 3985 pmulhrsw m6, m2 ; t3 3986 pmulhrsw m2, m10 ; t2 3987 pmulhrsw m4, m0 ; t0 3988 vpbroadcastd m11, [o(pw_2896_2896)] 3989 vpbroadcastd m12, [o(pw_m2896_2896)] 3990 vpbroadcastd m10, [o(pd_2048)] 3991 mova m0, m4 ; t1 3992 call m(inv_txfm_add_dct_dct_32x8_8bpc).main3 3993 vpbroadcastd m21, [o(pw_4076x8)] 3994 vpbroadcastd m8, [o(pw_401x8)] 3995 vpbroadcastd m18, [o(pw_m2598x8)] 3996 vpbroadcastd m9, [o(pw_3166x8)] 3997 vpbroadcastd m19, [o(pw_3612x8)] 3998 vpbroadcastd m11, [o(pw_1931x8)] 3999 vpbroadcastd m20, [o(pw_m1189x8)] 4000 vpbroadcastd m12, [o(pw_3920x8)] 4001 pmulhrsw m21, m14 ; t15a 4002 pmulhrsw m14, m8 ; t8a 4003 pmulhrsw m18, m17 ; t9a 4004 pmulhrsw m17, m9 ; t14a 4005 pmulhrsw m19, m16 ; t13a 4006 pmulhrsw m16, m11 ; t10a 4007 pmulhrsw m20, m15 ; t11a 4008 pmulhrsw m15, m12 ; t12a 4009 jmp .main2 4010 ALIGN function_align 4011 cglobal_label .main_oddhalf 4012 ITX_MULSUB_2W 14, 21, 8, 9, 10, 401, 4076 ; t8a, t15a 4013 ITX_MULSUB_2W 18, 17, 8, 9, 10, 3166, 2598 ; t9a, t14a 4014 ITX_MULSUB_2W 16, 19, 8, 9, 10, 1931, 3612 ; t10a, t13a 4015 ITX_MULSUB_2W 20, 15, 8, 9, 10, 3920, 1189 ; t11a, t12a 4016 .main2: 4017 paddsw m8, m20, m16 ; t11 4018 psubsw m20, m16 ; t10 4019 paddsw m16, m15, m19 ; t12 4020 psubsw m15, m19 ; t13 4021 psubsw m19, m14, m18 ; t9 4022 paddsw m14, m18 ; t8 4023 psubsw m18, m21, m17 ; t14 4024 paddsw m21, m17 ; t15 4025 .main3: 4026 vpbroadcastd m11, [o(pw_1567_3784)] 4027 vpbroadcastd m12, [o(pw_m3784_1567)] 4028 ITX_MULSUB_2W 18, 19, 9, 17, 10, 11, 12 ; t9a, t14a 4029 vpbroadcastd m11, [o(pw_m1567_m3784)] 4030 ITX_MULSUB_2W 15, 20, 9, 17, 10, 12, 11 ; t10a, t13a 4031 vpbroadcastd m11, [o(pw_2896_2896)] 4032 vpbroadcastd m12, [o(pw_m2896_2896)] 4033 psubsw m17, m14, m8 ; t11a 4034 paddsw m8, m14 ; t8a 4035 paddsw m14, m18, m15 ; t9 4036 psubsw m18, m15 ; t10 4037 psubsw m15, m19, m20 ; t13 4038 paddsw m19, m20 ; t14 4039 paddsw m20, m21, m16 ; t15a 4040 psubsw m16, m21, m16 ; t12a 4041 ITX_MULSUB_2W 15, 18, 9, 21, 10, 11, 12 ; t10a, t13a 4042 ITX_MULSUB_2W 16, 17, 9, 21, 10, 11, 12 ; t11, t12 4043 psubsw m21, m0, m20 ; out15 4044 paddsw m0, m20 ; out0 4045 psubsw m20, m1, m19 ; out14 4046 paddsw m1, m19 ; out1 4047 psubsw m19, m2, m18 ; out13 4048 paddsw m2, m18 ; out2 4049 psubsw m18, m3, m17 ; out12 4050 paddsw m3, m17 ; out3 4051 psubsw m17, m4, m16 ; out11 4052 paddsw m4, m16 ; out4 4053 psubsw m16, m5, m15 ; out10 4054 paddsw m5, m15 ; out5 4055 psubsw m15, m6, m14 ; out9 4056 paddsw m6, m14 ; out6 4057 psubsw m14, m7, m8 ; out8 4058 paddsw m7, m8 ; out7 4059 ret 4060 .transpose_round: 4061 punpcklwd m8, m0, m2 4062 punpckhwd m0, m2 4063 punpcklwd m2, m1, m3 4064 punpckhwd m1, m3 4065 punpcklwd m3, m4, m6 4066 punpckhwd m4, m6 4067 punpcklwd m6, m5, m7 4068 punpckhwd m5, m7 4069 punpcklwd m7, m14, m16 4070 punpckhwd m14, m16 4071 punpcklwd m16, m15, m17 4072 punpckhwd m15, m17 4073 punpcklwd m17, m19, m21 4074 punpckhwd m19, m21 4075 punpckhwd m21, m18, m20 4076 punpcklwd m18, m20 4077 punpcklwd m20, m8, m1 4078 punpckhwd m8, m1 4079 punpcklwd m1, m0, m2 4080 punpckhwd m0, m2 4081 punpcklwd m2, m3, m5 4082 punpckhwd m3, m5 4083 punpcklwd m5, m4, m6 4084 punpckhwd m4, m6 4085 REPX {pmulhrsw x, m9}, m20, m8, m1, m0 4086 punpcklwd m6, m7, m15 4087 punpckhwd m7, m15 4088 punpcklwd m15, m14, m16 4089 punpckhwd m14, m16 4090 REPX {pmulhrsw x, m9}, m2, m3, m5, m4 4091 punpckhwd m16, m18, m19 4092 punpcklwd m18, m19 4093 punpcklwd m19, m21, m17 4094 punpckhwd m21, m17 4095 REPX {pmulhrsw x, m9}, m6, m7, m15, m14 4096 punpcklwd m17, m8, m0 ; a2 a6 aa ae 4097 punpckhwd m8, m0 ; a3 a7 ab af 4098 punpcklwd m0, m20, m1 ; a0 a4 a8 ac 4099 punpckhwd m20, m1 ; a1 a5 a9 ad 4100 REPX {pmulhrsw x, m9}, m16, m18, m19, m21 4101 punpcklwd m1, m2, m5 ; b0 b4 b8 bc 4102 punpckhwd m2, m5 ; b1 b5 b9 bd 4103 punpcklwd m5, m3, m4 ; b2 b6 ba be 4104 punpckhwd m3, m4 ; b3 b7 bb bf 4105 punpcklwd m4, m6, m15 ; c0 c4 c8 cc 4106 punpckhwd m6, m15 ; c1 c5 c9 cd 4107 punpcklwd m15, m7, m14 ; c2 c6 ca ce 4108 punpckhwd m7, m14 ; c3 c7 cb cf 4109 punpcklwd m14, m18, m19 ; d0 d4 d8 dc 4110 punpckhwd m18, m19 ; d1 d5 d9 dd 4111 punpcklwd m9, m16, m21 ; d2 d6 da de 4112 punpckhwd m16, m21 ; d3 d7 db df 4113 vshufi32x4 m21, m0, m1, q3232 ; a8 ac b8 bc 4114 vinserti32x8 m0, ym1, 1 ; a0 a4 b0 b4 4115 vinserti32x8 m1, m17, ym5, 1 ; a2 a6 b2 b6 4116 vshufi32x4 m5, m17, m5, q3232 ; aa ae ba be 4117 vinserti32x8 m17, m8, ym3, 1 ; a3 a7 b3 b7 4118 vshufi32x4 m19, m8, m3, q3232 ; ab af bb bf 4119 vinserti32x8 m3, m4, ym14, 1 ; c0 c4 d0 d4 4120 vshufi32x4 m4, m14, q3232 ; c8 cc d8 dc 4121 vinserti32x8 m14, m20, ym2, 1 ; a1 a5 b1 b5 4122 vshufi32x4 m20, m2, q3232 ; a9 ad b9 bd 4123 vinserti32x8 m2, m6, ym18, 1 ; c1 c5 d1 d5 4124 vshufi32x4 m6, m18, q3232 ; c9 cd d9 dd 4125 vinserti32x8 m18, m15, ym9, 1 ; c2 c6 d2 d6 4126 vshufi32x4 m15, m9, q3232 ; ca ce da de 4127 vinserti32x8 m9, m7, ym16, 1 ; c3 c7 d3 d7 4128 vshufi32x4 m7, m16, q3232 ; cb cf db df 4129 ret 4130 4131 %macro IDTX_16x32 4 ; src/dst[1-4] 4132 pmulhrsw m%1, m15, [cq+64*%1] 4133 pmulhrsw m%2, m15, [cq+64*%2] 4134 pmulhrsw m%3, m15, [cq+64*%3] 4135 pmulhrsw m%4, m15, [cq+64*%4] 4136 pmulhrsw m18, m16, m%1 4137 pmulhrsw m19, m16, m%2 4138 pmulhrsw m20, m16, m%3 4139 pmulhrsw m21, m16, m%4 4140 REPX {pmulhrsw x, m17}, m18, m19, m20, m21 4141 paddsw m%1, m18 4142 paddsw m%2, m19 4143 paddsw m%3, m20 4144 paddsw m%4, m21 4145 %endmacro 4146 4147 %macro IDTX_16x32_STORE 2 ; src[1-2] 4148 mova xm17, [dstq+r3*0] 4149 vinserti128 ym17, [dstq+r3*4], 1 4150 vinserti32x4 m17, [dstq+r3*8], 2 4151 vinserti32x4 m17, [dstq+r4*8], 3 4152 mova [cq+64*(%1*2+0)], m18 4153 mova [cq+64*(%1*2+1)], m18 4154 punpcklbw m16, m17, m18 4155 punpckhbw m17, m18 4156 paddw m16, m%1 4157 paddw m17, m%2 4158 packuswb m16, m17 4159 mova [dstq+r3*0], xm16 4160 vextracti128 [dstq+r3*4], ym16, 1 4161 vextracti32x4 [dstq+r3*8], m16, 2 4162 vextracti32x4 [dstq+r4*8], m16, 3 4163 %if %1 != 7 4164 add dstq, strideq 4165 %endif 4166 %endmacro 4167 4168 cglobal inv_txfm_add_identity_identity_16x32_8bpc, 3, 5, 22, dst, stride, c 4169 vpbroadcastd m15, [pw_2896x8] 4170 vpbroadcastd m16, [pw_1697x16] 4171 vpbroadcastd m17, [pw_16384] 4172 IDTX_16x32 0, 1, 2, 3 4173 IDTX_16x32 4, 5, 6, 7 4174 IDTX_16x32 8, 9, 10, 11 4175 IDTX_16x32 12, 13, 14, 15 4176 vpbroadcastd m16, [pw_8192] 4177 call .transpose_2x8x8_round 4178 lea r3, [strideq*2] 4179 lea r4, [strideq*3] 4180 pxor m18, m18 4181 IDTX_16x32_STORE 0, 8 4182 IDTX_16x32_STORE 1, 9 4183 IDTX_16x32_STORE 2, 10 4184 IDTX_16x32_STORE 3, 11 4185 IDTX_16x32_STORE 4, 12 4186 IDTX_16x32_STORE 5, 13 4187 IDTX_16x32_STORE 6, 14 4188 IDTX_16x32_STORE 7, 15 4189 RET 4190 ALIGN function_align 4191 .transpose_2x8x8_round: 4192 punpckhwd m17, m4, m5 4193 punpcklwd m4, m5 4194 punpckhwd m5, m0, m1 4195 punpcklwd m0, m1 4196 punpckhwd m1, m6, m7 4197 punpcklwd m6, m7 4198 punpckhwd m7, m2, m3 4199 punpcklwd m2, m3 4200 punpckhdq m3, m0, m2 4201 punpckldq m0, m2 4202 punpckldq m2, m4, m6 4203 punpckhdq m4, m6 4204 punpckhdq m6, m5, m7 4205 punpckldq m5, m7 4206 punpckldq m7, m17, m1 4207 punpckhdq m17, m1 4208 REPX {pmulhrsw x, m16}, m0, m2, m3, m4, m5, m7, m6, m17 4209 punpckhqdq m1, m0, m2 4210 punpcklqdq m0, m2 4211 punpcklqdq m2, m3, m4 4212 punpckhqdq m3, m4 4213 punpcklqdq m4, m5, m7 4214 punpckhqdq m5, m7 4215 punpckhqdq m7, m6, m17 4216 punpcklqdq m6, m17 4217 punpckhwd m17, m12, m13 4218 punpcklwd m12, m13 4219 punpckhwd m13, m8, m9 4220 punpcklwd m8, m9 4221 punpckhwd m9, m14, m15 4222 punpcklwd m14, m15 4223 punpckhwd m15, m10, m11 4224 punpcklwd m10, m11 4225 punpckhdq m11, m8, m10 4226 punpckldq m8, m10 4227 punpckldq m10, m12, m14 4228 punpckhdq m12, m14 4229 punpckhdq m14, m13, m15 4230 punpckldq m13, m15 4231 punpckldq m15, m17, m9 4232 punpckhdq m17, m9 4233 REPX {pmulhrsw x, m16}, m8, m10, m11, m12, m13, m15, m14, m17 4234 punpckhqdq m9, m8, m10 4235 punpcklqdq m8, m10 4236 punpcklqdq m10, m11, m12 4237 punpckhqdq m11, m12 4238 punpcklqdq m12, m13, m15 4239 punpckhqdq m13, m15 4240 punpckhqdq m15, m14, m17 4241 punpcklqdq m14, m17 4242 ret 4243 4244 %macro IDTX_32x16 4 ; dst[1-4] 4245 pmulhrsw m%2, m12, [cq+32*(%1+ 0)] 4246 pmulhrsw m18, m12, [cq+32*(%1+16)] 4247 pmulhrsw m%4, m12, [cq+32*(%3+ 0)] 4248 pmulhrsw m19, m12, [cq+32*(%3+16)] 4249 REPX {paddsw x, x}, m%2, m18, m%4, m19 4250 mova m%1, m14 4251 vpermi2q m%1, m%2, m18 4252 vpermt2q m%2, m16, m18 4253 %if %3 != 14 4254 mova m%3, m14 4255 %endif 4256 vpermi2q m%3, m%4, m19 4257 vpermt2q m%4, m16, m19 4258 pmulhrsw m18, m17, m%1 4259 pmulhrsw m19, m17, m%2 4260 pmulhrsw m20, m17, m%3 4261 pmulhrsw m21, m17, m%4 4262 REPX {paddsw x, x}, m%1, m%2, m%3, m%4 4263 paddsw m%1, m18 4264 paddsw m%2, m19 4265 paddsw m%3, m20 4266 paddsw m%4, m21 4267 %endmacro 4268 4269 %macro IDTX_32x16_STORE 2-3 0 ; src[1-2], 32x32 4270 mova ym19, [dstq+strideq*0] 4271 vinserti32x8 m19, [dstq+strideq*8], 1 4272 %if %3 == 0 4273 mova [cq+64*(%1*2+0)], m20 4274 mova [cq+64*(%1*2+1)], m20 4275 %endif 4276 punpcklbw m18, m19, m20 4277 punpckhbw m19, m20 4278 paddw m18, m%1 4279 paddw m19, m%2 4280 packuswb m18, m19 4281 mova [dstq+strideq*0], ym18 4282 vextracti32x8 [dstq+strideq*8], m18, 1 4283 %if %3 || %1 != 7 4284 add dstq, strideq 4285 %endif 4286 %endmacro 4287 4288 cglobal inv_txfm_add_identity_identity_32x16_8bpc, 3, 3, 22, dst, stride, c 4289 vpbroadcastd m12, [pw_2896x8] 4290 movu m14, [permB+7] 4291 vpbroadcastd m17, [pw_1697x16] 4292 psrlq m16, m14, 4 4293 IDTX_32x16 0, 1, 2, 3 4294 IDTX_32x16 4, 5, 6, 7 4295 IDTX_32x16 8, 9, 10, 11 4296 IDTX_32x16 12, 13, 14, 15 4297 vpbroadcastd m16, [pw_2048] 4298 call m(inv_txfm_add_identity_identity_16x32_8bpc).transpose_2x8x8_round 4299 pxor m20, m20 4300 IDTX_32x16_STORE 0, 8 4301 IDTX_32x16_STORE 1, 9 4302 IDTX_32x16_STORE 2, 10 4303 IDTX_32x16_STORE 3, 11 4304 IDTX_32x16_STORE 4, 12 4305 IDTX_32x16_STORE 5, 13 4306 IDTX_32x16_STORE 6, 14 4307 IDTX_32x16_STORE 7, 15 4308 RET 4309 4310 %macro IDCT_32x32_END 4 ; src, mem, stride[1-2] 4311 pmovzxbw m10, [dstq+%3] 4312 pmovzxbw m11, [r3 +%4] 4313 %if %2 < 8 4314 paddsw m8, m%2, m%1 4315 psubsw m9, m%2, m%1 4316 %else 4317 mova m9, [cq+64*(%2*2-16)] 4318 paddsw m8, m9, m%1 4319 psubsw m9, m%1 4320 %endif 4321 pmulhrsw m8, m12 4322 pmulhrsw m9, m12 4323 %if %2 >= 8 4324 %if %2 == 8 4325 pxor m0, m0 4326 %endif 4327 mova [cq+64*(%2*2-16)], m0 4328 mova [cq+64*(%2*2-15)], m0 4329 %endif 4330 paddw m8, m10 4331 paddw m9, m11 4332 packuswb m8, m9 4333 vpermq m8, m13, m8 4334 mova [dstq+%3], ym8 4335 vextracti32x8 [r3 +%4], m8, 1 4336 %if %2 == 3 || %2 == 7 || %2 == 11 4337 add dstq, r5 4338 sub r3, r5 4339 %endif 4340 %endmacro 4341 4342 cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 6, 0, dst, stride, c, eob 4343 %undef cmp 4344 lea r5, [o_base] 4345 test eobd, eobd 4346 jz .dconly 4347 WIN64_SPILL_XMM 30 4348 cmp eobd, 136 4349 jb .fast 4350 mova m5, [cq+64*20] 4351 mova m3, [cq+64*12] 4352 mova m1, [cq+64* 4] 4353 mova m7, [cq+64*28] 4354 mova m2, [cq+64* 8] 4355 mova m6, [cq+64*24] 4356 mova m0, [cq+64* 0] 4357 mova m4, [cq+64*16] 4358 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 4359 mova m14, [cq+64* 2] 4360 mova m21, [cq+64*30] 4361 mova m18, [cq+64*18] 4362 mova m17, [cq+64*14] 4363 mova m16, [cq+64*10] 4364 mova m19, [cq+64*22] 4365 mova m20, [cq+64*26] 4366 mova m15, [cq+64* 6] 4367 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 4368 mova [cq+64* 0], m14 4369 mova [cq+64* 2], m15 4370 mova [cq+64* 4], m16 4371 mova [cq+64* 6], m17 4372 mova [cq+64* 8], m18 4373 mova [cq+64*10], m19 4374 mova [cq+64*12], m20 4375 mova [cq+64*14], m21 4376 mova m22, [cq+64* 1] 4377 mova m21, [cq+64*31] 4378 mova m14, [cq+64*17] 4379 mova m29, [cq+64*15] 4380 mova m26, [cq+64* 9] 4381 mova m17, [cq+64*23] 4382 mova m18, [cq+64*25] 4383 mova m25, [cq+64* 7] 4384 mova m24, [cq+64* 5] 4385 mova m19, [cq+64*27] 4386 mova m16, [cq+64*21] 4387 mova m27, [cq+64*11] 4388 mova m28, [cq+64*13] 4389 mova m15, [cq+64*19] 4390 mova m20, [cq+64*29] 4391 mova m23, [cq+64* 3] 4392 call .main_oddhalf 4393 vpbroadcastd m10, [o(pw_8192)] 4394 psubsw m13, m0, m29 ; 31 4395 paddsw m0, m29 ; 0 4396 psubsw m29, m1, m28 ; 30 4397 paddsw m1, m28 ; 1 4398 psubsw m28, m2, m27 ; 29 4399 paddsw m2, m27 ; 2 4400 psubsw m27, m3, m26 ; 28 4401 paddsw m3, m26 ; 3 4402 psubsw m26, m4, m25 ; 27 4403 paddsw m4, m25 ; 4 4404 psubsw m25, m5, m24 ; 26 4405 paddsw m5, m24 ; 5 4406 psubsw m24, m6, m23 ; 25 4407 paddsw m6, m23 ; 6 4408 psubsw m23, m7, m22 ; 24 4409 paddsw m7, m22 ; 7 4410 pxor m9, m9 4411 punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7 4412 punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3 4413 punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7 4414 punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3 4415 REPX {mova [cq+64*x], m9}, 16, 17, 18, 19 4416 punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7 4417 punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3 4418 punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7 4419 punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3 4420 REPX {mova [cq+64*x], m9}, 20, 21, 22, 23 4421 punpckhwd m3, m23, m24 4422 punpcklwd m23, m24 4423 punpckhwd m24, m25, m26 4424 punpcklwd m25, m26 4425 REPX {mova [cq+64*x], m9}, 24, 25, 26, 27 4426 punpckhwd m26, m27, m28 4427 punpcklwd m27, m28 4428 punpckhwd m28, m29, m13 4429 punpcklwd m29, m13 4430 REPX {mova [cq+64*x], m9}, 28, 29, 30, 31 4431 punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 4432 punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 4433 punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3 4434 punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1 4435 punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7 4436 punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5 4437 punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7 4438 punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5 4439 REPX {pmulhrsw x, m10}, m0, m4, m8, m22 4440 punpckhdq m13, m23, m25 4441 punpckldq m23, m25 4442 punpckhdq m25, m27, m29 4443 punpckldq m27, m29 4444 REPX {pmulhrsw x, m10}, m13, m23, m25, m27 4445 punpckhdq m9, m3, m24 4446 punpckldq m3, m24 4447 punpckhdq m24, m26, m28 4448 punpckldq m26, m28 4449 punpcklqdq m5, m23, m27 ; d00 d08 d16 d24 4450 punpckhqdq m23, m27 ; d01 d09 d17 d25 4451 punpckhqdq m27, m13, m25 ; d03 d11 d19 d27 4452 punpcklqdq m13, m25 ; d02 d10 d18 d26 4453 punpckhqdq m25, m3, m26 ; d05 d13 d21 d29 4454 punpcklqdq m3, m26 ; d04 d12 d20 d28 4455 punpckhqdq m26, m9, m24 ; d07 d15 d23 d31 4456 punpcklqdq m9, m24 ; d06 d14 d22 d30 4457 REPX {pmulhrsw x, m10}, m25, m3, m26 4458 mova [cq+64* 9], m23 4459 mova [cq+64*11], m27 4460 mova [cq+64*13], m25 4461 mova [cq+64*15], m26 4462 punpckhqdq m24, m8, m22 ; a05 a13 a21 a29 4463 punpcklqdq m8, m22 ; a04 a12 a20 a28 4464 punpckhqdq m22, m0, m4 ; a01 a09 a17 a25 4465 punpcklqdq m0, m4 ; a00 a08 a16 a24 4466 punpckhqdq m23, m7, m2 ; a03 a11 a19 a27 4467 punpcklqdq m7, m2 ; a02 a10 a18 a26 4468 punpckhqdq m25, m6, m1 ; a07 a15 a23 a31 4469 punpcklqdq m6, m1 ; a06 a14 a22 a30 4470 mova m2, [cq+64* 0] 4471 mova m11, [cq+64* 2] 4472 mova m12, [cq+64* 4] 4473 mova m29, [cq+64* 6] 4474 mova m27, [cq+64* 8] 4475 mova m26, [cq+64*10] 4476 mova m4, [cq+64*12] 4477 mova m28, [cq+64*14] 4478 psubsw m1, m2, m21 ; 23 4479 paddsw m2, m21 ; 8 4480 psubsw m21, m11, m20 ; 22 4481 paddsw m11, m20 ; 9 4482 psubsw m20, m12, m19 ; 21 4483 paddsw m12, m19 ; 10 4484 psubsw m19, m29, m18 ; 20 4485 paddsw m29, m18 ; 11 4486 psubsw m18, m27, m17 ; 19 4487 paddsw m27, m17 ; 12 4488 psubsw m17, m26, m16 ; 18 4489 paddsw m26, m16 ; 13 4490 paddsw m16, m4, m15 ; 14 4491 psubsw m4, m15 ; 17 4492 pmulhrsw m15, m6, m10 4493 psubsw m6, m28, m14 ; 16 4494 paddsw m28, m14 ; 15 4495 pmulhrsw m14, m7, m10 4496 punpcklwd m7, m6, m4 4497 punpckhwd m6, m4 4498 punpckhwd m4, m17, m18 4499 punpcklwd m17, m18 4500 punpckhwd m18, m19, m20 4501 punpcklwd m19, m20 4502 punpckhwd m20, m21, m1 4503 punpcklwd m21, m1 4504 punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7 4505 punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3 4506 punpckhwd m11, m12, m29 ; k4 l4 k5 l5 k6 l6 k7 l7 4507 punpcklwd m12, m29 ; k0 l0 k1 l1 k2 l2 k3 l3 4508 punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7 4509 punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3 4510 punpckhwd m26, m16, m28 ; o4 p4 o5 p5 o6 p6 o7 p7 4511 punpcklwd m16, m28 ; o0 p0 o1 p1 o2 p2 o3 p3 4512 pmulhrsw m23, m10 4513 pmulhrsw m25, m10 4514 punpckhdq m28, m2, m12 ; i2 j2 k2 l2 i3 j3 k3 l3 4515 punpckldq m2, m12 ; i0 j0 k0 l0 i1 j1 k1 l1 4516 punpckhdq m12, m27, m16 ; m2 n2 o2 p2 m3 n3 o3 p3 4517 punpckldq m27, m16 ; m0 n0 o0 p0 m1 n1 o1 p1 4518 REPX {pmulhrsw x, m10}, m28, m2, m12, m27 4519 punpckhdq m16, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7 4520 punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5 4521 punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7 4522 punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5 4523 REPX {pmulhrsw x, m10}, m16, m1, m11, m29 4524 punpckhdq m26, m19, m21 4525 punpckldq m19, m21 4526 punpckhdq m21, m6, m4 4527 punpckldq m6, m4 4528 REPX {pmulhrsw x, m10}, m26, m19, m21, m6 4529 punpckhdq m4, m18, m20 4530 punpckldq m18, m20 4531 punpckhdq m20, m7, m17 4532 punpckldq m7, m17 4533 REPX {pmulhrsw x, m10}, m4, m18, m20, m7 4534 punpcklqdq m17, m28, m12 ; b02 b10 b18 b26 4535 punpckhqdq m28, m12 ; b03 b11 b19 b27 4536 punpckhqdq m12, m2, m27 ; b01 b09 b17 b25 4537 punpcklqdq m2, m27 ; b00 b08 b16 b24 4538 punpckhqdq m27, m1, m29 ; b05 b13 b21 b29 4539 punpcklqdq m1, m29 ; b04 b12 b20 b28 4540 punpckhqdq m29, m16, m11 ; b07 b15 b23 b31 4541 punpcklqdq m16, m11 ; b06 b14 b22 b30 4542 mova [cq+64* 1], m12 4543 mova [cq+64* 3], m28 4544 mova [cq+64* 5], m27 4545 mova [cq+64* 7], m29 4546 punpckhqdq m27, m20, m26 ; c03 c11 c19 c27 4547 punpcklqdq m20, m26 ; c02 c10 c18 c26 4548 punpckhqdq m26, m7, m19 ; c01 c09 c17 c25 4549 punpcklqdq m7, m19 ; c00 c08 c16 c24 4550 punpckhqdq m28, m6, m18 ; c05 c13 c21 c29 4551 punpcklqdq m6, m18 ; c04 c12 c20 c28 4552 punpckhqdq m29, m21, m4 ; c07 c15 c23 c31 4553 punpcklqdq m21, m4 ; c06 c14 c22 c30 4554 pmulhrsw m19, m9, m10 4555 vshufi32x4 m4, m0, m2, q3232 ; a16 a24 b16 b24 4556 vinserti32x8 m0, ym2, 1 ; a00 a08 b00 b08 4557 vshufi32x4 m2, m7, m5, q3232 ; c16 c24 d16 d24 4558 vinserti32x8 m7, ym5, 1 ; c00 c08 d00 d08 4559 vshufi32x4 m5, m8, m1, q3232 ; a20 a28 b20 b28 4560 vinserti32x8 m1, m8, ym1, 1 ; a04 a12 b04 b12 4561 vshufi32x4 m8, m6, m3, q3232 ; c20 c28 d20 d28 4562 vinserti32x8 m6, ym3, 1 ; c04 c12 d04 d12 4563 vshufi32x4 m3, m1, m6, q3131 ; 12 4564 vshufi32x4 m1, m6, q2020 ; 4 4565 vshufi32x4 m6, m4, m2, q3131 ; 24 4566 vshufi32x4 m4, m2, q2020 ; 16 4567 vshufi32x4 m2, m0, m7, q3131 ; 8 4568 vshufi32x4 m0, m7, q2020 ; 0 4569 vshufi32x4 m7, m5, m8, q3131 ; 28 4570 vshufi32x4 m5, m8, q2020 ; 20 4571 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 4572 vshufi32x4 m18, m14, m17, q3232 ; a18 a26 b18 b26 4573 vinserti32x8 m14, ym17, 1 ; a02 a10 b02 b10 4574 vshufi32x4 m17, m20, m13, q3232 ; c18 c26 d18 d26 4575 vinserti32x8 m20, ym13, 1 ; c02 c10 d02 d10 4576 vshufi32x4 m13, m21, m19, q3232 ; c22 c30 d22 d30 4577 vinserti32x8 m21, ym19, 1 ; c06 c14 d06 d14 4578 vshufi32x4 m19, m15, m16, q3232 ; a22 a30 b22 b30 4579 vinserti32x8 m15, ym16, 1 ; a06 a14 b06 b14 4580 vshufi32x4 m16, m14, m20, q3131 ; 10 4581 vshufi32x4 m14, m20, q2020 ; 2 4582 vshufi32x4 m20, m18, m17, q3131 ; 26 4583 vshufi32x4 m18, m17, q2020 ; 18 4584 vshufi32x4 m17, m15, m21, q3131 ; 14 4585 vshufi32x4 m15, m21, q2020 ; 6 4586 vshufi32x4 m21, m19, m13, q3131 ; 30 4587 vshufi32x4 m19, m13, q2020 ; 22 4588 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 4589 mova [cq+64* 0], m14 4590 mova [cq+64* 2], m15 4591 mova [cq+64* 4], m16 4592 mova [cq+64* 6], m17 4593 mova [cq+64* 8], m18 4594 mova [cq+64*10], m19 4595 mova [cq+64*12], m20 4596 mova [cq+64*14], m21 4597 mova m15, [cq+64* 1] 4598 mova m16, [cq+64* 3] 4599 mova m17, [cq+64* 5] 4600 mova m19, [cq+64* 7] 4601 mova m20, [cq+64* 9] 4602 mova m21, [cq+64*11] 4603 mova m13, [cq+64*13] 4604 mova m18, [cq+64*15] 4605 vshufi32x4 m14, m22, m15, q3232 ; a17 a25 b17 b25 4606 vinserti32x8 m22, ym15, 1 ; a01 a09 b01 b09 4607 vshufi32x4 m15, m23, m16, q3232 ; a19 a27 b19 b27 4608 vinserti32x8 m23, ym16, 1 ; a03 a11 b03 b11 4609 vshufi32x4 m16, m24, m17, q3232 ; a21 a29 b21 b29 4610 vinserti32x8 m24, ym17, 1 ; a05 a13 b05 b13 4611 vshufi32x4 m17, m25, m19, q3232 ; a23 a31 b23 b31 4612 vinserti32x8 m25, ym19, 1 ; a07 a15 b07 b15 4613 vinserti32x8 m8, m26, ym20, 1 ; c01 c09 d01 d09 4614 vshufi32x4 m26, m20, q3232 ; c17 c25 d17 d25 4615 vinserti32x8 m9, m27, ym21, 1 ; c03 c11 d03 d11 4616 vshufi32x4 m27, m21, q3232 ; c19 c27 d19 d27 4617 vinserti32x8 m11, m28, ym13, 1 ; c05 c13 d05 d13 4618 vshufi32x4 m28, m13, q3232 ; c21 c29 d21 d29 4619 vinserti32x8 m12, m29, ym18, 1 ; c07 c15 d07 d15 4620 vshufi32x4 m29, m18, q3232 ; c23 c31 d23 d31 4621 vshufi32x4 m18, m14, m26, q3131 ; 25 4622 vshufi32x4 m14, m26, q2020 ; 17 4623 vshufi32x4 m19, m15, m27, q3131 ; 27 4624 vshufi32x4 m15, m27, q2020 ; 19 4625 vshufi32x4 m20, m16, m28, q3131 ; 29 4626 vshufi32x4 m16, m28, q2020 ; 21 4627 vshufi32x4 m21, m17, m29, q3131 ; 31 4628 vshufi32x4 m17, m29, q2020 ; 23 4629 vshufi32x4 m26, m22, m8, q3131 ; 9 4630 vshufi32x4 m22, m8, q2020 ; 1 4631 vshufi32x4 m27, m23, m9, q3131 ; 11 4632 vshufi32x4 m23, m9, q2020 ; 3 4633 vshufi32x4 m28, m24, m11, q3131 ; 13 4634 vshufi32x4 m24, m11, q2020 ; 5 4635 vshufi32x4 m29, m25, m12, q3131 ; 15 4636 vshufi32x4 m25, m12, q2020 ; 7 4637 call .main_oddhalf 4638 jmp .end 4639 .fast: ; bottom/right halves are zero 4640 mova m14, [o(dup16_perm)] 4641 pmovzxwd m9, [cq+64* 0] 4642 pmovzxwd m6, [cq+64* 8] 4643 vpermb m8, m14, [cq+64* 2] 4644 vpermb ym0, ym14, [cq+64*14] 4645 vpermb ym5, ym14, [cq+64*10] 4646 vpermb m1, m14, [cq+64* 6] 4647 vpermb m7, m14, [cq+64* 4] 4648 vpermb ym3, ym14, [cq+64*12] 4649 pslld m9, 16 4650 pslld m6, 16 4651 call m(idct_16x16_internal_8bpc).main_fast 4652 vpermb m21, m14, [cq+64* 1] 4653 vpermb ym17, ym14, [cq+64*15] 4654 vpermb ym20, ym14, [cq+64* 9] 4655 vpermb m15, m14, [cq+64* 7] 4656 vpermb m18, m14, [cq+64* 5] 4657 vpermb ym16, ym14, [cq+64*11] 4658 vpermb ym19, ym14, [cq+64*13] 4659 vpermb m14, m14, [cq+64* 3] 4660 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast 4661 vpbroadcastd m9, [o(pw_8192)] 4662 call m(inv_txfm_add_dct_dct_32x16_8bpc).transpose_round 4663 vshufi32x4 m22, m14, m2, q2020 ; 1 4664 vshufi32x4 m24, m14, m2, q3131 ; 5 4665 vshufi32x4 m23, m17, m9, q2020 ; 3 4666 vshufi32x4 m25, m17, m9, q3131 ; 7 4667 vshufi32x4 m16, m5, m15, q2020 ; 10 4668 vshufi32x4 m17, m5, m15, q3131 ; 14 4669 vshufi32x4 m14, m1, m18, q2020 ; 2 4670 vshufi32x4 m15, m1, m18, q3131 ; 6 4671 vshufi32x4 m1, m0, m3, q3131 ; 4 4672 vshufi32x4 m0, m3, q2020 ; 0 4673 vshufi32x4 m3, m21, m4, q3131 ; 12 4674 vshufi32x4 m2, m21, m4, q2020 ; 8 4675 vshufi32x4 m26, m20, m6, q2020 ; 9 4676 vshufi32x4 m28, m20, m6, q3131 ; 13 4677 vshufi32x4 m27, m19, m7, q2020 ; 11 4678 vshufi32x4 m29, m19, m7, q3131 ; 15 4679 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast 4680 mova [cq+64* 0], m14 4681 mova [cq+64* 2], m15 4682 mova [cq+64* 4], m16 4683 mova [cq+64* 6], m17 4684 mova [cq+64* 8], m18 4685 mova [cq+64*10], m19 4686 mova [cq+64*12], m20 4687 mova [cq+64*14], m21 4688 call .main_oddhalf_fast 4689 .end: 4690 lea r4, [strideq*3] 4691 vpbroadcastd m12, [o(pw_2048)] 4692 movshdup m13, [o(permD)] 4693 lea r3, [dstq+r4*8] 4694 lea r5, [strideq+r4] ; stride*4 4695 add r3, r5 ; dst+stride*28 4696 IDCT_32x32_END 29, 0, strideq*0, r4 4697 IDCT_32x32_END 28, 1, strideq*1, strideq*2 4698 IDCT_32x32_END 27, 2, strideq*2, strideq*1 4699 IDCT_32x32_END 26, 3, r4 , strideq*0 4700 IDCT_32x32_END 25, 4, strideq*0, r4 4701 IDCT_32x32_END 24, 5, strideq*1, strideq*2 4702 IDCT_32x32_END 23, 6, strideq*2, strideq*1 4703 IDCT_32x32_END 22, 7, r4 , strideq*0 4704 IDCT_32x32_END 21, 8, strideq*0, r4 4705 IDCT_32x32_END 20, 9, strideq*1, strideq*2 4706 IDCT_32x32_END 19, 10, strideq*2, strideq*1 4707 IDCT_32x32_END 18, 11, r4 , strideq*0 4708 IDCT_32x32_END 17, 12, strideq*0, r4 4709 IDCT_32x32_END 16, 13, strideq*1, strideq*2 4710 IDCT_32x32_END 15, 14, strideq*2, strideq*1 4711 IDCT_32x32_END 14, 15, r4 , strideq*0 4712 RET 4713 .dconly: 4714 movsx r6d, word [cq] 4715 mov [cq], eobd 4716 or r3d, 32 4717 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly2 4718 ALIGN function_align 4719 cglobal_label .main_oddhalf_fast3 ; bottom seven-eights are zero 4720 vpbroadcastd m21, [o(pw_4091x8)] 4721 vpbroadcastd m8, [o(pw_201x8)] 4722 vpbroadcastd m24, [o(pw_m601x8)] 4723 vpbroadcastd m12, [o(pw_4052x8)] 4724 pmulhrsw m21, m22 ; t31a 4725 pmulhrsw m22, m8 ; t16a 4726 pmulhrsw m24, m23 ; t23a 4727 pmulhrsw m23, m12 ; t24a 4728 4729 punpcklwd m9, m22, m21 4730 punpckhwd m8, m22, m21 4731 mova m15, m10 4732 vpdpwssd m15, m9, [o(pw_m4017_799)] {bcstd} 4733 mova m17, m10 4734 vpdpwssd m17, m8, [o(pw_m4017_799)] {bcstd} 4735 REPX {psrad x, 12}, m15, m17 4736 packssdw m15, m17 4737 mova m17, m10 4738 vpdpwssd m17, m8, [o(pw_799_4017)] {bcstd} 4739 mova m8, m10 4740 vpdpwssd m8, m9, [o(pw_799_4017)] {bcstd} 4741 REPX {psrad x, 12}, m17, m8 4742 packssdw m8, m17 4743 4744 punpcklwd m9, m24, m23 4745 punpckhwd m16, m24, m23 4746 mova m20, m10 4747 vpdpwssd m20, m9, [o(pw_m3406_m2276)] {bcstd} 4748 mova m17, m10 4749 vpdpwssd m17, m16, [o(pw_m3406_m2276)] {bcstd} 4750 REPX {psrad x, 12}, m20, m17 4751 packssdw m20, m17 4752 mova m17, m10 4753 vpdpwssd m17, m16, [o(pw_m2276_3406)] {bcstd} 4754 mova m16, m10 4755 vpdpwssd m16, m9, [o(pw_m2276_3406)] {bcstd} 4756 REPX {psrad x, 12}, m17, m16 4757 packssdw m16, m17 4758 4759 mova m17, m21 4760 mova m27, m15 4761 mova m25, m20 4762 mova m29, m8 4763 mova m18, m22 4764 mova m14, m24 4765 mova m28, m16 4766 mova m26, m23 4767 jmp .main4 4768 cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero 4769 vpbroadcastd m21, [o(pw_4091x8)] 4770 vpbroadcastd m8, [o(pw_201x8)] 4771 vpbroadcastd m18, [o(pw_m1380x8)] 4772 vpbroadcastd m9, [o(pw_3857x8)] 4773 vpbroadcastd m19, [o(pw_3973x8)] 4774 vpbroadcastd m11, [o(pw_995x8)] 4775 vpbroadcastd m28, [o(pw_m601x8)] 4776 vpbroadcastd m12, [o(pw_4052x8)] 4777 pmulhrsw m21, m22 ; t31a 4778 pmulhrsw m22, m8 ; t16a 4779 pmulhrsw m18, m25 ; t19a 4780 pmulhrsw m25, m9 ; t28a 4781 pmulhrsw m19, m24 ; t27a 4782 pmulhrsw m24, m11 ; t20a 4783 pmulhrsw m28, m23 ; t23a 4784 pmulhrsw m23, m12 ; t24a 4785 mova m15, m21 4786 mova m8, m22 4787 mova m14, m18 4788 mova m27, m25 4789 mova m29, m19 4790 mova m26, m24 4791 mova m16, m28 4792 mova m20, m23 4793 jmp .main3 4794 ALIGN function_align 4795 cglobal_label .main_oddhalf_fast ; bottom half is zero 4796 vpbroadcastd m21, [o(pw_4091x8)] 4797 vpbroadcastd m8, [o(pw_201x8)] 4798 vpbroadcastd m14, [o(pw_m2751x8)] 4799 vpbroadcastd m9, [o(pw_3035x8)] 4800 vpbroadcastd m17, [o(pw_3703x8)] 4801 vpbroadcastd m11, [o(pw_1751x8)] 4802 vpbroadcastd m18, [o(pw_m1380x8)] 4803 vpbroadcastd m12, [o(pw_3857x8)] 4804 pmulhrsw m21, m22 ; t31a 4805 vpbroadcastd m19, [o(pw_3973x8)] 4806 pmulhrsw m22, m8 ; t16a 4807 vpbroadcastd m8, [o(pw_995x8)] 4808 pmulhrsw m14, m29 ; t30a 4809 vpbroadcastd m16, [o(pw_m2106x8)] 4810 pmulhrsw m29, m9 ; t17a 4811 vpbroadcastd m9, [o(pw_3513x8)] 4812 pmulhrsw m17, m26 ; t29a 4813 vpbroadcastd m15, [o(pw_3290x8)] 4814 pmulhrsw m26, m11 ; t18a 4815 vpbroadcastd m11, [o(pw_2440x8)] 4816 pmulhrsw m18, m25 ; t19a 4817 vpbroadcastd m20, [o(pw_m601x8)] 4818 pmulhrsw m25, m12 ; t28a 4819 vpbroadcastd m12, [o(pw_4052x8)] 4820 pmulhrsw m19, m24 ; t27a 4821 pmulhrsw m24, m8 ; t20a 4822 pmulhrsw m16, m27 ; t21a 4823 pmulhrsw m27, m9 ; t26a 4824 pmulhrsw m15, m28 ; t25a 4825 pmulhrsw m28, m11 ; t22a 4826 pmulhrsw m20, m23 ; t23a 4827 pmulhrsw m23, m12 ; t24a 4828 jmp .main2 4829 ALIGN function_align 4830 cglobal_label .main_oddhalf 4831 ITX_MULSUB_2W 22, 21, 8, 9, 10, 201, 4091 ; t16a, t31a 4832 ITX_MULSUB_2W 14, 29, 8, 9, 10, 3035, 2751 ; t17a, t30a 4833 ITX_MULSUB_2W 26, 17, 8, 9, 10, 1751, 3703 ; t18a, t29a 4834 ITX_MULSUB_2W 18, 25, 8, 9, 10, 3857, 1380 ; t19a, t28a 4835 ITX_MULSUB_2W 24, 19, 8, 9, 10, 995, 3973 ; t20a, t27a 4836 ITX_MULSUB_2W 16, 27, 8, 9, 10, 3513, 2106 ; t21a, t26a 4837 ITX_MULSUB_2W 28, 15, 8, 9, 10, 2440, 3290 ; t22a, t25a 4838 ITX_MULSUB_2W 20, 23, 8, 9, 10, 4052, 601 ; t23a, t24a 4839 .main2: 4840 psubsw m8, m22, m14 ; t17 4841 paddsw m22, m14 ; t16 4842 paddsw m14, m18, m26 ; t19 4843 psubsw m18, m26 ; t18 4844 psubsw m26, m24, m16 ; t21 4845 paddsw m24, m16 ; t20 4846 psubsw m16, m20, m28 ; t22 4847 paddsw m28, m20 ; t23 4848 psubsw m20, m23, m15 ; t25 4849 paddsw m23, m15 ; t24 4850 psubsw m15, m21, m29 ; t30 4851 paddsw m21, m29 ; t31 4852 psubsw m29, m19, m27 ; t26 4853 paddsw m19, m27 ; t27 4854 paddsw m27, m25, m17 ; t28 4855 psubsw m25, m17 ; t29 4856 .main3: 4857 ITX_MULSUB_2W 15, 8, 9, 17, 10, 799, 4017 ; t17a, t30a 4858 ITX_MULSUB_2W 25, 18, 9, 17, 10, m4017, 799 ; t18a, t29a 4859 ITX_MULSUB_2W 29, 26, 9, 17, 10, 3406, 2276 ; t21a, t26a 4860 ITX_MULSUB_2W 20, 16, 9, 17, 10, m2276, 3406 ; t22a, t25a 4861 psubsw m17, m21, m27 ; t28a 4862 paddsw m21, m27 ; t31a 4863 psubsw m27, m15, m25 ; t18 4864 paddsw m15, m25 ; t17 4865 psubsw m25, m20, m29 ; t21 4866 paddsw m20, m29 ; t22 4867 psubsw m29, m8, m18 ; t29 4868 paddsw m8, m18 ; t30 4869 psubsw m18, m22, m14 ; t19a 4870 paddsw m22, m14 ; t16a 4871 psubsw m14, m28, m24 ; t20a 4872 paddsw m24, m28 ; t23a 4873 paddsw m28, m16, m26 ; t25 4874 psubsw m16, m26 ; t26 4875 psubsw m26, m23, m19 ; t27a 4876 paddsw m23, m19 ; t24a 4877 .main4: 4878 vpbroadcastd m12, [o(pw_m3784_1567)] 4879 vpbroadcastd m11, [o(pw_1567_3784)] 4880 ITX_MULSUB_2W 29, 27, 9, 19, 10, 11, 12 ; t18a, t29a 4881 ITX_MULSUB_2W 17, 18, 9, 19, 10, 11, 12 ; t19, t28 4882 vpbroadcastd m11, [o(pw_m1567_m3784)] 4883 ITX_MULSUB_2W 16, 25, 9, 19, 10, 12, 11 ; t21a, t26a 4884 ITX_MULSUB_2W 26, 14, 9, 19, 10, 12, 11 ; t20, t27 4885 vpbroadcastd m12, [o(pw_m2896_2896)] 4886 vpbroadcastd m11, [o(pw_2896_2896)] 4887 psubsw m19, m27, m25 ; t26 4888 paddsw m27, m25 ; t29 4889 psubsw m25, m17, m26 ; t20a 4890 paddsw m17, m26 ; t19a 4891 paddsw m26, m18, m14 ; t28a 4892 psubsw m18, m14 ; t27a 4893 paddsw m14, m22, m24 ; t16 4894 psubsw m22, m24 ; t23 4895 psubsw m24, m29, m16 ; t21 4896 paddsw m16, m29 ; t18 4897 paddsw m29, m21, m23 ; t31 4898 psubsw m21, m23 ; t24 4899 psubsw m23, m15, m20 ; t22a 4900 paddsw m15, m20 ; t17a 4901 psubsw m20, m8, m28 ; t25a 4902 paddsw m28, m8 ; t30a 4903 ITX_MULSUB_2W 18, 25, 8, 9, 10, 11, 12 ; t20, t27 4904 ITX_MULSUB_2W 19, 24, 8, 9, 10, 11, 12 ; t21a, t26a 4905 ITX_MULSUB_2W 21, 22, 8, 9, 10, 11, 12 ; t23a, t24a 4906 ITX_MULSUB_2W 20, 23, 8, 9, 10, 11, 12 ; t22, t25 4907 ret 4908 4909 %macro IDTX_32x32 2 ; dst[1-2] 4910 vmovdqa32 ym%1, [cq+64*(%1+ 0)] ; force EVEX encoding, which 4911 vmovdqa32 ym17, [cq+64*(%1+16)] ; reduces code size due to 4912 vmovdqa32 ym%2, [cq+64*(%2+ 0)] ; compressed displacements 4913 vmovdqa32 ym18, [cq+64*(%2+16)] 4914 vpermt2q m%1, m21, m17 4915 vpermt2q m%2, m21, m18 4916 %endmacro 4917 4918 cglobal inv_txfm_add_identity_identity_32x32_8bpc, 3, 3, 22, dst, stride, c 4919 movu m21, [permB+7] 4920 vpbroadcastd m16, [pw_8192] 4921 pxor m20, m20 4922 .loop: 4923 IDTX_32x32 0, 1 4924 IDTX_32x32 2, 3 4925 IDTX_32x32 4, 5 4926 IDTX_32x32 6, 7 4927 IDTX_32x32 8, 9 4928 IDTX_32x32 10, 11 4929 IDTX_32x32 12, 13 4930 IDTX_32x32 14, 15 4931 call m(inv_txfm_add_identity_identity_16x32_8bpc).transpose_2x8x8_round 4932 IDTX_32x16_STORE 0, 8, 1 4933 IDTX_32x16_STORE 1, 9, 1 4934 IDTX_32x16_STORE 2, 10, 1 4935 IDTX_32x16_STORE 3, 11, 1 4936 IDTX_32x16_STORE 4, 12, 1 4937 IDTX_32x16_STORE 5, 13, 1 4938 IDTX_32x16_STORE 6, 14, 1 4939 IDTX_32x16_STORE 7, 15, 1 4940 lea dstq, [dstq+strideq*8] 4941 btc cq, 5 4942 jnc .loop 4943 mov r0d, 8 4944 .zero_loop: 4945 mova [cq+64*0], m20 4946 mova [cq+64*1], m20 4947 mova [cq+64*2], m20 4948 mova [cq+64*3], m20 4949 add cq, 64*4 4950 dec r0d 4951 jg .zero_loop 4952 RET 4953 4954 cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 7, 0, dst, stride, c, eob 4955 %undef cmp 4956 lea r5, [o_base] 4957 test eobd, eobd 4958 jz .dconly 4959 WIN64_SPILL_XMM 30 4960 cmp eobd, 151 4961 jb .fast 4962 mova m5, [cq+64*10] 4963 mova m3, [cq+64* 6] 4964 mova m1, [cq+64* 2] 4965 mova m7, [cq+64*14] 4966 mova m2, [cq+64* 4] 4967 mova m6, [cq+64*12] 4968 mova m0, [cq+64* 0] 4969 mova m4, [cq+64* 8] 4970 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 4971 mova m14, [cq+64* 1] 4972 mova m21, [cq+64*15] 4973 mova m18, [cq+64* 9] 4974 mova m17, [cq+64* 7] 4975 mova m16, [cq+64* 5] 4976 mova m19, [cq+64*11] 4977 mova m20, [cq+64*13] 4978 mova m15, [cq+64* 3] 4979 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 4980 vpbroadcastd m9, [o(pw_8192)] 4981 %macro TRANSPOSE_8x4_ROUND 4 4982 punpckhwd m8, m%3, m%4 ; c4 d4 c5 d5 c6 d6 c7 d7 4983 punpcklwd m%3, m%4 ; c0 d0 c1 d1 c2 d2 c3 d3 4984 punpckhwd m%4, m%1, m%2 ; a4 b4 a5 b5 a6 b6 a7 b7 4985 punpcklwd m%1, m%2 ; a0 b0 a1 b1 a2 b2 a3 b3 4986 punpckhdq m%2, m%1, m%3 ; a2 b2 c2 d2 a3 b3 c3 d3 4987 punpckldq m%1, m%3 ; a0 b0 c0 d0 a1 b1 c1 d1 4988 punpckldq m%3, m%4, m8 ; a4 b4 c4 d4 a5 b5 c5 d5 4989 punpckhdq m%4, m8 ; a6 b6 c6 d6 a7 b7 c7 d7 4990 REPX {pmulhrsw x, m9}, m%2, m%1, m%3, m%4 4991 %endmacro 4992 TRANSPOSE_8x4_ROUND 0, 1, 2, 3 4993 TRANSPOSE_8x4_ROUND 4, 5, 6, 7 4994 TRANSPOSE_8x4_ROUND 14, 15, 16, 17 4995 TRANSPOSE_8x4_ROUND 18, 19, 20, 21 4996 vinserti32x8 m26, m0, ym4, 1 ; a0 a4 b0 b4 4997 vshufi32x4 m0, m4, q3232 ; a8 a12 b8 b12 4998 vinserti32x8 m27, m1, ym5, 1 ; a1 a5 b1 b5 4999 vshufi32x4 m1, m5, q3232 ; a9 a13 b9 b13 5000 vinserti32x8 m28, m2, ym6, 1 ; a2 a6 b2 b6 5001 vshufi32x4 m2, m6, q3232 ; a10 a14 b10 b14 5002 vinserti32x8 m29, m3, ym7, 1 ; a3 a7 b3 b7 5003 vshufi32x4 m8, m3, m7, q3232 ; a11 a15 b11 b15 5004 vinserti32x8 m4, m14, ym18, 1 ; c0 c4 d0 d4 5005 vshufi32x4 m14, m18, q3232 ; c8 c12 d8 d12 5006 vinserti32x8 m5, m15, ym19, 1 ; c1 c5 d1 d5 5007 vshufi32x4 m15, m19, q3232 ; c9 c13 d9 d13 5008 vinserti32x8 m6, m16, ym20, 1 ; c2 c6 d2 d6 5009 vshufi32x4 m16, m20, q3232 ; c10 c14 d10 d14 5010 vinserti32x8 m7, m17, ym21, 1 ; c3 c7 d3 d7 5011 vshufi32x4 m17, m21, q3232 ; c11 c15 d11 d15 5012 vshufi32x4 m22, m26, m4, q2020 ; 0 1 5013 vshufi32x4 m26, m4, q3131 ; 8 9 5014 vshufi32x4 m23, m27, m5, q2020 ; 2 3 5015 vshufi32x4 m27, m5, q3131 ; 10 11 5016 vshufi32x4 m24, m28, m6, q2020 ; 4 5 5017 vshufi32x4 m28, m6, q3131 ; 12 13 5018 vshufi32x4 m25, m29, m7, q2020 ; 6 7 5019 vshufi32x4 m29, m7, q3131 ; 14 15 5020 vshufi32x4 m4, m0, m14, q2020 ; 16 17 5021 vshufi32x4 m3, m0, m14, q3131 ; 24 25 5022 vshufi32x4 m20, m1, m15, q2020 ; 18 19 5023 vshufi32x4 m19, m1, m15, q3131 ; 26 27 5024 vshufi32x4 m5, m2, m16, q2020 ; 20 21 5025 vshufi32x4 m0, m2, m16, q3131 ; 28 29 5026 vshufi32x4 m16, m8, m17, q2020 ; 22 23 5027 vshufi32x4 m17, m8, m17, q3131 ; 30 31 5028 pxor m6, m6 5029 mova [cq+64* 0], m4 5030 mova [cq+64* 2], m5 5031 mova [cq+64* 4], m3 5032 mova [cq+64* 6], m0 5033 punpcklwd m8, m24, m24 ; 4 5034 punpcklwd m0, m0 ; 28 5035 punpcklwd m5, m5 ; 20 5036 punpcklwd m1, m28, m28 ; 12 5037 punpcklwd m7, m26, m26 ; 8 5038 punpcklwd m3, m3 ; 24 5039 punpcklwd m9, m6, m22 ; __ 0 5040 punpcklwd m6, m4 ; __ 16 5041 call m(idct_16x16_internal_8bpc).main_fast3 5042 mova [cq+64* 1], m20 5043 mova [cq+64* 3], m16 5044 mova [cq+64* 5], m19 5045 mova [cq+64* 7], m17 5046 punpcklwd m21, m23, m23 ; 2 5047 punpcklwd m17, m17 ; 30 5048 punpcklwd m20, m20 ; 18 5049 punpcklwd m15, m29, m29 ; 14 5050 punpcklwd m18, m27, m27 ; 10 5051 punpcklwd m16, m16 ; 22 5052 punpcklwd m19, m19 ; 26 5053 punpcklwd m14, m25, m25 ; 6 5054 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast 5055 mova [cq+64* 8], m14 5056 mova [cq+64* 9], m15 5057 mova [cq+64*10], m16 5058 mova [cq+64*11], m17 5059 mova [cq+64*12], m18 5060 mova [cq+64*13], m19 5061 mova [cq+64*14], m20 5062 mova [cq+64*15], m21 5063 mova m21, [cq+64* 7] 5064 mova m14, [cq+64* 0] 5065 mova m17, [cq+64* 3] 5066 mova m18, [cq+64* 4] 5067 mova m19, [cq+64* 5] 5068 mova m16, [cq+64* 2] 5069 mova m15, [cq+64* 1] 5070 mova m20, [cq+64* 6] 5071 REPX {punpckhwd x, x}, m22, m21, m14, m29, m26, m17, m18, m25, \ 5072 m24, m19, m16, m27, m28, m15, m20, m23 5073 call .main_oddhalf 5074 jmp .end 5075 .fast: ; right half is zero 5076 mova ym8, [cq+64*15] 5077 vinserti32x8 m8, [cq+64* 1], 1 5078 mova m2, [o(int16_perm)] 5079 mova ym9, [cq+64* 8] 5080 vinserti32x8 m9, [cq+64* 0], 1 5081 mova ym0, [cq+64* 7] 5082 vinserti32x8 m0, [cq+64* 9], 1 5083 mova ym7, [cq+64*14] 5084 vinserti32x8 m7, [cq+64* 2], 1 5085 mova ym1, [cq+64* 3] 5086 vinserti32x8 m1, [cq+64*13], 1 5087 mova ym3, [cq+64* 6] 5088 vinserti32x8 m3, [cq+64*10], 1 5089 mova ym5, [cq+64*11] 5090 vinserti32x8 m5, [cq+64* 5], 1 5091 mova ym6, [cq+64*12] 5092 vinserti32x8 m6, [cq+64* 4], 1 5093 REPX {vpermb x, m2, x}, m8, m9, m0, m7, m1, m3, m5, m6 5094 call m(idct_16x16_internal_8bpc).main2 5095 vbroadcasti32x4 m8, [o(int_shuf3)] 5096 vbroadcasti32x4 m9, [o(int_shuf4)] 5097 vpbroadcastd m11, [o(pw_8192)] 5098 pshufb m0, m8 5099 pshufb m1, m9 5100 pshufb m2, m8 5101 pshufb m3, m9 5102 REPX {pmulhrsw x, m11}, m0, m1, m2, m3 5103 pshufb m4, m8 5104 pshufb m5, m9 5105 pshufb m6, m8 5106 pshufb m7, m9 5107 REPX {pmulhrsw x, m11}, m4, m5, m6, m7 5108 punpckhdq m28, m0, m1 5109 punpckldq m0, m1 5110 punpckhdq m27, m2, m3 5111 punpckldq m2, m3 5112 punpckhdq m22, m4, m5 5113 punpckldq m4, m5 5114 punpckhdq m23, m6, m7 5115 punpckldq m6, m7 5116 vinserti32x8 m14, m0, ym2, 1 5117 vshufi32x4 m15, m0, m2, q3232 5118 vinserti32x8 m2, m4, ym6, 1 5119 vshufi32x4 m4, m6, q3232 5120 vshufi32x4 m21, m14, m2, q2020 ; 0 2 5121 vshufi32x4 m14, m2, q3131 ; 4 6 5122 vshufi32x4 m18, m15, m4, q2020 ; 8 10 5123 vshufi32x4 m15, m4, q3131 ; 12 14 5124 pxor m9, m9 5125 punpcklwd m8, m14, m14 ; 4 5126 punpcklwd m1, m15, m15 ; 12 5127 punpcklwd m7, m18, m18 ; 8 5128 punpcklwd m9, m21 ; __ 0 5129 call m(idct_16x16_internal_8bpc).main_fast4 5130 punpckhwd m21, m21 ; 2 5131 punpckhwd m15, m15 ; 14 5132 punpckhwd m18, m18 ; 10 5133 punpckhwd m14, m14 ; 6 5134 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 5135 vinserti32x8 m24, m28, ym27, 1 5136 vshufi32x4 m28, m27, q3232 5137 vinserti32x8 m27, m22, ym23, 1 5138 vshufi32x4 m22, m23, q3232 5139 vshufi32x4 m23, m24, m27, q2020 ; 1 3 5140 vshufi32x4 m24, m27, q3131 ; 5 7 5141 vshufi32x4 m27, m28, m22, q2020 ; 9 11 5142 vshufi32x4 m28, m22, q3131 ; 13 15 5143 punpcklwd m22, m23, m23 ; 1 5144 punpckhwd m29, m28, m28 ; 15 5145 punpcklwd m26, m27, m27 ; 9 5146 punpckhwd m25, m24, m24 ; 7 5147 mova [cq+64* 8], m14 5148 mova [cq+64* 9], m15 5149 mova [cq+64*10], m16 5150 mova [cq+64*11], m17 5151 punpcklwd m24, m24 ; 5 5152 punpckhwd m27, m27 ; 11 5153 punpcklwd m28, m28 ; 13 5154 punpckhwd m23, m23 ; 3 5155 mova [cq+64*12], m18 5156 mova [cq+64*13], m19 5157 mova [cq+64*14], m20 5158 mova [cq+64*15], m21 5159 call .main_oddhalf_fast 5160 .end: 5161 imul r6, strideq, 60 5162 mova m10, [o(end_16x32p)] 5163 vpbroadcastd m11, [o(pw_2048)] 5164 lea r3, [strideq*3] 5165 pxor m12, m12 5166 add r6, dstq ; dst+stride*60 5167 psrldq m13, m10, 1 5168 lea r4, [strideq+r3] ; stride*4 5169 %macro IDCT_16x64_END 3 ; idct32, idct64, tmp 5170 %if %1 & 1 5171 %define %%s0 r3 5172 %define %%s1 strideq*2 5173 %define %%s2 strideq*1 5174 %define %%s3 strideq*0 5175 %else 5176 %define %%s0 strideq*0 5177 %define %%s1 strideq*1 5178 %define %%s2 strideq*2 5179 %define %%s3 r3 5180 %if %1 5181 add dstq, r4 5182 sub r6, r4 5183 %endif 5184 %endif 5185 %if %1 < 8 5186 pmulhrsw m8, m11, m%1 5187 pmulhrsw m9, m11, m%2 5188 %else 5189 mova m9, [cq+64*%1] 5190 paddsw m8, m9, m%2 ; out 0+n, 1+n 5191 psubsw m9, m%2 ; out 63-n, 62-n 5192 pmulhrsw m8, m11 5193 pmulhrsw m9, m11 5194 %endif 5195 mova xm29, [dstq+%%s0] 5196 vinserti128 ym29, [dstq+%%s1], 1 5197 mova xm%3, [r6 +%%s3] 5198 vinserti128 ym%3, [r6 +%%s2], 1 5199 vpermb m29, m10, m29 5200 vpermb m%3, m10, m%3 5201 mova [cq+64*%1], m12 5202 paddw m29, m8 5203 paddw m%3, m9 5204 packuswb m29, m%3 5205 vpermd m29, m13, m29 5206 mova [dstq+%%s0], xm29 5207 vextracti128 [dstq+%%s1], ym29, 1 5208 vextracti32x4 [r6 +%%s2], m29, 2 5209 vextracti32x4 [r6 +%%s3], m29, 3 5210 %endmacro 5211 IDCT_16x64_END 0, 29, 0 5212 IDCT_16x64_END 1, 28, 28 5213 IDCT_16x64_END 2, 27, 28 5214 IDCT_16x64_END 3, 26, 28 5215 IDCT_16x64_END 4, 25, 28 5216 IDCT_16x64_END 5, 24, 28 5217 IDCT_16x64_END 6, 23, 28 5218 IDCT_16x64_END 7, 22, 28 5219 IDCT_16x64_END 8, 21, 28 5220 IDCT_16x64_END 9, 20, 28 5221 IDCT_16x64_END 10, 19, 28 5222 IDCT_16x64_END 11, 18, 28 5223 IDCT_16x64_END 12, 17, 28 5224 IDCT_16x64_END 13, 16, 28 5225 IDCT_16x64_END 14, 15, 28 5226 IDCT_16x64_END 15, 14, 28 5227 RET 5228 .dconly: 5229 movsx r6d, word [cq] 5230 mov [cq], eobd 5231 or r3d, 64 5232 imul r6d, 181 5233 add r6d, 128+512 5234 sar r6d, 8+2 5235 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3 5236 ALIGN function_align 5237 cglobal_label .main_oddhalf_fast ; bottom three-quarters are zero 5238 vpbroadcastd m8, [o(pw_101_4095x8)] 5239 vpbroadcastd m21, [o(pw_m1474_3822x8)] 5240 vpbroadcastd m14, [o(pw_897_3996x8)] 5241 vpbroadcastd m17, [o(pw_m700_4036x8)] 5242 vpbroadcastd m18, [o(pw_501_4065x8)] 5243 vpbroadcastd m19, [o(pw_m1092_3948x8)] 5244 vpbroadcastd m16, [o(pw_1285_3889x8)] 5245 vpbroadcastd m15, [o(pw_m301_4085x8)] 5246 pmulhrsw m8, m22 ; t32a t63a 5247 pmulhrsw m21, m29 ; t35a t60a 5248 pmulhrsw m14, m26 ; t36a t59a 5249 pmulhrsw m17, m25 ; t39a t56 5250 pmulhrsw m18, m24 ; t40a t55a 5251 pmulhrsw m19, m27 ; t43a t52a 5252 pmulhrsw m16, m28 ; t44a t51a 5253 pmulhrsw m15, m23 ; t47a t48a 5254 mova m22, m8 5255 mova m29, m21 5256 mova m26, m14 5257 mova m25, m17 5258 mova m24, m18 5259 mova m27, m19 5260 mova m28, m16 5261 mova m20, m15 5262 jmp .main_oddhalf2 5263 ALIGN function_align 5264 cglobal_label .main_oddhalf 5265 vpbroadcastd m8, [o(pw_101_4095x8)] 5266 vpbroadcastd m9, [o(pw_m2824_2967x8)] 5267 vpbroadcastd m11, [o(pw_1660_3745x8)] 5268 vpbroadcastd m12, [o(pw_m1474_3822x8)] 5269 pmulhrsw m22, m8 ; t32a t63a 5270 vpbroadcastd m8, [o(pw_897_3996x8)] 5271 pmulhrsw m21, m9 ; t33a t62a 5272 vpbroadcastd m9, [o(pw_m2191_3461x8)] 5273 pmulhrsw m14, m11 ; t34a t61a 5274 vpbroadcastd m11, [o(pw_2359_3349x8)] 5275 pmulhrsw m29, m12 ; t35a t60a 5276 vpbroadcastd m12, [o(pw_m700_4036x8)] 5277 pmulhrsw m26, m8 ; t36a t59a 5278 vpbroadcastd m8, [o(pw_501_4065x8)] 5279 pmulhrsw m17, m9 ; t37a t58a 5280 vpbroadcastd m9, [o(pw_m2520_3229x8)] 5281 pmulhrsw m18, m11 ; t38a t57a 5282 vpbroadcastd m11, [o(pw_2019_3564x8)] 5283 pmulhrsw m25, m12 ; t39a t56a 5284 vpbroadcastd m12, [o(pw_m1092_3948x8)] 5285 pmulhrsw m24, m8 ; t40a t55a 5286 vpbroadcastd m8, [o(pw_1285_3889x8)] 5287 pmulhrsw m19, m9 ; t41a t54a 5288 vpbroadcastd m9, [o(pw_m1842_3659x8)] 5289 pmulhrsw m16, m11 ; t42a t53a 5290 vpbroadcastd m11, [o(pw_2675_3102x8)] 5291 pmulhrsw m27, m12 ; t43a t52a 5292 vpbroadcastd m12, [o(pw_m301_4085x8)] 5293 pmulhrsw m28, m8 ; t44a t51a 5294 pmulhrsw m15, m9 ; t45a t50a 5295 pmulhrsw m20, m11 ; t46a t49a 5296 pmulhrsw m23, m12 ; t47a t48a 5297 psubsw m8, m22, m21 ; t33 t62 5298 paddsw m22, m21 ; t32 t63 5299 psubsw m21, m29, m14 ; t34 t61 5300 paddsw m29, m14 ; t35 t60 5301 psubsw m14, m26, m17 ; t37 t58 5302 paddsw m26, m17 ; t36 t59 5303 psubsw m17, m25, m18 ; t38 t57 5304 paddsw m25, m18 ; t39 t56 5305 psubsw m18, m24, m19 ; t41 t54 5306 paddsw m24, m19 ; t40 t55 5307 psubsw m19, m27, m16 ; t42 t53 5308 paddsw m27, m16 ; t43 t52 5309 psubsw m16, m28, m15 ; t45 t50 5310 paddsw m28, m15 ; t44 t51 5311 psubsw m15, m23, m20 ; t46 t49 5312 paddsw m20, m23 ; t47 t48 5313 .main_oddhalf2: 5314 ITX_MUL2X_PACK 8, 9, 23, 10, 401, 4076, 5 ; t33a t62a 5315 ITX_MUL2X_PACK 21, 9, 23, 10, m4076, 401, 5 ; t34a t61a 5316 ITX_MUL2X_PACK 14, 9, 23, 10, 3166, 2598, 5 ; t37a t58a 5317 ITX_MUL2X_PACK 17, 9, 23, 10, m2598, 3166, 5 ; t38a t57a 5318 ITX_MUL2X_PACK 18, 9, 23, 10, 1931, 3612, 5 ; t41a t54a 5319 ITX_MUL2X_PACK 19, 9, 23, 10, m3612, 1931, 5 ; t42a t53a 5320 ITX_MUL2X_PACK 16, 9, 23, 10, 3920, 1189, 5 ; t45a t50a 5321 ITX_MUL2X_PACK 15, 9, 23, 10, m1189, 3920, 5 ; t46a t49a 5322 vpbroadcastd m11, [o(pw_m4017_799)] 5323 psubsw m23, m25, m26 ; t36a t59a 5324 paddsw m25, m26 ; t39a t56a 5325 psubsw m26, m24, m27 ; t43a t52a 5326 paddsw m27, m24 ; t40a t55a 5327 psubsw m24, m20, m28 ; t44a t51a 5328 paddsw m20, m28 ; t47a t48a 5329 psubsw m28, m8, m21 ; t34 t61 5330 paddsw m8, m21 ; t33 t62 5331 psubsw m21, m17, m14 ; t37 t58 5332 paddsw m17, m14 ; t38 t57 5333 psubsw m14, m18, m19 ; t42 t53 5334 paddsw m18, m19 ; t41 t54 5335 psubsw m19, m15, m16 ; t45 t50 5336 paddsw m15, m16 ; t46 t49 5337 psubsw m16, m22, m29 ; t35a t60a 5338 paddsw m22, m29 ; t32a t63a 5339 ITX_MUL2X_PACK 16, 9, 29, 10, 799_4017, 11, 20 ; t35 t60 5340 ITX_MUL2X_PACK 28, 9, 29, 10, 799_4017, 11, 20 ; t34a t61a 5341 ITX_MUL2X_PACK 23, 9, 29, 10, 11, m799_m4017, 36 ; t36 t59 5342 ITX_MUL2X_PACK 21, 9, 29, 10, 11, m799_m4017, 36 ; t37a t58a 5343 vpbroadcastd m11, [o(pw_m2276_3406)] 5344 ITX_MUL2X_PACK 26, 9, 29, 10, 3406_2276, 11, 20 ; t43 t52 5345 ITX_MUL2X_PACK 14, 9, 29, 10, 3406_2276, 11, 20 ; t42a t53a 5346 ITX_MUL2X_PACK 24, 9, 29, 10, 11, m3406_m2276, 36 ; t44 t51 5347 ITX_MUL2X_PACK 19, 9, 29, 10, 11, m3406_m2276, 36 ; t45a t50a 5348 vpbroadcastd m11, [o(pw_1567_3784)] 5349 vpbroadcastd m12, [o(pw_m3784_1567)] 5350 psubsw m29, m22, m25 ; t39 t56 5351 paddsw m22, m25 ; t32 t63 5352 psubsw m25, m20, m27 ; t40 t55 5353 paddsw m20, m27 ; t47 t48 5354 psubsw m27, m8, m17 ; t38a t57a 5355 paddsw m8, m17 ; t33a t62a 5356 psubsw m17, m15, m18 ; t41a t54a 5357 paddsw m15, m18 ; t46a t49a 5358 paddsw m18, m16, m23 ; t35a t60a 5359 psubsw m16, m23 ; t36a t59a 5360 psubsw m23, m24, m26 ; t43a t52a 5361 paddsw m24, m26 ; t44a t51a 5362 paddsw m26, m28, m21 ; t34 t61 5363 psubsw m28, m21 ; t37 t58 5364 psubsw m21, m19, m14 ; t42 t53 5365 paddsw m19, m14 ; t45 t50 5366 ITX_MUL2X_PACK 29, 9, 14, 10, 11, 12, 4 ; t39a t56a 5367 ITX_MUL2X_PACK 27, 9, 14, 10, 11, 12, 4 ; t38 t57 5368 ITX_MUL2X_PACK 16, 9, 14, 10, 11, 12, 4 ; t36 t59 5369 ITX_MUL2X_PACK 28, 9, 14, 10, 11, 12, 4 ; t37a t58a 5370 vpbroadcastd m11, [o(pw_m1567_m3784)] 5371 ITX_MUL2X_PACK 25, 9, 14, 10, 12, 11, 4 ; t40a t55a 5372 ITX_MUL2X_PACK 17, 9, 14, 10, 12, 11, 4 ; t41 t54 5373 ITX_MUL2X_PACK 23, 9, 14, 10, 12, 11, 4 ; t43 t52 5374 ITX_MUL2X_PACK 21, 9, 14, 10, 12, 11, 4 ; t42a t53a 5375 vbroadcasti32x4 m13, [o(deint_shuf)] 5376 vpbroadcastd m11, [o(pw_2896_2896)] 5377 vpbroadcastd m12, [o(pw_m2896_2896)] 5378 paddsw m14, m22, m20 ; t32a t63a 5379 psubsw m22, m20 ; t47a t48a 5380 psubsw m20, m8, m15 ; t46 t49 5381 paddsw m8, m15 ; t33 t62 5382 paddsw m15, m18, m24 ; t35 t60 5383 psubsw m18, m24 ; t44 t51 5384 psubsw m24, m26, m19 ; t45a t50a 5385 paddsw m26, m19 ; t34a t61a 5386 REPX {pshufb x, m13}, m14, m8, m15, m26 5387 psubsw m19, m29, m25 ; t40 t55 5388 paddsw m25, m29 ; t39 t56 5389 psubsw m29, m27, m17 ; t41a t54a 5390 paddsw m27, m17 ; t38a t57a 5391 psubsw m17, m16, m23 ; t43a t52a 5392 paddsw m16, m23 ; t36a t59a 5393 psubsw m9, m28, m21 ; t42 t53 5394 paddsw m28, m21 ; t37 t58 5395 REPX {pshufb x, m13}, m25, m27, m16, m28 5396 ITX_MUL2X_PACK 22, 13, 21, 10, 11, 12, 8 ; t47 t48 5397 ITX_MUL2X_PACK 20, 23, 22, 10, 11, 12, 8 ; t46a t49a 5398 packssdw m21, m22 ; t47 t46a 5399 packssdw m13, m23 ; t48 t49a 5400 ITX_MUL2X_PACK 18, 22, 20, 10, 11, 12, 8 ; t44a t51a 5401 ITX_MUL2X_PACK 24, 23, 18, 10, 11, 12, 8 ; t45 t50 5402 packssdw m20, m18 ; t44a t45 5403 packssdw m22, m23 ; t51a t50 5404 ITX_MUL2X_PACK 19, 24, 18, 10, 11, 12, 8 ; t40a t55a 5405 ITX_MUL2X_PACK 29, 23, 19, 10, 11, 12, 8 ; t41 t54 5406 packssdw m18, m19 ; t40a t41 5407 packssdw m24, m23 ; t55a t54 5408 ITX_MUL2X_PACK 17, 23, 19, 10, 11, 12, 8 ; t43 t52 5409 ITX_MUL2X_PACK 9, 29, 17, 10, 11, 12, 8 ; t42a t53a 5410 packssdw m19, m17 ; t43 t42a 5411 packssdw m23, m29 ; t52 t53a 5412 punpcklqdq m17, m25, m27 ; t39 t38a 5413 punpckhqdq m25, m27 ; t56 t57a 5414 punpckhqdq m27, m15, m26 ; t60 t61a 5415 punpcklqdq m15, m26 ; t35 t34a 5416 punpckhqdq m26, m16, m28 ; t59a t58 5417 punpcklqdq m16, m28 ; t36a t37 5418 punpckhqdq m28, m14, m8 ; t63a t62 5419 punpcklqdq m14, m8 ; t32a t33 5420 psubsw m29, m0, m28 ; out63 out62 5421 paddsw m0, m28 ; out0 out1 5422 psubsw m28, m1, m27 ; out60 out61 5423 paddsw m1, m27 ; out3 out2 5424 psubsw m27, m2, m26 ; out59 out58 5425 paddsw m2, m26 ; out4 out5 5426 psubsw m26, m3, m25 ; out56 out57 5427 paddsw m3, m25 ; out7 out6 5428 psubsw m25, m4, m24 ; out55 out54 5429 paddsw m4, m24 ; out8 out9 5430 psubsw m24, m5, m23 ; out52 out53 5431 paddsw m5, m23 ; out11 out10 5432 psubsw m23, m6, m22 ; out51 out50 5433 paddsw m6, m22 ; out12 out13 5434 psubsw m22, m7, m13 ; out48 out49 5435 paddsw m7, m13 ; out15 out14 5436 ret 5437 5438 cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 7, 0, dst, stride, c, eob 5439 %undef cmp 5440 lea r5, [o_base] 5441 test eobd, eobd 5442 jnz .normal 5443 movsx r6d, word [cq] 5444 mov [cq], eobd 5445 or r3d, 16 5446 .dconly: 5447 imul r6d, 181 5448 add r6d, 128+512 5449 sar r6d, 8+2 5450 .dconly2: 5451 imul r6d, 181 5452 add r6d, 128+2048 5453 sar r6d, 8+4 5454 pxor m2, m2 5455 vpbroadcastw m3, r6d 5456 .dconly_loop: 5457 mova m1, [dstq] 5458 punpcklbw m0, m1, m2 5459 punpckhbw m1, m2 5460 paddw m0, m3 5461 paddw m1, m3 5462 packuswb m0, m1 5463 mova [dstq], m0 5464 add dstq, strideq 5465 dec r3d 5466 jg .dconly_loop 5467 RET 5468 .normal: 5469 WIN64_SPILL_XMM 31 5470 mova m19, [o(dup16_perm)] 5471 mova m24, [cq+64* 2] 5472 mova m28, [cq+64* 6] 5473 mova m26, [cq+64* 4] 5474 mova m22, [cq+64* 0] 5475 mova m23, [cq+64* 1] 5476 mova m29, [cq+64* 7] 5477 mova m27, [cq+64* 5] 5478 mova m25, [cq+64* 3] 5479 vpermb m8, m19, m24 ; 4 5480 vpermb m1, m19, m28 ; 12 5481 vpermb m7, m19, m26 ; 8 5482 vpermb m9, m19, m22 ; __ 0 5483 vpermb m21, m19, m23 ; 2 5484 vpermb m15, m19, m29 ; 14 5485 vpermb m18, m19, m27 ; 10 5486 vpermb m14, m19, m25 ; 6 5487 pslld m9, 16 5488 vpord m30, m19, [o(pb_32)] {1to16} 5489 REPX {vpermb x, m30, x}, m22, m29, m26, m25, m24, m27, m28, m23 5490 cmp eobd, 151 5491 jb .fast 5492 vpermb m0, m19, [cq+64*14] ; 28 5493 vpermb m5, m19, [cq+64*10] ; 20 5494 vpermb m3, m19, [cq+64*12] ; 24 5495 vpermb m6, m19, [cq+64* 8] ; __ 16 5496 pslld m6, 16 5497 call m(idct_16x16_internal_8bpc).main_fast 5498 vpermb m17, m19, [cq+64*15] ; 30 5499 vpermb m20, m19, [cq+64* 9] ; 18 5500 vpermb m16, m19, [cq+64*11] ; 22 5501 vpermb m19, m19, [cq+64*13] ; 26 5502 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast 5503 mova [cq+64* 0], m14 5504 mova [cq+64* 1], m15 5505 mova [cq+64* 2], m16 5506 mova [cq+64* 3], m17 5507 mova [cq+64* 4], m18 5508 mova [cq+64* 5], m19 5509 mova [cq+64* 6], m20 5510 mova [cq+64* 7], m21 5511 vpermb m21, m30, [cq+64*15] 5512 vpermb m14, m30, [cq+64* 8] 5513 vpermb m17, m30, [cq+64*11] 5514 vpermb m18, m30, [cq+64*12] 5515 vpermb m19, m30, [cq+64*13] 5516 vpermb m16, m30, [cq+64*10] 5517 vpermb m15, m30, [cq+64* 9] 5518 vpermb m20, m30, [cq+64*14] 5519 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf 5520 jmp .end 5521 .fast: ; bottom half is zero 5522 call m(idct_16x16_internal_8bpc).main_fast2 5523 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 5524 mova [cq+64* 0], m14 5525 mova [cq+64* 1], m15 5526 mova [cq+64* 2], m16 5527 mova [cq+64* 3], m17 5528 mova [cq+64* 4], m18 5529 mova [cq+64* 5], m19 5530 mova [cq+64* 6], m20 5531 mova [cq+64* 7], m21 5532 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast 5533 .end: 5534 mova [cq+64* 8], m4 5535 mova [cq+64* 9], m5 5536 mova [cq+64*10], m6 5537 mova [cq+64*11], m7 5538 mova [cq+64*12], m26 5539 mova [cq+64*13], m27 5540 mova [cq+64*14], m28 5541 mova [cq+64*15], m29 5542 vpbroadcastd m13, [o(pw_8192)] 5543 call .pass1_end 5544 call .pass2 5545 mova [cq+64* 0], m0 5546 mova [cq+64* 1], m1 5547 mova [cq+64* 2], m2 5548 mova [cq+64* 3], m3 5549 mova [cq+64* 4], m4 5550 mova [cq+64* 5], m5 5551 mova [cq+64* 6], m6 5552 mova [cq+64* 7], m7 5553 pmulhrsw m0, m13, [cq+64* 8] 5554 pmulhrsw m1, m13, [cq+64* 9] 5555 pmulhrsw m2, m13, [cq+64*10] 5556 pmulhrsw m3, m13, [cq+64*11] 5557 vpbroadcastd m30, [o(pw_2048)] 5558 pmulhrsw m4, m13, m22 5559 pmulhrsw m5, m13, m23 5560 pmulhrsw m6, m13, m24 5561 pmulhrsw m7, m13, m25 5562 pmulhrsw m22, m30, m14 5563 pmulhrsw m14, m13, m26 5564 pmulhrsw m23, m30, m15 5565 pmulhrsw m15, m13, m27 5566 pmulhrsw m24, m30, m16 5567 pmulhrsw m16, m13, m28 5568 pmulhrsw m25, m30, m17 5569 pmulhrsw m17, m13, m29 5570 pmulhrsw m26, m30, m18 5571 pmulhrsw m18, m13, [cq+64*12] 5572 pmulhrsw m27, m30, m19 5573 pmulhrsw m19, m13, [cq+64*13] 5574 pmulhrsw m28, m30, m20 5575 pmulhrsw m20, m13, [cq+64*14] 5576 pmulhrsw m29, m30, m21 5577 pmulhrsw m21, m13, [cq+64*15] 5578 call .transpose_round 5579 call .pass2 5580 pxor m10, m10 5581 lea r3, [strideq*3] 5582 %macro IDCT_64x16_END 4 5583 mova m9, [dstq+%4] 5584 %if %1 < 8 5585 pmulhrsw m%3, m30, [cq+64*%1] 5586 %endif 5587 pmulhrsw m%2, m30 5588 mova [cq+64*%1], m10 5589 punpcklbw m8, m9, m10 5590 punpckhbw m9, m10 5591 paddw m8, m%3 5592 paddw m9, m%2 5593 packuswb m8, m9 5594 mova [dstq+%4], m8 5595 %if %1 == 3 || %1 == 7 || %1 == 11 5596 lea dstq, [dstq+strideq*4] 5597 %endif 5598 %endmacro 5599 IDCT_64x16_END 0, 0, 11, strideq*0 5600 IDCT_64x16_END 1, 1, 11, strideq*1 5601 IDCT_64x16_END 2, 2, 11, strideq*2 5602 IDCT_64x16_END 3, 3, 11, r3 5603 IDCT_64x16_END 4, 4, 11, strideq*0 5604 IDCT_64x16_END 5, 5, 11, strideq*1 5605 IDCT_64x16_END 6, 6, 11, strideq*2 5606 IDCT_64x16_END 7, 7, 11, r3 5607 IDCT_64x16_END 8, 14, 22, strideq*0 5608 IDCT_64x16_END 9, 15, 23, strideq*1 5609 IDCT_64x16_END 10, 16, 24, strideq*2 5610 IDCT_64x16_END 11, 17, 25, r3 5611 IDCT_64x16_END 12, 18, 26, strideq*0 5612 IDCT_64x16_END 13, 19, 27, strideq*1 5613 IDCT_64x16_END 14, 20, 28, strideq*2 5614 IDCT_64x16_END 15, 21, 29, r3 5615 RET 5616 ALIGN function_align 5617 .pass1_end: 5618 mova m4, [cq+64* 0] 5619 mova m5, [cq+64* 1] 5620 mova m6, [cq+64* 2] 5621 mova m7, [cq+64* 3] 5622 mova m8, [cq+64* 4] 5623 mova m9, [cq+64* 5] 5624 mova m11, [cq+64* 6] 5625 mova m12, [cq+64* 7] 5626 psubsw m29, m4, m21 ; out47 out46 5627 paddsw m4, m21 ; out16 out17 5628 psubsw m28, m5, m20 ; out44 out45 5629 paddsw m5, m20 ; out19 out18 5630 REPX {pmulhrsw x, m13}, m0, m1, m2, m3 5631 psubsw m27, m6, m19 ; out43 out42 5632 paddsw m6, m19 ; out20 out21 5633 psubsw m26, m7, m18 ; out40 out41 5634 paddsw m7, m18 ; out23 out22 5635 pmulhrsw m18, m13, m22 5636 pmulhrsw m19, m13, m23 5637 pmulhrsw m20, m13, m24 5638 pmulhrsw m21, m13, m25 5639 paddsw m25, m12, m14 ; out31 out30 5640 psubsw m14, m12, m14 ; out32 out33 5641 paddsw m24, m11, m15 ; out28 out29 5642 psubsw m15, m11, m15 ; out35 out34 5643 REPX {pmulhrsw x, m13}, m4, m5, m6, m7 5644 paddsw m23, m9, m16 ; out27 out26 5645 psubsw m16, m9, m16 ; out36 out37 5646 paddsw m22, m8, m17 ; out24 out25 5647 psubsw m17, m8, m17 ; out39 out38 5648 REPX {pmulhrsw x, m13}, m14, m15, m16, m17 5649 .transpose_round: 5650 %macro TRANSPOSE_8x4_PACKED 4 5651 punpckhwd m8, m%1, m%3 ; b0 f0 b1 f1 b2 f2 b3 f3 5652 punpcklwd m%1, m%3 ; a0 e0 a1 e1 a2 e2 a3 e3 5653 punpcklwd m%3, m%2, m%4 ; d0 h0 d1 h1 d2 h2 d3 h3 5654 punpckhwd m%2, m%4 ; c0 g0 c1 g1 c2 g2 c3 g3 5655 punpckhwd m%4, m%1, m%2 ; a2 c2 e2 g2 a3 c3 e3 g3 5656 punpcklwd m%1, m%2 ; a0 c0 e0 g0 a1 c1 e1 g1 5657 punpckhwd m%2, m8, m%3 ; b2 d2 f2 h2 b3 d3 f3 h3 5658 punpcklwd m8, m%3 ; b0 d0 f0 h0 b1 d1 f1 h1 5659 punpcklwd m%3, m%4, m%2 ; 2 5660 punpckhwd m%4, m%2 ; 3 5661 punpckhwd m%2, m%1, m8 ; 1 5662 punpcklwd m%1, m8 ; 0 5663 %endmacro 5664 TRANSPOSE_8x4_PACKED 0, 1, 2, 3 5665 TRANSPOSE_8x4_PACKED 18, 19, 20, 21 5666 TRANSPOSE_8x4_PACKED 4, 5, 6, 7 5667 TRANSPOSE_8x4_PACKED 14, 15, 16, 17 5668 vshufi32x4 m8, m0, m4, q3232 ; a02 a03 b02 b03 5669 vinserti32x8 m0, ym4, 1 ; a00 a01 b00 b01 5670 vshufi32x4 m4, m1, m5, q3232 ; a12 a13 b12 b13 5671 vinserti32x8 m9, m1, ym5, 1 ; a10 a11 b10 b11 5672 vshufi32x4 m5, m2, m6, q3232 ; a22 a23 b22 b23 5673 vinserti32x8 m1, m2, ym6, 1 ; a20 a21 b20 b21 5674 vshufi32x4 m6, m3, m7, q3232 ; a32 a33 b32 b33 5675 vinserti32x8 m11, m3, ym7, 1 ; a30 a31 b30 b31 5676 vshufi32x4 m2, m14, m18, q3232 ; c02 c03 d02 d03 5677 vinserti32x8 m3, m14, ym18, 1 ; c00 c01 d00 d01 5678 vshufi32x4 m18, m15, m19, q3232 ; c12 c13 d12 d13 5679 vinserti32x8 m15, ym19, 1 ; c10 c11 d10 d11 5680 vshufi32x4 m19, m16, m20, q3232 ; c22 c23 d22 d23 5681 vinserti32x8 m16, ym20, 1 ; c20 c21 d20 d21 5682 vshufi32x4 m20, m17, m21, q3232 ; c32 c33 d32 d33 5683 vinserti32x8 m17, ym21, 1 ; c30 c31 d30 d31 5684 ret 5685 .pass2: 5686 vshufi32x4 m7, m5, m19, q3131 ; 14 5687 vshufi32x4 m5, m19, q2020 ; 10 5688 vshufi32x4 m21, m6, m20, q3131 ; 15 5689 vshufi32x4 m19, m6, m20, q2020 ; 11 5690 vshufi32x4 m20, m4, m18, q3131 ; 13 5691 vshufi32x4 m18, m4, m18, q2020 ; 9 5692 vshufi32x4 m6, m8, m2, q3131 ; 12 5693 vshufi32x4 m4, m8, m2, q2020 ; 8 5694 vshufi32x4 m2, m0, m3, q3131 ; 4 5695 vshufi32x4 m0, m3, q2020 ; 0 5696 vshufi32x4 m3, m1, m16, q3131 ; 6 5697 vshufi32x4 m1, m16, q2020 ; 2 5698 vshufi32x4 m16, m9, m15, q3131 ; 5 5699 vshufi32x4 m14, m9, m15, q2020 ; 1 5700 vshufi32x4 m15, m11, m17, q2020 ; 3 5701 vshufi32x4 m17, m11, m17, q3131 ; 7 5702 call m(inv_txfm_add_dct_dct_32x8_8bpc).main2 5703 jmp m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 5704 5705 cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 7, 0, dst, stride, c, eob 5706 lea r5, [o_base] 5707 test eobd, eobd 5708 jz .dconly 5709 PROLOGUE 0, 9, 30, 64*32, dst, stride, c, eob 5710 vpbroadcastd m23, [o(pw_2896x8)] 5711 %undef cmp 5712 cmp eobd, 136 5713 jb .fast 5714 pmulhrsw m5, m23, [cq+64*20] 5715 pmulhrsw m3, m23, [cq+64*12] 5716 pmulhrsw m1, m23, [cq+64* 4] 5717 pmulhrsw m7, m23, [cq+64*28] 5718 pmulhrsw m2, m23, [cq+64* 8] 5719 pmulhrsw m6, m23, [cq+64*24] 5720 pmulhrsw m0, m23, [cq+64* 0] 5721 pmulhrsw m4, m23, [cq+64*16] 5722 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 5723 pmulhrsw m14, m23, [cq+64* 2] 5724 pmulhrsw m21, m23, [cq+64*30] 5725 pmulhrsw m18, m23, [cq+64*18] 5726 pmulhrsw m17, m23, [cq+64*14] 5727 pmulhrsw m16, m23, [cq+64*10] 5728 pmulhrsw m19, m23, [cq+64*22] 5729 pmulhrsw m20, m23, [cq+64*26] 5730 pmulhrsw m15, m23, [cq+64* 6] 5731 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 5732 mova [cq+64* 0], m14 5733 mova [cq+64* 2], m15 5734 mova [cq+64* 4], m16 5735 mova [cq+64* 6], m17 5736 mova [cq+64* 8], m18 5737 mova [cq+64*10], m19 5738 mova [cq+64*12], m20 5739 mova [cq+64*14], m21 5740 pmulhrsw m22, m23, [cq+64* 1] 5741 pmulhrsw m21, m23, [cq+64*31] 5742 pmulhrsw m14, m23, [cq+64*17] 5743 pmulhrsw m29, m23, [cq+64*15] 5744 pmulhrsw m26, m23, [cq+64* 9] 5745 pmulhrsw m17, m23, [cq+64*23] 5746 pmulhrsw m18, m23, [cq+64*25] 5747 pmulhrsw m25, m23, [cq+64* 7] 5748 pmulhrsw m24, m23, [cq+64* 5] 5749 pmulhrsw m19, m23, [cq+64*27] 5750 pmulhrsw m16, m23, [cq+64*21] 5751 pmulhrsw m27, m23, [cq+64*11] 5752 pmulhrsw m28, m23, [cq+64*13] 5753 pmulhrsw m15, m23, [cq+64*19] 5754 pmulhrsw m20, m23, [cq+64*29] 5755 pmulhrsw m23, [cq+64* 3] 5756 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf 5757 vpbroadcastd m12, [o(pw_16384)] 5758 psubsw m13, m0, m29 ; 31 5759 paddsw m0, m29 ; 0 5760 psubsw m29, m1, m28 ; 30 5761 paddsw m1, m28 ; 1 5762 psubsw m28, m2, m27 ; 29 5763 paddsw m2, m27 ; 2 5764 psubsw m27, m3, m26 ; 28 5765 paddsw m3, m26 ; 3 5766 psubsw m26, m4, m25 ; 27 5767 paddsw m4, m25 ; 4 5768 psubsw m25, m5, m24 ; 26 5769 paddsw m5, m24 ; 5 5770 psubsw m24, m6, m23 ; 25 5771 paddsw m6, m23 ; 6 5772 psubsw m23, m7, m22 ; 24 5773 paddsw m7, m22 ; 7 5774 pxor m9, m9 5775 punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7 5776 punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3 5777 punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7 5778 punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3 5779 REPX {mova [cq+64*x], m9}, 16, 17, 18, 19 5780 punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7 5781 punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3 5782 punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7 5783 punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3 5784 REPX {mova [cq+64*x], m9}, 20, 21, 22, 23 5785 punpckhwd m3, m23, m24 5786 punpcklwd m23, m24 5787 punpckhwd m24, m25, m26 5788 punpcklwd m25, m26 5789 REPX {mova [cq+64*x], m9}, 24, 25, 26, 27 5790 punpckhwd m26, m27, m28 5791 punpcklwd m27, m28 5792 punpckhwd m28, m29, m13 5793 punpcklwd m29, m13 5794 REPX {mova [cq+64*x], m9}, 28, 29, 30, 31 5795 punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 5796 punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 5797 punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3 5798 punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1 5799 REPX {pmulhrsw x, m12}, m7, m0, m2, m4 5800 punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7 5801 punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5 5802 punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7 5803 punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5 5804 REPX {pmulhrsw x, m12}, m6, m8, m1, m22 5805 punpckhdq m13, m23, m25 5806 punpckldq m23, m25 5807 punpckhdq m25, m27, m29 5808 punpckldq m27, m29 5809 REPX {pmulhrsw x, m12}, m13, m23, m25, m27 5810 punpckhdq m9, m3, m24 5811 punpckldq m3, m24 5812 punpckhdq m24, m26, m28 5813 punpckldq m26, m28 5814 REPX {pmulhrsw x, m12}, m9, m3, m24, m26 5815 punpckhqdq m5, m23, m27 ; d01 d09 d17 d25 5816 punpcklqdq m23, m27 ; d00 d08 d16 d24 5817 punpcklqdq m27, m13, m25 ; d02 d10 d18 d26 5818 punpckhqdq m13, m25 ; d03 d11 d19 d27 5819 punpcklqdq m25, m3, m26 ; d04 d12 d20 d28 5820 punpckhqdq m3, m26 ; d05 d13 d21 d29 5821 punpcklqdq m26, m9, m24 ; d06 d14 d22 d30 5822 punpckhqdq m9, m24 ; d07 d15 d23 d31 5823 mova [cq+64* 3], m23 5824 mova [cq+64*13], m27 5825 mova [cq+64* 7], m25 5826 mova [cq+64*15], m26 5827 punpckhqdq m24, m8, m22 ; a05 a13 a21 a29 5828 punpcklqdq m8, m22 ; a04 a12 a20 a28 5829 punpckhqdq m22, m0, m4 ; a01 a09 a17 a25 5830 punpcklqdq m0, m4 ; a00 a08 a16 a24 5831 punpckhqdq m23, m7, m2 ; a03 a11 a19 a27 5832 punpcklqdq m7, m2 ; a02 a10 a18 a26 5833 punpckhqdq m25, m6, m1 ; a07 a15 a23 a31 5834 punpcklqdq m6, m1 ; a06 a14 a22 a30 5835 mova [cq+64* 1], m0 5836 mova [cq+64* 9], m7 5837 mova [cq+64* 5], m8 5838 mova [cq+64*11], m6 5839 mova m2, [cq+64* 0] 5840 mova m11, [cq+64* 2] 5841 mova m8, [cq+64* 4] 5842 mova m29, [cq+64* 6] 5843 mova m27, [cq+64* 8] 5844 mova m26, [cq+64*10] 5845 mova m4, [cq+64*12] 5846 mova m28, [cq+64*14] 5847 psubsw m1, m2, m21 ; 23 5848 paddsw m2, m21 ; 8 5849 psubsw m21, m11, m20 ; 22 5850 paddsw m11, m20 ; 9 5851 psubsw m20, m8, m19 ; 21 5852 paddsw m8, m19 ; 10 5853 psubsw m19, m29, m18 ; 20 5854 paddsw m29, m18 ; 11 5855 psubsw m18, m27, m17 ; 19 5856 paddsw m27, m17 ; 12 5857 psubsw m17, m26, m16 ; 18 5858 paddsw m26, m16 ; 13 5859 psubsw m16, m4, m15 ; 17 5860 paddsw m4, m15 ; 14 5861 psubsw m15, m28, m14 ; 16 5862 paddsw m28, m14 ; 15 5863 punpcklwd m14, m15, m16 5864 punpckhwd m15, m16 5865 punpckhwd m16, m17, m18 5866 punpcklwd m17, m18 5867 punpckhwd m18, m19, m20 5868 punpcklwd m19, m20 5869 punpckhwd m20, m21, m1 5870 punpcklwd m21, m1 5871 punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7 5872 punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3 5873 punpckhwd m11, m8, m29 ; k4 l4 k5 l5 k6 l6 k7 l7 5874 punpcklwd m8, m29 ; k0 l0 k1 l1 k2 l2 k3 l3 5875 punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7 5876 punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3 5877 punpckhwd m26, m4, m28 ; o4 p4 o5 p5 o6 p6 o7 p7 5878 punpcklwd m4, m28 ; o0 p0 o1 p1 o2 p2 o3 p3 5879 punpckhdq m28, m2, m8 ; i2 j2 k2 l2 i3 j3 k3 l3 5880 punpckldq m2, m8 ; i0 j0 k0 l0 i1 j1 k1 l1 5881 punpckhdq m8, m27, m4 ; m2 n2 o2 p2 m3 n3 o3 p3 5882 punpckldq m27, m4 ; m0 n0 o0 p0 m1 n1 o1 p1 5883 REPX {pmulhrsw x, m12}, m28, m2, m8, m27 5884 punpckhdq m4, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7 5885 punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5 5886 punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7 5887 punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5 5888 REPX {pmulhrsw x, m12}, m4, m1, m11, m29 5889 punpckhdq m26, m19, m21 5890 punpckldq m19, m21 5891 punpckhdq m21, m15, m16 5892 punpckldq m15, m16 5893 REPX {pmulhrsw x, m12}, m26, m19, m21, m15 5894 punpckhdq m16, m18, m20 5895 punpckldq m18, m20 5896 punpckhdq m20, m14, m17 5897 punpckldq m14, m17 5898 REPX {pmulhrsw x, m12}, m16, m18, m20, m14 5899 punpckhqdq m17, m28, m8 ; b03 b11 b19 b27 5900 punpcklqdq m28, m8 ; b02 b10 b18 b26 5901 punpckhqdq m8, m2, m27 ; b01 b09 b17 b25 5902 punpcklqdq m2, m27 ; b00 b08 b16 b24 5903 punpcklqdq m27, m1, m29 ; b04 b12 b20 b28 5904 punpckhqdq m1, m29 ; b05 b13 b21 b29 5905 punpcklqdq m29, m4, m11 ; b06 b14 b22 b30 5906 punpckhqdq m4, m11 ; b07 b15 b23 b31 5907 mova [cq+64* 0], m2 5908 mova [cq+64* 8], m28 5909 mova [cq+64* 4], m27 5910 mova [cq+64*10], m29 5911 punpckhqdq m27, m20, m26 ; c03 c11 c19 c27 5912 punpcklqdq m20, m26 ; c02 c10 c18 c26 5913 punpckhqdq m26, m14, m19 ; c01 c09 c17 c25 5914 punpcklqdq m14, m19 ; c00 c08 c16 c24 5915 punpckhqdq m28, m15, m18 ; c05 c13 c21 c29 5916 punpcklqdq m15, m18 ; c04 c12 c20 c28 5917 punpckhqdq m29, m21, m16 ; c07 c15 c23 c31 5918 punpcklqdq m21, m16 ; c06 c14 c22 c30 5919 mova [cq+64* 2], m14 5920 mova [cq+64*12], m20 5921 mova [cq+64* 6], m15 5922 mova [cq+64*14], m21 5923 vshufi32x4 m14, m22, m8, q3232 ; a17 a25 b17 b25 5924 vinserti32x8 m22, ym8, 1 ; a01 a09 b01 b09 5925 vshufi32x4 m15, m23, m17, q3232 ; a19 a27 b19 b27 5926 vinserti32x8 m23, ym17, 1 ; a03 a11 b03 b11 5927 vshufi32x4 m16, m24, m1, q3232 ; a21 a29 b21 b29 5928 vinserti32x8 m24, ym1, 1 ; a05 a13 b05 b13 5929 vshufi32x4 m17, m25, m4, q3232 ; a23 a31 b23 b31 5930 vinserti32x8 m25, ym4, 1 ; a07 a15 b07 b15 5931 vinserti32x8 m19, m26, ym5, 1 ; c01 c09 d01 d09 5932 vshufi32x4 m26, m5, q3232 ; c17 c25 d17 d25 5933 vinserti32x8 m20, m27, ym13, 1 ; c03 c11 d03 d11 5934 vshufi32x4 m27, m13, q3232 ; c19 c27 d19 d27 5935 vinserti32x8 m21, m28, ym3, 1 ; c05 c13 d05 d13 5936 vshufi32x4 m28, m3, q3232 ; c21 c29 d21 d29 5937 vinserti32x8 m18, m29, ym9, 1 ; c07 c15 d07 d15 5938 vshufi32x4 m29, m9, q3232 ; c23 c31 d23 d31 5939 mov r4, rsp 5940 vshufi32x4 m0, m22, m19, q2020 ; 1 5941 vshufi32x4 m1, m17, m29, q3131 ; 31 5942 vshufi32x4 m2, m14, m26, q2020 ; 17 5943 vshufi32x4 m3, m25, m18, q3131 ; 15 5944 call .main_part1 5945 vshufi32x4 m0, m25, m18, q2020 ; 7 5946 vshufi32x4 m1, m14, m26, q3131 ; 25 5947 vshufi32x4 m2, m17, m29, q2020 ; 23 5948 vshufi32x4 m3, m22, m19, q3131 ; 9 5949 call .main_part1 5950 vshufi32x4 m0, m24, m21, q2020 ; 5 5951 vshufi32x4 m1, m15, m27, q3131 ; 27 5952 vshufi32x4 m2, m16, m28, q2020 ; 21 5953 vshufi32x4 m3, m23, m20, q3131 ; 11 5954 call .main_part1 5955 vshufi32x4 m0, m23, m20, q2020 ; 3 5956 vshufi32x4 m1, m16, m28, q3131 ; 29 5957 vshufi32x4 m2, m15, m27, q2020 ; 19 5958 vshufi32x4 m3, m24, m21, q3131 ; 13 5959 call .main_part1 5960 call .main_part2 5961 mova m0, [cq+64* 1] ; a0 5962 mova m15, [cq+64* 0] ; b0 5963 mova m3, [cq+64* 2] ; c0 5964 mova m16, [cq+64* 3] ; d0 5965 mova m14, [cq+64* 5] ; a4 5966 mova m8, [cq+64* 4] ; b4 5967 mova m17, [cq+64* 6] ; c4 5968 mova m1, [cq+64* 7] ; d4 5969 vshufi32x4 m2, m0, m15, q3232 ; a16 a24 b16 b24 5970 vinserti32x8 m0, ym15, 1 ; a00 a08 b00 b08 5971 vshufi32x4 m15, m3, m16, q3232 ; c16 c24 d16 d24 5972 vinserti32x8 m3, ym16, 1 ; c00 c08 d00 d08 5973 vshufi32x4 m16, m14, m8, q3232 ; a20 a28 b20 b28 5974 vinserti32x8 m14, ym8, 1 ; a04 a12 b04 b12 5975 vshufi32x4 m8, m17, m1, q3232 ; c20 c28 d20 d28 5976 vinserti32x8 m17, ym1, 1 ; c04 c12 d04 d12 5977 vshufi32x4 m1, m0, m3, q3131 ; 8 5978 vshufi32x4 m0, m3, q2020 ; 0 5979 vshufi32x4 m3, m2, m15, q3131 ; 24 5980 vshufi32x4 m2, m15, q2020 ; 16 5981 vshufi32x4 m15, m14, m17, q3131 ; 12 5982 vshufi32x4 m14, m17, q2020 ; 4 5983 vshufi32x4 m17, m16, m8, q3131 ; 28 5984 vshufi32x4 m16, m8, q2020 ; 20 5985 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast 5986 mova m8, [cq+64* 8] 5987 mova m9, [cq+64*12] 5988 mova m11, [cq+64*10] 5989 mova m12, [cq+64*14] 5990 mova [cq+64* 0], m14 5991 mova [cq+64* 2], m15 5992 mova [cq+64* 4], m16 5993 mova [cq+64* 6], m17 5994 mova [cq+64* 8], m18 5995 mova [cq+64*10], m19 5996 mova [cq+64*12], m20 5997 mova [cq+64*14], m21 5998 mova m22, [cq+64* 9] 5999 mova m27, [cq+64*13] 6000 mova m23, [cq+64*11] 6001 mova m24, [cq+64*15] 6002 vshufi32x4 m26, m22, m8, q3232 ; a18 a26 b18 b26 6003 vinserti32x8 m22, ym8, 1 ; a02 a10 b02 b10 6004 vshufi32x4 m8, m9, m27, q3232 ; c18 c26 d18 d26 6005 vinserti32x8 m9, ym27, 1 ; c02 c10 d02 d10 6006 vshufi32x4 m27, m23, m11, q3232 ; a22 a30 b22 b30 6007 vinserti32x8 m23, ym11, 1 ; a06 a14 b06 b14 6008 vshufi32x4 m11, m12, m24, q3232 ; c22 c30 d22 d30 6009 vinserti32x8 m12, ym24, 1 ; c06 c14 d06 d14 6010 vshufi32x4 m28, m26, m8, q3131 ; 26 6011 vshufi32x4 m26, m8, q2020 ; 18 6012 vshufi32x4 m24, m22, m9, q3131 ; 10 6013 vshufi32x4 m22, m9, q2020 ; 2 6014 vshufi32x4 m29, m27, m11, q3131 ; 30 6015 vshufi32x4 m27, m11, q2020 ; 22 6016 vshufi32x4 m25, m23, m12, q3131 ; 14 6017 vshufi32x4 m23, m12, q2020 ; 6 6018 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast 6019 jmp .end 6020 .fast: ; bottom/right halves are zero 6021 pmulhrsw ym9, ym23, [cq+64* 0] 6022 pmulhrsw ym6, ym23, [cq+64* 8] 6023 mova m14, [o(dup16_perm)] 6024 pmulhrsw ym8, ym23, [cq+64* 2] 6025 pmulhrsw xm0, xm23, [cq+64*14] 6026 pmulhrsw xm5, xm23, [cq+64*10] 6027 pmulhrsw ym1, ym23, [cq+64* 6] 6028 pmulhrsw ym7, ym23, [cq+64* 4] 6029 pmulhrsw xm3, xm23, [cq+64*12] 6030 pmovzxwd m9, ym9 6031 pmovzxwd m6, ym6 6032 vpermb m8, m14, m8 6033 punpcklwd xm0, xm0 6034 vpermb ym5, ym14, ym5 6035 vpermb m1, m14, m1 6036 vpermb m7, m14, m7 6037 punpcklwd xm3, xm3 6038 pslld m9, 16 6039 pslld m6, 16 6040 call m(idct_16x16_internal_8bpc).main_fast 6041 vpmulhrsw ym21, ym23, [cq+64* 1] 6042 {evex}vpmulhrsw xm17, xm23, [cq+64*15] ; force EVEX encoding, which 6043 {evex}vpmulhrsw xm20, xm23, [cq+64* 9] ; reduces code size due to 6044 {evex}vpmulhrsw ym15, ym23, [cq+64* 7] ; compressed displacements 6045 {evex}vpmulhrsw ym18, ym23, [cq+64* 5] 6046 {evex}vpmulhrsw xm16, xm23, [cq+64*11] 6047 {evex}vpmulhrsw xm19, xm23, [cq+64*13] 6048 {evex}vpmulhrsw ym23, [cq+64* 3] 6049 vpermb m21, m14, m21 6050 punpcklwd xm17, xm17 6051 vpermb ym20, ym14, ym20 6052 vpermb m15, m14, m15 6053 vpermb m18, m14, m18 6054 vpermb ym16, ym14, ym16 6055 punpcklwd xm19, xm19 6056 vpermb m14, m14, m23 6057 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast 6058 vpbroadcastd m9, [o(pw_16384)] 6059 call m(inv_txfm_add_dct_dct_32x16_8bpc).transpose_round 6060 vshufi32x4 m16, m0, m3, q2020 ; 0 6061 vshufi32x4 m26, m0, m3, q3131 ; 4 6062 vshufi32x4 m0, m14, m2, q2020 ; 1 6063 vshufi32x4 m14, m2, q3131 ; 5 6064 vshufi32x4 m3, m19, m7, q3131 ; 15 6065 vshufi32x4 m19, m7, q2020 ; 11 6066 vshufi32x4 m27, m17, m9, q2020 ; 3 6067 vshufi32x4 m17, m9, q3131 ; 7 6068 vshufi32x4 m28, m20, m6, q2020 ; 9 6069 vshufi32x4 m20, m6, q3131 ; 13 6070 vshufi32x4 m22, m1, m18, q2020 ; 2 6071 vshufi32x4 m23, m1, m18, q3131 ; 6 6072 vshufi32x4 m24, m5, m15, q2020 ; 10 6073 vshufi32x4 m25, m5, m15, q3131 ; 14 6074 vshufi32x4 m15, m21, m4, q3131 ; 12 6075 vshufi32x4 m21, m21, m4, q2020 ; 8 6076 mov r4, rsp 6077 call .main_part1_fast 6078 mova m0, m17 6079 mova m3, m28 6080 call .main_part1_fast 6081 mova m0, m14 6082 mova m3, m19 6083 call .main_part1_fast 6084 mova m0, m27 6085 mova m3, m20 6086 call .main_part1_fast 6087 call .main_part2 6088 mova m0, m16 6089 mova m1, m21 6090 mova m14, m26 6091 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 6092 mova [cq+64*14], m21 6093 mova [cq+64* 0], m14 6094 mova [cq+64* 6], m17 6095 mova [cq+64* 8], m18 6096 mova [cq+64*10], m19 6097 mova [cq+64* 4], m16 6098 mova [cq+64* 2], m15 6099 mova [cq+64*12], m20 6100 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 6101 .end: 6102 lea r4, [strideq*3] 6103 vpbroadcastd m12, [o(pw_2048)] 6104 movshdup m13, [o(permD)] 6105 lea r5, [r4+strideq] ; stride*4 6106 lea r3, [dstq+r4*8] 6107 lea r6, [strideq+r5*8] ; stride*33 6108 lea r8, [r4+r5*8] ; stride*35 6109 add r3, r5 ; dst+stride*28 6110 lea r7, [r6+strideq] ; stride*34 6111 %macro IDCT_32x64_END 6 ; src, mem, stride[1-4] 6112 %if %2 < 8 6113 paddsw m10, m%2, m%1 6114 psubsw m11, m%2, m%1 6115 %else 6116 mova m11, [cq+64*(%2*2-16)] 6117 paddsw m10, m11, m%1 6118 psubsw m11, m%1 6119 %endif 6120 mova m9, [rsp+64*(31-%2)] 6121 mova m%1, [rsp+64*%2] 6122 paddsw m8, m10, m9 6123 psubsw m10, m9 6124 paddsw m9, m11, m%1 6125 pmovzxbw m0, [dstq+%3] 6126 psubsw m11, m%1 6127 pmovzxbw m%1, [r3 +%4] 6128 REPX {pmulhrsw x, m12}, m8, m10, m9, m11 6129 paddw m8, m0 6130 pmovzxbw m0, [r3 +%5] 6131 paddw m10, m%1 6132 pmovzxbw m%1, [dstq+%6] 6133 paddw m9, m0 6134 paddw m11, m%1 6135 %if %2 >= 8 6136 %if %2 == 8 6137 pxor m1, m1 6138 %endif 6139 mova [cq+64*(%2*2-16)], m1 6140 mova [cq+64*(%2*2-15)], m1 6141 %endif 6142 packuswb m8, m10 6143 packuswb m9, m11 6144 vpermq m8, m13, m8 6145 vpermq m9, m13, m9 6146 mova [dstq+%3], ym8 6147 vextracti32x8 [r3 +%4], m8, 1 6148 mova [r3 +%5], ym9 6149 vextracti32x8 [dstq+%6], m9, 1 6150 %if %2 == 3 || %2 == 7 || %2 == 11 6151 add dstq, r5 6152 sub r3, r5 6153 %endif 6154 %endmacro 6155 IDCT_32x64_END 29, 0, strideq*0, r8, r4 , r5*8 6156 IDCT_32x64_END 28, 1, strideq*1, r7, strideq*2, r6 6157 IDCT_32x64_END 27, 2, strideq*2, r6, strideq*1, r7 6158 IDCT_32x64_END 26, 3, r4 , r5*8, strideq*0, r8 6159 IDCT_32x64_END 25, 4, strideq*0, r8, r4 , r5*8 6160 IDCT_32x64_END 24, 5, strideq*1, r7, strideq*2, r6 6161 IDCT_32x64_END 23, 6, strideq*2, r6, strideq*1, r7 6162 IDCT_32x64_END 22, 7, r4 , r5*8, strideq*0, r8 6163 IDCT_32x64_END 21, 8, strideq*0, r8, r4 , r5*8 6164 IDCT_32x64_END 20, 9, strideq*1, r7, strideq*2, r6 6165 IDCT_32x64_END 19, 10, strideq*2, r6, strideq*1, r7 6166 IDCT_32x64_END 18, 11, r4 , r5*8, strideq*0, r8 6167 IDCT_32x64_END 17, 12, strideq*0, r8, r4 , r5*8 6168 IDCT_32x64_END 16, 13, strideq*1, r7, strideq*2, r6 6169 IDCT_32x64_END 15, 14, strideq*2, r6, strideq*1, r7 6170 IDCT_32x64_END 14, 15, r4 , r5*8, strideq*0, r8 6171 RET 6172 .dconly: 6173 movsx r6d, word [cq] 6174 mov [cq], eobd 6175 or r3d, 64 6176 imul r6d, 181 6177 add r6d, 128 6178 sar r6d, 8 6179 imul r6d, 181 6180 add r6d, 128+256 6181 sar r6d, 8+1 6182 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3 6183 ALIGN function_align ; bottom three-quarters are zero 6184 cglobal_label .main_part1_fast2 6185 vpbroadcastd m7, [o(idct64_mul+4*0)] 6186 vpbroadcastd m8, [o(idct64_mul+4*1)] 6187 pmulhrsw m7, m0 ; t63a 6188 pmulhrsw m0, m8 ; t32a 6189 6190 punpcklwd m4, m0, m7 6191 punpckhwd m6, m0, m7 6192 mova m1, m10 6193 vpdpwssd m1, m4, [o(idct64_mul+4*9)] {bcstd} 6194 mova m9, m10 6195 vpdpwssd m9, m6, [o(idct64_mul+4*9)] {bcstd} 6196 REPX {psrad x, 12}, m1, m9 6197 packssdw m1, m9 6198 mova m9, m10 6199 vpdpwssd m9, m6, [o(idct64_mul+4*8)] {bcstd} 6200 mova m6, m10 6201 vpdpwssd m6, m4, [o(idct64_mul+4*8)] {bcstd} 6202 REPX {psrad x, 12}, m9, m6 6203 packssdw m6, m9 6204 6205 mova m4, m0 6206 mova m3, m7 6207 mova m5, m1 6208 mova m2, m6 6209 jmp .main_part1c 6210 cglobal_label .main_part1_fast 6211 vpbroadcastd m1, [o(idct64_mul+4*0)] 6212 vpbroadcastd m8, [o(idct64_mul+4*1)] 6213 vpbroadcastd m2, [o(idct64_mul+4*6)] 6214 vpbroadcastd m9, [o(idct64_mul+4*7)] 6215 pmulhrsw m1, m0 ; t63a 6216 pmulhrsw m0, m8 ; t32a 6217 pmulhrsw m2, m3 ; t60a 6218 pmulhrsw m3, m9 ; t35a 6219 mova m8, m0 6220 mova m7, m1 6221 mova m6, m3 6222 mova m5, m2 6223 jmp .main_part1b 6224 cglobal_label .main_part1 6225 ; idct64 steps 1-5: 6226 ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a 6227 ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a 6228 ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a 6229 ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a 6230 vpbroadcastd m7, [o(idct64_mul+4*0)] 6231 vpbroadcastd m8, [o(idct64_mul+4*1)] 6232 vpbroadcastd m6, [o(idct64_mul+4*2)] 6233 vpbroadcastd m9, [o(idct64_mul+4*3)] 6234 pmulhrsw m7, m0 ; t63a 6235 vpbroadcastd m5, [o(idct64_mul+4*4)] 6236 pmulhrsw m0, m8 ; t32a 6237 vpbroadcastd m8, [o(idct64_mul+4*5)] 6238 pmulhrsw m6, m1 ; t62a 6239 vpbroadcastd m4, [o(idct64_mul+4*6)] 6240 pmulhrsw m1, m9 ; t33a 6241 vpbroadcastd m9, [o(idct64_mul+4*7)] 6242 pmulhrsw m5, m2 ; t61a 6243 pmulhrsw m2, m8 ; t34a 6244 pmulhrsw m4, m3 ; t60a 6245 pmulhrsw m3, m9 ; t35a 6246 psubsw m8, m0, m1 ; t33 6247 paddsw m0, m1 ; t32 6248 psubsw m1, m7, m6 ; t62 6249 paddsw m7, m6 ; t63 6250 psubsw m6, m3, m2 ; t34 6251 paddsw m3, m2 ; t35 6252 psubsw m2, m4, m5 ; t61 6253 paddsw m5, m4 ; t60 6254 .main_part1b: 6255 vpbroadcastd m11, [o(idct64_mul+4*8)] 6256 vpbroadcastd m12, [o(idct64_mul+4*9)] 6257 ITX_MULSUB_2W 1, 8, 4, 9, 10, 11, 12 ; t33a, t62a 6258 vpbroadcastd m11, [o(idct64_mul+4*10)] 6259 ITX_MULSUB_2W 2, 6, 4, 9, 10, 12, 11 ; t34a, t61a 6260 psubsw m4, m0, m3 ; t35a 6261 paddsw m0, m3 ; t32a 6262 psubsw m3, m7, m5 ; t60a 6263 paddsw m7, m5 ; t63a 6264 psubsw m5, m1, m2 ; t34 6265 paddsw m1, m2 ; t33 6266 psubsw m2, m8, m6 ; t61 6267 paddsw m6, m8 ; t62 6268 .main_part1c: 6269 vpbroadcastd m11, [o(idct64_mul+4*11)] 6270 vpbroadcastd m12, [o(idct64_mul+4*12)] 6271 add r5, 4*13 6272 ITX_MULSUB_2W 3, 4, 8, 9, 10, 11, 12 ; t35, t60 6273 ITX_MULSUB_2W 2, 5, 8, 9, 10, 11, 12 ; t34a, t61a 6274 mova [r4+64*0], m0 6275 mova [r4+64*7], m7 6276 mova [r4+64*1], m1 6277 mova [r4+64*6], m6 6278 mova [r4+64*3], m3 6279 mova [r4+64*4], m4 6280 mova [r4+64*2], m2 6281 mova [r4+64*5], m5 6282 add r4, 64*8 6283 ret 6284 cglobal_label .main_part2 6285 vpbroadcastd m11, [o(pw_1567_3784 -16*13)] 6286 vpbroadcastd m12, [o(pw_m3784_1567 -16*13)] 6287 lea r6, [r4+64*7] 6288 vpbroadcastd m17, [o(pw_m1567_m3784-16*13)] 6289 vpbroadcastd m18, [o(pw_2896_2896 -16*13)] 6290 vpbroadcastd m19, [o(pw_m2896_2896 -16*13)] 6291 sub r5, 16*13 6292 .main_part2_loop: 6293 mova m0, [r4-64*32] ; t32a 6294 mova m1, [r6-64*24] ; t39a 6295 mova m2, [r6-64*32] ; t63a 6296 mova m3, [r4-64*24] ; t56a 6297 mova m4, [r4-64*16] ; t40a 6298 mova m5, [r6-64* 8] ; t47a 6299 mova m6, [r6-64*16] ; t55a 6300 mova m7, [r4-64* 8] ; t48a 6301 psubsw m8, m0, m1 ; t39 6302 paddsw m0, m1 ; t32 6303 psubsw m1, m2, m3 ; t56 6304 paddsw m2, m3 ; t63 6305 psubsw m3, m5, m4 ; t40 6306 paddsw m5, m4 ; t47 6307 psubsw m4, m7, m6 ; t55 6308 paddsw m7, m6 ; t48 6309 ITX_MULSUB_2W 1, 8, 6, 9, 10, 11, 12 ; t39a, t56a 6310 ITX_MULSUB_2W 4, 3, 6, 9, 10, 12, 17 ; t40a, t55a 6311 psubsw m6, m2, m7 ; t48a 6312 paddsw m2, m7 ; t63a 6313 psubsw m7, m0, m5 ; t47a 6314 paddsw m0, m5 ; t32a 6315 psubsw m5, m8, m3 ; t55 6316 paddsw m8, m3 ; t56 6317 psubsw m3, m1, m4 ; t40 6318 paddsw m1, m4 ; t39 6319 ITX_MULSUB_2W 6, 7, 4, 9, 10, 18, 19 ; t47, t48 6320 ITX_MULSUB_2W 5, 3, 4, 9, 10, 18, 19 ; t40a, t55a 6321 mova [r6-64* 8], m2 6322 mova [r4-64*32], m0 6323 mova [r4-64* 8], m8 6324 mova [r6-64*32], m1 6325 mova [r6-64*24], m6 6326 mova [r4-64*16], m7 6327 mova [r4-64*24], m5 6328 mova [r6-64*16], m3 6329 add r4, 64 6330 sub r6, 64 6331 cmp r4, r6 6332 jb .main_part2_loop 6333 ret 6334 6335 cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 7, 0, dst, stride, c, eob 6336 lea r5, [o_base] 6337 test eobd, eobd 6338 jz .dconly 6339 PROLOGUE 0, 7, 30, 64*32, dst, stride, c, eob 6340 vpbroadcastd m23, [o(pw_2896x8)] 6341 %undef cmp 6342 cmp eobd, 136 6343 jb .fast 6344 pmulhrsw m0, m23, [cq+64* 1] 6345 pmulhrsw m1, m23, [cq+64*31] 6346 pmulhrsw m2, m23, [cq+64*17] 6347 pmulhrsw m3, m23, [cq+64*15] 6348 vpbroadcastd m10, [o(pd_2048)] 6349 mov r4, rsp 6350 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 6351 pmulhrsw m0, m23, [cq+64* 7] 6352 pmulhrsw m1, m23, [cq+64*25] 6353 pmulhrsw m2, m23, [cq+64*23] 6354 pmulhrsw m3, m23, [cq+64* 9] 6355 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 6356 pmulhrsw m0, m23, [cq+64* 5] 6357 pmulhrsw m1, m23, [cq+64*27] 6358 pmulhrsw m2, m23, [cq+64*21] 6359 pmulhrsw m3, m23, [cq+64*11] 6360 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 6361 pmulhrsw m0, m23, [cq+64* 3] 6362 pmulhrsw m1, m23, [cq+64*29] 6363 pmulhrsw m2, m23, [cq+64*19] 6364 pmulhrsw m3, m23, [cq+64*13] 6365 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 6366 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 6367 pmulhrsw m3, m23, [cq+64*24] 6368 pmulhrsw m1, m23, [cq+64* 8] 6369 pmulhrsw m2, m23, [cq+64*16] 6370 pmulhrsw m0, m23, [cq+64* 0] 6371 pmulhrsw m14, m23, [cq+64* 4] 6372 pmulhrsw m17, m23, [cq+64*28] 6373 pmulhrsw m16, m23, [cq+64*20] 6374 pmulhrsw m15, m23, [cq+64*12] 6375 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast 6376 pmulhrsw m22, m23, [cq+64* 2] 6377 pmulhrsw m29, m23, [cq+64*30] 6378 pmulhrsw m26, m23, [cq+64*18] 6379 pmulhrsw m25, m23, [cq+64*14] 6380 pmulhrsw m24, m23, [cq+64*10] 6381 pmulhrsw m27, m23, [cq+64*22] 6382 pmulhrsw m28, m23, [cq+64*26] 6383 pmulhrsw m23, [cq+64* 6] 6384 mova [cq+64* 0], m14 6385 mova [cq+64* 1], m15 6386 mova [cq+64* 2], m16 6387 mova [cq+64* 3], m17 6388 mova [cq+64* 4], m18 6389 mova [cq+64* 5], m19 6390 mova [cq+64* 6], m20 6391 mova [cq+64* 7], m21 6392 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast 6393 vpbroadcastd m13, [o(pw_16384)] 6394 call .pass1_end_part1 6395 mova [cq+64*16], m1 6396 mova [cq+64*17], m3 6397 mova [cq+64*18], m5 6398 mova [cq+64*19], m7 6399 mova [cq+64*24], m23 6400 mova [cq+64*25], m25 6401 mova [cq+64*26], m27 6402 mova [cq+64*27], m29 6403 pmulhrsw m23, m13, m0 ; a0 6404 pmulhrsw m25, m13, m2 ; a2 6405 pmulhrsw m27, m13, m4 ; a4 6406 pmulhrsw m29, m13, m6 ; a6 6407 REPX {pmulhrsw x, m13}, m22, m24, m26, m28 ; e0 e2 e4 e6 6408 call .pass1_end_part2 6409 mova [cq+64*20], m15 6410 mova [cq+64*21], m17 6411 mova [cq+64*22], m19 6412 mova [cq+64*23], m21 6413 mova [cq+64*28], m1 6414 mova [cq+64*29], m3 6415 mova [cq+64*30], m5 6416 mova [cq+64*31], m7 6417 REPX {pmulhrsw x, m13}, m14, m16, m18, m20 ; c0 c2 c4 c6 6418 REPX {pmulhrsw x, m13}, m0, m2, m4, m6 ; g0 g2 g4 g6 6419 vinserti32x8 m3, m23, ym14, 1 ; a00 a01 c00 c01 6420 vshufi32x4 m23, m14, q3232 ; a02 a03 c02 c03 6421 vinserti32x8 m15, m22, ym0, 1 ; e00 e01 g00 g01 6422 vshufi32x4 m22, m0, q3232 ; e02 e03 g02 g03 6423 vinserti32x8 m1, m27, ym18, 1 ; a40 a41 c40 c41 6424 vshufi32x4 m27, m18, q3232 ; a42 a43 c42 c43 6425 vinserti32x8 m18, m26, ym4, 1 ; e40 e41 g40 g41 6426 vshufi32x4 m26, m4, q3232 ; e42 e43 g42 g43 6427 vinserti32x8 m14, m25, ym16, 1 ; a20 a21 c20 c21 6428 vshufi32x4 m25, m16, q3232 ; a22 a23 c22 c23 6429 vinserti32x8 m17, m24, ym2, 1 ; e20 e21 g20 g21 6430 vshufi32x4 m24, m2, q3232 ; e22 e23 g22 g23 6431 vinserti32x8 m19, m29, ym20, 1 ; a60 a61 c60 c61 6432 vshufi32x4 m29, m20, q3232 ; a62 a63 c62 c63 6433 vinserti32x8 m20, m28, ym6, 1 ; e60 e61 g60 g61 6434 vshufi32x4 m28, m6, q3232 ; e62 e63 g62 g63 6435 vshufi32x4 m2, m3, m15, q3131 ; 8 6436 vshufi32x4 m0, m3, m15, q2020 ; 0 6437 vshufi32x4 m6, m23, m22, q3131 ; 24 6438 vshufi32x4 m4, m23, m22, q2020 ; 16 6439 vshufi32x4 m3, m1, m18, q3131 ; 12 6440 vshufi32x4 m1, m18, q2020 ; 4 6441 vshufi32x4 m7, m27, m26, q3131 ; 28 6442 vshufi32x4 m5, m27, m26, q2020 ; 20 6443 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 6444 vshufi32x4 m16, m14, m17, q3131 ; 10 6445 vshufi32x4 m14, m17, q2020 ; 2 6446 vshufi32x4 m17, m19, m20, q3131 ; 14 6447 vshufi32x4 m15, m19, m20, q2020 ; 6 6448 vshufi32x4 m20, m25, m24, q3131 ; 26 6449 vshufi32x4 m18, m25, m24, q2020 ; 18 6450 vshufi32x4 m21, m29, m28, q3131 ; 30 6451 vshufi32x4 m19, m29, m28, q2020 ; 22 6452 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 6453 pmulhrsw m22, m13, [cq+64*16] ; a1 6454 pmulhrsw m23, m13, [cq+64*20] ; c1 6455 pmulhrsw m24, m13, [cq+64*24] ; e1 6456 pmulhrsw m25, m13, [cq+64*28] ; g1 6457 pmulhrsw m26, m13, [cq+64*17] ; a3 6458 pmulhrsw m27, m13, [cq+64*21] ; c3 6459 pmulhrsw m28, m13, [cq+64*25] ; e3 6460 pmulhrsw m29, m13, [cq+64*29] ; g3 6461 mova [cq+64* 8], m14 6462 mova [cq+64* 9], m15 6463 mova [cq+64*10], m16 6464 mova [cq+64*11], m17 6465 mova [cq+64*12], m18 6466 mova [cq+64*13], m19 6467 mova [cq+64*14], m20 6468 mova [cq+64*15], m21 6469 pmulhrsw m14, m13, [cq+64*18] ; a5 6470 pmulhrsw m15, m13, [cq+64*22] ; c5 6471 pmulhrsw m16, m13, [cq+64*26] ; e5 6472 pmulhrsw m17, m13, [cq+64*30] ; g5 6473 pmulhrsw m18, m13, [cq+64*19] ; a7 6474 pmulhrsw m19, m13, [cq+64*23] ; c7 6475 pmulhrsw m20, m13, [cq+64*27] ; e7 6476 pmulhrsw m21, m13, [cq+64*31] ; g7 6477 vinserti32x8 m8, m22, ym23, 1 ; a10 a11 c10 c11 6478 vshufi32x4 m22, m23, q3232 ; a12 a13 c12 c13 6479 vinserti32x8 m9, m24, ym25, 1 ; e10 e11 g10 g11 6480 vshufi32x4 m24, m25, q3232 ; e12 e13 g12 g13 6481 vinserti32x8 m23, m26, ym27, 1 ; a30 a31 c30 c31 6482 vshufi32x4 m26, m27, q3232 ; a32 a33 c32 c33 6483 vinserti32x8 m11, m28, ym29, 1 ; e30 e31 g30 g31 6484 vshufi32x4 m28, m29, q3232 ; e32 e33 g32 g33 6485 mova [cq+64* 0], m0 6486 mova [cq+64* 1], m1 6487 mova [cq+64* 2], m2 6488 mova [cq+64* 3], m3 6489 mova [cq+64* 4], m4 6490 mova [cq+64* 5], m5 6491 mova [cq+64* 6], m6 6492 mova [cq+64* 7], m7 6493 vinserti32x8 m12, m14, ym15, 1 ; a50 a51 c50 c51 6494 vshufi32x4 m14, m15, q3232 ; a52 a53 c52 c53 6495 vinserti32x8 m13, m16, ym17, 1 ; e50 e51 g50 g51 6496 vshufi32x4 m16, m17, q3232 ; e52 e53 g52 g53 6497 vinserti32x8 m25, m18, ym19, 1 ; a70 a71 c70 c71 6498 vshufi32x4 m18, m19, q3232 ; a72 a73 c72 c73 6499 vinserti32x8 m17, m20, ym21, 1 ; e70 e71 g70 g71 6500 vshufi32x4 m20, m21, q3232 ; e72 e73 g72 g73 6501 vshufi32x4 m27, m23, m11, q3131 ; 11 m27 6502 vshufi32x4 m23, m11, q2020 ; 3 m23 6503 vshufi32x4 m19, m26, m28, q3131 ; 27 m19 6504 vshufi32x4 m15, m26, m28, q2020 ; 19 m15 6505 vshufi32x4 m29, m25, m17, q3131 ; 15 m29 6506 vshufi32x4 m25, m17, q2020 ; 7 m25 6507 vshufi32x4 m21, m18, m20, q3131 ; 31 m21 6508 vshufi32x4 m17, m18, m20, q2020 ; 23 m17 6509 vshufi32x4 m20, m14, m16, q3131 ; 29 m20 6510 vshufi32x4 m16, m14, m16, q2020 ; 21 m16 6511 vshufi32x4 m18, m22, m24, q3131 ; 25 m18 6512 vshufi32x4 m14, m22, m24, q2020 ; 17 m14 6513 vshufi32x4 m26, m8, m9, q3131 ; 9 m26 6514 vshufi32x4 m22, m8, m9, q2020 ; 1 m22 6515 vshufi32x4 m28, m12, m13, q3131 ; 13 m28 6516 vshufi32x4 m24, m12, m13, q2020 ; 5 m24 6517 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf 6518 vpbroadcastd m13, [o(pw_16384)] 6519 pmulhrsw m0, m13, [r4-64*21] 6520 pmulhrsw m1, m13, [r4-64*22] 6521 pmulhrsw m2, m13, [r4-64*23] 6522 pmulhrsw m3, m13, [r4-64*24] 6523 pmulhrsw m4, m13, [r4-64*25] 6524 pmulhrsw m5, m13, [r4-64*26] 6525 pmulhrsw m6, m13, [r4-64*27] 6526 pmulhrsw m7, m13, [r4-64*28] 6527 mova [cq+64*16], m14 6528 mova [cq+64*17], m15 6529 mova [cq+64*18], m16 6530 mova [cq+64*19], m17 6531 mova [cq+64*20], m18 6532 mova [cq+64*21], m19 6533 mova [cq+64*22], m20 6534 mova [cq+64*23], m21 6535 pmulhrsw m14, m13, [r4-64*12] 6536 pmulhrsw m15, m13, [r4-64*11] 6537 pmulhrsw m16, m13, [r4-64*10] 6538 pmulhrsw m17, m13, [r4-64* 9] 6539 pmulhrsw m18, m13, [r4-64* 8] 6540 pmulhrsw m19, m13, [r4-64* 7] 6541 pmulhrsw m20, m13, [r4-64* 6] 6542 pmulhrsw m21, m13, [r4-64* 5] 6543 mova [cq+64*24], m22 6544 mova [cq+64*25], m23 6545 mova [cq+64*26], m24 6546 mova [cq+64*27], m25 6547 mova [cq+64*28], m26 6548 mova [cq+64*29], m27 6549 mova [cq+64*30], m28 6550 mova [cq+64*31], m29 6551 call .transpose_2x8x8_lo 6552 mova [r4-64*12], m1 6553 mova [r4-64*11], m3 6554 mova [r4-64*10], m5 6555 mova [r4-64* 9], m7 6556 mova [r4-64* 8], m15 6557 mova [r4-64* 7], m17 6558 mova [r4-64* 6], m19 6559 mova [r4-64* 5], m21 6560 vinserti32x8 m22, m0, ym14, 1 ; f00 f01 h00 h01 6561 vshufi32x4 m23, m0, m14, q3232 ; f02 f03 h02 h03 6562 vinserti32x8 m24, m2, ym16, 1 ; f20 f21 h20 h21 6563 vshufi32x4 m25, m2, m16, q3232 ; f22 f23 h22 h23 6564 vinserti32x8 m26, m4, ym18, 1 ; f40 f41 h40 h41 6565 vshufi32x4 m27, m4, m18, q3232 ; f42 f43 h42 h43 6566 vinserti32x8 m28, m6, ym20, 1 ; f60 f61 h60 h61 6567 vshufi32x4 m29, m6, m20, q3232 ; f62 f63 h62 h63 6568 pmulhrsw m0, m13, [r4-64*20] 6569 pmulhrsw m1, m13, [r4-64*19] 6570 pmulhrsw m2, m13, [r4-64*18] 6571 pmulhrsw m3, m13, [r4-64*17] 6572 pmulhrsw m4, m13, [r4-64*16] 6573 pmulhrsw m5, m13, [r4-64*15] 6574 pmulhrsw m6, m13, [r4-64*14] 6575 pmulhrsw m7, m13, [r4-64*13] 6576 pmulhrsw m14, m13, [r4-64*29] 6577 pmulhrsw m15, m13, [r4-64*30] 6578 pmulhrsw m16, m13, [r4-64*31] 6579 pmulhrsw m17, m13, [r4-64*32] 6580 pmulhrsw m18, m13, [r4-64*33] 6581 pmulhrsw m19, m13, [r4-64*34] 6582 pmulhrsw m20, m13, [r4-64*35] 6583 pmulhrsw m21, m13, [r4-64*36] 6584 call .transpose_2x8x8_lo 6585 mova [r4-64*20], m1 6586 mova [r4-64*19], m3 6587 mova [r4-64*18], m5 6588 mova [r4-64*17], m7 6589 mova [r4-64*16], m15 6590 mova [r4-64*15], m17 6591 mova [r4-64*14], m19 6592 mova [r4-64*13], m21 6593 vinserti32x8 m1, m4, ym18, 1 ; b40 b41 d40 d41 6594 vshufi32x4 m5, m4, m18, q3232 ; b42 b43 d42 d43 6595 vshufi32x4 m4, m0, m14, q3232 ; b02 b03 d02 d03 6596 vinserti32x8 m0, ym14, 1 ; b00 b01 d00 d01 6597 vinserti32x8 m14, m2, ym16, 1 ; b20 b21 d20 d21 6598 vshufi32x4 m18, m2, m16, q3232 ; b22 b23 d22 d23 6599 vinserti32x8 m15, m6, ym20, 1 ; b60 b61 d60 d61 6600 vshufi32x4 m19, m6, m20, q3232 ; b62 b63 d62 d63 6601 vshufi32x4 m2, m0, m22, q3131 ; 8 6602 vshufi32x4 m0, m22, q2020 ; 0 6603 vshufi32x4 m3, m1, m26, q3131 ; 12 6604 vshufi32x4 m1, m26, q2020 ; 4 6605 vshufi32x4 m6, m4, m23, q3131 ; 24 6606 vshufi32x4 m4, m23, q2020 ; 16 6607 vshufi32x4 m7, m5, m27, q3131 ; 28 6608 vshufi32x4 m5, m27, q2020 ; 20 6609 call m(inv_txfm_add_dct_dct_32x8_8bpc).main 6610 vshufi32x4 m16, m14, m24, q3131 ; 10 6611 vshufi32x4 m14, m24, q2020 ; 2 6612 vshufi32x4 m17, m15, m28, q3131 ; 14 6613 vshufi32x4 m15, m28, q2020 ; 6 6614 vshufi32x4 m20, m18, m25, q3131 ; 26 6615 vshufi32x4 m18, m25, q2020 ; 18 6616 vshufi32x4 m21, m19, m29, q3131 ; 30 6617 vshufi32x4 m19, m29, q2020 ; 22 6618 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf 6619 mova m22, [r4-64*20] 6620 mova m26, [r4-64*16] 6621 mova m23, [r4-64*19] 6622 mova m27, [r4-64*15] 6623 mova m24, [r4-64*18] 6624 mova m28, [r4-64*14] 6625 mova m25, [r4-64*17] 6626 mova m29, [r4-64*13] 6627 mova [r4-64*20], m14 6628 mova [r4-64*19], m15 6629 mova [r4-64*18], m16 6630 mova [r4-64*17], m17 6631 mova [r4-64*16], m18 6632 mova [r4-64*15], m19 6633 mova [r4-64*14], m20 6634 mova [r4-64*13], m21 6635 mova m19, [r4-64*12] 6636 mova m11, [r4-64* 8] 6637 mova m20, [r4-64*11] 6638 mova m12, [r4-64* 7] 6639 mova m21, [r4-64*10] 6640 mova m8, [r4-64* 6] 6641 mova m9, [r4-64* 9] 6642 mova m18, [r4-64* 5] 6643 vshufi32x4 m14, m22, m26, q3232 ; b12 b13 d12 d13 6644 vinserti32x8 m22, ym26, 1 ; b10 b11 d10 d11 6645 vshufi32x4 m15, m23, m27, q3232 ; b32 b33 d32 d33 6646 vinserti32x8 m23, ym27, 1 ; b30 b31 d30 d31 6647 vshufi32x4 m16, m24, m28, q3232 ; b52 b53 d52 d53 6648 vinserti32x8 m24, ym28, 1 ; b50 b51 d50 d51 6649 vshufi32x4 m17, m25, m29, q3232 ; b72 b73 d72 d73 6650 vinserti32x8 m25, ym29, 1 ; b70 b71 d70 d71 6651 vinserti32x8 m27, m19, ym11, 1 ; f10 f11 h10 h11 6652 vshufi32x4 m19, m11, q3232 ; f12 f13 h12 h13 6653 vinserti32x8 m28, m20, ym12, 1 ; f30 f31 h30 h31 6654 vshufi32x4 m20, m12, q3232 ; f32 f33 h32 h33 6655 vinserti32x8 m29, m21, ym8, 1 ; f50 f51 h50 h51 6656 vshufi32x4 m21, m8, q3232 ; f52 f53 h52 h53 6657 vinserti32x8 m8, m9, ym18, 1 ; f70 f71 h70 h71 6658 vshufi32x4 m9, m18, q3232 ; f72 f73 h72 h73 6659 vshufi32x4 m26, m22, m27, q3131 ; 9 6660 vshufi32x4 m22, m27, q2020 ; 1 6661 vshufi32x4 m27, m23, m28, q3131 ; 11 6662 vshufi32x4 m23, m28, q2020 ; 3 6663 vshufi32x4 m28, m24, m29, q3131 ; 13 6664 vshufi32x4 m24, m29, q2020 ; 5 6665 vshufi32x4 m29, m25, m8, q3131 ; 15 6666 vshufi32x4 m25, m8, q2020 ; 7 6667 vshufi32x4 m18, m14, m19, q3131 ; 25 6668 vshufi32x4 m14, m19, q2020 ; 17 6669 vshufi32x4 m19, m15, m20, q3131 ; 27 6670 vshufi32x4 m15, m20, q2020 ; 19 6671 vshufi32x4 m20, m16, m21, q3131 ; 29 6672 vshufi32x4 m16, m21, q2020 ; 21 6673 vshufi32x4 m21, m17, m9, q3131 ; 31 6674 vshufi32x4 m17, m9, q2020 ; 23 6675 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf 6676 jmp .end 6677 .fast: ; bottom/right halves are zero 6678 {evex}vpmulhrsw ym8, ym23, [cq+64* 4] 6679 {evex}vpmulhrsw xm1, xm23, [cq+64*12] 6680 mova m28, [o(dup16_perm)] 6681 {evex}vpmulhrsw ym7, ym23, [cq+64* 8] 6682 vpmulhrsw ym22, ym23, [cq+64* 0] 6683 vpermb m8, m28, m8 6684 vpermb ym1, ym28, ym1 6685 vpermb m7, m28, m7 6686 pmovzxwd m9, ym22 6687 pslld m9, 16 6688 call m(idct_16x16_internal_8bpc).main_fast2 6689 {evex}vpmulhrsw ym21, ym23, [cq+64* 2] 6690 {evex}vpmulhrsw xm15, xm23, [cq+64*14] 6691 {evex}vpmulhrsw xm18, xm23, [cq+64*10] 6692 {evex}vpmulhrsw ym14, ym23, [cq+64* 6] 6693 vpermb m21, m28, m21 6694 punpcklwd xm15, xm15 6695 vpermb ym18, ym28, ym18 6696 vpermb m14, m28, m14 6697 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 6698 vpmulhrsw ym22, ym23, [cq+64* 1] 6699 {evex}vpmulhrsw xm29, xm23, [cq+64*15] 6700 {evex}vpmulhrsw xm26, xm23, [cq+64* 9] 6701 {evex}vpmulhrsw ym25, ym23, [cq+64* 7] 6702 {evex}vpmulhrsw ym24, ym23, [cq+64* 5] 6703 {evex}vpmulhrsw xm27, xm23, [cq+64*11] 6704 {evex}vpmulhrsw xm8, xm23, [cq+64*13] 6705 {evex}vpmulhrsw ym23, [cq+64* 3] 6706 vpermb m22, m28, m22 6707 punpcklwd xm29, xm29 6708 vpermb ym26, ym28, ym26 6709 vpermb m25, m28, m25 6710 mova [cq+64* 0], m14 6711 mova [cq+64* 1], m15 6712 mova [cq+64* 2], m16 6713 mova [cq+64* 3], m17 6714 REPX {vpermb x, m28, x}, m24, m27, m23 6715 punpcklwd xm28, xm8, xm8 6716 mova [cq+64* 4], m18 6717 mova [cq+64* 5], m19 6718 mova [cq+64* 6], m20 6719 mova [cq+64* 7], m21 6720 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast 6721 mov r4, rsp 6722 vpbroadcastd m13, [o(pw_16384)] 6723 mova [r4+64*16], m4 6724 mova [r4+64*17], m5 6725 mova [r4+64*18], m6 6726 mova [r4+64*19], m7 6727 mova [r4+64*28], m26 6728 mova [r4+64*29], m27 6729 mova [r4+64*30], m28 6730 mova [r4+64*31], m29 6731 call m(inv_txfm_add_dct_dct_64x16_8bpc).pass1_end 6732 mova [r4+64*20], m22 6733 mova [r4+64*21], m23 6734 mova [r4+64*22], m24 6735 mova [r4+64*23], m25 6736 mova [r4+64*24], m26 6737 mova [r4+64*25], m27 6738 mova [r4+64*26], m28 6739 mova [r4+64*27], m29 6740 call .pass2_fast 6741 mova [cq+64* 8], m14 6742 mova [cq+64* 9], m15 6743 mova [cq+64*10], m16 6744 mova [cq+64*11], m17 6745 mova [cq+64*12], m18 6746 mova [cq+64*13], m19 6747 mova [cq+64*14], m20 6748 mova [cq+64*15], m21 6749 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast 6750 mova [cq+64* 0], m0 6751 mova [cq+64* 1], m1 6752 mova [cq+64* 2], m2 6753 mova [cq+64* 3], m3 6754 mova [cq+64* 4], m4 6755 mova [cq+64* 5], m5 6756 mova [cq+64* 6], m6 6757 mova [cq+64* 7], m7 6758 pmulhrsw m0, m13, [r4+64*16] 6759 pmulhrsw m1, m13, [r4+64*17] 6760 pmulhrsw m2, m13, [r4+64*18] 6761 pmulhrsw m3, m13, [r4+64*19] 6762 pmulhrsw m4, m13, [r4+64*20] 6763 pmulhrsw m5, m13, [r4+64*21] 6764 pmulhrsw m6, m13, [r4+64*22] 6765 pmulhrsw m7, m13, [r4+64*23] 6766 mova [cq+64*16], m14 6767 mova [cq+64*17], m15 6768 mova [cq+64*18], m16 6769 mova [cq+64*19], m17 6770 mova [cq+64*20], m18 6771 mova [cq+64*21], m19 6772 mova [cq+64*22], m20 6773 mova [cq+64*23], m21 6774 pmulhrsw m14, m13, [r4+64*24] 6775 pmulhrsw m15, m13, [r4+64*25] 6776 pmulhrsw m16, m13, [r4+64*26] 6777 pmulhrsw m17, m13, [r4+64*27] 6778 pmulhrsw m18, m13, [r4+64*28] 6779 pmulhrsw m19, m13, [r4+64*29] 6780 pmulhrsw m20, m13, [r4+64*30] 6781 pmulhrsw m21, m13, [r4+64*31] 6782 mova [cq+64*24], m22 6783 mova [cq+64*25], m23 6784 mova [cq+64*26], m24 6785 mova [cq+64*27], m25 6786 mova [cq+64*28], m26 6787 mova [cq+64*29], m27 6788 mova [cq+64*30], m28 6789 mova [cq+64*31], m29 6790 call m(inv_txfm_add_dct_dct_64x16_8bpc).transpose_round 6791 call .pass2_fast 6792 mova [r4+64*16], m14 6793 mova [r4+64*17], m15 6794 mova [r4+64*18], m16 6795 mova [r4+64*19], m17 6796 mova [r4+64*20], m18 6797 mova [r4+64*21], m19 6798 mova [r4+64*22], m20 6799 mova [r4+64*23], m21 6800 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast 6801 .end: 6802 vpbroadcastd m13, [o(pw_2048)] 6803 lea r5, [strideq*3] 6804 pxor m12, m12 6805 lea r3, [dstq+r5*8] 6806 lea r6, [strideq+r5] ; stride*4 6807 add r3, r6 ; dst+stride*28 6808 %macro IDCT_64x32_END 5 ; src16, src32, mem, off_lo, off_hi 6809 mova m11, [cq+64*( %3)] ; 0 6810 mova m9, [cq+64*(31-%3)] ; 31 6811 %if %3 >= 8 6812 mova m%1, [rsp+64*(%1+16)] 6813 %endif 6814 mova m10, [dstq+%4] 6815 paddsw m8, m11, m9 6816 psubsw m11, m9 6817 paddsw m9, m%1, m%2 6818 psubsw m%1, m%2 6819 punpcklbw m%2, m10, m12 6820 punpckhbw m10, m12 6821 pmulhrsw m8, m13 6822 pmulhrsw m9, m13 6823 paddw m8, m%2 6824 paddw m9, m10 6825 mova m10, [r3+%5] 6826 pmulhrsw m11, m13 6827 pmulhrsw m%1, m13 6828 mova [cq+64*( %3)], m12 6829 mova [cq+64*(31-%3)], m12 6830 punpcklbw m%2, m10, m12 6831 punpckhbw m10, m12 6832 packuswb m8, m9 6833 paddw m11, m%2 6834 paddw m%1, m10 6835 packuswb m11, m%1 6836 mova [dstq+%4], m8 6837 mova [r3 +%5], m11 6838 %if %3 == 3 || %3 == 7 || %3 == 11 6839 add dstq, r6 6840 sub r3, r6 6841 %endif 6842 %endmacro 6843 IDCT_64x32_END 0, 29, 0, strideq*0, r5 6844 IDCT_64x32_END 1, 28, 1, strideq*1, strideq*2 6845 IDCT_64x32_END 2, 27, 2, strideq*2, strideq*1 6846 IDCT_64x32_END 3, 26, 3, r5 , strideq*0 6847 IDCT_64x32_END 4, 25, 4, strideq*0, r5 6848 IDCT_64x32_END 5, 24, 5, strideq*1, strideq*2 6849 IDCT_64x32_END 6, 23, 6, strideq*2, strideq*1 6850 IDCT_64x32_END 7, 22, 7, r5 , strideq*0 6851 IDCT_64x32_END 0, 21, 8, strideq*0, r5 6852 IDCT_64x32_END 1, 20, 9, strideq*1, strideq*2 6853 IDCT_64x32_END 2, 19, 10, strideq*2, strideq*1 6854 IDCT_64x32_END 3, 18, 11, r5 , strideq*0 6855 IDCT_64x32_END 4, 17, 12, strideq*0, r5 6856 IDCT_64x32_END 5, 16, 13, strideq*1, strideq*2 6857 IDCT_64x32_END 6, 15, 14, strideq*2, strideq*1 6858 IDCT_64x32_END 7, 14, 15, r5 , strideq*0 6859 RET 6860 ALIGN function_align 6861 .dconly: 6862 movsx r6d, word [cq] 6863 mov [cq], eobd 6864 or r3d, 32 6865 imul r6d, 181 6866 add r6d, 128 6867 sar r6d, 8 6868 imul r6d, 181 6869 add r6d, 128+256 6870 sar r6d, 8+1 6871 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly2 6872 ALIGN function_align 6873 .pass1_end_part1: 6874 %macro IDCT_64x32_PASS1_END 3 ; src16, src32, src64 6875 %if %1 != %3 6876 mova m%1, [cq+64*%1] 6877 %endif 6878 mova m9, [r4+64*(%3-36)] ; idct64 32+n 6879 mova m11, [r4+64*(-5-%3)] ; idct64 63-n 6880 psubsw m8, m%1, m%2 ; idct32 31-n 6881 paddsw m%1, m%2 ; idct32 0+n 6882 %if %1 == %3 6883 psubsw m%2, m8, m9 ; out 32+n e 6884 paddsw m8, m9 ; out 31-n d 6885 psubsw m9, m%1, m11 ; out 63-n h 6886 paddsw m%1, m11 ; out 0+n a 6887 %else 6888 paddsw m%2, m8, m9 ; out 23-n c 6889 psubsw m8, m9 ; out 40+n f 6890 paddsw m9, m%1, m11 ; out 8+n b 6891 psubsw m%1, m11 ; out 55-n g 6892 %endif 6893 mova [r4+64*(%3-36)], m8 6894 mova [r4+64*(-5-%3)], m9 6895 %endmacro 6896 IDCT_64x32_PASS1_END 0, 29, 0 6897 IDCT_64x32_PASS1_END 1, 28, 1 6898 IDCT_64x32_PASS1_END 2, 27, 2 6899 IDCT_64x32_PASS1_END 3, 26, 3 6900 IDCT_64x32_PASS1_END 4, 25, 4 6901 IDCT_64x32_PASS1_END 5, 24, 5 6902 IDCT_64x32_PASS1_END 6, 23, 6 6903 IDCT_64x32_PASS1_END 7, 22, 7 6904 .transpose_2x8x8_hi: ; m0-m7 + m22-m29 (inverted) 6905 punpcklwd m8, m25, m24 ; e0 f0 e1 f1 e2 f2 e3 f3 6906 punpckhwd m25, m24 ; e4 f4 e5 f5 e6 f6 e7 f7 6907 punpcklwd m24, m23, m22 ; g0 h0 g1 h1 g2 h2 g3 h3 6908 punpckhwd m23, m22 ; g4 h4 g5 h5 g6 h6 g7 h7 6909 punpcklwd m22, m29, m28 ; a0 b0 a1 b1 a2 b2 a3 b3 6910 punpckhwd m29, m28 ; a4 b4 a5 b5 a6 b6 a7 b7 6911 punpcklwd m28, m27, m26 ; c0 d0 c1 d1 c2 d2 c3 d3 6912 punpckhwd m27, m26 ; c4 d4 c5 d5 c6 d6 c7 d7 6913 punpckldq m26, m29, m27 ; a4 b4 c4 d4 a5 b5 c5 d5 6914 punpckhdq m29, m27 ; a6 b6 c6 d6 a7 b7 c7 d7 6915 punpckldq m27, m8, m24 ; e0 f0 g0 h0 e1 f1 g1 h1 6916 punpckhdq m8, m24 ; e2 f2 g2 h2 e3 f3 g3 h3 6917 punpckhdq m24, m22, m28 ; a2 b2 c2 d2 a3 b3 c3 d3 6918 punpckldq m22, m28 ; a0 b0 c0 d0 a1 b1 c1 d1 6919 punpckldq m28, m25, m23 ; e4 f4 g4 h4 e5 f5 g5 h5 6920 punpckhdq m25, m23 ; e6 f6 g6 h6 e7 f7 g7 h7 6921 punpckhqdq m23, m22, m27 ; 1 23 6922 punpcklqdq m22, m27 ; 0 22 6923 punpckhqdq m27, m26, m28 ; 5 27 6924 punpcklqdq m26, m28 ; 4 26 6925 punpcklqdq m28, m29, m25 ; 6 28 6926 punpckhqdq m29, m25 ; 7 29 6927 punpckhqdq m25, m24, m8 ; 3 25 6928 punpcklqdq m24, m8 ; 2 24 6929 .transpose_8x8: 6930 punpckhwd m8, m4, m5 6931 punpcklwd m4, m5 6932 punpckhwd m5, m0, m1 6933 punpcklwd m0, m1 6934 punpckhwd m1, m6, m7 6935 punpcklwd m6, m7 6936 punpckhwd m7, m2, m3 6937 punpcklwd m2, m3 6938 punpckhdq m3, m0, m2 6939 punpckldq m0, m2 6940 punpckldq m2, m4, m6 6941 punpckhdq m4, m6 6942 punpckhdq m6, m5, m7 6943 punpckldq m5, m7 6944 punpckldq m7, m8, m1 6945 punpckhdq m8, m1 6946 punpckhqdq m1, m0, m2 6947 punpcklqdq m0, m2 6948 punpcklqdq m2, m3, m4 6949 punpckhqdq m3, m4 6950 punpcklqdq m4, m5, m7 6951 punpckhqdq m5, m7 6952 punpckhqdq m7, m6, m8 6953 punpcklqdq m6, m8 6954 ret 6955 .pass1_end_part2: 6956 IDCT_64x32_PASS1_END 0, 21, 8 6957 IDCT_64x32_PASS1_END 1, 20, 9 6958 IDCT_64x32_PASS1_END 2, 19, 10 6959 IDCT_64x32_PASS1_END 3, 18, 11 6960 IDCT_64x32_PASS1_END 4, 17, 12 6961 IDCT_64x32_PASS1_END 5, 16, 13 6962 IDCT_64x32_PASS1_END 6, 15, 14 6963 IDCT_64x32_PASS1_END 7, 14, 15 6964 .transpose_2x8x8_lo: ; m0-m7 (inverted) + m14-m21 6965 punpcklwd m8, m3, m2 6966 punpckhwd m3, m2 6967 punpcklwd m2, m1, m0 6968 punpckhwd m1, m0 6969 punpcklwd m0, m7, m6 6970 punpckhwd m7, m6 6971 punpcklwd m6, m5, m4 6972 punpckhwd m5, m4 6973 punpckldq m4, m7, m5 6974 punpckhdq m7, m5 6975 punpckldq m5, m8, m2 6976 punpckhdq m8, m2 6977 punpckhdq m2, m0, m6 6978 punpckldq m0, m6 6979 punpckldq m6, m3, m1 6980 punpckhdq m3, m1 6981 punpckhqdq m1, m0, m5 6982 punpcklqdq m0, m5 6983 punpckhqdq m5, m4, m6 6984 punpcklqdq m4, m6 6985 punpcklqdq m6, m7, m3 6986 punpckhqdq m7, m3 6987 punpckhqdq m3, m2, m8 6988 punpcklqdq m2, m8 6989 punpckhwd m8, m18, m19 6990 punpcklwd m18, m19 6991 punpckhwd m19, m14, m15 6992 punpcklwd m14, m15 6993 punpckhwd m15, m20, m21 6994 punpcklwd m20, m21 6995 punpckhwd m21, m16, m17 6996 punpcklwd m16, m17 6997 punpckhdq m17, m14, m16 6998 punpckldq m14, m16 6999 punpckldq m16, m18, m20 7000 punpckhdq m18, m20 7001 punpckhdq m20, m19, m21 7002 punpckldq m19, m21 7003 punpckldq m21, m8, m15 7004 punpckhdq m8, m15 7005 punpckhqdq m15, m14, m16 7006 punpcklqdq m14, m16 7007 punpcklqdq m16, m17, m18 7008 punpckhqdq m17, m18 7009 punpcklqdq m18, m19, m21 7010 punpckhqdq m19, m21 7011 punpckhqdq m21, m20, m8 7012 punpcklqdq m20, m8 7013 ret 7014 .pass2_fast: 7015 vshufi32x4 m24, m9, m15, q3131 ; 5 7016 vshufi32x4 m22, m9, m15, q2020 ; 1 7017 vshufi32x4 m15, m1, m16, q3131 ; 6 7018 vshufi32x4 m14, m1, m16, q2020 ; 2 7019 vshufi32x4 m1, m0, m3, q3131 ; 4 7020 vshufi32x4 m0, m3, q2020 ; 0 7021 vshufi32x4 m3, m8, m2, q3131 ; 12 7022 vshufi32x4 m2, m8, m2, q2020 ; 8 7023 vshufi32x4 m25, m11, m17, q3131 ; 7 7024 vshufi32x4 m23, m11, m17, q2020 ; 3 7025 vshufi32x4 m17, m5, m19, q3131 ; 14 7026 vshufi32x4 m16, m5, m19, q2020 ; 10 7027 vshufi32x4 m29, m6, m20, q3131 ; 15 7028 vshufi32x4 m27, m6, m20, q2020 ; 11 7029 vshufi32x4 m28, m4, m18, q3131 ; 13 7030 vshufi32x4 m26, m4, m18, q2020 ; 9 7031 jmp m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast 7032 7033 cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 7, 0, dst, stride, c, eob 7034 lea r5, [o_base] 7035 test eobd, eobd 7036 jz .dconly 7037 PROLOGUE 0, 7, 30, 64*96, dst, stride, c, eob 7038 %undef cmp 7039 cmp eobd, 136 7040 jb .fast 7041 mova m0, [cq+64* 1] 7042 mova m1, [cq+64*31] 7043 mova m2, [cq+64*17] 7044 mova m3, [cq+64*15] 7045 vpbroadcastd m10, [o(pd_2048)] 7046 mov r4, rsp 7047 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7048 mova m0, [cq+64* 7] 7049 mova m1, [cq+64*25] 7050 mova m2, [cq+64*23] 7051 mova m3, [cq+64* 9] 7052 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7053 mova m0, [cq+64* 5] 7054 mova m1, [cq+64*27] 7055 mova m2, [cq+64*21] 7056 mova m3, [cq+64*11] 7057 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7058 mova m0, [cq+64* 3] 7059 mova m1, [cq+64*29] 7060 mova m2, [cq+64*19] 7061 mova m3, [cq+64*13] 7062 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7063 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 7064 mova m0, [cq+64* 0] 7065 mova m1, [cq+64* 8] 7066 mova m2, [cq+64*16] 7067 mova m3, [cq+64*24] 7068 mova m14, [cq+64* 4] 7069 mova m15, [cq+64*12] 7070 mova m16, [cq+64*20] 7071 mova m17, [cq+64*28] 7072 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast 7073 mova m22, [cq+64* 2] 7074 mova m29, [cq+64*30] 7075 mova m26, [cq+64*18] 7076 mova m25, [cq+64*14] 7077 mova m24, [cq+64*10] 7078 mova m27, [cq+64*22] 7079 mova m28, [cq+64*26] 7080 mova m23, [cq+64* 6] 7081 mova [cq+64* 0], m14 7082 mova [cq+64* 1], m15 7083 mova [cq+64* 2], m16 7084 mova [cq+64* 3], m17 7085 mova [cq+64* 4], m18 7086 mova [cq+64* 5], m19 7087 mova [cq+64* 6], m20 7088 mova [cq+64* 7], m21 7089 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast 7090 vpbroadcastd m13, [o(pw_8192)] 7091 call m(inv_txfm_add_dct_dct_64x32_8bpc).pass1_end_part1 7092 mova [r4+64*36], m1 7093 mova [r4+64*37], m3 7094 mova [r4+64*38], m5 7095 mova [r4+64*39], m7 7096 mova [r4+64*44], m23 7097 mova [r4+64*45], m25 7098 mova [r4+64*46], m27 7099 mova [r4+64*47], m29 7100 pmulhrsw m23, m13, m0 ; a0 7101 pmulhrsw m25, m13, m2 ; a2 7102 pmulhrsw m27, m13, m4 ; a4 7103 pmulhrsw m29, m13, m6 ; a6 7104 call m(inv_txfm_add_dct_dct_64x32_8bpc).pass1_end_part2 7105 lea r6, [r4-64*4] 7106 add r4, 64*28 7107 call .pass2_end 7108 mov r4, rsp 7109 mova m0, [r4+64*23] 7110 mova m1, [r4+64*22] 7111 mova m2, [r4+64*21] 7112 mova m3, [r4+64*20] 7113 mova m4, [r4+64*19] 7114 mova m5, [r4+64*18] 7115 mova m6, [r4+64*17] 7116 mova m7, [r4+64*16] 7117 mova m22, [r4+64*15] 7118 mova m23, [r4+64*14] 7119 mova m24, [r4+64*13] 7120 mova m25, [r4+64*12] 7121 mova m26, [r4+64*11] 7122 mova m27, [r4+64*10] 7123 mova m28, [r4+64* 9] 7124 mova m29, [r4+64* 8] 7125 call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_2x8x8_hi 7126 vpbroadcastd m13, [o(pw_8192)] 7127 mova [r4+64* 8], m1 7128 mova [r4+64* 9], m3 7129 mova [r4+64*10], m5 7130 mova [r4+64*11], m7 7131 mova [r4+64*16], m23 7132 mova [r4+64*17], m25 7133 mova [r4+64*18], m27 7134 mova [r4+64*19], m29 7135 pmulhrsw m23, m13, m0 ; b0 7136 pmulhrsw m25, m13, m2 ; b2 7137 pmulhrsw m27, m13, m4 ; b4 7138 pmulhrsw m29, m13, m6 ; b6 7139 mova m0, [r4+64*31] 7140 mova m1, [r4+64*30] 7141 mova m2, [r4+64*29] 7142 mova m3, [r4+64*28] 7143 mova m4, [r4+64*27] 7144 mova m5, [r4+64*26] 7145 mova m6, [r4+64*25] 7146 mova m7, [r4+64*24] 7147 mova m14, [r4+64* 7] 7148 mova m15, [r4+64* 6] 7149 mova m16, [r4+64* 5] 7150 mova m17, [r4+64* 4] 7151 mova m18, [r4+64* 3] 7152 mova m19, [r4+64* 2] 7153 mova m20, [r4+64* 1] 7154 mova m21, [r4+64* 0] 7155 call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_2x8x8_lo 7156 mov r6, cq 7157 call .pass2_end 7158 jmp .end 7159 .fast: ; bottom/right halves are zero 7160 mova m28, [o(dup16_perm)] 7161 pmovzxwd m9, [cq+64* 0] 7162 vpermb m8, m28, [cq+64* 4] 7163 vpermb ym1, ym28, [cq+64*12] 7164 vpermb m7, m28, [cq+64* 8] 7165 pslld m9, 16 7166 call m(idct_16x16_internal_8bpc).main_fast2 7167 vpermb m21, m28, [cq+64* 2] 7168 vpermb ym15, ym28, [cq+64*14] 7169 vpermb ym18, ym28, [cq+64*10] 7170 vpermb m14, m28, [cq+64* 6] 7171 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 7172 vpermb m22, m28, [cq+64* 1] 7173 vpermb ym29, ym28, [cq+64*15] 7174 vpermb ym26, ym28, [cq+64* 9] 7175 vpermb m25, m28, [cq+64* 7] 7176 vpermb m24, m28, [cq+64* 5] 7177 vpermb ym27, ym28, [cq+64*11] 7178 vpermb m23, m28, [cq+64* 3] 7179 vpermb ym28, ym28, [cq+64*13] 7180 mova [cq+64* 0], m14 7181 mova [cq+64* 1], m15 7182 mova [cq+64* 2], m16 7183 mova [cq+64* 3], m17 7184 mova [cq+64* 4], m18 7185 mova [cq+64* 5], m19 7186 mova [cq+64* 6], m20 7187 mova [cq+64* 7], m21 7188 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast 7189 vpbroadcastd m13, [o(pw_8192)] 7190 mova [cq+64*16], m4 7191 mova [cq+64*17], m5 7192 mova [cq+64*18], m6 7193 mova [cq+64*19], m7 7194 mova [cq+64*28], m26 7195 mova [cq+64*29], m27 7196 mova [cq+64*30], m28 7197 mova [cq+64*31], m29 7198 call m(inv_txfm_add_dct_dct_64x16_8bpc).pass1_end 7199 mova [cq+64*20], m22 7200 mova [cq+64*21], m23 7201 mova [cq+64*22], m24 7202 mova [cq+64*23], m25 7203 mova [cq+64*24], m26 7204 mova [cq+64*25], m27 7205 mova [cq+64*26], m28 7206 mova [cq+64*27], m29 7207 lea r4, [rsp+64*64] 7208 lea r3, [rsp+64*32] 7209 call .pass2_fast 7210 pmulhrsw m0, m13, [cq+64*16] 7211 pmulhrsw m1, m13, [cq+64*17] 7212 pmulhrsw m2, m13, [cq+64*18] 7213 pmulhrsw m3, m13, [cq+64*19] 7214 pmulhrsw m4, m13, [cq+64*20] 7215 pmulhrsw m5, m13, [cq+64*21] 7216 pmulhrsw m6, m13, [cq+64*22] 7217 pmulhrsw m7, m13, [cq+64*23] 7218 pmulhrsw m14, m13, [cq+64*24] 7219 pmulhrsw m15, m13, [cq+64*25] 7220 pmulhrsw m16, m13, [cq+64*26] 7221 pmulhrsw m17, m13, [cq+64*27] 7222 pmulhrsw m18, m13, [cq+64*28] 7223 pmulhrsw m19, m13, [cq+64*29] 7224 pmulhrsw m20, m13, [cq+64*30] 7225 pmulhrsw m21, m13, [cq+64*31] 7226 call m(inv_txfm_add_dct_dct_64x16_8bpc).transpose_round 7227 mov r4, rsp 7228 mov r3, cq 7229 call .pass2_fast 7230 .end: 7231 vpbroadcastd m17, [o(pw_2048)] 7232 lea r5, [strideq*8] 7233 mov r3, dstq 7234 pxor m16, m16 7235 sub r4, 64*5 ; rsp+64*31 7236 mov r6, rsp 7237 .end_loop: 7238 mova m2, [r6+64*32] ; idct16 0+n lo 7239 mova m7, [r6+64*48] ; idct32 31-n lo 7240 mova m6, [cq+64* 0] ; idct16 0+n hi 7241 mova m0, [cq+64*16] ; idct32 31-n hi 7242 mova m4, [r4+64*64] ; idct64 63-n lo 7243 mova m1, [r4+64* 0] ; idct64 63-n hi 7244 mova m5, [r6+64*64] ; idct64 32+n lo 7245 mova m8, [r6+64* 0] ; idct64 32+n hi 7246 sub r3, strideq 7247 paddsw m3, m2, m7 ; idct32 0+n lo 7248 mova m12, [dstq+r5*0] 7249 psubsw m2, m7 ; idct32 31-n lo 7250 mova m15, [r3 +r5*8] 7251 paddsw m7, m6, m0 ; idct32 0+n hi 7252 mova m13, [r3 +r5*4] 7253 psubsw m6, m0 ; idct32 31-n hi 7254 mova m14, [dstq+r5*4] 7255 paddsw m0, m3, m4 ; out 0+n lo 7256 add r6, 64 7257 psubsw m3, m4 ; out 63-n lo 7258 sub r4, 64 7259 paddsw m4, m7, m1 ; out 0+n hi 7260 mova [cq+64* 0], m16 7261 psubsw m7, m1 ; out 63-n hi 7262 mova [cq+64*16], m16 7263 paddsw m1, m2, m5 ; out 31-n lo 7264 add cq, 64 7265 psubsw m2, m5 ; out 32+n lo 7266 paddsw m5, m6, m8 ; out 31-n hi 7267 psubsw m6, m8 ; out 32+n hi 7268 pmulhrsw m0, m17 7269 punpcklbw m8, m12, m16 7270 pmulhrsw m4, m17 7271 punpckhbw m12, m16 7272 pmulhrsw m3, m17 7273 punpcklbw m11, m15, m16 7274 pmulhrsw m7, m17 7275 punpckhbw m15, m16 7276 pmulhrsw m1, m17 7277 punpcklbw m9, m13, m16 7278 pmulhrsw m5, m17 7279 punpckhbw m13, m16 7280 pmulhrsw m2, m17 7281 punpcklbw m10, m14, m16 7282 pmulhrsw m6, m17 7283 punpckhbw m14, m16 7284 paddw m0, m8 7285 paddw m4, m12 7286 packuswb m0, m4 7287 paddw m3, m11 7288 paddw m7, m15 7289 packuswb m3, m7 7290 paddw m1, m9 7291 paddw m5, m13 7292 packuswb m1, m5 7293 paddw m2, m10 7294 paddw m6, m14 7295 packuswb m2, m6 7296 mova [dstq+r5*0], m0 7297 mova [r3 +r5*8], m3 7298 mova [r3 +r5*4], m1 7299 mova [dstq+r5*4], m2 7300 add dstq, strideq 7301 cmp r6, r4 7302 jb .end_loop 7303 RET 7304 .dconly: 7305 movsx r6d, word [cq] 7306 mov [cq], eobd 7307 or r3d, 64 7308 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly 7309 ALIGN function_align 7310 .pass2_end: 7311 REPX {pmulhrsw x, m13}, m22, m24, m26, m28, m14, m16, m18, m20, m0, m2, m4, m6 7312 mova [r4+64*20], m1 7313 mova [r4+64*21], m3 7314 mova [r4+64*22], m5 7315 mova [r4+64*23], m7 7316 vinserti32x8 m1, m23, ym14, 1 ; a00 a01 c00 c01 7317 vshufi32x4 m3, m23, m14, q3232 ; a02 a03 c02 c03 7318 vinserti32x8 m5, m22, ym0, 1 ; e00 e01 g00 g01 7319 vshufi32x4 m14, m22, m0, q3232 ; e02 e03 g02 g03 7320 mova [r4+64*12], m15 7321 mova [r4+64*13], m17 7322 mova [r4+64*14], m19 7323 mova [r4+64*15], m21 7324 vinserti32x8 m15, m27, ym18, 1 ; a40 a41 c40 c41 7325 vshufi32x4 m17, m27, m18, q3232 ; a42 a43 c42 c43 7326 vinserti32x8 m18, m26, ym4, 1 ; e40 e41 g40 g41 7327 vshufi32x4 m19, m26, m4, q3232 ; e42 e43 g42 g43 7328 vinserti32x8 m22, m25, ym16, 1 ; a20 a21 c20 c21 7329 vshufi32x4 m26, m25, m16, q3232 ; a22 a23 c22 c23 7330 vinserti32x8 m25, m24, ym2, 1 ; e20 e21 g20 g21 7331 vshufi32x4 m27, m24, m2, q3232 ; e22 e23 g22 g23 7332 vinserti32x8 m23, m29, ym20, 1 ; a60 a61 c60 c61 7333 vshufi32x4 m29, m20, q3232 ; a62 a63 c62 c63 7334 vshufi32x4 m13, m28, m6, q3232 ; e62 e63 g62 g63 7335 vinserti32x8 m28, ym6, 1 ; e60 e61 g60 g61 7336 vshufi32x4 m0, m1, m5, q2020 ; 0 7337 vshufi32x4 m1, m5, q3131 ; 8 7338 vshufi32x4 m2, m3, m14, q2020 ; 16 7339 vshufi32x4 m3, m14, q3131 ; 24 7340 vshufi32x4 m14, m15, m18, q2020 ; 4 7341 vshufi32x4 m15, m18, q3131 ; 12 7342 vshufi32x4 m16, m17, m19, q2020 ; 20 7343 vshufi32x4 m17, m19, q3131 ; 28 7344 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast 7345 vshufi32x4 m24, m22, m25, q3131 ; 10 7346 vshufi32x4 m22, m25, q2020 ; 2 7347 vshufi32x4 m25, m23, m28, q3131 ; 14 7348 vshufi32x4 m23, m28, q2020 ; 6 7349 vshufi32x4 m28, m26, m27, q3131 ; 26 7350 vshufi32x4 m26, m27, q2020 ; 18 7351 vshufi32x4 m27, m29, m13, q2020 ; 22 7352 vshufi32x4 m29, m13, q3131 ; 30 7353 mova [r6+64* 0], m0 7354 mova [r6+64* 1], m1 7355 mova [r6+64* 2], m2 7356 mova [r6+64* 3], m3 7357 mova [r6+64* 4], m4 7358 mova [r6+64* 5], m5 7359 mova [r6+64* 6], m6 7360 mova [r6+64* 7], m7 7361 mova [r6+64* 8], m14 7362 mova [r6+64* 9], m15 7363 mova [r6+64*10], m16 7364 mova [r6+64*11], m17 7365 mova [r6+64*12], m18 7366 mova [r6+64*13], m19 7367 mova [r6+64*14], m20 7368 mova [r6+64*15], m21 7369 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast 7370 vpbroadcastd m13, [o(pw_8192)] 7371 mova [r6+64*16], m29 7372 mova [r6+64*17], m28 7373 mova [r6+64*18], m27 7374 mova [r6+64*19], m26 7375 mova [r6+64*20], m25 7376 mova [r6+64*21], m24 7377 mova [r6+64*22], m23 7378 mova [r6+64*23], m22 7379 mova [r6+64*24], m21 7380 mova [r6+64*25], m20 7381 mova [r6+64*26], m19 7382 mova [r6+64*27], m18 7383 mova [r6+64*28], m17 7384 mova [r6+64*29], m16 7385 mova [r6+64*30], m15 7386 mova [r6+64*31], m14 7387 pmulhrsw m15, m13, [r4+64* 8] ; 1 9 17 25 7388 pmulhrsw m16, m13, [r4+64*12] 7389 pmulhrsw m17, m13, [r4+64*16] 7390 pmulhrsw m18, m13, [r4+64*20] 7391 pmulhrsw m19, m13, [r4+64*11] ; 7 15 23 31 7392 pmulhrsw m20, m13, [r4+64*15] 7393 pmulhrsw m21, m13, [r4+64*19] 7394 pmulhrsw m22, m13, [r4+64*23] 7395 vinserti32x8 m14, m15, ym16, 1 ; a1 a9 c1 c9 7396 vshufi32x4 m15, m16, q3232 ; a17 a25 c17 c25 7397 vinserti32x8 m16, m17, ym18, 1 ; e1 e9 g1 g9 7398 vshufi32x4 m17, m18, q3232 ; e17 e25 g17 g25 7399 pmulhrsw m23, m13, [r4+64*10] ; 5 13 21 29 7400 pmulhrsw m24, m13, [r4+64*14] 7401 pmulhrsw m25, m13, [r4+64*18] 7402 pmulhrsw m26, m13, [r4+64*22] 7403 vinserti32x8 m18, m19, ym20, 1 ; a7 a15 c7 c15 7404 vshufi32x4 m19, m20, q3232 ; a23 a31 c23 c31 7405 vinserti32x8 m20, m21, ym22, 1 ; e7 e15 g7 g15 7406 vshufi32x4 m21, m22, q3232 ; e23 e31 g23 g31 7407 pmulhrsw m27, m13, [r4+64* 9] ; 3 11 19 27 7408 pmulhrsw m28, m13, [r4+64*13] 7409 pmulhrsw m29, m13, [r4+64*17] 7410 pmulhrsw m13, [r4+64*21] 7411 vshufi32x4 m0, m14, m16, q2020 ; 1 7412 vshufi32x4 m1, m19, m21, q3131 ; 31 7413 vshufi32x4 m2, m15, m17, q2020 ; 17 7414 vshufi32x4 m3, m18, m20, q3131 ; 15 7415 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7416 vshufi32x4 m0, m18, m20, q2020 ; 7 7417 vshufi32x4 m1, m15, m17, q3131 ; 25 7418 vshufi32x4 m2, m19, m21, q2020 ; 23 7419 vshufi32x4 m3, m14, m16, q3131 ; 9 7420 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7421 vinserti32x8 m22, m23, ym24, 1 ; a5 a13 c5 c13 7422 vshufi32x4 m23, m24, q3232 ; a21 a29 c21 c29 7423 vinserti32x8 m24, m25, ym26, 1 ; e5 e13 g5 g13 7424 vshufi32x4 m25, m26, q3232 ; e21 e29 g21 g29 7425 vinserti32x8 m26, m27, ym28, 1 ; a3 a11 c3 c11 7426 vshufi32x4 m27, m28, q3232 ; a19 a27 c19 c27 7427 vinserti32x8 m28, m29, ym13, 1 ; e3 e11 g3 g11 7428 vshufi32x4 m29, m13, q3232 ; e19 e17 g19 g27 7429 vshufi32x4 m0, m22, m24, q2020 ; 5 7430 vshufi32x4 m1, m27, m29, q3131 ; 27 7431 vshufi32x4 m2, m23, m25, q2020 ; 21 7432 vshufi32x4 m3, m26, m28, q3131 ; 11 7433 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7434 vshufi32x4 m0, m26, m28, q2020 ; 3 7435 vshufi32x4 m1, m23, m25, q3131 ; 29 7436 vshufi32x4 m2, m27, m29, q2020 ; 19 7437 vshufi32x4 m3, m22, m24, q3131 ; 13 7438 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 7439 jmp m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 7440 ALIGN function_align 7441 .pass2_fast: 7442 vshufi32x4 m23, m1, m16, q3131 ; 6 7443 vshufi32x4 m22, m1, m16, q2020 ; 2 7444 vshufi32x4 m14, m0, m3, q3131 ; 4 7445 vshufi32x4 m26, m0, m3, q2020 ; 0 7446 vshufi32x4 m28, m9, m15, q3131 ; 5 7447 vshufi32x4 m0, m9, m15, q2020 ; 1 7448 vshufi32x4 m16, m11, m17, q3131 ; 7 7449 vshufi32x4 m29, m11, m17, q2020 ; 3 7450 vshufi32x4 m15, m8, m2, q3131 ; 12 7451 vshufi32x4 m27, m8, m2, q2020 ; 8 7452 vshufi32x4 m25, m5, m19, q3131 ; 14 7453 vshufi32x4 m24, m5, m19, q2020 ; 10 7454 vshufi32x4 m3, m6, m20, q3131 ; 15 7455 vshufi32x4 m19, m6, m20, q2020 ; 11 7456 vshufi32x4 m17, m4, m18, q3131 ; 13 7457 vshufi32x4 m18, m4, m18, q2020 ; 9 7458 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast 7459 mova m0, m16 7460 mova m3, m18 7461 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast 7462 mova m0, m28 7463 mova m3, m19 7464 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast 7465 mova m0, m29 7466 mova m3, m17 7467 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast 7468 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 7469 mova m0, m26 7470 mova m1, m27 7471 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 7472 mova [r3+64* 0], m0 7473 mova [r3+64* 1], m1 7474 mova [r3+64* 2], m2 7475 mova [r3+64* 3], m3 7476 mova [r3+64* 4], m4 7477 mova [r3+64* 5], m5 7478 mova [r3+64* 6], m6 7479 mova [r3+64* 7], m7 7480 mova [r3+64* 8], m14 7481 mova [r3+64* 9], m15 7482 mova [r3+64*10], m16 7483 mova [r3+64*11], m17 7484 mova [r3+64*12], m18 7485 mova [r3+64*13], m19 7486 mova [r3+64*14], m20 7487 mova [r3+64*15], m21 7488 call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 7489 mova [r3+64*16], m29 7490 mova [r3+64*17], m28 7491 mova [r3+64*18], m27 7492 mova [r3+64*19], m26 7493 mova [r3+64*20], m25 7494 mova [r3+64*21], m24 7495 mova [r3+64*22], m23 7496 mova [r3+64*23], m22 7497 mova [r3+64*24], m21 7498 mova [r3+64*25], m20 7499 mova [r3+64*26], m19 7500 mova [r3+64*27], m18 7501 mova [r3+64*28], m17 7502 mova [r3+64*29], m16 7503 mova [r3+64*30], m15 7504 mova [r3+64*31], m14 7505 ret 7506 7507 %endif ; ARCH_X86_64