tx_priv.h (17874B)
1 /* 2 * This file is part of FFmpeg. 3 * 4 * FFmpeg is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU Lesser General Public 6 * License as published by the Free Software Foundation; either 7 * version 2.1 of the License, or (at your option) any later version. 8 * 9 * FFmpeg is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * Lesser General Public License for more details. 13 * 14 * You should have received a copy of the GNU Lesser General Public 15 * License along with FFmpeg; if not, write to the Free Software 16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 19 #ifndef AVUTIL_TX_PRIV_H 20 #define AVUTIL_TX_PRIV_H 21 22 #include "tx.h" 23 #include "thread.h" 24 #include "mem_internal.h" 25 #include "common.h" 26 #include "attributes.h" 27 28 #ifdef TX_FLOAT 29 #define TX_TAB(x) x ## _float 30 #define TX_NAME(x) x ## _float_c 31 #define TX_NAME_STR(x) NULL_IF_CONFIG_SMALL(x "_float_c") 32 #define TX_TYPE(x) AV_TX_FLOAT_ ## x 33 #define TX_FN_NAME(fn, suffix) ff_tx_ ## fn ## _float_ ## suffix 34 #define TX_FN_NAME_STR(fn, suffix) NULL_IF_CONFIG_SMALL(#fn "_float_" #suffix) 35 #define MULT(x, m) ((x) * (m)) 36 #define SCALE_TYPE float 37 typedef float TXSample; 38 typedef float TXUSample; 39 typedef AVComplexFloat TXComplex; 40 #elif defined(TX_DOUBLE) 41 #define TX_TAB(x) x ## _double 42 #define TX_NAME(x) x ## _double_c 43 #define TX_NAME_STR(x) NULL_IF_CONFIG_SMALL(x "_double_c") 44 #define TX_TYPE(x) AV_TX_DOUBLE_ ## x 45 #define TX_FN_NAME(fn, suffix) ff_tx_ ## fn ## _double_ ## suffix 46 #define TX_FN_NAME_STR(fn, suffix) NULL_IF_CONFIG_SMALL(#fn "_double_" #suffix) 47 #define MULT(x, m) ((x) * (m)) 48 #define SCALE_TYPE double 49 typedef double TXSample; 50 typedef double TXUSample; 51 typedef AVComplexDouble TXComplex; 52 #elif defined(TX_INT32) 53 #define TX_TAB(x) x ## _int32 54 #define TX_NAME(x) x ## _int32_c 55 #define TX_NAME_STR(x) NULL_IF_CONFIG_SMALL(x "_int32_c") 56 #define TX_TYPE(x) AV_TX_INT32_ ## x 57 #define TX_FN_NAME(fn, suffix) ff_tx_ ## fn ## _int32_ ## suffix 58 #define TX_FN_NAME_STR(fn, suffix) NULL_IF_CONFIG_SMALL(#fn "_int32_" #suffix) 59 #define MULT(x, m) (((((int64_t)(x)) * (int64_t)(m)) + 0x40000000) >> 31) 60 #define SCALE_TYPE float 61 typedef int32_t TXSample; 62 typedef uint32_t TXUSample; 63 typedef AVComplexInt32 TXComplex; 64 #else 65 typedef void TXComplex; 66 #endif 67 68 #define TX_DECL_FN(fn, suffix) \ 69 void TX_FN_NAME(fn, suffix)(AVTXContext *s, void *o, void *i, ptrdiff_t st); 70 71 #define TX_DEF(fn, tx_type, len_min, len_max, f1, f2, \ 72 p, init_fn, suffix, cf, cd_flags, cf2) \ 73 &(const FFTXCodelet){ \ 74 .name = TX_FN_NAME_STR(fn, suffix), \ 75 .function = TX_FN_NAME(fn, suffix), \ 76 .type = TX_TYPE(tx_type), \ 77 .flags = FF_TX_ALIGNED | FF_TX_OUT_OF_PLACE | cd_flags, \ 78 .factors = { (f1), (f2) }, \ 79 .nb_factors = !!(f1) + !!(f2), \ 80 .min_len = len_min, \ 81 .max_len = len_max, \ 82 .init = init_fn, \ 83 .cpu_flags = cf2 | AV_CPU_FLAG_ ## cf, \ 84 .prio = p, \ 85 } 86 87 #if defined(TX_FLOAT) || defined(TX_DOUBLE) 88 89 #define CMUL(dre, dim, are, aim, bre, bim) \ 90 do { \ 91 (dre) = (are) * (bre) - (aim) * (bim); \ 92 (dim) = (are) * (bim) + (aim) * (bre); \ 93 } while (0) 94 95 #define SMUL(dre, dim, are, aim, bre, bim) \ 96 do { \ 97 (dre) = (are) * (bre) - (aim) * (bim); \ 98 (dim) = (are) * (bim) - (aim) * (bre); \ 99 } while (0) 100 101 #define UNSCALE(x) (x) 102 #define RESCALE(x) (x) 103 104 #define FOLD(a, b) ((a) + (b)) 105 106 #define BF(x, y, a, b) \ 107 do { \ 108 x = (a) - (b); \ 109 y = (a) + (b); \ 110 } while (0) 111 112 #elif defined(TX_INT32) 113 114 /* Properly rounds the result */ 115 #define CMUL(dre, dim, are, aim, bre, bim) \ 116 do { \ 117 int64_t accu; \ 118 (accu) = (int64_t)(bre) * (are); \ 119 (accu) -= (int64_t)(bim) * (aim); \ 120 (dre) = (int)(((accu) + 0x40000000) >> 31); \ 121 (accu) = (int64_t)(bim) * (are); \ 122 (accu) += (int64_t)(bre) * (aim); \ 123 (dim) = (int)(((accu) + 0x40000000) >> 31); \ 124 } while (0) 125 126 #define SMUL(dre, dim, are, aim, bre, bim) \ 127 do { \ 128 int64_t accu; \ 129 (accu) = (int64_t)(bre) * (are); \ 130 (accu) -= (int64_t)(bim) * (aim); \ 131 (dre) = (int)(((accu) + 0x40000000) >> 31); \ 132 (accu) = (int64_t)(bim) * (are); \ 133 (accu) -= (int64_t)(bre) * (aim); \ 134 (dim) = (int)(((accu) + 0x40000000) >> 31); \ 135 } while (0) 136 137 #define UNSCALE(x) ((double)(x)/2147483648.0) 138 #define RESCALE(x) (av_clip64(llrintf((x) * 2147483648.0), INT32_MIN, INT32_MAX)) 139 140 #define FOLD(x, y) ((int32_t)((x) + (unsigned)(y) + 32) >> 6) 141 142 #define BF(x, y, a, b) \ 143 do { \ 144 x = (a) - (unsigned)(b); \ 145 y = (a) + (unsigned)(b); \ 146 } while (0) 147 148 #endif /* TX_INT32 */ 149 150 #define CMUL3(c, a, b) CMUL((c).re, (c).im, (a).re, (a).im, (b).re, (b).im) 151 152 /* Codelet flags, used to pick codelets. Must be a superset of enum AVTXFlags, 153 * but if it runs out of bits, it can be made separate. */ 154 #define FF_TX_OUT_OF_PLACE (1ULL << 63) /* Can be OR'd with AV_TX_INPLACE */ 155 #define FF_TX_ALIGNED (1ULL << 62) /* Cannot be OR'd with AV_TX_UNALIGNED */ 156 #define FF_TX_PRESHUFFLE (1ULL << 61) /* Codelet expects permuted coeffs */ 157 #define FF_TX_INVERSE_ONLY (1ULL << 60) /* For non-orthogonal inverse-only transforms */ 158 #define FF_TX_FORWARD_ONLY (1ULL << 59) /* For non-orthogonal forward-only transforms */ 159 #define FF_TX_ASM_CALL (1ULL << 58) /* For asm->asm functions only */ 160 161 typedef enum FFTXCodeletPriority { 162 FF_TX_PRIO_BASE = 0, /* Baseline priority */ 163 164 /* For SIMD, set base prio to the register size in bits and increment in 165 * steps of 64 depending on faster/slower features, like FMA. */ 166 167 FF_TX_PRIO_MIN = -131072, /* For naive implementations */ 168 FF_TX_PRIO_MAX = 32768, /* For custom implementations/ASICs */ 169 } FFTXCodeletPriority; 170 171 typedef enum FFTXMapDirection { 172 /* No map. Make a map up. */ 173 FF_TX_MAP_NONE = 0, 174 175 /* Lookup table must be applied via dst[i] = src[lut[i]]; */ 176 FF_TX_MAP_GATHER, 177 178 /* Lookup table must be applied via dst[lut[i]] = src[i]; */ 179 FF_TX_MAP_SCATTER, 180 } FFTXMapDirection; 181 182 /* Codelet options */ 183 typedef struct FFTXCodeletOptions { 184 /* Request a specific lookup table direction. Codelets MUST put the 185 * direction in AVTXContext. If the codelet does not respect this, a 186 * conversion will be performed. */ 187 FFTXMapDirection map_dir; 188 } FFTXCodeletOptions; 189 190 /* Maximum number of factors a codelet may have. Arbitrary. */ 191 #define TX_MAX_FACTORS 16 192 193 /* Maximum amount of subtransform functions, subtransforms and factors. Arbitrary. */ 194 #define TX_MAX_SUB 4 195 196 /* Maximum number of returned results for ff_tx_decompose_length. Arbitrary. */ 197 #define TX_MAX_DECOMPOSITIONS 512 198 199 typedef struct FFTXCodelet { 200 const char *name; /* Codelet name, for debugging */ 201 av_tx_fn function; /* Codelet function, != NULL */ 202 enum AVTXType type; /* Type of codelet transform */ 203 #define TX_TYPE_ANY INT32_MAX /* Special type to allow all types */ 204 205 uint64_t flags; /* A combination of AVTXFlags and codelet 206 * flags that describe its properties. */ 207 208 int factors[TX_MAX_FACTORS]; /* Length factors. MUST be coprime. */ 209 #define TX_FACTOR_ANY -1 /* When used alone, signals that the codelet 210 * supports all factors. Otherwise, if other 211 * factors are present, it signals that whatever 212 * remains will be supported, as long as the 213 * other factors are a component of the length */ 214 215 int nb_factors; /* Minimum number of factors that have to 216 * be a modulo of the length. Must not be 0. */ 217 218 int min_len; /* Minimum length of transform, must be >= 1 */ 219 int max_len; /* Maximum length of transform */ 220 #define TX_LEN_UNLIMITED -1 /* Special length value to permit all lengths */ 221 222 int (*init)(AVTXContext *s, /* Optional callback for current context initialization. */ 223 const struct FFTXCodelet *cd, 224 uint64_t flags, 225 FFTXCodeletOptions *opts, 226 int len, int inv, 227 const void *scale); 228 229 int (*uninit)(AVTXContext *s); /* Optional callback for uninitialization. */ 230 231 int cpu_flags; /* CPU flags. If any negative flags like 232 * SLOW are present, will avoid picking. 233 * 0x0 to signal it's a C codelet */ 234 #define FF_TX_CPU_FLAGS_ALL 0x0 /* Special CPU flag for C */ 235 236 int prio; /* < 0 = least, 0 = no pref, > 0 = prefer */ 237 } FFTXCodelet; 238 239 struct AVTXContext { 240 /* Fields the root transform and subtransforms use or may use. 241 * NOTE: This section is used by assembly, do not reorder or change */ 242 int len; /* Length of the transform */ 243 int inv; /* If transform is inverse */ 244 int *map; /* Lookup table(s) */ 245 TXComplex *exp; /* Any non-pre-baked multiplication factors, 246 * or extra temporary buffer */ 247 TXComplex *tmp; /* Temporary buffer, if needed */ 248 249 AVTXContext *sub; /* Subtransform context(s), if needed */ 250 av_tx_fn fn[TX_MAX_SUB]; /* Function(s) for the subtransforms */ 251 int nb_sub; /* Number of subtransforms. 252 * The reason all of these are set here 253 * rather than in each separate context 254 * is to eliminate extra pointer 255 * dereferences. */ 256 257 /* Fields mainly useul/applicable for the root transform or initialization. 258 * Fields below are not used by assembly code. */ 259 const FFTXCodelet *cd[TX_MAX_SUB]; /* Subtransform codelets */ 260 const FFTXCodelet *cd_self; /* Codelet for the current context */ 261 enum AVTXType type; /* Type of transform */ 262 uint64_t flags; /* A combination of AVTXFlags and 263 * codelet flags used when creating */ 264 FFTXMapDirection map_dir; /* Direction of AVTXContext->map */ 265 float scale_f; 266 double scale_d; 267 void *opaque; /* Free to use by implementations */ 268 }; 269 270 /* This function embeds a Ruritanian PFA input map into an existing lookup table 271 * to avoid double permutation. This allows for compound factors to be 272 * synthesized as fast PFA FFTs and embedded into either other or standalone 273 * transforms. 274 * The output CRT map must still be pre-baked into the transform. */ 275 #define TX_EMBED_INPUT_PFA_MAP(map, tot_len, d1, d2) \ 276 do { \ 277 int mtmp[(d1)*(d2)]; \ 278 for (int k = 0; k < tot_len; k += (d1)*(d2)) { \ 279 memcpy(mtmp, &map[k], (d1)*(d2)*sizeof(*mtmp)); \ 280 for (int m = 0; m < (d2); m++) \ 281 for (int n = 0; n < (d1); n++) \ 282 map[k + m*(d1) + n] = mtmp[(m*(d1) + n*(d2)) % ((d1)*(d2))]; \ 283 } \ 284 } while (0) 285 286 /* This function generates a Ruritanian PFA input map into s->map. */ 287 int ff_tx_gen_pfa_input_map(AVTXContext *s, FFTXCodeletOptions *opts, 288 int d1, int d2); 289 290 /* Create a subtransform in the current context with the given parameters. 291 * The flags parameter from FFTXCodelet.init() should be preserved as much 292 * as that's possible. 293 * MUST be called during the sub() callback of each codelet. */ 294 int ff_tx_init_subtx(AVTXContext *s, enum AVTXType type, 295 uint64_t flags, FFTXCodeletOptions *opts, 296 int len, int inv, const void *scale); 297 298 /* Clear the context by freeing all tables, maps and subtransforms. */ 299 void ff_tx_clear_ctx(AVTXContext *s); 300 301 /* Attempt to factorize a length into 2 integers such that 302 * len / dst1 == dst2, where dst1 and dst2 are coprime. */ 303 int ff_tx_decompose_length(int dst[TX_MAX_DECOMPOSITIONS], enum AVTXType type, 304 int len, int inv); 305 306 /* Generate a default map (0->len or 0, (len-1)->1 for inverse transforms) 307 * for a context. */ 308 int ff_tx_gen_default_map(AVTXContext *s, FFTXCodeletOptions *opts); 309 310 /* 311 * Generates the PFA permutation table into AVTXContext->pfatab. The end table 312 * is appended to the start table. 313 * The `inv` flag should only be enabled if the lookup tables of subtransforms 314 * won't get flattened. 315 */ 316 int ff_tx_gen_compound_mapping(AVTXContext *s, FFTXCodeletOptions *opts, 317 int inv, int n, int m); 318 319 /* 320 * Generates a standard-ish (slightly modified) Split-Radix revtab into 321 * AVTXContext->map. Invert lookup changes how the mapping needs to be applied. 322 * If it's set to 0, it has to be applied like out[map[i]] = in[i], otherwise 323 * if it's set to 1, has to be applied as out[i] = in[map[i]] 324 */ 325 int ff_tx_gen_ptwo_revtab(AVTXContext *s, FFTXCodeletOptions *opts); 326 327 /* 328 * Generates an index into AVTXContext->inplace_idx that if followed in the 329 * specific order, allows the revtab to be done in-place. The sub-transform 330 * and its map should already be initialized. 331 */ 332 int ff_tx_gen_inplace_map(AVTXContext *s, int len); 333 334 /* 335 * This generates a parity-based revtab of length len and direction inv. 336 * 337 * Parity means even and odd complex numbers will be split, e.g. the even 338 * coefficients will come first, after which the odd coefficients will be 339 * placed. For example, a 4-point transform's coefficients after reordering: 340 * z[0].re, z[0].im, z[2].re, z[2].im, z[1].re, z[1].im, z[3].re, z[3].im 341 * 342 * The basis argument is the length of the largest non-composite transform 343 * supported, and also implies that the basis/2 transform is supported as well, 344 * as the split-radix algorithm requires it to be. 345 * 346 * The dual_stride argument indicates that both the basis, as well as the 347 * basis/2 transforms support doing two transforms at once, and the coefficients 348 * will be interleaved between each pair in a split-radix like so (stride == 2): 349 * tx1[0], tx1[2], tx2[0], tx2[2], tx1[1], tx1[3], tx2[1], tx2[3] 350 * A non-zero number switches this on, with the value indicating the stride 351 * (how many values of 1 transform to put first before switching to the other). 352 * Must be a power of two or 0. Must be less than the basis. 353 * Value will be clipped to the transform size, so for a basis of 16 and a 354 * dual_stride of 8, dual 8-point transforms will be laid out as if dual_stride 355 * was set to 4. 356 * Usually you'll set this to half the complex numbers that fit in a single 357 * register or 0. This allows to reuse SSE functions as dual-transform 358 * functions in AVX mode. 359 * 360 * If length is smaller than basis/2 this function will not do anything. 361 * 362 * If inv_lookup is set to 1, it will flip the lookup from out[map[i]] = src[i] 363 * to out[i] = src[map[i]]. 364 */ 365 int ff_tx_gen_split_radix_parity_revtab(AVTXContext *s, int len, int inv, 366 FFTXCodeletOptions *opts, 367 int basis, int dual_stride); 368 369 /* Typed init function to initialize shared tables. Will initialize all tables 370 * for all factors of a length. */ 371 void ff_tx_init_tabs_float (int len); 372 void ff_tx_init_tabs_double(int len); 373 void ff_tx_init_tabs_int32 (int len); 374 375 /* Typed init function to initialize an MDCT exptab in a context. 376 * If pre_tab is set, duplicates the entire table, with the first 377 * copy being shuffled according to pre_tab, and the second copy 378 * being the original. */ 379 int ff_tx_mdct_gen_exp_float (AVTXContext *s, int *pre_tab); 380 int ff_tx_mdct_gen_exp_double(AVTXContext *s, int *pre_tab); 381 int ff_tx_mdct_gen_exp_int32 (AVTXContext *s, int *pre_tab); 382 383 /* Lists of codelets */ 384 extern const FFTXCodelet * const ff_tx_codelet_list_float_c []; 385 extern const FFTXCodelet * const ff_tx_codelet_list_float_x86 []; 386 extern const FFTXCodelet * const ff_tx_codelet_list_float_aarch64 []; 387 388 extern const FFTXCodelet * const ff_tx_codelet_list_double_c []; 389 390 extern const FFTXCodelet * const ff_tx_codelet_list_int32_c []; 391 392 #endif /* AVUTIL_TX_PRIV_H */