test-sets.yml (14302B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 --- 5 # Each key in this file specifies a set of tests to run. Different test sets 6 # may, for example, be bound to different test platforms. 7 # 8 # Note that set does not depend on the tree; tree-dependent job selection 9 # should be performed in the target task selection phase of task-graph 10 # generation. 11 # 12 # A test set has a name, and a list of tests that it contains. 13 # 14 # Test names given here reference tests.yml. 15 16 ### Feature-specific test groups 17 18 webgpu: 19 - mochitest-webgpu 20 - web-platform-tests-webgpu 21 - web-platform-tests-webgpu-long 22 - web-platform-tests-webgpu-backlog 23 - web-platform-tests-webgpu-backlog-long 24 25 web-platform-tests: 26 - web-platform-tests 27 - web-platform-tests-crashtest 28 - web-platform-tests-print-reftest 29 - web-platform-tests-reftest 30 - web-platform-tests-wdspec 31 - web-platform-tests-canvas 32 - web-platform-tests-webcodecs 33 - web-platform-tests-eme 34 35 web-platform-tests-2404-tsan: 36 - web-platform-tests 37 - web-platform-tests-crashtest 38 - web-platform-tests-print-reftest 39 - web-platform-tests-reftest 40 - web-platform-tests-wdspec 41 - web-platform-tests-canvas 42 - web-platform-tests-webcodecs 43 - web-platform-tests-eme 44 45 web-platform-tests-backlog: 46 - web-platform-tests-backlog 47 - web-platform-tests-reftest-backlog 48 49 mochitest-devtools-chrome: 50 - mochitest-devtools-chrome 51 52 talos: 53 - talos-bcv 54 - talos-chrome 55 - talos-damp-inspector 56 - talos-damp-webconsole 57 - talos-damp-other 58 - talos-dromaeojs 59 - talos-g1 60 - talos-g3 61 - talos-g4 62 - talos-g5 63 - talos-other 64 - talos-sessionrestore-many-windows 65 - talos-svgr 66 - talos-tp5o 67 - talos-perf-reftest 68 - talos-perf-reftest-singletons 69 - talos-tabswitch 70 - talos-realworld-webextensions 71 - talos-webgl 72 - talos-pdfpaint 73 # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions 74 # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions 75 76 talos-profiling: 77 - talos-bcv-profiling 78 - talos-chrome-profiling 79 # - talos-damp-profiling - Disabled b/c of Bug 1597734 80 - talos-dromaeojs-profiling 81 - talos-g1-profiling 82 - talos-g3-profiling 83 - talos-g4-profiling 84 - talos-g5-profiling 85 - talos-motionmark-profiling 86 - talos-other-profiling 87 - talos-sessionrestore-many-windows-profiling 88 - talos-perf-reftest-profiling 89 - talos-perf-reftest-singletons-profiling 90 - talos-svgr-profiling 91 - talos-tp5o-profiling 92 - talos-tabswitch-profiling 93 - talos-realworld-webextensions-profiling 94 - talos-webgl-profiling 95 96 browsertime: 97 - browsertime-tp6 98 - browsertime-tp7 99 - browsertime-tp6-essential 100 - browsertime-tp6-bytecode 101 - browsertime-tp6-live 102 - browsertime-tp6-live-sheriffed 103 - browsertime-tp6-webextensions 104 - browsertime-benchmark 105 - browsertime-benchmark-wasm 106 - browsertime-youtube-playback 107 - browsertime-youtube-playback-power 108 - browsertime-custom 109 - browsertime-first-install 110 - browsertime-responsiveness 111 - browsertime-regression-tests 112 - browsertime-upload 113 - browsertime-speculative 114 - browsertime-throttled 115 - browsertime-pageload-benchmark 116 - browsertime-indexeddb 117 - browsertime-video-playback-latency 118 - browsertime-network-bench 119 - browsertime-webcodecs 120 - browsertime-trr-performance 121 122 browsertime-clang: 123 - browsertime-tp6 124 - browsertime-tp6-essential 125 - browsertime-tp6-bytecode 126 - browsertime-tp6-live 127 - browsertime-tp6-live-sheriffed 128 - browsertime-tp6-webextensions 129 - browsertime-benchmark 130 - browsertime-benchmark-wasm 131 - browsertime-youtube-playback 132 - browsertime-responsiveness 133 134 browsertime-profiling: 135 - browsertime-tp6-profiling 136 137 browsertime-high-priority: 138 - browsertime-benchmark 139 - browsertime-tp6 140 141 browsertime-ref-hw: 142 - browsertime-benchmark 143 144 awsy: 145 - awsy 146 - awsy-base 147 - awsy-dmd 148 - awsy-base-dmd 149 - awsy-tp6 150 151 ccov-code-coverage-tests: 152 - test-coverage 153 - test-coverage-wpt 154 155 mochitest-valgrind: 156 - mochitest-valgrind 157 158 desktop-screenshot-capture: 159 - mochitest-browser-screenshots 160 161 ### Specific platform test sets 162 163 # Linux 164 linux-tests: 165 - cppunittest 166 - crashtest 167 - firefox-ui-functional 168 - gtest 169 # - jittest # See comment in linux-ccov-tests set. 170 - jsreftest 171 - marionette-integration 172 - marionette-unittest 173 - mochitest-a11y 174 - mochitest-browser-a11y 175 - mochitest-browser-chrome 176 - mochitest-browser-media 177 - mochitest-browser-translations 178 - mochitest-chrome 179 - mochitest-chrome-gpu 180 - mochitest-devtools-chrome 181 - mochitest-media 182 - mochitest-plain 183 - mochitest-plain-gpu 184 - mochitest-remote 185 - mochitest-webgl1-core 186 - mochitest-webgl1-ext 187 - mochitest-webgl2-core 188 - mochitest-webgl2-ext 189 - reftest 190 - telemetry-tests-client 191 - test-verify 192 - test-verify-gpu 193 - test-verify-wpt 194 - xpcshell 195 196 linux-2404-tsan-tests: 197 - cppunittest 198 - crashtest 199 - firefox-ui-functional 200 - gtest 201 # - jittest # See comment in linux-ccov-tests set. 202 - jsreftest 203 - marionette-integration 204 - marionette-unittest 205 - mochitest-a11y 206 - mochitest-browser-a11y 207 - mochitest-browser-chrome 208 - mochitest-browser-media 209 - mochitest-browser-translations 210 - mochitest-chrome 211 - mochitest-chrome-gpu 212 - mochitest-devtools-chrome 213 - mochitest-media 214 - mochitest-plain 215 - mochitest-plain-gpu 216 - mochitest-remote 217 - mochitest-webgl1-core 218 - mochitest-webgl1-ext 219 - mochitest-webgl2-core 220 - mochitest-webgl2-ext 221 - reftest 222 - telemetry-tests-client 223 - test-verify 224 - test-verify-gpu 225 - test-verify-wpt 226 - xpcshell 227 228 linux-ccov-tests: 229 # jittest run on regular linux builds is redundant with SM(p), so we omit it 230 # from the linux2404-tests set. But, we still want 231 # to run it on ccov builds, so we have this set that has it and which runs 232 # on ccov builds. 233 - jittest 234 235 linux2204-wayland-tests: 236 - mochitest-plain 237 - cppunittest 238 - crashtest 239 - firefox-ui-functional 240 - gtest 241 - jsreftest 242 - marionette-integration 243 - marionette-unittest 244 - mochitest-a11y 245 - mochitest-browser-a11y 246 - mochitest-browser-chrome 247 - mochitest-browser-chrome-failures 248 - mochitest-browser-media 249 - mochitest-browser-translations 250 - mochitest-chrome 251 - mochitest-chrome-gpu 252 - mochitest-devtools-chrome 253 - mochitest-media 254 - mochitest-plain 255 - mochitest-plain-gpu 256 - mochitest-remote 257 - mochitest-webgl1-core 258 - mochitest-webgl1-ext 259 - mochitest-webgl2-core 260 - mochitest-webgl2-ext 261 - reftest 262 - telemetry-tests-client 263 - test-verify 264 - test-verify-gpu 265 - test-verify-wpt 266 - xpcshell 267 268 # Windows 269 windows32-tests: 270 - cppunittest 271 - crashtest 272 - gtest 273 - firefox-ui-functional 274 - jsreftest 275 - marionette-integration 276 - marionette-unittest 277 - mochitest-a11y 278 - mochitest-browser-a11y 279 - mochitest-browser-chrome 280 - mochitest-browser-media 281 - mochitest-chrome 282 - mochitest-media 283 - reftest 284 - test-verify 285 - test-verify-gpu 286 - test-verify-wpt 287 - xpcshell 288 - web-platform-tests 289 - web-platform-tests-crashtest 290 291 windows-aarch64-tests: 292 - cppunittest 293 294 windows10-2009-tests: 295 - cppunittest 296 - gtest 297 - mochitest-chrome 298 - mochitest-chrome-gpu 299 - mochitest-plain 300 - mochitest-plain-gpu 301 - xpcshell 302 303 windows11-64-2009-ccov-tests: 304 - jittest 305 - test-coverage 306 - web-platform-tests 307 - web-platform-tests-crashtest 308 - web-platform-tests-print-reftest 309 - web-platform-tests-reftest 310 - web-platform-tests-wdspec 311 - web-platform-tests-webcodecs 312 - web-platform-tests-eme 313 314 windows11-24h2-tests: 315 - cppunittest 316 - crashtest 317 - firefox-ui-functional 318 - gtest 319 - jsreftest 320 - marionette-integration 321 - marionette-unittest 322 - mochitest-a11y 323 - mochitest-browser-a11y 324 - mochitest-browser-chrome 325 - mochitest-browser-media 326 - mochitest-browser-translations 327 - mochitest-chrome 328 - mochitest-chrome-gpu 329 - mochitest-devtools-chrome 330 - mochitest-media 331 - mochitest-plain 332 - mochitest-plain-gpu 333 - mochitest-remote 334 - mochitest-webgl1-core 335 - mochitest-webgl1-ext 336 - mochitest-webgl2-core 337 - mochitest-webgl2-ext 338 - reftest 339 - telemetry-tests-client 340 - test-verify 341 - test-verify-gpu 342 - test-verify-wpt 343 - xpcshell 344 345 mingwclang-tests-debug: 346 - cppunittest 347 - firefox-ui-functional 348 - mochitest-a11y 349 - mochitest-chrome-gpu 350 - mochitest-plain-gpu 351 - mochitest-webgl1-core 352 - mochitest-webgl1-ext 353 - mochitest-webgl2-core 354 - mochitest-webgl2-ext 355 - telemetry-tests-client 356 - reftest 357 358 # The following is the same as the above block, minus reftests. 359 # Reftests were disabled due to frequent hangs; see bug 1642719 360 # for re-enabling. 361 mingwclang-win32-tests-debug: 362 - cppunittest 363 - firefox-ui-functional 364 - mochitest-a11y 365 - mochitest-chrome-gpu 366 - mochitest-plain-gpu 367 - mochitest-webgl1-core 368 - mochitest-webgl1-ext 369 - mochitest-webgl2-core 370 - mochitest-webgl2-ext 371 # - reftest - disabled, see bug 1642719 372 - telemetry-tests-client 373 374 mingwclang-tests-opt: 375 - cppunittest 376 - mochitest-chrome-gpu 377 - mochitest-plain-gpu 378 379 windows-talos-webgl: 380 - talos-webgl 381 382 windows-talos: 383 - talos-bcv 384 - talos-chrome 385 - talos-damp-inspector 386 - talos-damp-webconsole 387 - talos-damp-other 388 - talos-dromaeojs 389 - talos-g1 390 - talos-g4 391 - talos-g5 392 - talos-other 393 - talos-sessionrestore-many-windows 394 - talos-perf-reftest 395 - talos-perf-reftest-singletons 396 - talos-svgr 397 - talos-tp5o 398 - talos-tabswitch 399 - talos-realworld-webextensions 400 - talos-pdfpaint 401 402 windows-ref-hw-tests: 403 - browsertime-video-playback-latency 404 - mochitest-media 405 406 talos-xperf: 407 - talos-xperf 408 409 # macOS 410 macosx1500-64-jittests: 411 - jittest 412 413 macosx1500-64-tests: 414 - crashtest 415 - jsreftest 416 - mochitest-browser-a11y 417 - mochitest-browser-chrome 418 - mochitest-browser-media 419 - mochitest-browser-translations 420 - mochitest-chrome-gpu 421 - mochitest-media 422 - mochitest-webgl1-core 423 - mochitest-webgl1-ext 424 - mochitest-webgl2-core 425 - mochitest-webgl2-ext 426 - reftest 427 - xpcshell 428 - web-platform-tests-webcodecs 429 - web-platform-tests-eme 430 431 macosx1500-aarch64-tests: 432 - cppunittest 433 - mochitest-media 434 435 macosx1470-64-tests: 436 - cppunittest 437 - crashtest 438 - firefox-ui-functional 439 - gtest 440 - jittest 441 - jsreftest 442 - marionette-integration 443 - marionette-unittest 444 - mochitest-a11y 445 - mochitest-browser-a11y 446 - mochitest-browser-media 447 - mochitest-browser-translations 448 - mochitest-chrome 449 - mochitest-chrome-gpu 450 - mochitest-devtools-chrome 451 - mochitest-media 452 - mochitest-plain 453 - mochitest-plain-gpu 454 - mochitest-remote 455 - mochitest-webgl1-core 456 - mochitest-webgl1-ext 457 - mochitest-webgl2-core 458 - mochitest-webgl2-ext 459 - reftest 460 - telemetry-tests-client 461 - test-verify 462 - test-verify-gpu 463 - test-verify-wpt 464 - web-platform-tests 465 - web-platform-tests-canvas 466 - web-platform-tests-crashtest 467 - web-platform-tests-eme 468 - web-platform-tests-print-reftest 469 - web-platform-tests-reftest 470 - web-platform-tests-wdspec 471 - web-platform-tests-webcodecs 472 - xpcshell 473 474 macosx1015-64-qr-tests: 475 - cppunittest 476 - crashtest 477 - jittest 478 - mochitest-browser-chrome 479 - mochitest-media 480 - mochitest-webgl1-core 481 - mochitest-webgl1-ext 482 - mochitest-webgl2-core 483 - mochitest-webgl2-ext 484 - xpcshell 485 486 macosx64-talos: 487 - talos-bcv 488 - talos-chrome 489 - talos-damp-inspector 490 - talos-damp-webconsole 491 - talos-damp-other 492 - talos-dromaeojs 493 - talos-g1 494 - talos-g4 495 - talos-g5 496 - talos-other 497 - talos-sessionrestore-many-windows 498 - talos-svgr 499 - talos-tp5o 500 - talos-perf-reftest 501 - talos-perf-reftest-singletons 502 - talos-realworld-webextensions 503 - talos-webgl 504 - talos-tabswitch 505 - talos-pdfpaint 506 # - talos-h1 # too long to unpack profile- Bug 1442893 507 508 macosx1500-64-perftests: 509 - browsertime-benchmark 510 511 # Android 512 513 android14-x86_64-tests: 514 - cppunittest 515 - crashtest 516 - geckoview-junit 517 - gtest 518 - mochitest-media 519 - mochitest-plain 520 - mochitest-plain-gpu 521 # - mochitest-webgl1-core # Harness issues? 522 # - mochitest-webgl1-ext 523 # - mochitest-webgl2-core # Emulator not creating ES3? 524 # - mochitest-webgl2-ext 525 - reftest 526 - test-verify 527 - web-platform-tests 528 - web-platform-tests-crashtest 529 - web-platform-tests-reftest 530 - web-platform-tests-wdspec 531 - web-platform-tests-webcodecs 532 - web-platform-tests-eme 533 - xpcshell 534 535 android-hw-aarch64-opt-unittests: 536 - jittest 537 538 android-hw-arm7-pgo-unittests: 539 - jittest-all 540 541 android-hw-aarch64-pgo-unittests: 542 - jittest 543 544 android-hw-arm7-debug-unittests: 545 - jittest 546 547 android-hw-aarch64-debug-unittests: 548 - jittest 549 550 android-hw-aarch64: 551 - crashtest-qr 552 - reftest-qr 553 - mochitest-media 554 - mochitest-webgl1-core 555 - mochitest-webgl1-ext 556 - mochitest-webgl2-core 557 558 android-hw-browsertime: 559 - browsertime-tp6m 560 - browsertime-tp6m-essential 561 - browsertime-tp6m-webextensions 562 - browsertime-tp6m-live 563 - browsertime-benchmark-speedometer-mobile 564 - browsertime-benchmark-speedometer3-mobile 565 - browsertime-youtube-playback-mobile 566 - browsertime-benchmark-unity-webgl-mobile 567 - browsertime-benchmark-motionmark-1-3 568 - browsertime-benchmark-jetstream2 569 - browsertime-benchmark-jetstream3 570 - browsertime-video-playback-latency-mobile 571 - browsertime-youtube-playback-power-mobile 572 - browsertime-trr-performance-m 573 574 android-hw-browsertime-profiling: 575 - browsertime-tp6m-profiling 576 577 android-hw-speedometer-tests: 578 - browsertime-benchmark-speedometer-mobile 579 - browsertime-benchmark-speedometer3-mobile 580 581 android-hw-power-tests: 582 - browsertime-youtube-playback-power-mobile