mobile.yml (18304B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 --- 5 task-defaults: 6 max-run-time: 3600 7 suite: raptor 8 variants: 9 - geckoview-no-fission 10 run-on-projects: [] 11 tier: 12 by-app: 13 fenix: 1 14 default: 3 15 test-manifest-loader: null # don't load tests in the taskgraph 16 target: 17 by-app: 18 fenix: 19 by-test-platform: 20 android-hw.*-aarch64.*/.*: 21 upstream-task: signing-apk 22 name: public/build/target.arm64-v8a.apk 23 default: 24 upstream-task: signing-apk 25 name: public/build/target.armeabi-v7a.apk 26 refbrow: 27 by-test-platform: 28 android-hw.*-aarch64.*/.*: 29 index: mobile.v2.reference-browser.raptor.latest.arm64-v8a 30 name: target.arm64-v8a.apk 31 default: 32 index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a 33 name: target.armeabi-v7a.apk 34 default: geckoview_example.apk 35 virtualization: hardware 36 raptor: 37 activity: 38 by-app: 39 fenix: org.mozilla.fenix.IntentReceiverActivity 40 geckoview: org.mozilla.geckoview_example.GeckoViewActivity 41 refbrow: org.mozilla.reference.browser.BrowserTestActivity 42 default: None 43 binary-path: 44 by-app: 45 fenix: org.mozilla.fenix 46 geckoview: org.mozilla.geckoview_example 47 refbrow: org.mozilla.reference.browser.raptor 48 default: None 49 mozharness: 50 script: raptor_script.py 51 config: 52 - raptor/android_hw_config.py 53 extra-options: 54 # Bug 1635749, disable window recorder temporarily 55 - --browsertime-no-ffwindowrecorder 56 optimization: 57 skip-unless-android-perftest-backstop: null 58 limit-platforms: 59 by-app: 60 chrome-m: 61 - .*shippable.* 62 cstm-car-m: 63 - .*shippable.* 64 default: [] 65 fetches: 66 toolchain: 67 by-app: 68 cstm-car-m: 69 - android-custom-car 70 - linux64-minidump-stackwalk 71 - linux64-fix-stacks 72 default: 73 - linux64-minidump-stackwalk 74 - linux64-fix-stacks 75 dependencies: 76 by-app: 77 fenix: 78 signing-apk: signing-apk-fenix-nightly-simulation 79 default: {} 80 81 browsertime-tp6m: 82 <<: &tp6m-defaults 83 description: Raptor (browsertime) tp6 page-load tests on android 84 max-run-time: 85 by-subtest: 86 reddit: 87 by-app: 88 geckoview: 7200 89 default: 3600 90 imdb: 91 by-app: 92 geckoview: 7200 93 default: 3600 94 default: 3600 95 mozharness: 96 extra-options: 97 - --chimera 98 # Bug 1884667 - Fresh profiles cause high frequency intermittents 99 - --conditioned-profile=settled 100 tier: 101 by-app: 102 geckoview: 1 103 fenix: 1 104 refbrow: 2 105 default: 3 106 treeherder-symbol: Btime(tp6m) 107 raptor: 108 apps: [geckoview, fenix, chrome-m, refbrow, cstm-car-m] 109 run-visual-metrics: true 110 test: tp6m 111 subtests: 112 - amazon 113 - bing 114 - [bing-search-restaurants, bing-s-r] 115 - booking 116 - cnn 117 - [cnn-ampstories, cnn-amp] 118 - dailymail 119 - [ebay-kleinanzeigen, ebay-k] 120 - [ebay-kleinanzeigen-search, ebay-k-s] 121 - [facebook-cristiano, fb-cris] 122 - [google-maps, gmaps] 123 - [google-search-restaurants, gsearch-r] 124 - instagram 125 - imdb 126 - reddit 127 - sina 128 - [stackoverflow, stacko] 129 - bild-de 130 - wikipedia 131 - youtube 132 run-on-projects: 133 by-app: 134 geckoview: [] 135 fenix: 136 by-variant: 137 geckoview-no-fission: 138 by-test-platform: 139 android-hw-a55.*shippable.*: [trunk-only] 140 default: [] 141 default: [mozilla-central] 142 default: [] 143 144 browsertime-tp6m-essential: 145 <<: *tp6m-defaults 146 raptor: 147 apps: [geckoview, fenix, chrome-m, refbrow, cstm-car-m] 148 run-visual-metrics: true 149 test: tp6m 150 subtests: 151 - allrecipes 152 - [amazon-search, amazon-s] 153 - espn 154 # Bug 1682027 - Disabled due to high failure rate 155 # - facebook 156 - google 157 - [microsoft-support, micros-sup] 158 - [youtube-watch, youtube-w] 159 run-on-projects: 160 by-app: 161 geckoview: [] 162 fenix: 163 by-test-platform: 164 android-hw-a55.*shippable.*: [trunk-only] 165 default: [] 166 default: [] 167 tier: 168 by-app: 169 geckoview: 1 170 fenix: 1 171 refbrow: 2 172 default: 3 173 174 browsertime-tp6m-webextensions: 175 <<: *tp6m-defaults 176 description: Raptor (browsertime) tp6 page-load tests with webextensions installed on android 177 treeherder-symbol: Btime-webext(tp6m) 178 mozharness: 179 extra-options: 180 - --chimera 181 - --browsertime-no-ffwindowrecorder 182 - --conditioned-profile=settled-webext 183 - --browser-cycles=5 184 raptor: 185 apps: [fenix] 186 run-visual-metrics: true 187 test: tp6m 188 subtests: 189 - [amazon-search, amazon-s] 190 fetches: 191 fetch: 192 - firefox-addons 193 run-on-projects: 194 by-app: 195 fenix: 196 by-test-platform: 197 android-hw-a55.*shippable.*: [mozilla-central] 198 default: [] 199 default: [] 200 # TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3. 201 tier: 3 202 203 browsertime-youtube-playback-mobile: 204 description: Browsertime YouTube Playback on Android 205 treeherder-symbol: Btime(ytp) 206 raptor: 207 apps: [fenix, geckoview, refbrow] 208 subtests: 209 by-app: 210 fenix: 211 by-test-platform: 212 android-hw-a55.*: 213 - [youtube-playback-av1-sfr, ytp-av1-sfr] 214 - [youtube-playback-hfr, ytp-hfr] 215 - [youtube-playback-vp9-sfr, ytp-vp9-sfr] 216 # Bug 1699469 - Disabled until a crash is fixed. 217 # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] 218 # - [youtube-playback-widevine-hfr, ytp-widevine-hfr] 219 default: [] 220 default: 221 by-test-platform: 222 android-hw-a55.*: 223 - [youtube-playback-h264-sfr, ytp-h264-sfr] 224 - [youtube-playback-hfr, ytp-hfr] 225 # Bug 1699469 - Disabled until a crash is fixed. 226 # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr] 227 # - [youtube-playback-widevine-hfr, ytp-widevine-hfr] 228 default: [] 229 test-url-param: 230 by-subtest: 231 youtube-playback-h264-sfr: # remove high resolution tests 232 exclude=1,2, 233 23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38 234 youtube-playback-hfr: 235 by-test-platform: 236 android-hw-a55.*: # remove VP9(1-34) and AV1(51-74) tests 237 exclude=1,2, 238 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34, 239 51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74 240 default: "" 241 youtube-playback-widevine-hfr: 242 by-test-platform: 243 android-hw-a55.*: # remove VP9(1-34) tests 244 exclude=1,2, 245 3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34 246 default: "" 247 default: "" 248 run-on-projects: 249 by-app: 250 fenix: 251 by-test-platform: 252 android-hw-a55.*shippable.*: [mozilla-central] 253 default: [] 254 geckoview: [] 255 default: [] 256 tier: 257 by-app: 258 fenix: 1 259 default: 3 260 max-run-time: 3600 261 262 browsertime-youtube-playback-power-mobile: 263 description: Browsertime YouTube Playback on Android for Power Usage Testing 264 treeherder-symbol: Btime(ytp) 265 tier: 266 by-app: 267 fenix: 1 268 geckoview: 2 269 default: 3 270 raptor: 271 apps: [fenix, geckoview, chrome-m] 272 subtests: 273 - [youtube-playback-h264-720p60, ytp-h264-720] 274 - [youtube-playback-vp9-720p60, ytp-vp9-720] 275 run-on-projects: 276 by-app: 277 fenix: 278 by-test-platform: 279 android-hw-a55.*shippable.*: [autoland, mozilla-central] 280 default: [] 281 geckoview: [] 282 default: [] 283 max-run-time: 1800 284 mozharness: 285 extra-options: 286 - --power-test 287 288 browsertime-tp6m-live: 289 <<: *tp6m-defaults 290 raptor: 291 apps: [chrome-m, fenix, geckoview] 292 run-visual-metrics: true 293 test: tp6m 294 subtests: 295 - allrecipes 296 - amazon 297 - [amazon-search, amazon-s] 298 - bing 299 - [bing-search-restaurants, bing-s-r] 300 - booking 301 - booking-sf 302 - cnn 303 - [cnn-ampstories, cnn-amp] 304 - dailymail 305 - discord 306 - [ebay-kleinanzeigen, ebay-k] 307 - [ebay-kleinanzeigen-search, ebay-k-s] 308 - espn 309 - expedia 310 - facebook 311 - [facebook-cristiano, fb-cris] 312 - [fashionbeans, fashbea] 313 - google 314 - [google-accounts, gacc] 315 - [google-maps, gmaps] 316 - [google-search-restaurants, gsearch-r] 317 - instagram 318 - imdb 319 - [imdb-firefox, imdb-f] 320 - [medium-article, medium] 321 - [microsoft-support, micros-sup] 322 - nytimes 323 - [people-article, people] 324 - reddit 325 - [reddit-thread, reddit-thr] 326 - rumble-fox 327 - [stackoverflow, stacko] 328 - [stackoverflow-question, stack-q] 329 - [urbandictionary-define, urbict-def] 330 - bild-de 331 - [wikia-marvel, wikia-m] 332 - wikipedia 333 - youtube 334 - [youtube-watch, youtube-w] 335 mozharness: 336 extra-options: 337 - --live-sites 338 - --chimera 339 run-on-projects: 340 by-subtest: 341 cnn-ampstories: 342 by-app: 343 geckoview: [] 344 default: [] 345 default: [] 346 tier: 347 by-app: 348 fenix: 1 349 default: 3 350 treeherder-symbol: Btime-live(tp6m) 351 352 browsertime-tp6m-profiling: 353 description: Raptor (browsertime) tp6 page-load tests on android with Gecko profiling 354 raptor: 355 apps: [geckoview] 356 test: tp6m 357 subtests: [amazon, youtube] 358 treeherder-symbol: Btime-Prof(tp6m) 359 mozharness: 360 script: raptor_script.py 361 config: 362 - raptor/android_hw_config.py 363 extra-options: 364 - --chimera 365 - --gecko-profile 366 run-on-projects: 367 by-test-platform: 368 android-hw-a55.*shippable.*/opt: [mozilla-central] 369 default: [] 370 tier: 371 by-app: 372 fenix: 1 373 default: 3 374 375 browsertime-video-playback-latency-mobile: 376 <<: *tp6m-defaults 377 description: Browsertime Video Playback Latency on Android 378 treeherder-symbol: Btime(vpl) 379 raptor: 380 apps: [chrome-m, fenix, geckoview] 381 run-visual-metrics: true 382 test: video-playback-latency 383 subtests: 384 - vpl-av1 385 - vpl-h264 386 - vpl-vp9 387 mozharness: 388 extra-options: 389 - --chimera 390 - --browsertime-no-ffwindowrecorder 391 - --extra-profiler-run 392 run-on-projects: 393 by-app: 394 chrome-m: [] 395 geckoview: [] 396 default: 397 by-test-platform: 398 android-hw-a55.*shippable.*: [autoland, mozilla-central] 399 default: [] 400 tier: 401 by-app: 402 fenix: 1 403 default: 3 404 405 browsertime-benchmark-speedometer-mobile: 406 description: Browsertime Speedometer on Android 407 raptor: 408 apps: [geckoview, fenix, chrome-m, cstm-car-m] 409 tier: 410 by-app: 411 fenix: 1 412 default: 3 413 run-on-projects: 414 by-app: 415 fenix: 416 by-test-platform: 417 android-hw-a55.*shippable.*: [trunk] 418 default: [] 419 geckoview: [] 420 default: [] 421 treeherder-symbol: Btime(sp) 422 mozharness: 423 extra-options: 424 - --test=speedometer 425 426 browsertime-benchmark-speedometer3-mobile: 427 description: Browsertime Speedometer 3 on Android 428 429 raptor: 430 apps: [geckoview, fenix, chrome-m, cstm-car-m] 431 tier: 432 by-app: 433 fenix: 1 434 geckoview: 2 435 cstm-car-m: 2 436 default: 3 437 run-on-projects: 438 by-app: 439 fenix: 440 by-test-platform: 441 android-hw-a55.*shippable.*: [trunk-only] 442 android-hw-p6.*shippable.*: [mozilla-central] 443 android-hw-s24.*shippable.*: [mozilla-central] 444 default: [] 445 geckoview: [] 446 chrome-m: 447 by-test-platform: 448 android-hw-p6.*shippable.*: [mozilla-central] 449 android-hw-s24.*shippable.*: [mozilla-central] 450 default: [] 451 default: [] 452 treeherder-symbol: Btime(sp3) 453 mozharness: 454 extra-options: 455 - --test=speedometer3 456 457 browsertime-benchmark-motionmark-1-3: 458 description: Raptor (browsertime) Motionmark 1.3 benchmark tests 459 raptor: 460 apps: [geckoview, fenix, chrome-m, cstm-car-m] 461 subtests: 462 - [motionmark-1-3, mm-1.3] 463 - [motionmark-htmlsuite-1-3, mm-h-1.3] 464 run-on-projects: 465 by-app: 466 chrome-m: [] 467 geckoview: [] 468 default: 469 by-test-platform: 470 android-hw-a55.*shippable.*: [mozilla-central] 471 default: [] 472 treeherder-symbol: Btime() 473 tier: 474 by-app: 475 fenix: 1 476 geckoview: 2 477 default: 3 478 479 browsertime-benchmark-jetstream2: 480 description: Raptor (browsertime) jetstream 2 benchmark test 481 raptor: 482 apps: [geckoview, fenix, chrome-m, cstm-car-m] 483 subtests: 484 - [jetstream2, js] 485 run-on-projects: 486 by-test-platform: 487 android-hw-a55.*shippable.*: 488 by-app: 489 fenix: [mozilla-central] 490 default: [] 491 default: [] 492 treeherder-symbol: Btime() 493 tier: 494 by-app: 495 fenix: 1 496 default: 3 497 498 browsertime-benchmark-jetstream3: 499 description: Raptor (browsertime) jetstream 3 benchmark test 500 raptor: 501 # Bug 1954124 not quite ready for Fenix and GVE 502 apps: [chrome-m, cstm-car-m, fenix] 503 subtests: 504 - [jetstream3, js3] 505 run-on-projects: 506 by-test-platform: 507 android-hw-a55.*shippable.*: 508 by-app: 509 fenix: [trunk-only] 510 default: [] 511 default: [] 512 treeherder-symbol: Btime() 513 tier: 514 by-app: 515 fenix: 2 516 default: 3 517 518 browsertime-benchmark-unity-webgl-mobile: 519 description: Browsertime Unity WebGL on Android 520 raptor: 521 apps: [geckoview, refbrow, fenix, chrome-m] 522 tier: 523 by-app: 524 fenix: 1 525 geckoview: 2 526 default: 3 527 treeherder-symbol: Btime(ugl) 528 run-on-projects: 529 by-app: 530 refbrow: [] 531 geckoview: [] 532 default: 533 by-test-platform: 534 android-hw-a55.*shippable.*: [mozilla-central] 535 default: [] 536 max-run-time: 900 537 mozharness: 538 extra-options: 539 - --test=unity-webgl 540 541 browsertime-power: 542 description: Browsertime Power Usage Tests on Android 543 raptor: 544 apps: [geckoview, fenix, refbrow] 545 subtests: 546 - idle 547 - idle-bg 548 - [speedometer, sp] 549 tier: 550 by-app: 551 fenix: 1 552 default: 2 553 run-on-projects: [] 554 treeherder-symbol: Btime-P(power) 555 mozharness: 556 extra-options: 557 - --power-test 558 559 browsertime-trr-performance-m: 560 raptor: 561 apps: [fenix] 562 run-visual-metrics: false 563 subtests: 564 - trr-clou-m 565 - trr-next-m 566 - trr-natv-m 567 - trr-multm 568 - trr-w-cl-m 569 - trr-w-ne-m 570 571 description: Raptor (browsertime) trr performance tests, mobile 572 max-run-time: 600 573 run-on-projects: 574 by-app: 575 fenix: 576 by-test-platform: 577 android-hw-a55.*shippable.*: [mozilla-central] 578 default: [] 579 tier: 3 580 treeherder-symbol: Btime(trrperfm)