test_skipfails.py (58009B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import json 6 import os 7 from pathlib import Path 8 9 import pytest 10 from mozunit import main 11 from skipfails import Kind, Skipfails, SkipfailsMode, read_json 12 13 DATA_PATH = Path(__file__).with_name("data") 14 15 16 def test_from_flags(): 17 carryover_mode: bool = True 18 known_intermittents_mode: bool = True 19 new_failures_mode: bool = True 20 replace_tbd_mode: bool = True 21 mode: int = 10 22 with pytest.raises(Exception) as e: 23 mode = SkipfailsMode.from_flags( 24 carryover_mode, 25 known_intermittents_mode, 26 new_failures_mode, 27 replace_tbd_mode, 28 ) 29 assert ( 30 str(e.value) 31 == "may not specifiy more than one mode: --carryover --known-intermittents --new-failures --replace-tbd" 32 ) 33 34 carryover_mode = False 35 known_intermittents_mode = False 36 new_failures_mode = False 37 mode = SkipfailsMode.from_flags( 38 carryover_mode, known_intermittents_mode, new_failures_mode, replace_tbd_mode 39 ) 40 assert mode == SkipfailsMode.REPLACE_TBD 41 42 43 def test_get_revision(): 44 """Test get_revision""" 45 46 sf = Skipfails() 47 with pytest.raises(ValueError) as e_info: 48 sf.get_revision("") 49 assert str(e_info.value) == "try_url scheme not https" 50 51 with pytest.raises(ValueError) as e_info: 52 sf.get_revision("https://foo.bar") 53 assert str(e_info.value) == "try_url server not treeherder.mozilla.org" 54 55 with pytest.raises(ValueError) as e_info: 56 sf.get_revision("https://treeherder.mozilla.org") 57 assert str(e_info.value) == "try_url query missing" 58 59 with pytest.raises(ValueError) as e_info: 60 sf.get_revision("https://treeherder.mozilla.org?a=1") 61 assert str(e_info.value) == "try_url query missing revision" 62 63 revision, repo = sf.get_revision( 64 "https://treeherder.mozilla.org/jobs?repo=try&revision=5b1738d0af571777199ff3c694b1590ff574946b" 65 ) 66 assert revision == "5b1738d0af571777199ff3c694b1590ff574946b" 67 assert repo == "try" 68 69 70 def test_get_tasks(): 71 """Test get_tasks import of mozci""" 72 73 from mozci.push import Push 74 75 revision = "5b1738d0af571777199ff3c694b1590ff574946b" 76 repo = "try" 77 push = Push(revision, repo) 78 assert push is not None 79 80 81 def get_failures( 82 tasks_name, exp_f_name, task_details=None, error_summary=None, implicit_vars=False 83 ): 84 """Runs Skipfails.get_failures on tasks to compare with failures""" 85 sf = Skipfails(implicit_vars=implicit_vars) 86 assert sf.implicit_vars == implicit_vars 87 if task_details is not None: # preload task details cache, if needed 88 if isinstance(task_details, str): # read file 89 task_details = read_json(DATA_PATH.joinpath(task_details)) 90 sf.tasks = task_details 91 if error_summary is not None: # preload task details cache, if needed 92 if isinstance(error_summary, str): # read file 93 error_summary = read_json(DATA_PATH.joinpath(error_summary)) 94 sf.error_summary = error_summary 95 tasks = sf.read_tasks(DATA_PATH.joinpath(tasks_name)) 96 exp_f = sf.read_failures(DATA_PATH.joinpath(exp_f_name)) 97 expected_failures = json.dumps(exp_f, indent=2, sort_keys=True).strip() 98 failures = sf.get_failures(tasks) 99 actual_failures = json.dumps(failures, indent=2, sort_keys=True).strip() 100 assert actual_failures == expected_failures 101 102 103 def test_get_failures_1(): 104 """Test get_failures 1""" 105 task_details = {"dwOJ8M9ERSmk6oI2KXg6hg": {}} 106 get_failures("wayland-tasks-1.json", "wayland-failures-1.json", task_details) 107 108 109 def test_get_failures_2(): 110 """Test get_failures 2""" 111 task_details = { 112 "Y7r1q2xWSu-2bRAofEfeBw": {}, 113 "Z7r1q2xWSu-2bRAofEfeBw": {}, 114 "X7r1q2xWSu-2bRAofEfeBw": {}, 115 } 116 get_failures("wayland-tasks-2.json", "wayland-failures-2.json", task_details) 117 118 119 def test_get_failures_3(): 120 """Test get_failures 3""" 121 task_details = { 122 "b7_ahjGtQ_-ZMNBG_hUZUw": {}, 123 "WVczuxkuSRKZg_jMiGyQsA": {}, 124 "UOZUIVAaTZKmRwArq5WkDw": {}, 125 } 126 get_failures("wayland-tasks-3.json", "wayland-failures-3.json", task_details) 127 128 129 def test_get_failures_4(): 130 """Test get_failures 4""" 131 task_details = { 132 "bxMVPbPMTru_bfAivc1sPA": {}, 133 "EDql3NKPR3W6OEU3mLeKbg": {}, 134 "FDql3NKPR3W6OEU3mLeKbg": {}, 135 } 136 get_failures("wayland-tasks-4.json", "wayland-failures-4.json", task_details) 137 138 139 def test_get_failures_5(): 140 """Test get_failures 5""" 141 142 task_details = { 143 "Bgc6We1sSjakIo3V9crldw": { 144 "expires": "2024-01-09T16:05:56.825Z", 145 "extra": { 146 "suite": "mochitest-browser", 147 "test-setting": { 148 "build": {"type": "opt"}, 149 "platform": { 150 "arch": "64", 151 "os": {"name": "linux", "version": "22.04"}, 152 "display": "wayland", 153 }, 154 "runtime": {}, 155 }, 156 }, 157 } 158 } 159 get_failures("wayland-tasks-5.json", "wayland-failures-5.json", task_details) 160 161 162 def test_get_failures_6(): 163 """Test get_failures 6""" 164 165 task_details = { 166 "AKYqxtoWStigj_5yHVqAeg": { 167 "expires": "2024-03-19T03:29:11.050Z", 168 "extra": { 169 "suite": "web-platform-tests", 170 "test-setting": { 171 "build": { 172 "type": "opt", 173 "shippable": True, 174 }, 175 "platform": { 176 "arch": "32", 177 "os": {"name": "linux", "version": "1804"}, 178 }, 179 "runtime": {}, 180 }, 181 }, 182 } 183 } 184 get_failures("wpt-tasks-1.json", "wpt-failures-1.json", task_details) 185 186 187 def test_get_failures_7(): 188 """Test get_failures 7""" 189 190 get_failures( 191 "reftest-tasks-1.json", 192 "reftest-failures-1.json", 193 "reftest-extra-1.json", 194 "reftest-summary-1.json", 195 True, 196 ) 197 198 199 def test_get_bug_by_id(): 200 """Test get_bug_by_id""" 201 202 sf = Skipfails() 203 id = 1682371 204 # preload bug cache 205 bugs_filename = f"bug-{id}.json" 206 sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename)) 207 # function under test 208 bug = sf.get_bug_by_id(id) 209 assert bug is not None 210 assert bug.id == id 211 assert bug.product == "Testing" 212 assert bug.component == "General" 213 assert ( 214 bug.summary 215 == "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests" 216 ) 217 218 219 def test_get_bugs_by_summary(): 220 """Test get_bugs_by_summary""" 221 222 sf = Skipfails() 223 id = 1682371 224 # preload bug cache 225 bugs_filename = f"bug-{id}.json" 226 sf.bugs = sf.read_bugs(DATA_PATH.joinpath(bugs_filename)) 227 # function under test 228 summary = "create tool to quickly parse and identify all failures from a try push and ideally annotate manifests" 229 bugs = sf.get_bugs_by_summary(summary) 230 assert len(bugs) == 1 231 assert bugs[0].id == id 232 assert bugs[0].product == "Testing" 233 assert bugs[0].component == "General" 234 assert bugs[0].summary == summary 235 236 237 def test_get_variants(): 238 """Test get_variants""" 239 240 sf = Skipfails() 241 variants = sf.get_variants() 242 assert "1proc" in variants 243 assert variants["1proc"] == "e10s" 244 assert "webrender-sw" in variants 245 assert variants["webrender-sw"] == "swgl" 246 assert "aab" in variants 247 assert variants["aab"] == "aab" 248 249 250 def test_task_to_skip_if(): 251 """Test task_to_skip_if""" 252 253 # Windows 2009 task 254 sf = Skipfails() 255 task_id = "UP-t3xrGSDWvUNjFGIt_aQ" 256 task_details = { 257 "expires": "2024-01-09T16:05:56.825Z", 258 "extra": { 259 "suite": "mochitest-plain", 260 "test-setting": { 261 "build": {"type": "debug"}, 262 "platform": { 263 "arch": "32", 264 "os": {"build": "24h2", "name": "windows", "version": "11"}, 265 }, 266 "runtime": {}, 267 }, 268 }, 269 } 270 sf.tasks[task_id] = task_details 271 sf.platform_permutations = { 272 "test-manifest": { 273 "win": { 274 "11.26100": { 275 "x86": {"debug": {"no_variant": {}}, "opt": {"no_variant": {}}} 276 } 277 } 278 } 279 } 280 # function under test 281 skip_if = sf.task_to_skip_if( 282 "test-manifest", task_id, Kind.TOML, "test-path", False 283 ) 284 assert ( 285 skip_if == "os == 'win' && os_version == '11.26100' && arch == 'x86' && debug" 286 ) 287 288 # Failed task on specific runtime on x86_64 289 sf = Skipfails() 290 task_id = "I3iXyGDATDSDyzGh4YfNJw" 291 task_details = { 292 "expires": "2024-01-09T16:05:56.825Z", 293 "extra": { 294 "suite": "web-platform-tests-crashtest", 295 "test-setting": { 296 "build": {"type": "debug"}, 297 "platform": { 298 "arch": "64", 299 "os": {"name": "macosx", "version": "1015"}, 300 }, 301 "runtime": {"webrender-sw": True}, 302 }, 303 }, 304 } 305 sf.tasks[task_id] = task_details 306 sf.platform_permutations = { 307 "test-manifest": { 308 "mac": { 309 "10.15": { 310 "x86_64": { 311 "debug": {"swgl": {}, "no_variant": {}}, 312 "opt": {"no_variant": {}}, 313 } 314 } 315 } 316 } 317 } 318 # function under test 319 skip_if = sf.task_to_skip_if( 320 "test-manifest", task_id, Kind.TOML, "test-path", False 321 ) 322 assert ( 323 skip_if 324 == "os == 'mac' && os_version == '10.15' && arch == 'x86_64' && debug && swgl" 325 ) 326 327 # Failed task on specific runtime on aarch64 328 sf = Skipfails() 329 task_id = "bAkMaQIVQp6oeEIW6fzBDw" 330 task_details = { 331 "expires": "2024-01-09T16:05:56.825Z", 332 "extra": { 333 "suite": "mochitest-media", 334 "test-setting": { 335 "build": {"type": "debug"}, 336 "platform": { 337 "arch": "aarch64", 338 "os": {"name": "macosx", "version": "1015"}, 339 }, 340 "runtime": {"webrender-sw": True}, 341 }, 342 }, 343 } 344 sf.tasks[task_id] = task_details 345 sf.platform_permutations = { 346 "test-manifest": { 347 "mac": { 348 "10.15": { 349 "aarch64": { 350 "debug": {"swgl": {}, "no_variant": {}}, 351 "opt": {"no_variant": {}}, 352 } 353 } 354 } 355 } 356 } 357 # function under test 358 skip_if = sf.task_to_skip_if( 359 "test-manifest", task_id, Kind.TOML, "test-path", False 360 ) 361 assert ( 362 skip_if 363 == "os == 'mac' && os_version == '10.15' && arch == 'aarch64' && debug && swgl" 364 ) 365 366 # Hacks for macosx 11 367 sf = Skipfails() 368 task_id = "bAkMaQIVQp6oeEIW6fzBDw" 369 task_details = { 370 "expires": "2024-01-09T16:05:56.825Z", 371 "extra": { 372 "suite": "mochitest-media", 373 "test-setting": { 374 "build": {"type": "debug"}, 375 "platform": { 376 "arch": "aarch64", 377 "os": {"name": "macosx", "version": "1100"}, 378 }, 379 "runtime": {"webrender-sw": True}, 380 }, 381 }, 382 } 383 sf.tasks[task_id] = task_details 384 sf.platform_permutations = { 385 "test-manifest": { 386 "mac": { 387 "11.20": { 388 "aarch64": { 389 "debug": {"swgl": {}, "no_variant": {}}, 390 "opt": {"no_variant": {}}, 391 } 392 } 393 } 394 } 395 } 396 # function under test 397 skip_if = sf.task_to_skip_if( 398 "test-manifest", task_id, Kind.TOML, "test-path", False 399 ) 400 assert ( 401 skip_if 402 == "os == 'mac' && os_version == '11.20' && arch == 'aarch64' && debug && swgl" 403 ) 404 405 ## The test below is now altered - we WILL use the build_type and test variant 406 ## regardless of other result permutations (part of deprecating FailedPlatorm) 407 # Do not include build type or test variant if everything failed 408 sf = Skipfails() 409 task_id = "AKYqxtoWStigj_5yHVqAeg" 410 task_details = { 411 "expires": "2024-03-19T03:29:11.050Z", 412 "extra": { 413 "suite": "web-platform-tests", 414 "test-setting": { 415 "build": { 416 "type": "opt", 417 }, 418 "platform": { 419 "arch": "32", 420 "os": {"name": "linux", "version": "1804"}, 421 }, 422 "runtime": {}, 423 }, 424 }, 425 } 426 sf.tasks[task_id] = task_details 427 sf.platform_permutations = { 428 "test-manifest": {"linux": {"18.04": {"x86": {"opt": {"no_variant": {}}}}}} 429 } 430 # function under test 431 skip_if = sf.task_to_skip_if( 432 "test-manifest", task_id, Kind.TOML, "test-path", False 433 ) 434 # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" 435 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" 436 437 ## The test below is now altered - we WILL use the build_type and test variant 438 ## regardless of other result permutations (part of deprecating FailedPlatorm) 439 sf = Skipfails() 440 task_id = "QFo2jGFvTKGVcoqHCBpMGw" 441 task_details = { 442 "expires": "2024-03-19T03:29:11.050Z", 443 "extra": { 444 "suite": "web-platform-tests", 445 "test-setting": { 446 "build": { 447 "type": "opt", 448 }, 449 "platform": { 450 "arch": "32", 451 "os": {"name": "linux", "version": "1804"}, 452 }, 453 "runtime": {"xorigin": True}, 454 }, 455 }, 456 } 457 sf.tasks[task_id] = task_details 458 sf.platform_permutations = { 459 "test-manifest": {"linux": {"18.04": {"x86": {"opt": {"xorigin": {}}}}}} 460 } 461 # function under test 462 skip_if = sf.task_to_skip_if( 463 "test-manifest", task_id, Kind.TOML, "test-path", False 464 ) 465 # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" 466 assert ( 467 skip_if 468 == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && xorigin" 469 ) 470 471 ## The test below is now altered - we WILL NOT negate other variants 472 ## (part of deprecating FailedPlatorm) 473 # Only the test without variant failed 474 sf = Skipfails() 475 task_id = "Xvdt2gbEQ3iDVAZPddY9PQ" 476 task_details = { 477 "expires": "2024-03-19T03:29:11.050Z", 478 "extra": { 479 "suite": "web-platform-tests", 480 "test-setting": { 481 "build": { 482 "type": "opt", 483 }, 484 "platform": { 485 "arch": "32", 486 "os": {"name": "linux", "version": "1804"}, 487 }, 488 "runtime": {}, 489 }, 490 }, 491 } 492 sf.tasks[task_id] = task_details 493 sf.platform_permutations = { 494 "test-manifest": { 495 "linux": {"18.04": {"x86": {"opt": {"no_variant": {}, "xorigin": {}}}}} 496 } 497 } 498 # function under test 499 skip_if = sf.task_to_skip_if( 500 "test-manifest", task_id, Kind.TOML, "test-path", False 501 ) 502 # assert ( 503 # skip_if 504 # == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && !xorigin" 505 # ) 506 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" 507 508 # Missing platform permutation for the task 509 sf = Skipfails() 510 task_id = "czj2mQwqQv6PwON5aijPJg" 511 task_details = { 512 "expires": "2024-03-19T03:29:11.050Z", 513 "extra": { 514 "suite": "web-platform-tests", 515 "test-setting": { 516 "build": { 517 "type": "opt", 518 }, 519 "platform": { 520 "arch": "32", 521 "os": {"name": "linux", "version": "1804"}, 522 }, 523 "runtime": {}, 524 }, 525 }, 526 } 527 sf.tasks[task_id] = task_details 528 sf.platform_permutations = {} 529 # function under test 530 skip_if = sf.task_to_skip_if( 531 "test-manifest", task_id, Kind.TOML, "test-path", False 532 ) 533 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" 534 535 sf = Skipfails() 536 task_id = "czj2mQwqQv6PwON5aijPJg" 537 task_details = { 538 "expires": "2024-03-19T03:29:11.050Z", 539 "extra": { 540 "suite": "web-platform-tests", 541 "test-setting": { 542 "build": { 543 "type": "opt", 544 }, 545 "platform": { 546 "arch": "32", 547 "os": {"name": "linux", "version": "1804"}, 548 }, 549 "runtime": {"xorigin": True}, 550 }, 551 }, 552 } 553 sf.tasks[task_id] = task_details 554 sf.platform_permutations = {} 555 # function under test 556 skip_if = sf.task_to_skip_if( 557 "test-manifest", task_id, Kind.TOML, "test-path", False 558 ) 559 assert ( 560 skip_if 561 == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && xorigin" 562 ) 563 564 # Full fail with several tasks 565 sf = Skipfails() 566 sf.platform_permutations = { 567 "test-manifest": { 568 "linux": { 569 "18.04": { 570 "x86": { 571 "opt": {"no_variant": {}, "xorigin": {}}, 572 "debug": {"no_variant": {}}, 573 } 574 } 575 } 576 } 577 } 578 task_id = "PPWic3zuRIyGdzUXC-XCvw" 579 task_details = { 580 "expires": "2024-03-19T03:29:11.050Z", 581 "extra": { 582 "suite": "web-platform-tests", 583 "test-setting": { 584 "build": { 585 "type": "opt", 586 }, 587 "platform": { 588 "arch": "32", 589 "os": {"name": "linux", "version": "1804"}, 590 }, 591 "runtime": {"xorigin": True}, 592 }, 593 }, 594 } 595 sf.tasks[task_id] = task_details 596 # function under test 597 skip_if = sf.task_to_skip_if( 598 "test-manifest", task_id, Kind.TOML, "test-path", False 599 ) 600 assert ( 601 skip_if 602 == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && xorigin" 603 ) 604 605 task_id = "c_OXt3mESB-G-aElu0hoxg" 606 task_details = { 607 "expires": "2024-03-19T03:29:11.050Z", 608 "extra": { 609 "suite": "web-platform-tests", 610 "test-setting": { 611 "build": { 612 "type": "opt", 613 }, 614 "platform": { 615 "arch": "32", 616 "os": {"name": "linux", "version": "1804"}, 617 }, 618 "runtime": {}, 619 }, 620 }, 621 } 622 sf.tasks[task_id] = task_details 623 # function under test 624 skip_if = sf.task_to_skip_if( 625 "test-manifest", task_id, Kind.TOML, "test-path", False 626 ) 627 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" 628 629 ## The test below is now altered - we WILL include the build_type 630 ## regardless of other result permutations (part of deprecating FailedPlatorm) 631 task_id = "ShPeY1F8SY6Gm-1VSjsyUA" 632 task_details = { 633 "expires": "2024-03-19T03:29:11.050Z", 634 "extra": { 635 "suite": "web-platform-tests", 636 "test-setting": { 637 "build": { 638 "type": "debug", 639 }, 640 "platform": { 641 "arch": "32", 642 "os": {"name": "linux", "version": "1804"}, 643 }, 644 "runtime": {}, 645 }, 646 }, 647 } 648 sf.tasks[task_id] = task_details 649 # function under test 650 skip_if = sf.task_to_skip_if( 651 "test-manifest", task_id, Kind.TOML, "test-path", False 652 ) 653 # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" 654 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && debug" 655 656 # Multiple failed tasks allowing for optimized skip if 657 sf = Skipfails() 658 sf.platform_permutations = { 659 "test-manifest": { 660 "linux": { 661 "18.04": { 662 "x86": { 663 "opt": {"no_variant": {}}, 664 "debug": {"no_variant": {}, "xorigin": {}}, 665 } 666 } 667 } 668 } 669 } 670 task_id = "Zc17K1IQRXOsGoSgegA_kA" 671 task_details = { 672 "expires": "2024-03-19T03:29:11.050Z", 673 "extra": { 674 "suite": "web-platform-tests", 675 "test-setting": { 676 "build": { 677 "type": "debug", 678 }, 679 "platform": { 680 "arch": "32", 681 "os": {"name": "linux", "version": "1804"}, 682 }, 683 "runtime": {"xorigin": True}, 684 }, 685 }, 686 } 687 sf.tasks[task_id] = task_details 688 # function under test 689 skip_if = sf.task_to_skip_if( 690 "test-manifest", task_id, Kind.TOML, "test-path", False 691 ) 692 assert ( 693 skip_if 694 == "os == 'linux' && os_version == '18.04' && arch == 'x86' && debug && xorigin" 695 ) 696 697 task_id = "ChOXnndsQQODAGpDqscbMg" 698 task_details = { 699 "expires": "2024-03-19T03:29:11.050Z", 700 "extra": { 701 "suite": "web-platform-tests", 702 "test-setting": { 703 "build": { 704 "type": "debug", 705 }, 706 "platform": { 707 "arch": "32", 708 "os": {"name": "linux", "version": "1804"}, 709 }, 710 "runtime": {}, 711 }, 712 }, 713 } 714 sf.tasks[task_id] = task_details 715 # function under test 716 skip_if = sf.task_to_skip_if( 717 "test-manifest", task_id, Kind.TOML, "test-path", False 718 ) 719 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && debug" 720 721 ## The test below is now altered - we WILL include the build_type 722 ## regardless of other result permutations (part of deprecating FailedPlatorm) 723 task_id = "caDMGUmnT7muCqNWj6w3nQ" 724 task_details = { 725 "expires": "2024-03-19T03:29:11.050Z", 726 "extra": { 727 "suite": "web-platform-tests", 728 "test-setting": { 729 "build": { 730 "type": "opt", 731 }, 732 "platform": { 733 "arch": "32", 734 "os": {"name": "linux", "version": "1804"}, 735 }, 736 "runtime": {}, 737 }, 738 }, 739 } 740 sf.tasks[task_id] = task_details 741 # function under test 742 skip_if = sf.task_to_skip_if( 743 "test-manifest", task_id, Kind.TOML, "test-path", False 744 ) 745 # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" 746 assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" 747 748 # for 3 or more variants, elide all variants 749 task_id = "PpkXyfUVRNiU0qRYczlhyw" 750 task_details = { 751 "expires": "2025-09-19T03:29:11.050Z", 752 "extra": { 753 "suite": "xpcshell", 754 "test-setting": { 755 "build": { 756 "type": "debug", 757 }, 758 "runtime": { 759 "no-fission": True, 760 "socketprocess_networking": True, 761 "http3": True, 762 }, 763 "platform": { 764 "arch": "64", 765 "os": {"name": "linux", "version": "2404"}, 766 }, 767 }, 768 }, 769 } 770 sf.tasks[task_id] = task_details 771 # function under test 772 skip_if = sf.task_to_skip_if( 773 "test-manifest", task_id, Kind.TOML, "test-path", False 774 ) 775 assert ( 776 skip_if == "os == 'linux' && os_version == '24.04' && arch == 'x86_64' && debug" 777 ) 778 779 780 def test_task_to_skip_if_high_freq(): 781 """Test task_to_skip_if with high freq flag""" 782 783 # Do not add skip if line if failed less than 7 times 784 sf = Skipfails() 785 task_id = "UP-t3xrGSDWvUNjFGIt_aQ" 786 task_details = { 787 "expires": "2024-01-09T16:05:56.825Z", 788 "extra": { 789 "suite": "mochitest-plain", 790 "test-setting": { 791 "build": {"type": "debug"}, 792 "platform": { 793 "arch": "64", 794 "os": {"name": "linux", "version": "2204"}, 795 }, 796 "runtime": {}, 797 }, 798 }, 799 } 800 sf.tasks[task_id] = task_details 801 sf.platform_permutations = { 802 "test-manifest": { 803 "linux": { 804 "22.04": { 805 "x86_64": {"debug": {"no_variant": {}}, "opt": {"no_variant": {}}} 806 } 807 } 808 } 809 } 810 for i in range(0, 6): 811 skip_if = sf.task_to_skip_if( 812 "test-manifest", task_id, Kind.TOML, "test-path", True 813 ) 814 assert skip_if is None 815 # Skip whole platform with build type if there are at least 7 failures 816 skip_if = sf.task_to_skip_if("test-manifest", task_id, Kind.TOML, "test-path", True) 817 assert ( 818 skip_if == "os == 'linux' && os_version == '22.04' && arch == 'x86_64' && debug" 819 ) 820 821 # Same if we add failures to other build types 822 task_id = "caDMGUmnT7muCqNWj6w3nQ" 823 task_details = { 824 "expires": "2024-01-09T16:05:56.825Z", 825 "extra": { 826 "suite": "mochitest-plain", 827 "test-setting": { 828 "build": {"type": "opt"}, 829 "platform": { 830 "arch": "64", 831 "os": {"name": "linux", "version": "2204"}, 832 }, 833 "runtime": {}, 834 }, 835 }, 836 } 837 sf.tasks[task_id] = task_details 838 for i in range(0, 6): 839 skip_if = sf.task_to_skip_if( 840 "test-manifest", task_id, Kind.TOML, "test-path", True 841 ) 842 assert skip_if is None 843 # Skip whole platform if all build types failed 7 times or more 844 skip_if = sf.task_to_skip_if("test-manifest", task_id, Kind.TOML, "test-path", True) 845 assert skip_if == "os == 'linux' && os_version == '22.04' && arch == 'x86_64'" 846 847 # Only skip a specific test variant if it represents more than 75% of failures 848 sf = Skipfails() 849 task_id = "UP-t3xrGSDWvUNjFGIt_aQ" 850 task_details = { 851 "expires": "2024-01-09T16:05:56.825Z", 852 "extra": { 853 "suite": "mochitest-plain", 854 "test-setting": { 855 "build": {"type": "debug"}, 856 "platform": { 857 "arch": "64", 858 "os": {"name": "linux", "version": "2204"}, 859 }, 860 "runtime": {"webrender-sw": "swgl"}, 861 }, 862 }, 863 } 864 sf.tasks[task_id] = task_details 865 sf.platform_permutations = { 866 "test-manifest": { 867 "linux": { 868 "22.04": { 869 "x86_64": { 870 "debug": {"no_variant": {}, "swgl": {}}, 871 "opt": {"no_variant": {}}, 872 } 873 } 874 } 875 } 876 } 877 for i in range(0, 6): 878 skip_if = sf.task_to_skip_if( 879 "test-manifest", task_id, Kind.TOML, "test-path", True 880 ) 881 assert skip_if is None 882 # Skip whole platform with build type and test variant 883 skip_if = sf.task_to_skip_if("test-manifest", task_id, Kind.TOML, "test-path", True) 884 assert ( 885 skip_if 886 == "os == 'linux' && os_version == '22.04' && arch == 'x86_64' && debug && swgl" 887 ) 888 889 # Skip only build type if there are less than 75% failures on a test variant 890 task_id = "caDMGUmnT7muCqNWj6w3nQ" 891 task_details = { 892 "expires": "2024-01-09T16:05:56.825Z", 893 "extra": { 894 "suite": "mochitest-plain", 895 "test-setting": { 896 "build": {"type": "debug"}, 897 "platform": { 898 "arch": "64", 899 "os": {"name": "linux", "version": "2204"}, 900 }, 901 "runtime": {}, 902 }, 903 }, 904 } 905 sf.tasks[task_id] = task_details 906 for i in range(0, 2): 907 skip_if = sf.task_to_skip_if( 908 "test-manifest", task_id, Kind.TOML, "test-path", True 909 ) 910 assert ( 911 skip_if 912 == "os == 'linux' && os_version == '22.04' && arch == 'x86_64' && debug && swgl" 913 ) 914 skip_if = sf.task_to_skip_if("test-manifest", task_id, Kind.TOML, "test-path", True) 915 assert ( 916 skip_if == "os == 'linux' && os_version == '22.04' && arch == 'x86_64' && debug" 917 ) 918 919 920 def test_task_to_skip_if_wpt(): 921 """Test task_to_skip_if_wpt""" 922 923 # preload task cache 924 task_id = "AKYqxtoWStigj_5yHVqAeg" 925 task_details = { 926 "expires": "2024-03-19T03:29:11.050Z", 927 "extra": { 928 "suite": "web-platform-tests", 929 "test-setting": { 930 "build": { 931 "type": "opt", 932 "shippable": True, 933 }, 934 "platform": { 935 "arch": "32", 936 "os": {"name": "linux", "version": "1804"}, 937 }, 938 "runtime": {}, 939 }, 940 }, 941 } 942 sf = Skipfails() 943 sf.tasks[task_id] = task_details 944 sf.platform_permutations = { 945 "test-manifest": { 946 "linux": { 947 "18.04": { 948 "x86": {"debug": {"no_variant": {}}, "opt": {"no_variant": {}}} 949 } 950 } 951 } 952 } 953 # function under test 954 skip_if = sf.task_to_skip_if("test-manifest", task_id, Kind.WPT, "test-path", False) 955 assert ( 956 skip_if == 'os == "linux" and os_version == "18.04" and arch == "x86" and opt' 957 ) 958 959 960 def test_task_to_skip_if_reftest(): 961 """Test task_to_skip_if_reftest""" 962 963 # preload task cache 964 task_id = "AKYqxtoWStigj_5yHVqAeg" 965 task_details = { 966 "expires": "2024-03-19T03:29:11.050Z", 967 "extra": { 968 "suite": "reftest", 969 "test-setting": { 970 "build": { 971 "type": "opt", 972 "shippable": True, 973 }, 974 "platform": { 975 "arch": "32", 976 "os": {"name": "linux", "version": "1804"}, 977 }, 978 "runtime": {}, 979 }, 980 }, 981 } 982 sf = Skipfails(implicit_vars=True) 983 sf.tasks[task_id] = task_details 984 # function under test 985 skip_if = sf.task_to_skip_if("", task_id, Kind.LIST, "test-path", False) 986 assert skip_if == "gtkWidget&&optimized&&!is64Bit" 987 988 989 def test_task_to_skip_if_reftest2(): 990 """Test task_to_skip_if_reftest2""" 991 992 # preload task cache 993 task_id = "ajp7DRgGQbyfnIAklKA7Tw" 994 task_details = { 995 "expires": "2024-03-19T03:29:11.050Z", 996 "extra": { 997 "suite": "reftest", 998 "test-setting": { 999 "build": {"tsan": True, "type": "opt"}, 1000 "runtime": {"webrender-sw": True}, 1001 "platform": {"os": {"name": "linux", "version": "1804"}, "arch": "64"}, 1002 }, 1003 }, 1004 } 1005 sf = Skipfails(implicit_vars=True) 1006 sf.tasks[task_id] = task_details 1007 # function under test 1008 skip_if = sf.task_to_skip_if("", task_id, Kind.LIST, "test-path", False) 1009 assert skip_if == "gtkWidget&&ThreadSanitizer&&swgl" 1010 1011 1012 def test_task_to_skip_if_reftest3(): 1013 """Test task_to_skip_if_reftest3""" 1014 1015 # preload task cache 1016 task_id = "UP-t3xrGSDWvUNjFGIt_aQ" 1017 task_details = { 1018 "expires": "2024-01-09T16:05:56.825Z", 1019 "extra": { 1020 "suite": "mochitest-plain", 1021 "test-setting": { 1022 "build": {"type": "debug"}, 1023 "platform": { 1024 "arch": "32", 1025 "os": {"build": "24h2", "name": "windows", "version": "11"}, 1026 }, 1027 "runtime": {}, 1028 }, 1029 }, 1030 } 1031 sf = Skipfails(implicit_vars=False) 1032 sf.tasks[task_id] = task_details 1033 # function under test 1034 skip_if = sf.task_to_skip_if("", task_id, Kind.LIST, "test-path", False) 1035 assert skip_if == "winWidget&&isDebugBuild&&fission&&!is64Bit&&!swgl&&!nogpu" 1036 1037 1038 def test_task_to_skip_if_reftest4(): 1039 """Test task_to_skip_if_reftest4""" 1040 1041 # preload task cache 1042 task_id = "ajp7DRgGQbyfnIAklKA7Tw" 1043 task_details = { 1044 "expires": "2024-03-19T03:29:11.050Z", 1045 "extra": { 1046 "suite": "reftest", 1047 "test-setting": { 1048 "build": {"tsan": True, "type": "opt"}, 1049 "runtime": {}, 1050 "platform": {"os": {"name": "linux", "version": "1804"}, "arch": "64"}, 1051 }, 1052 }, 1053 } 1054 sf = Skipfails(implicit_vars=False) 1055 sf.tasks[task_id] = task_details 1056 # function under test 1057 skip_if = sf.task_to_skip_if("", task_id, Kind.LIST, "test-path", False) 1058 assert skip_if == "gtkWidget&&ThreadSanitizer&&fission&&!swgl&&!nogpu" 1059 1060 1061 def test_wpt_add_skip_if(): 1062 """Test wpt_add_skip_if""" 1063 1064 sf = Skipfails() 1065 manifest_before1 = "" 1066 anyjs = {} 1067 filename = "myfile.html" 1068 anyjs[filename] = False 1069 skip_if = 'os == "linux" and arch == "x86" and not debug' 1070 bug_reference = "Bug 123" 1071 disabled = " disabled:\n" 1072 condition = " if " + skip_if + ": " + bug_reference + "\n" 1073 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1074 manifest_before1, anyjs, skip_if, bug_reference 1075 ) 1076 manifest_expected1 = "[myfile.html]\n" + disabled + condition + "\n" 1077 assert manifest_str == manifest_expected1 1078 manifest_before2 = """[myfile.html] 1079 expected: 1080 if fission: [OK, FAIL] 1081 [< [Test 5\\] 1 out of 2 assertions were failed.] 1082 expected: FAIL 1083 """ 1084 manifest_expected2 = ( 1085 """[myfile.html] 1086 expected: 1087 if fission: [OK, FAIL] 1088 """ 1089 + disabled 1090 + condition 1091 + """ 1092 [< [Test 5\\] 1 out of 2 assertions were failed.] 1093 expected: FAIL 1094 """ 1095 ) 1096 anyjs[filename] = False 1097 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1098 manifest_before2, anyjs, skip_if, bug_reference 1099 ) 1100 assert manifest_str == manifest_expected2 1101 anyjs[filename] = False 1102 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1103 manifest_expected2, anyjs, skip_if, bug_reference 1104 ) 1105 assert manifest_str == manifest_expected2 1106 manifest_before4 = """;https: //bugzilla.mozilla.org/show_bug.cgi?id=1838684 1107 expected: [FAIL, PASS] 1108 [custom-highlight-painting-overlapping-highlights-002.html] 1109 expected: 1110 [PASS, FAIL] 1111 """ 1112 anyjs[filename] = False 1113 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1114 manifest_before4, anyjs, skip_if, bug_reference 1115 ) 1116 manifest_expected4 = manifest_before4 + "\n" + manifest_expected1 1117 assert manifest_str == manifest_expected4 1118 manifest_before5 = """[myfile.html] 1119 disabled: 1120 if os == 'win' && os_version == '11.26100' && arch == '32' && debug: Bug 456 1121 """ 1122 anyjs[filename] = False 1123 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1124 manifest_before5, anyjs, skip_if, bug_reference 1125 ) 1126 manifest_expected5 = manifest_before5 + condition + "\n" 1127 assert manifest_str == manifest_expected5 1128 manifest_before6 = """[myfile.html] 1129 [Window Size] 1130 expected: 1131 if product == "firefox_android": FAIL 1132 """ 1133 anyjs[filename] = False 1134 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1135 manifest_before6, anyjs, skip_if, bug_reference 1136 ) 1137 manifest_expected6 = ( 1138 """[myfile.html] 1139 disabled: 1140 """ 1141 + condition 1142 + """ 1143 [Window Size] 1144 expected: 1145 if product == "firefox_android": FAIL 1146 """ 1147 ) 1148 assert manifest_str == manifest_expected6 1149 manifest_before7 = """[myfile.html] 1150 fuzzy: 1151 if os == "android": maxDifference=0-255;totalPixels=0-105 # bug 1392254 1152 if os == "linux": maxDifference=0-255;totalPixels=0-136 # bug 1599638 1153 maxDifference=0-1;totalPixels=0-80 1154 disabled: 1155 if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1314684 1156 """ 1157 anyjs[filename] = False 1158 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1159 manifest_before7, anyjs, skip_if, bug_reference 1160 ) 1161 manifest_expected7 = manifest_before7 + condition + "\n" 1162 assert manifest_str == manifest_expected7 1163 manifest_before8 = """[myfile.html] 1164 disabled: 1165 if os == "linux" and os_version == "22.04" and arch == "x86_64" and debug and display == "wayland": Bug TBD 1166 expected: 1167 if swgl and (os == "linux") and debug and not fission: [PASS, FAIL] 1168 1169 """ 1170 anyjs[filename] = False 1171 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1172 manifest_before8, anyjs, skip_if, bug_reference 1173 ) 1174 manifest_expected8 = ( 1175 """[myfile.html] 1176 disabled: 1177 if os == "linux" and os_version == "22.04" and arch == "x86_64" and debug and display == "wayland": Bug TBD 1178 """ 1179 + condition 1180 + """ expected: 1181 if swgl and (os == "linux") and debug and not fission: [PASS, FAIL] 1182 1183 """ 1184 ) 1185 assert manifest_str == manifest_expected8 1186 manifest_before9 = """[myfile.html] 1187 disabled: 1188 if os == "linux" and os_version == "22.04" and arch == "x86_64" and debug and display == "wayland": Bug TBD 1189 1190 """ 1191 anyjs[filename] = False 1192 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1193 manifest_before9, anyjs, skip_if, bug_reference 1194 ) 1195 manifest_expected9 = ( 1196 """[myfile.html] 1197 disabled: 1198 if os == "linux" and os_version == "22.04" and arch == "x86_64" and debug and display == "wayland": Bug TBD 1199 """ 1200 + condition 1201 + "\n" 1202 ) 1203 assert manifest_str == manifest_expected9 1204 manifest_before10 = """[myfile.html] 1205 disabled: 1206 if os == "linux" and os_version == "22.04" and arch == "x86_64" and not debug and display == "wayland": Bug TBD 1207 1208 [3P fetch: Cross site window setting HTTP cookies] 1209 """ 1210 anyjs[filename] = False 1211 manifest_str, additional_comment_ = sf.wpt_add_skip_if( 1212 manifest_before10, anyjs, skip_if, bug_reference 1213 ) 1214 manifest_expected10 = ( 1215 """[myfile.html] 1216 disabled: 1217 if os == "linux" and os_version == "22.04" and arch == "x86_64" and not debug and display == "wayland": Bug TBD 1218 """ 1219 + condition 1220 + """\n [3P fetch: Cross site window setting HTTP cookies] 1221 """ 1222 ) 1223 assert manifest_str == manifest_expected10 1224 1225 1226 def test_get_filename_in_manifest(): 1227 """Test get_filename_in_manifest""" 1228 1229 sf = Skipfails() 1230 1231 assert ( 1232 sf.get_filename_in_manifest( 1233 "browser/components/sessionstore/test/browser.toml", 1234 "browser/components/sessionstore/test/browser_closed_tabs_windows.js", 1235 ) 1236 == "browser_closed_tabs_windows.js" 1237 ) 1238 assert ( 1239 sf.get_filename_in_manifest( 1240 "browser/base/content/test/webrtc/gracePeriod/browser.toml", 1241 "browser/base/content/test/webrtc/browser_devices_get_user_media_grace.js", 1242 ) 1243 == "../browser_devices_get_user_media_grace.js" 1244 ) 1245 assert ( 1246 sf.get_filename_in_manifest( 1247 "dom/animation/test/mochitest.toml", 1248 "dom/animation/test/document-timeline/test_document-timeline.html", 1249 ) 1250 == "document-timeline/test_document-timeline.html" 1251 ) 1252 1253 1254 def test_reftest_add_fuzzy_if(): 1255 """Test reftest_add_fuzzy_if""" 1256 1257 sf = Skipfails(implicit_vars=True) 1258 manifest_path = DATA_PATH.joinpath("reftest-1-before.list") 1259 manifest_str = open(manifest_path, encoding="utf-8").read() 1260 after_path = DATA_PATH.joinpath("reftest-1-after.list") 1261 after_str = open(after_path, encoding="utf-8").read() 1262 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1263 manifest_str, 1264 "fuzzy(0-11,0-7155) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg", 1265 "gtkWidget", 1266 [10, 12], 1267 [7000, 8000], 1268 2, 1269 False, 1270 "Bug TBD", 1271 ) 1272 assert additional_comment == "" 1273 assert manifest_str == after_str 1274 1275 1276 def test_reftest_add_fuzzy_if2(): 1277 """Test reftest_add_fuzzy_if2""" 1278 1279 sf = Skipfails(implicit_vars=True) 1280 manifest_path = DATA_PATH.joinpath("reftest-2-before.list") 1281 manifest_str = open(manifest_path, encoding="utf-8").read() 1282 after_path = DATA_PATH.joinpath("reftest-2-after.list") 1283 after_str = open(after_path, encoding="utf-8").read() 1284 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1285 manifest_str, 1286 "fuzzy(0-11,0-7155) HTTP(..) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg", 1287 "gtkWidget", 1288 [10, 12], 1289 [7000, 8000], 1290 2, 1291 False, 1292 "Bug TBD", 1293 ) 1294 assert additional_comment == "" 1295 assert manifest_str == after_str 1296 1297 1298 def test_reftest_add_fuzzy_if3(): 1299 """Test reftest_add_fuzzy_if3""" 1300 1301 sf = Skipfails(implicit_vars=True) 1302 manifest_path = DATA_PATH.joinpath("reftest-2-after.list") 1303 manifest_str = open(manifest_path, encoding="utf-8").read() 1304 after_path = DATA_PATH.joinpath("reftest-3-after.list") 1305 after_str = open(after_path, encoding="utf-8").read() 1306 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1307 manifest_str, 1308 "fuzzy(0-11,0-7155) fuzzy-if(gtkWidget,10-12,7000-8400) HTTP(..) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg", 1309 "gtkWidget", 1310 [9, 13], 1311 [7500, 7900], 1312 2, 1313 False, 1314 "Bug TBD", 1315 ) 1316 assert additional_comment == "" 1317 assert manifest_str == after_str 1318 1319 1320 def test_reftest_add_fuzzy_if4(): 1321 """Test reftest_add_fuzzy_if4""" 1322 1323 # Now with the relaxed merge policy the additional condition has been merged 1324 # instead of added.... 1325 # fuzzy-if(winWidget,0-177,0-1) fuzzy-if(winWidget&&!is64Bit&&isDebugBuild,0-152,0-1) == 23605-5.html 23605-5-ref.html # Bug TBD 1326 1327 sf = Skipfails(implicit_vars=True) 1328 manifest_path = DATA_PATH.joinpath("reftest-4-before.list") 1329 manifest_str = open(manifest_path, encoding="utf-8").read() 1330 after_path = DATA_PATH.joinpath("reftest-4-after.list") 1331 after_str = open(after_path, encoding="utf-8").read() 1332 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1333 manifest_str, 1334 "fuzzy-if(winWidget,0-177,0-1) == 23605-5.html 23605-5-ref.html", 1335 "winWidget&&!is64Bit&&isDebugBuild", 1336 [139, 145], 1337 [1, 1], 1338 2, 1339 True, 1340 "Bug TBD", 1341 ) 1342 assert additional_comment == "" 1343 assert manifest_str == after_str 1344 # assert ( 1345 # additional_comment 1346 # == "NOTE: more than one fuzzy-if for the OS = winWidget ==> may require manual review" 1347 # ) 1348 1349 1350 def test_reftest_add_fuzzy_if5(): 1351 """Test reftest_add_fuzzy_if5""" 1352 1353 sf = Skipfails(implicit_vars=True) 1354 manifest_path = DATA_PATH.joinpath("reftest-2-after.list") 1355 manifest_str = open(manifest_path, encoding="utf-8").read() 1356 after_path = DATA_PATH.joinpath("reftest-5-after.list") 1357 after_str = open(after_path, encoding="utf-8").read() 1358 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1359 manifest_str, 1360 "fuzzy(0-11,0-7155) fuzzy-if(gtkWidget,10-12,7000-8400) HTTP(..) == blur-inside-clipPath.svg blur-inside-clipPath-ref.svg", 1361 "gtkWidget&&!is64Bit", 1362 [5, 6], 1363 [3000, 4000], 1364 2, 1365 False, 1366 "Bug TBD", 1367 ) 1368 assert additional_comment == "" 1369 assert manifest_str == after_str 1370 1371 1372 def test_reftest_add_fuzzy_if6(): 1373 """Test reftest_add_fuzzy_if6""" 1374 1375 sf = Skipfails(implicit_vars=True) 1376 manifest_path = DATA_PATH.joinpath("reftest-6-before.list") 1377 manifest_str = open(manifest_path, encoding="utf-8").read() 1378 after_path = DATA_PATH.joinpath("reftest-6-after.list") 1379 after_str = open(after_path, encoding="utf-8").read() 1380 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1381 manifest_str, 1382 "fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(cocoaWidget,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png", 1383 "Android&&isDebugBuild&&!fission&&swgl", 1384 [1, 32], 1385 [1, 1760], 1386 2, 1387 True, 1388 "Bug TBD", 1389 ) 1390 assert additional_comment == "" 1391 assert manifest_str == after_str 1392 1393 1394 def test_reftest_add_fuzzy_if7(): 1395 """Test reftest_add_fuzzy_if7""" 1396 1397 sf = Skipfails(implicit_vars=False) 1398 manifest_path = DATA_PATH.joinpath("reftest-7-before.list") 1399 manifest_str = open(manifest_path, encoding="utf-8").read() 1400 after_path = DATA_PATH.joinpath("reftest-7-after.list") 1401 after_str = open(after_path, encoding="utf-8").read() 1402 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1403 manifest_str, 1404 "skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(appleSilicon,30-48,1835-187409) fuzzy-if(cocoaWidget,30-32,187326-187407) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm", 1405 "cocoaWidget", 1406 [], 1407 [], 1408 2, 1409 False, 1410 "Bug TBD", 1411 ) 1412 assert additional_comment == "" 1413 assert manifest_str == after_str 1414 1415 1416 def test_reftest_add_fuzzy_if8(): 1417 """Test reftest_add_fuzzy_if8""" 1418 1419 sf = Skipfails(implicit_vars=True) 1420 manifest_path = DATA_PATH.joinpath("reftest-6-after.list") 1421 manifest_str = open(manifest_path, encoding="utf-8").read() 1422 after_path = DATA_PATH.joinpath("reftest-8-after.list") 1423 after_str = open(after_path, encoding="utf-8").read() 1424 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1425 manifest_str, 1426 "fuzzy(16-51,5234-5622) fuzzy-if(swgl,32-38,1600-91746) fuzzy-if(useDrawSnapshot,16-16,11600-11600) fuzzy-if(Android&&isDebugBuild&&!fission&&swgl,0-33,0-1848) fuzzy-if(cocoaWidget,16-73,5212-5622) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm ../reftest_img.html?src=color_quads/720p.png", 1427 "cocoaWidget", 1428 [0, 0], 1429 [0, 0], 1430 2, 1431 True, 1432 "Bug TBD", 1433 ) 1434 assert additional_comment == "" 1435 assert manifest_str == after_str 1436 1437 1438 def test_reftest_add_fuzzy_if9(): 1439 """Test reftest_add_fuzzy_if9""" 1440 1441 sf = Skipfails(implicit_vars=True) 1442 manifest_path = DATA_PATH.joinpath("reftest-9-before.list") 1443 manifest_str = open(manifest_path, encoding="utf-8").read() 1444 after_path = DATA_PATH.joinpath("reftest-9-after.list") 1445 after_str = open(after_path, encoding="utf-8").read() 1446 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1447 manifest_str, 1448 "fuzzy(0-201,0-1486) fuzzy-if(Android&&(fission||!fission),201-267,42-1560) fuzzy-if(Android&&!fission&&(swgl||!swgl),201-267,40-1560) fuzzy-if(Android&&isDebugBuild&&!fission&&swgl,198-207,1439-1510) fuzzy-if(cocoaWidget&&isDebugBuild&&swgl,198-267,40-1510) fuzzy-if(gtkWidget&&swgl,198-267,40-1510) fuzzy-if(gtkWidget&&!fission&&swgl,0-267,0-1510) fuzzy-if(gtkWidget&&!fission&&(swgl||!swgl),201-267,40-1560) == element-paint-transform-02.html element-paint-transform-02-ref.html", 1449 "gtkWidget&&isDebugBuild&&!fission&&swgl", 1450 [198], 1451 [1439], 1452 2, 1453 True, 1454 "Bug TBD", 1455 ) 1456 assert manifest_str == after_str 1457 1458 1459 def test_reftest_add_fuzzy_if10(): 1460 """Test reftest_add_fuzzy_if10""" 1461 1462 sf = Skipfails(implicit_vars=False) 1463 manifest_path = DATA_PATH.joinpath("reftest-10-before.list") 1464 manifest_str = open(manifest_path, encoding="utf-8").read() 1465 after_path = DATA_PATH.joinpath("reftest-10-after.list") 1466 after_str = open(after_path, encoding="utf-8").read() 1467 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1468 manifest_str, 1469 "skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(appleSilicon,30-48,1835-187409) fuzzy-if(cocoaWidget,0-0,0-0) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm", 1470 "cocoaWidget", 1471 [], 1472 [], 1473 2, 1474 True, 1475 "Bug TBD", 1476 ) 1477 assert additional_comment == "fuzzy-if removed as calculated range is 0-0,0-0" 1478 assert manifest_str == after_str 1479 1480 1481 def test_reftest_add_fuzzy_if11(): 1482 """Test reftest_add_fuzzy_if11""" 1483 1484 sf = Skipfails(implicit_vars=False) 1485 manifest_path = DATA_PATH.joinpath("reftest-10-before.list") 1486 manifest_str = open(manifest_path, encoding="utf-8").read() 1487 manifest_str, additional_comment = sf.reftest_add_fuzzy_if( 1488 manifest_str, 1489 "skip-if(winWidget&&isCoverageBuild) fuzzy(0-16,75-1941) fuzzy-if(Android,28-255,273680-359920) fuzzy-if(cocoaWidget,30-32,187326-187407) fuzzy-if(appleSilicon,30-48,1835-187409) == ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.h264.mp4 ../reftest_video.html?src=color_quads/720p.png.bt709.bt709.tv.yuv420p.av1.webm", 1490 "winWidget", 1491 [], 1492 [], 1493 2, 1494 True, 1495 "Bug TBD", 1496 ) 1497 assert additional_comment == "fuzzy-if not added as calculated range is 0-0,0-0" 1498 assert manifest_str == "" 1499 1500 1501 def test_reftest_get_lineno(): 1502 """Test reftest_get_lineno""" 1503 1504 sf = Skipfails() 1505 mods = [ 1506 "pref(gfx.font_rendering.colr_v1.enabled,true)", 1507 "fuzzy(0-8,0-10100)", 1508 "==", 1509 "colrv1-01.html#C", 1510 "colrv1-01-ref.html#C", 1511 ] 1512 allmods = " ".join(mods) 1513 lineno = sf.reftest_find_lineno( 1514 DATA_PATH.joinpath("fontface_reftest.list"), mods, allmods 1515 ) 1516 assert lineno == 171 1517 1518 1519 def test_reftest_get_lineno2(): 1520 """Test reftest_get_lineno2""" 1521 1522 sf = Skipfails() 1523 mods = [ 1524 "pref(image.downscale-during-decode.enabled,true)", 1525 "fuzzy(0-53,0-6391)", 1526 "fuzzy-if(appleSilicon,0-20,0-11605)", 1527 "fuzzy-if(gtkWidget,18-19,5502-5568)", 1528 "skip-if(Android)", 1529 "==", 1530 "downscale-moz-icon-1.html", 1531 "downscale-moz-icon-1-ref.html", 1532 ] 1533 allmods = " ".join(mods) 1534 lineno = sf.reftest_find_lineno( 1535 DATA_PATH.joinpath("downscaling_reftest.list"), mods, allmods 1536 ) 1537 assert lineno == 183 1538 1539 1540 def test_reftest_get_lineno3(): 1541 """Test reftest_get_lineno3""" 1542 1543 sf = Skipfails() 1544 mods = [ 1545 "pref(webgl.force-enabled,true)", 1546 "skip-if(Android)", 1547 "fuzzy(0-235,0-3104)", 1548 "==", 1549 "1177726-text-stroke-bounds.html", 1550 "1177726-text-stroke-bounds-ref.html", 1551 ] 1552 allmods = " ".join(mods) 1553 lineno = sf.reftest_find_lineno( 1554 DATA_PATH.joinpath("dom_canvas_reftest.list"), mods, allmods 1555 ) 1556 assert lineno == 233 1557 1558 1559 def test_reftest_skip_failure_win_32(capsys): 1560 """Test reftest_skip_failure_win_32""" 1561 1562 sf = Skipfails(verbose=True, bugzilla="disable", implicit_vars=True, dry_run=True) 1563 manifest = "layout/reftests/svg/reftest.list" 1564 kind = Kind.LIST 1565 path = "fuzzy(0-1,0-5) fuzzy-if(winWidget,0-96,0-21713) skip-if(winWidget&&isCoverageBuild) fuzzy-if(Android&&device,0-4,0-946) == radialGradient-basic-03.svg radialGradient-basic-03-ref.html" 1566 anyjs = None 1567 differences = [68] 1568 pixels = [21668] 1569 lineno = 419 1570 status = "FAIL" 1571 label = "test-windows11-32-24h2-qr/debug-reftest-1" 1572 classification = "disable_recommended" 1573 task_id = "BpoP8I2CRZekXUKoSIZjUQ" 1574 try_url = "https://treeherder.mozilla.org/jobs?repo=try&tier=1%2C2%2C3&revision=3e54b0b81de7d6a3e6a2c3408892ffd6430bc137&selectedTaskRun=BpoP8I2CRZekXUKoSIZjUQ.0" 1575 revision = "3e54b0b81de7d6a3e6a2c3408892ffd6430bc137" 1576 repo = "try" 1577 meta_bug_id = None 1578 task_details = { # pre-cache task details 1579 "expires": "2024-01-09T16:05:56.825Z", 1580 "extra": { 1581 "suite": "reftest", 1582 "test-setting": { 1583 "build": {"type": "debug"}, 1584 "platform": { 1585 "arch": "32", 1586 "os": {"build": "24h2", "name": "windows", "version": "11"}, 1587 }, 1588 "runtime": {}, 1589 }, 1590 }, 1591 } 1592 sf.tasks[task_id] = task_details 1593 1594 sf.skip_failure( 1595 manifest, 1596 kind, 1597 path, 1598 task_id, 1599 None, 1600 None, 1601 False, 1602 anyjs, 1603 differences, 1604 pixels, 1605 lineno, 1606 status, 1607 label, 1608 classification, 1609 try_url, 1610 revision, 1611 repo, 1612 meta_bug_id, 1613 ) 1614 capsys.readouterr() 1615 # assert ( 1616 # captured.err.find( 1617 # "Skipping failures for Windows 32-bit are temporarily disabled" 1618 # ) 1619 # > 0 1620 # ) 1621 1622 1623 def test_reftest_skip_failure_reorder(capsys): 1624 """Test reftest_skip_failure_reorder""" 1625 1626 manifest_before_path = DATA_PATH.joinpath("reftest-reorder-before.list") 1627 manifest_before = open(manifest_before_path, encoding="utf-8").read() 1628 manifest_path = DATA_PATH.joinpath("reftest-reorder.list") 1629 manifest_fp = open(manifest_path, "w", encoding="utf-8") 1630 manifest_fp.write(manifest_before) 1631 manifest_fp.close() 1632 manifest_after_path = DATA_PATH.joinpath("reftest-reorder-after.list") 1633 manifest_after = open(manifest_after_path, encoding="utf-8").read() 1634 sf = Skipfails(verbose=True, bugzilla="disable", implicit_vars=True, dry_run=False) 1635 manifest = DATA_PATH.joinpath("reftest-reorder.list") 1636 kind = Kind.LIST 1637 path = "fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) skip-if(Android) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html" 1638 anyjs = None 1639 differences = [59, 61] 1640 pixels = [2032, 2133] 1641 lineno = 2 1642 status = "FAIL" 1643 label = "test-linux1804-64-asan-qr/opt-reftest-nofis-7" 1644 classification = "disable_failure" 1645 task_id = "a9Pz5yFJTxOL_uM-BEvGoQ" 1646 try_url = "https://treeherder.mozilla.org/jobs?repo=try&tier=1%2C2%2C3&revision=3e54b0b81de7d6a3e6a2c3408892ffd6430bc137&selectedTaskRun=BpoP8I2CRZekXUKoSIZjUQ.0" 1647 revision = "3e54b0b81de7d6a3e6a2c3408892ffd6430bc137" 1648 repo = "try" 1649 meta_bug_id = 1971610 1650 task_details = { # pre-cache task details 1651 "expires": "2024-01-09T16:05:56.825Z", 1652 "extra": { 1653 "suite": "reftest", 1654 "test-setting": { 1655 "build": {"type": "opt"}, 1656 "platform": { 1657 "arch": "64", 1658 "os": {"name": "linux", "version": "1804"}, 1659 }, 1660 "runtime": {"no-fission": True}, 1661 }, 1662 }, 1663 } 1664 sf.tasks[task_id] = task_details 1665 1666 sf.skip_failure( 1667 manifest, 1668 kind, 1669 path, 1670 task_id, 1671 None, 1672 None, 1673 False, 1674 anyjs, 1675 differences, 1676 pixels, 1677 lineno, 1678 status, 1679 label, 1680 classification, 1681 try_url, 1682 revision, 1683 repo, 1684 meta_bug_id, 1685 ) 1686 assert os.path.exists(manifest_path) 1687 manifest_str = open(manifest_path, encoding="utf-8").read() 1688 assert manifest_str == manifest_after 1689 os.remove(manifest_path) 1690 # captured = capsys.readouterr() 1691 # assert ( 1692 # captured.err.find( 1693 # "NOTE: more than one fuzzy-if for the OS = gtkWidget ==> may require manual review" 1694 # ) 1695 # > 0 1696 # ) 1697 1698 1699 def test_find_known_intermittent(): 1700 sf = Skipfails() 1701 1702 repo = "try" 1703 revision = "70613a6f11801e19478238c014b7ca61c28571d0" 1704 task_id = "B64K59oVTlir3gUcdZQiew" 1705 push_id = "1741497" 1706 job_id = "530457469" 1707 sf.push_ids[revision] = push_id # pre-cache push_id 1708 sf.job_ids[f"{push_id}:{task_id}"] = job_id # pre-cache job_id 1709 suggestions_path = DATA_PATH.joinpath(f"suggest-{job_id}.json") 1710 suggestions = read_json(suggestions_path) 1711 sf.suggestions[job_id] = suggestions # pre-cache suggestions 1712 manifest = "dom/midi/tests/mochitest.toml" 1713 filename = "test_midi_device_sysex.html" 1714 skip_if = "os == 'android' && os_version == '14' && arch == 'x86_64' && isolated_process && xorigin" 1715 1716 (bugid, comment, line_number) = sf.find_known_intermittent( 1717 repo, revision, task_id, manifest, filename, skip_if 1718 ) 1719 assert bugid == 1814775 1720 assert ( 1721 comment 1722 == "Intermittent failure in manifest: \"dom/midi/tests/mochitest.toml\"\n in test: \"[test_midi_device_sysex.html]\"\n added skip-if: \"os == 'android' && os_version == '14' && arch == 'x86_64' && isolated_process && xorigin\"\nError log line 2546: https://treeherder.mozilla.org/logviewer?repo=try&job_id=530457469&lineNumber=2546" 1723 ) 1724 assert line_number == 2546 1725 1726 1727 class Index: 1728 def __init__(self): 1729 self.index = 1 1730 1731 def expected(self): 1732 self.index += 1 1733 return self.index - 1 1734 1735 1736 @pytest.fixture(scope="session") 1737 def index(): 1738 yield Index() 1739 1740 1741 @pytest.mark.parametrize( 1742 "test_index, bug_reference, bugid", # test_index for convenience 1743 [ 1744 ( 1745 1, 1746 "# Bug 123456", 1747 "123456", 1748 ), 1749 ( 1750 2, 1751 "# Bug TBD", 1752 "TBD", 1753 ), 1754 ( 1755 3, 1756 " Bug 123456 ", 1757 "123456", 1758 ), 1759 ( 1760 4, 1761 "junk bUG123456_asdf", 1762 "123456", 1763 ), 1764 ], 1765 ) 1766 def test_bugid_from_reference( 1767 index: Index, test_index: int, bug_reference: str, bugid: str 1768 ): 1769 sf = Skipfails() 1770 bug = sf.bugid_from_reference(bug_reference) 1771 assert test_index == index.expected() 1772 assert bug == bugid 1773 1774 1775 if __name__ == "__main__": 1776 main()