web_platform_tests.py (28714B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 import copy 6 import gzip 7 import json 8 import os 9 import sys 10 from datetime import datetime, timedelta 11 12 # load modules from parent dir 13 sys.path.insert(1, os.path.dirname(sys.path[0])) 14 15 import mozinfo 16 from mozharness.base.errors import BaseErrorList 17 from mozharness.base.log import INFO 18 from mozharness.base.script import PreScriptAction 19 from mozharness.base.vcs.vcsbase import MercurialScript 20 from mozharness.mozilla.automation import TBPL_RETRY 21 from mozharness.mozilla.structuredlog import StructuredOutputParser 22 from mozharness.mozilla.testing.android import AndroidMixin 23 from mozharness.mozilla.testing.codecoverage import ( 24 CodeCoverageMixin, 25 code_coverage_config_options, 26 ) 27 from mozharness.mozilla.testing.errors import WptHarnessErrorList 28 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 29 30 31 class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): 32 config_options = ( 33 [ 34 [ 35 ["--test-type"], 36 { 37 "action": "extend", 38 "dest": "test_type", 39 "help": "Specify the test types to run.", 40 }, 41 ], 42 [ 43 ["--disable-e10s"], 44 { 45 "action": "store_false", 46 "dest": "e10s", 47 "default": True, 48 "help": "Run without e10s enabled", 49 }, 50 ], 51 [ 52 ["--disable-fission"], 53 { 54 "action": "store_true", 55 "dest": "disable_fission", 56 "default": False, 57 "help": "Run without fission enabled", 58 }, 59 ], 60 [ 61 ["--total-chunks"], 62 { 63 "action": "store", 64 "dest": "total_chunks", 65 "help": "Number of total chunks", 66 }, 67 ], 68 [ 69 ["--this-chunk"], 70 { 71 "action": "store", 72 "dest": "this_chunk", 73 "help": "Number of this chunk", 74 }, 75 ], 76 [ 77 ["--allow-software-gl-layers"], 78 { 79 "action": "store_true", 80 "dest": "allow_software_gl_layers", 81 "default": False, 82 "help": "Permits a software GL implementation (such as LLVMPipe) " 83 "to use the GL compositor.", 84 }, 85 ], 86 [ 87 ["--headless"], 88 { 89 "action": "store_true", 90 "dest": "headless", 91 "default": False, 92 "help": "Run tests in headless mode.", 93 }, 94 ], 95 [ 96 ["--headless-width"], 97 { 98 "action": "store", 99 "dest": "headless_width", 100 "default": "1600", 101 "help": "Specify headless virtual screen width (default: 1600).", 102 }, 103 ], 104 [ 105 ["--headless-height"], 106 { 107 "action": "store", 108 "dest": "headless_height", 109 "default": "1200", 110 "help": "Specify headless virtual screen height (default: 1200).", 111 }, 112 ], 113 [ 114 ["--setpref"], 115 { 116 "action": "append", 117 "metavar": "PREF=VALUE", 118 "dest": "extra_prefs", 119 "default": [], 120 "help": "Defines an extra user preference.", 121 }, 122 ], 123 [ 124 ["--skip-implementation-status"], 125 { 126 "action": "extend", 127 "dest": "skip_implementation_status", 128 "default": [], 129 "help": "Defines a way to not run a specific implementation status " 130 " (i.e. not implemented).", 131 }, 132 ], 133 [ 134 ["--backlog"], 135 { 136 "action": "store_true", 137 "dest": "backlog", 138 "default": False, 139 "help": "Defines if test category is backlog.", 140 }, 141 ], 142 [ 143 ["--skip-timeout"], 144 { 145 "action": "store_true", 146 "dest": "skip_timeout", 147 "default": False, 148 "help": "Ignore tests that are expected status of TIMEOUT", 149 }, 150 ], 151 [ 152 ["--skip-crash"], 153 { 154 "action": "store_true", 155 "dest": "skip_crash", 156 "default": False, 157 "help": "Ignore tests that are expected status of CRASH", 158 }, 159 ], 160 [ 161 ["--default-exclude"], 162 { 163 "action": "store_true", 164 "dest": "default_exclude", 165 "default": False, 166 "help": "Only run the tests explicitly given in arguments", 167 }, 168 ], 169 [ 170 ["--include"], 171 { 172 "action": "append", 173 "dest": "include", 174 "default": [], 175 "help": "Add URL prefix to include.", 176 }, 177 ], 178 [ 179 ["--exclude"], 180 { 181 "action": "append", 182 "dest": "exclude", 183 "default": [], 184 "help": "Add URL prefix to exclude.", 185 }, 186 ], 187 [ 188 ["--tag"], 189 { 190 "action": "append", 191 "dest": "tag", 192 "default": [], 193 "help": "Add test tag (which includes URL prefix) to include.", 194 }, 195 ], 196 [ 197 ["--exclude-tag"], 198 { 199 "action": "append", 200 "dest": "exclude_tag", 201 "default": [], 202 "help": "Add test tag (which includes URL prefix) to exclude.", 203 }, 204 ], 205 [ 206 ["--repeat"], 207 { 208 "action": "store", 209 "dest": "repeat", 210 "default": 0, 211 "type": int, 212 "help": "Repeat tests (used for confirm-failures) X times.", 213 }, 214 ], 215 [ 216 ["--timeout-multiplier"], 217 { 218 "action": "store", 219 "dest": "timeout_multiplier", 220 "type": float, 221 "help": "Sets the timeout multiplier (0.25 for `--backlog` tests by default)", 222 }, 223 ], 224 [ 225 ["--no-update-status-on-crash"], 226 { 227 "action": "store_false", 228 "dest": "update_status_on_crash", 229 "default": True, 230 "help": "Sets whether to update the test status if a crash dump is detected", 231 }, 232 ], 233 ] 234 + copy.deepcopy(testing_config_options) 235 + copy.deepcopy(code_coverage_config_options) 236 ) 237 238 def __init__(self, require_config_file=True): 239 super().__init__( 240 config_options=self.config_options, 241 all_actions=[ 242 "clobber", 243 "download-and-extract", 244 "download-and-process-manifest", 245 "create-virtualenv", 246 "pull", 247 "start-emulator", 248 "verify-device", 249 "install", 250 "run-tests", 251 ], 252 require_config_file=require_config_file, 253 config={"require_test_zip": True}, 254 ) 255 256 # Surely this should be in the superclass 257 c = self.config 258 self.installer_url = c.get("installer_url") 259 self.test_url = c.get("test_url") 260 self.test_packages_url = c.get("test_packages_url") 261 self.installer_path = c.get("installer_path") 262 self.binary_path = c.get("binary_path") 263 self.repeat = c.get("repeat") 264 self.abs_app_dir = None 265 self.xre_path = None 266 if self.is_emulator: 267 self.device_serial = "emulator-5554" 268 269 def query_abs_app_dir(self): 270 """We can't set this in advance, because OSX install directories 271 change depending on branding and opt/debug. 272 """ 273 if self.abs_app_dir: 274 return self.abs_app_dir 275 if not self.binary_path: 276 self.fatal("Can't determine abs_app_dir (binary_path not set!)") 277 self.abs_app_dir = os.path.dirname(self.binary_path) 278 return self.abs_app_dir 279 280 def query_abs_dirs(self): 281 if self.abs_dirs: 282 return self.abs_dirs 283 abs_dirs = super().query_abs_dirs() 284 285 dirs = {} 286 dirs["abs_app_install_dir"] = os.path.join( 287 abs_dirs["abs_work_dir"], "application" 288 ) 289 dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") 290 dirs["abs_test_bin_dir"] = os.path.join(dirs["abs_test_install_dir"], "bin") 291 dirs["abs_wpttest_dir"] = os.path.join( 292 dirs["abs_test_install_dir"], "web-platform" 293 ) 294 dirs["abs_blob_upload_dir"] = os.path.join( 295 abs_dirs["abs_work_dir"], "blobber_upload_dir" 296 ) 297 dirs["abs_test_extensions_dir"] = os.path.join( 298 dirs["abs_test_install_dir"], "extensions" 299 ) 300 work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"] 301 if self.is_android: 302 dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils") 303 if self.is_emulator: 304 dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux") 305 dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device") 306 dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar") 307 # AndroidMixin uses this when launching the emulator. We only want 308 # GLES3 if we're running WebRender (default) 309 self.use_gles3 = True 310 311 abs_dirs.update(dirs) 312 self.abs_dirs = abs_dirs 313 314 return self.abs_dirs 315 316 @PreScriptAction("create-virtualenv") 317 def _pre_create_virtualenv(self, action): 318 dirs = self.query_abs_dirs() 319 320 requirements = os.path.join( 321 dirs["abs_test_install_dir"], "config", "marionette_requirements.txt" 322 ) 323 324 self.register_virtualenv_module(requirements=[requirements]) 325 326 webtransport_requirements = os.path.join( 327 dirs["abs_test_install_dir"], 328 "config", 329 "wpt_ci_requirements.txt", 330 ) 331 332 self.register_virtualenv_module(requirements=[webtransport_requirements]) 333 334 def _query_geckodriver(self): 335 path = None 336 c = self.config 337 dirs = self.query_abs_dirs() 338 repl_dict = {} 339 repl_dict.update(dirs) 340 path = c.get("geckodriver", "geckodriver") 341 if path: 342 path = path % repl_dict 343 return path 344 345 def _query_cmd(self, test_types): 346 if not self.binary_path: 347 self.fatal("Binary path could not be determined") 348 # And exit 349 350 c = self.config 351 run_file_name = "runtests.py" 352 353 dirs = self.query_abs_dirs() 354 abs_app_dir = self.query_abs_app_dir() 355 str_format_values = { 356 "binary_path": self.binary_path, 357 "test_path": dirs["abs_wpttest_dir"], 358 "test_install_path": dirs["abs_test_install_dir"], 359 "abs_app_dir": abs_app_dir, 360 "abs_work_dir": dirs["abs_work_dir"], 361 "xre_path": self.xre_path, 362 } 363 364 cmd = [self.query_python_path("python"), "-u"] 365 cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name)) 366 367 mozinfo.find_and_update_from_json(dirs["abs_test_install_dir"]) 368 369 raw_log_file, error_summary_file = self.get_indexed_logs( 370 dirs["abs_blob_upload_dir"], "wpt" 371 ) 372 373 cmd += [ 374 "--log-raw=-", 375 "--log-wptreport=%s" 376 % os.path.join(dirs["abs_blob_upload_dir"], "wptreport.json"), 377 "--log-errorsummary=%s" % error_summary_file, 378 "--symbols-path=%s" % self.symbols_path, 379 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(), 380 "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin"), 381 "--no-pause-after-test", 382 "--instrument-to-file=%s" 383 % os.path.join(dirs["abs_blob_upload_dir"], "wpt_instruments.txt"), 384 "--specialpowers-path=%s" 385 % os.path.join( 386 dirs["abs_test_extensions_dir"], "specialpowers@mozilla.org.xpi" 387 ), 388 # Ensure that we don't get a Python traceback from handlers that will be 389 # added to the log summary 390 "--suppress-handler-traceback", 391 ] 392 393 if self.repeat > 0: 394 # repeat should repeat the original test, so +1 for first run 395 cmd.append("--repeat=%s" % (self.repeat + 1)) 396 397 if ( 398 self.is_android 399 or mozinfo.info["tsan"] 400 or "wdspec" in test_types 401 or not c["disable_fission"] 402 # reftest on osx needs to be 1 process 403 or "reftest" in test_types 404 and sys.platform.startswith("darwin") 405 ): 406 processes = 1 407 else: 408 processes = 2 409 cmd.append("--processes=%s" % processes) 410 411 if self.is_android: 412 cmd += [ 413 "--device-serial=%s" % self.device_serial, 414 "--package-name=%s" % self.query_package_name(), 415 "--product=firefox_android", 416 ] 417 else: 418 cmd += ["--binary=%s" % self.binary_path, "--product=firefox"] 419 420 cmd += ["--no-install-fonts"] 421 422 for test_type in test_types: 423 cmd.append("--test-type=%s" % test_type) 424 425 if c["extra_prefs"]: 426 cmd.extend([f"--setpref={p}" for p in c["extra_prefs"]]) 427 428 if c["disable_fission"]: 429 cmd.append("--disable-fission") 430 431 if not c["e10s"]: 432 cmd.append("--disable-e10s") 433 434 if c["skip_timeout"]: 435 cmd.append("--skip-timeout") 436 437 if c["skip_crash"]: 438 cmd.append("--skip-crash") 439 440 if c["default_exclude"]: 441 cmd.append("--default-exclude") 442 443 for implementation_status in c["skip_implementation_status"]: 444 cmd.append("--skip-implementation-status=%s" % implementation_status) 445 446 # Bug 1643177 - reduce timeout multiplier for web-platform-tests backlog 447 if "timeout_multiplier" in c: 448 cmd.append("--timeout-multiplier=%s" % c["timeout_multiplier"]) 449 elif c["backlog"]: 450 cmd.append("--timeout-multiplier=0.25") 451 452 if c["update_status_on_crash"]: 453 cmd.append("--update-status-on-crash") 454 else: 455 cmd.append("--no-update-status-on-crash") 456 457 test_paths = set() 458 if not (self.verify_enabled or self.per_test_coverage): 459 # mozharness_test_paths is a set of test groups (directories) to run 460 # if we have confirm_paths, this is a specific path we want to run and ignore the group 461 mozharness_test_paths = json.loads( 462 os.environ.get("MOZHARNESS_TEST_PATHS", '""') 463 ) 464 confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) 465 466 if mozharness_test_paths: 467 if confirm_paths: 468 mozharness_test_paths = confirm_paths 469 470 path = os.path.join(dirs["abs_fetches_dir"], "wpt_tests_by_group.json") 471 472 if not os.path.exists(path): 473 self.critical("Unable to locate web-platform-test groups file.") 474 475 cmd.append(f"--test-groups={path}") 476 477 for key in mozharness_test_paths.keys(): 478 if "web-platform" not in key: 479 self.info(f"Ignoring test_paths for {key} harness") 480 continue 481 paths = mozharness_test_paths.get(key, []) 482 for p in paths: 483 if not p.startswith("/"): 484 # Assume this is a filesystem path rather than a test id 485 path = os.path.relpath(p, "testing/web-platform") 486 if ".." in path: 487 self.fatal(f"Invalid WPT path: {path}") 488 path = os.path.join(dirs["abs_wpttest_dir"], path) 489 else: 490 path = p 491 492 test_paths.add(path) 493 else: 494 # As per WPT harness, the --run-by-dir flag is incompatible with 495 # the --test-groups flag. 496 cmd.append("--run-by-dir=%i" % (3 if not mozinfo.info["asan"] else 0)) 497 for opt in ["total_chunks", "this_chunk"]: 498 val = c.get(opt) 499 if val: 500 cmd.append("--%s=%s" % (opt.replace("_", "-"), val)) 501 502 options = list(c.get("options", [])) 503 504 if "wdspec" in test_types: 505 geckodriver_path = self._query_geckodriver() 506 if not geckodriver_path or not os.path.isfile(geckodriver_path): 507 self.fatal( 508 "Unable to find geckodriver binary " 509 "in common test package: %s" % str(geckodriver_path) 510 ) 511 cmd.append("--webdriver-binary=%s" % geckodriver_path) 512 cmd.append("--webdriver-arg=-vv") # enable trace logs 513 514 test_type_suite = { 515 "testharness": "web-platform-tests", 516 "crashtest": "web-platform-tests-crashtest", 517 "print-reftest": "web-platform-tests-print-reftest", 518 "reftest": "web-platform-tests-reftest", 519 "wdspec": "web-platform-tests-wdspec", 520 } 521 for test_type in test_types: 522 try_options, try_tests = self.try_args(test_type_suite[test_type]) 523 524 cmd.extend( 525 self.query_options( 526 options, try_options, str_format_values=str_format_values 527 ) 528 ) 529 cmd.extend( 530 self.query_tests_args(try_tests, str_format_values=str_format_values) 531 ) 532 533 for url_prefix in c["include"]: 534 cmd.append(f"--include={url_prefix}") 535 for url_prefix in c["exclude"]: 536 cmd.append(f"--exclude={url_prefix}") 537 for tag in c["tag"]: 538 cmd.append(f"--tag={tag}") 539 for tag in c["exclude_tag"]: 540 cmd.append(f"--exclude-tag={tag}") 541 542 if mozinfo.info["os"] == "win": 543 # Because of a limit on the length of CLI command line string length in Windows, we 544 # should prefer to pass paths by a file instead. 545 import tempfile 546 547 with tempfile.NamedTemporaryFile(delete=False) as tmp: 548 tmp.write("\n".join(test_paths).encode()) 549 cmd.append(f"--include-file={tmp.name}") 550 else: 551 cmd.extend(test_paths) 552 553 return cmd 554 555 def download_and_extract(self): 556 super().download_and_extract( 557 extract_dirs=[ 558 "mach", 559 "bin/*", 560 "config/*", 561 "extensions/*", 562 "mozbase/*", 563 "marionette/*", 564 "tools/*", 565 "web-platform/*", 566 "mozpack/*", 567 "mozbuild/*", 568 ], 569 suite_categories=["web-platform"], 570 ) 571 dirs = self.query_abs_dirs() 572 if self.is_android: 573 self.xre_path = dirs["abs_xre_dir"] 574 # Make sure that the logging directory exists 575 if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1: 576 self.fatal("Could not create blobber upload directory") 577 # Exit 578 579 def download_and_process_manifest(self): 580 """Downloads the tests-by-manifest JSON mapping generated by the decision task. 581 582 web-platform-tests are chunked in the decision task as of Bug 1608837 583 and this means tests are resolved by the TestResolver as part of this process. 584 585 The manifest file contains tests keyed by the groups generated in 586 TestResolver.get_wpt_group(). 587 588 Upon successful call, a JSON file containing only the web-platform test 589 groups are saved in the fetch directory. 590 591 Bug: 592 1634554 593 """ 594 dirs = self.query_abs_dirs() 595 url = os.environ.get("TESTS_BY_MANIFEST_URL", "") 596 if not url: 597 self.fatal("TESTS_BY_MANIFEST_URL not defined.") 598 599 artifact_name = url.split("/")[-1] 600 601 # Save file to the MOZ_FETCHES dir. 602 self.download_file( 603 url, file_name=artifact_name, parent_dir=dirs["abs_fetches_dir"] 604 ) 605 606 with gzip.open(os.path.join(dirs["abs_fetches_dir"], artifact_name), "r") as f: 607 tests_by_manifest = json.loads(f.read()) 608 609 # We need to filter out non-web-platform-tests without knowing what the 610 # groups are. Fortunately, all web-platform test 'manifests' begin with a 611 # forward slash. 612 test_groups = { 613 key: tests_by_manifest[key] 614 for key in tests_by_manifest.keys() 615 if key.startswith("/") 616 } 617 618 outfile = os.path.join(dirs["abs_fetches_dir"], "wpt_tests_by_group.json") 619 with open(outfile, "w+") as f: 620 json.dump(test_groups, f, indent=2, sort_keys=True) 621 622 def install(self): 623 if self.is_android: 624 self.install_android_app(self.installer_path) 625 else: 626 super().install() 627 628 def _install_fonts(self): 629 if self.is_android: 630 return 631 # Ensure the Ahem font is available 632 dirs = self.query_abs_dirs() 633 634 if not sys.platform.startswith("darwin"): 635 font_path = os.path.join(os.path.dirname(self.binary_path), "fonts") 636 else: 637 font_path = os.path.join( 638 os.path.dirname(self.binary_path), 639 os.pardir, 640 "Resources", 641 "res", 642 "fonts", 643 ) 644 if not os.path.exists(font_path): 645 os.makedirs(font_path) 646 ahem_src = os.path.join(dirs["abs_wpttest_dir"], "tests", "fonts", "Ahem.ttf") 647 ahem_dest = os.path.join(font_path, "Ahem.ttf") 648 with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest: 649 dest.write(src.read()) 650 651 def run_tests(self): 652 dirs = self.query_abs_dirs() 653 654 parser = StructuredOutputParser( 655 config=self.config, 656 log_obj=self.log_obj, 657 log_compact=True, 658 error_list=BaseErrorList + WptHarnessErrorList, 659 allow_crashes=True, 660 ) 661 662 env = {"MINIDUMP_SAVE_PATH": dirs["abs_blob_upload_dir"]} 663 env["RUST_BACKTRACE"] = "full" 664 665 if self.config["allow_software_gl_layers"]: 666 env["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1" 667 if self.config["headless"]: 668 env["MOZ_HEADLESS"] = "1" 669 env["MOZ_HEADLESS_WIDTH"] = self.config["headless_width"] 670 env["MOZ_HEADLESS_HEIGHT"] = self.config["headless_height"] 671 672 if self.is_android: 673 env["ADB_PATH"] = self.adb_path 674 675 env["MOZ_GMP_PATH"] = os.pathsep.join( 676 os.path.join(dirs["abs_test_bin_dir"], "plugins", p, "1.0") 677 for p in ("gmp-fake", "gmp-fakeopenh264") 678 ) 679 680 env = self.query_env(partial_env=env, log_level=INFO) 681 682 start_time = datetime.now() 683 max_per_test_time = timedelta(minutes=60) 684 max_per_test_tests = 10 685 if self.per_test_coverage: 686 max_per_test_tests = 30 687 executed_tests = 0 688 executed_too_many_tests = False 689 690 if self.per_test_coverage or self.verify_enabled: 691 suites = self.query_per_test_category_suites(None, None) 692 if "wdspec" in suites: 693 # geckodriver is required for wdspec, but not always available 694 geckodriver_path = self._query_geckodriver() 695 if not geckodriver_path or not os.path.isfile(geckodriver_path): 696 suites.remove("wdspec") 697 self.info("Skipping 'wdspec' tests - no geckodriver") 698 else: 699 test_types = self.config.get("test_type", []) 700 suites = [None] 701 for suite in suites: 702 if executed_too_many_tests and not self.per_test_coverage: 703 continue 704 705 if suite: 706 test_types = [suite] 707 708 summary = {} 709 for per_test_args in self.query_args(suite): 710 # Make sure baseline code coverage tests are never 711 # skipped and that having them run has no influence 712 # on the max number of actual tests that are to be run. 713 is_baseline_test = ( 714 "baselinecoverage" in per_test_args[-1] 715 if self.per_test_coverage 716 else False 717 ) 718 if executed_too_many_tests and not is_baseline_test: 719 continue 720 721 if not is_baseline_test: 722 if (datetime.now() - start_time) > max_per_test_time: 723 # Running tests has run out of time. That is okay! Stop running 724 # them so that a task timeout is not triggered, and so that 725 # (partial) results are made available in a timely manner. 726 self.info( 727 "TinderboxPrint: Running tests took too long: Not all tests " 728 "were executed.<br/>" 729 ) 730 return 731 if executed_tests >= max_per_test_tests: 732 # When changesets are merged between trees or many tests are 733 # otherwise updated at once, there probably is not enough time 734 # to run all tests, and attempting to do so may cause other 735 # problems, such as generating too much log output. 736 self.info( 737 "TinderboxPrint: Too many modified tests: Not all tests " 738 "were executed.<br/>" 739 ) 740 executed_too_many_tests = True 741 742 executed_tests = executed_tests + 1 743 744 cmd = self._query_cmd(test_types) 745 cmd.extend(per_test_args) 746 747 final_env = copy.copy(env) 748 749 if self.per_test_coverage: 750 self.set_coverage_env(final_env, is_baseline_test) 751 752 return_code = self.run_command( 753 cmd, 754 cwd=dirs["abs_work_dir"], 755 output_timeout=1000, 756 output_parser=parser, 757 env=final_env, 758 ) 759 760 if self.per_test_coverage: 761 self.add_per_test_coverage_report( 762 final_env, suite, per_test_args[-1] 763 ) 764 765 tbpl_status, log_level, summary = parser.evaluate_parser( 766 return_code, previous_summary=summary 767 ) 768 self.record_status(tbpl_status, level=log_level) 769 770 if len(per_test_args) > 0: 771 self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) 772 if tbpl_status == TBPL_RETRY: 773 self.info("Per-test run abandoned due to RETRY status") 774 return 775 776 777 # main {{{1 778 if __name__ == "__main__": 779 web_platform_tests = WebPlatformTest() 780 web_platform_tests.run_and_exit()