mach_commands.py (21324B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 # Integrates the web-platform-tests test runner with mach. 6 7 import os 8 import sys 9 10 from mach.decorators import Command 11 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt 12 from mozbuild.base import MachCommandConditions as conditions 13 from mozbuild.base import MozbuildObject 14 15 here = os.path.abspath(os.path.dirname(__file__)) 16 INTEROP_REQUIREMENTS_PATH = os.path.join(here, "interop_requirements.txt") 17 18 19 class WebPlatformTestsRunnerSetup(MozbuildObject): 20 default_log_type = "mach" 21 22 def __init__(self, *args, **kwargs): 23 super().__init__(*args, **kwargs) 24 self._here = os.path.join(self.topsrcdir, "testing", "web-platform") 25 kwargs["tests_root"] = os.path.join(self._here, "tests") 26 sys.path.insert(0, kwargs["tests_root"]) 27 build_path = os.path.join(self.topobjdir, "build") 28 if build_path not in sys.path: 29 sys.path.append(build_path) 30 31 def kwargs_common(self, kwargs): 32 """Setup kwargs relevant for all browser products""" 33 34 tests_src_path = os.path.join(self._here, "tests") 35 36 if kwargs["product"] in {"firefox", "firefox_android"}: 37 if kwargs["specialpowers_path"] is None: 38 kwargs["specialpowers_path"] = os.path.join( 39 self.distdir, "xpi-stage", "specialpowers@mozilla.org.xpi" 40 ) 41 42 if kwargs["config"] is None: 43 kwargs["config"] = os.path.join( 44 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini" 45 ) 46 47 if ( 48 kwargs["exclude"] is None 49 and kwargs["include"] is None 50 and not sys.platform.startswith("linux") 51 ): 52 kwargs["exclude"] = ["css"] 53 54 if kwargs["ssl_type"] in (None, "pregenerated"): 55 cert_root = os.path.join(tests_src_path, "tools", "certs") 56 if kwargs["ca_cert_path"] is None: 57 kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem") 58 59 if kwargs["host_key_path"] is None: 60 kwargs["host_key_path"] = os.path.join( 61 cert_root, "web-platform.test.key" 62 ) 63 64 if kwargs["host_cert_path"] is None: 65 kwargs["host_cert_path"] = os.path.join( 66 cert_root, "web-platform.test.pem" 67 ) 68 69 if kwargs["reftest_screenshot"] is None: 70 kwargs["reftest_screenshot"] = "fail" 71 72 kwargs["capture_stdio"] = True 73 74 return kwargs 75 76 def kwargs_firefox(self, kwargs): 77 """Setup kwargs specific to running Firefox and other gecko browsers""" 78 from wptrunner import wptcommandline 79 80 kwargs = self.kwargs_common(kwargs) 81 82 if kwargs["binary"] is None: 83 kwargs["binary"] = self.get_binary_path() 84 85 if kwargs["webdriver_binary"] is None: 86 kwargs["webdriver_binary"] = self.find_webdriver_binary() 87 88 if kwargs["certutil_binary"] is None: 89 kwargs["certutil_binary"] = self.get_binary_path("certutil") 90 91 if kwargs["install_fonts"] is None: 92 kwargs["install_fonts"] = False 93 94 if kwargs["preload_browser"] is None: 95 kwargs["preload_browser"] = False 96 97 if kwargs["prefs_root"] is None: 98 kwargs["prefs_root"] = os.path.join(self.topsrcdir, "testing", "profiles") 99 100 if kwargs["stackfix_dir"] is None: 101 kwargs["stackfix_dir"] = self.bindir 102 103 if kwargs["symbols_path"] is None: 104 kwargs["symbols_path"] = os.path.join(self.distdir, "crashreporter-symbols") 105 106 kwargs["gmp_path"] = os.pathsep.join( 107 os.path.join(self.distdir, "bin", p, "1.0") 108 for p in ("gmp-fake", "gmp-fakeopenh264") 109 ) 110 111 kwargs = wptcommandline.check_args(kwargs) 112 113 return kwargs 114 115 def kwargs_firefox_android(self, kwargs): 116 from wptrunner import wptcommandline 117 118 kwargs = self.kwargs_common(kwargs) 119 120 # package_name may be different in the future 121 package_name = kwargs["package_name"] 122 if not package_name: 123 kwargs["package_name"] = package_name = "org.mozilla.geckoview.test_runner" 124 125 # Note that this import may fail in non-firefox-for-android trees 126 from mozrunner.devices.android_device import ( 127 InstallIntent, 128 get_adb_path, 129 verify_android_device, 130 ) 131 132 kwargs["adb_binary"] = get_adb_path(self) 133 install = InstallIntent.NO if kwargs.pop("no_install") else InstallIntent.YES 134 verify_android_device( 135 self, install=install, verbose=False, xre=True, app=package_name 136 ) 137 138 if kwargs["webdriver_binary"] is None: 139 kwargs["webdriver_binary"] = self.find_webdriver_binary() 140 141 if kwargs["certutil_binary"] is None: 142 kwargs["certutil_binary"] = os.path.join( 143 os.environ.get("MOZ_HOST_BIN"), "certutil" 144 ) 145 146 if kwargs["install_fonts"] is None: 147 kwargs["install_fonts"] = False 148 149 if not kwargs["device_serial"]: 150 kwargs["device_serial"] = ["emulator-5554"] 151 152 kwargs = wptcommandline.check_args(kwargs) 153 154 return kwargs 155 156 def kwargs_wptrun(self, kwargs): 157 """Setup kwargs for wpt-run which is only used for non-gecko browser products""" 158 from tools.wpt import run, virtualenv 159 160 kwargs = self.kwargs_common(kwargs) 161 162 # Our existing kwargs corresponds to the wptrunner command line arguments. 163 # `wpt run` extends this with some additional arguments that are consumed by 164 # the frontend. Copy over the default values of these extra arguments so they 165 # are present when we call into that frontend. 166 run_parser = run.create_parser() 167 run_kwargs = run_parser.parse_args([kwargs["product"], kwargs["test_list"]]) 168 169 for key, value in vars(run_kwargs).items(): 170 if key not in kwargs: 171 kwargs[key] = value 172 173 # Install the deps 174 # We do this explicitly to avoid calling pip with options that aren't 175 # supported in the in-tree version 176 wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner") 177 browser_cls = run.product_setup[kwargs["product"]].browser_cls 178 requirements = ["requirements.txt"] 179 if ( 180 hasattr(browser_cls, "requirements") 181 and browser_cls.requirements is not None 182 ): 183 requirements.append(browser_cls.requirements) 184 185 for filename in requirements: 186 path = os.path.join(wptrunner_path, filename) 187 if os.path.exists(path): 188 self.virtualenv_manager.install_pip_requirements( 189 path, require_hashes=False 190 ) 191 192 venv = virtualenv.Virtualenv( 193 self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True 194 ) 195 try: 196 browser_cls, kwargs = run.setup_wptrunner(venv, **kwargs) 197 except run.WptrunError as e: 198 print(e, file=sys.stderr) 199 sys.exit(1) 200 201 # This is kind of a hack; override the metadata paths so we don't use 202 # gecko metadata for non-gecko products 203 for url_base, test_root in kwargs["test_paths"].items(): 204 meta_suffix = url_base.strip("/") 205 meta_dir = os.path.join( 206 self._here, "products", kwargs["product"].name, meta_suffix 207 ) 208 test_root.metadata_path = meta_dir 209 if not os.path.exists(meta_dir): 210 os.makedirs(meta_dir) 211 return kwargs 212 213 def setup_fonts_firefox(self): 214 # Ensure the Ahem font is available 215 if not sys.platform.startswith("darwin"): 216 font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts") 217 else: 218 font_path = os.path.join( 219 os.path.dirname(self.get_binary_path()), 220 os.pardir, 221 "Resources", 222 "res", 223 "fonts", 224 ) 225 ahem_src = os.path.join( 226 self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf" 227 ) 228 ahem_dest = os.path.join(font_path, "Ahem.ttf") 229 if not os.path.exists(ahem_dest) and os.path.exists(ahem_src): 230 with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest: 231 dest.write(src.read()) 232 233 def find_webdriver_binary(self): 234 ext = ".exe" if sys.platform in ["win32", "msys", "cygwin"] else "" 235 try_paths = [ 236 self.get_binary_path("geckodriver", validate_exists=False), 237 os.path.join(self.topobjdir, "dist", "host", "bin", f"geckodriver{ext}"), 238 ] 239 240 for build_type in ["release", "debug"]: 241 try_paths.append( 242 os.path.join(self.topsrcdir, "target", build_type, f"geckodriver{ext}") 243 ) 244 found_paths = [] 245 for path in try_paths: 246 if os.path.exists(path): 247 found_paths.append(path) 248 249 if found_paths: 250 # Pick the most recently modified version 251 found_paths.sort(key=os.path.getmtime) 252 return found_paths[-1] 253 254 255 class WebPlatformTestsServeRunner(MozbuildObject): 256 def run(self, **kwargs): 257 sys.path.insert(0, os.path.join(here, "tests")) 258 sys.path.insert(0, os.path.join(here, "tests", "tools")) 259 import logging 260 261 import manifestupdate 262 from serve import serve 263 from wptrunner import wptcommandline 264 265 logger = logging.getLogger("web-platform-tests") 266 267 src_root = self.topsrcdir 268 obj_root = self.topobjdir 269 src_wpt_dir = os.path.join(src_root, "testing", "web-platform") 270 271 config_path = manifestupdate.generate_config( 272 logger, 273 src_root, 274 src_wpt_dir, 275 os.path.join(obj_root, "_tests", "web-platform"), 276 False, 277 ) 278 279 test_paths = wptcommandline.get_test_paths( 280 wptcommandline.config.read(config_path) 281 ) 282 283 def get_route_builder(*args, **kwargs): 284 route_builder = serve.get_route_builder(*args, **kwargs) 285 286 for url_base, paths in test_paths.items(): 287 if url_base != "/": 288 route_builder.add_mount_point(url_base, paths.tests_path) 289 290 return route_builder 291 292 return 0 if serve.run(route_builder=get_route_builder, **kwargs) else 1 293 294 295 class WebPlatformTestsUpdater(MozbuildObject): 296 """Update web platform tests.""" 297 298 def setup_logging(self, **kwargs): 299 import update 300 301 return update.setup_logging(kwargs, {"mach": sys.stdout}) 302 303 def update_manifest(self, logger, **kwargs): 304 import manifestupdate 305 306 return manifestupdate.run( 307 logger=logger, src_root=self.topsrcdir, obj_root=self.topobjdir, **kwargs 308 ) 309 310 def run_update(self, logger, **kwargs): 311 import update 312 from update import updatecommandline 313 314 self.update_manifest(logger, **kwargs) 315 316 if kwargs["config"] is None: 317 kwargs["config"] = os.path.join( 318 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini" 319 ) 320 if kwargs["product"] is None: 321 kwargs["product"] = "firefox" 322 323 kwargs["store_state"] = False 324 325 kwargs = updatecommandline.check_args(kwargs) 326 327 try: 328 update.run_update(logger, **kwargs) 329 except Exception: 330 import traceback 331 332 traceback.print_exc() 333 334 335 class WebPlatformTestsUnittestRunner(MozbuildObject): 336 def run(self, **kwargs): 337 import unittestrunner 338 339 return unittestrunner.run(self.topsrcdir, **kwargs) 340 341 342 class WebPlatformTestsTestPathsRunner(MozbuildObject): 343 """Update web platform tests.""" 344 345 def run(self, **kwargs): 346 sys.path.insert( 347 0, 348 os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")), 349 ) 350 import logging 351 352 import localpaths # noqa: F401 353 import manifestupdate 354 from manifest import testpaths 355 from wptrunner import wptcommandline 356 357 logger = logging.getLogger("web-platform-tests") 358 359 src_root = self.topsrcdir 360 obj_root = self.topobjdir 361 src_wpt_dir = os.path.join(src_root, "testing", "web-platform") 362 363 config_path = manifestupdate.generate_config( 364 logger, 365 src_root, 366 src_wpt_dir, 367 os.path.join(obj_root, "_tests", "web-platform"), 368 False, 369 ) 370 371 test_paths = wptcommandline.get_test_paths( 372 wptcommandline.config.read(config_path) 373 ) 374 results = {} 375 for url_base, paths in test_paths.items(): 376 results.update( 377 testpaths.get_paths( 378 path=paths.manifest_path, 379 src_root=src_root, 380 tests_root=paths.tests_path, 381 update=kwargs["update"], 382 rebuild=kwargs["rebuild"], 383 url_base=url_base, 384 cache_root=kwargs["cache_root"], 385 test_ids=kwargs["test_ids"], 386 ) 387 ) 388 testpaths.write_output(results, kwargs["json"]) 389 return True 390 391 392 def create_parser_update(): 393 from update import updatecommandline 394 395 return updatecommandline.create_parser() 396 397 398 def create_parser_manifest_update(): 399 import manifestupdate 400 401 return manifestupdate.create_parser() 402 403 404 def create_parser_metadata_summary(): 405 import metasummary 406 407 return metasummary.create_parser() 408 409 410 def create_parser_metadata_merge(): 411 import metamerge 412 413 return metamerge.get_parser() 414 415 416 def create_parser_serve(): 417 sys.path.insert( 418 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")) 419 ) 420 import serve 421 422 return serve.serve.get_parser() 423 424 425 def create_parser_unittest(): 426 import unittestrunner 427 428 return unittestrunner.get_parser() 429 430 431 def create_parser_fetch_logs(): 432 import interop 433 434 return interop.get_parser_fetch_logs() 435 436 437 def create_parser_interop_score(): 438 import interop 439 440 return interop.get_parser_interop_score() 441 442 443 def create_parser_testpaths(): 444 import argparse 445 446 from mach.util import get_state_dir 447 448 parser = argparse.ArgumentParser() 449 parser.add_argument( 450 "--no-update", 451 dest="update", 452 action="store_false", 453 default=True, 454 help="Don't update manifest before continuing", 455 ) 456 parser.add_argument( 457 "-r", 458 "--rebuild", 459 action="store_true", 460 default=False, 461 help="Force a full rebuild of the manifest.", 462 ) 463 parser.add_argument( 464 "--cache-root", 465 action="store", 466 default=os.path.join(get_state_dir(), "cache", "wpt"), 467 help="Path in which to store any caches (default <tests_root>/.wptcache/)", 468 ) 469 parser.add_argument( 470 "test_ids", action="store", nargs="+", help="Test ids for which to get paths" 471 ) 472 parser.add_argument( 473 "--json", action="store_true", default=False, help="Output as JSON" 474 ) 475 return parser 476 477 478 @Command( 479 "web-platform-tests", 480 category="testing", 481 conditions=[conditions.is_firefox_or_android], 482 description="Run web-platform-tests.", 483 parser=create_parser_wpt, 484 virtualenv_name="wpt", 485 ) 486 def run_web_platform_tests(command_context, **params): 487 if params["product"] is None: 488 if conditions.is_android(command_context): 489 params["product"] = "firefox_android" 490 else: 491 params["product"] = "firefox" 492 if "test_objects" in params: 493 include = [] 494 test_types = set() 495 for item in params["test_objects"]: 496 include.append(item["name"]) 497 test_types.add(item.get("subsuite")) 498 if None not in test_types: 499 params["test_types"] = list(test_types) 500 params["include"] = include 501 del params["test_objects"] 502 # subsuite coming from `mach test` means something more like `test type`, so remove that argument 503 if "subsuite" in params: 504 del params["subsuite"] 505 if params.get("debugger", None): 506 import mozdebug 507 508 if not mozdebug.get_debugger_info(params.get("debugger")): 509 sys.exit(1) 510 511 wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup) 512 wpt_setup._mach_context = command_context._mach_context 513 wpt_setup._virtualenv_name = command_context._virtualenv_name 514 wpt_runner = WebPlatformTestsRunner(wpt_setup) 515 516 logger = wpt_runner.setup_logging(**params) 517 # wptrunner already handles setting any log parameter from 518 # mach test to the logger, so it's OK to remove that argument now 519 if "log" in params: 520 del params["log"] 521 522 if ( 523 conditions.is_android(command_context) 524 and params["product"] != "firefox_android" 525 ): 526 logger.warning("Must specify --product=firefox_android in Android environment.") 527 528 return wpt_runner.run(logger, **params) 529 530 531 @Command( 532 "wpt", 533 category="testing", 534 conditions=[conditions.is_firefox_or_android], 535 description="Run web-platform-tests.", 536 parser=create_parser_wpt, 537 virtualenv_name="wpt", 538 ) 539 def run_wpt(command_context, **params): 540 return run_web_platform_tests(command_context, **params) 541 542 543 @Command( 544 "web-platform-tests-update", 545 category="testing", 546 description="Update web-platform-test metadata.", 547 parser=create_parser_update, 548 virtualenv_name="wpt", 549 ) 550 def update_web_platform_tests(command_context, **params): 551 wpt_updater = command_context._spawn(WebPlatformTestsUpdater) 552 logger = wpt_updater.setup_logging(**params) 553 return wpt_updater.run_update(logger, **params) 554 555 556 @Command( 557 "wpt-update", 558 category="testing", 559 description="Update web-platform-test metadata.", 560 parser=create_parser_update, 561 virtualenv_name="wpt", 562 ) 563 def update_wpt(command_context, **params): 564 return update_web_platform_tests(command_context, **params) 565 566 567 @Command( 568 "wpt-manifest-update", 569 category="testing", 570 description="Update web-platform-test manifests.", 571 parser=create_parser_manifest_update, 572 virtualenv_name="wpt", 573 ) 574 def wpt_manifest_update(command_context, **params): 575 wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup) 576 wpt_runner = WebPlatformTestsRunner(wpt_setup) 577 logger = wpt_runner.setup_logging(**params) 578 logger.warning( 579 "The wpt manifest is now automatically updated, " 580 "so running this command is usually unnecessary" 581 ) 582 return 0 if wpt_runner.update_manifest(logger, **params) else 1 583 584 585 @Command( 586 "wpt-serve", 587 category="testing", 588 description="Run the wpt server", 589 parser=create_parser_serve, 590 virtualenv_name="wpt", 591 ) 592 def wpt_serve(command_context, **params): 593 import logging 594 595 logger = logging.getLogger("web-platform-tests") 596 logger.addHandler(logging.StreamHandler(sys.stdout)) 597 wpt_serve = command_context._spawn(WebPlatformTestsServeRunner) 598 return wpt_serve.run(**params) 599 600 601 @Command( 602 "wpt-metadata-summary", 603 category="testing", 604 description="Create a json summary of the wpt metadata", 605 parser=create_parser_metadata_summary, 606 virtualenv_name="wpt", 607 ) 608 def wpt_summary(command_context, **params): 609 import metasummary 610 611 wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup) 612 return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params) 613 614 615 @Command( 616 "wpt-metadata-merge", 617 category="testing", 618 parser=create_parser_metadata_merge, 619 virtualenv_name="wpt", 620 ) 621 def wpt_meta_merge(command_context, **params): 622 import metamerge 623 624 if params["dest"] is None: 625 params["dest"] = params["current"] 626 return metamerge.run(**params) 627 628 629 @Command( 630 "wpt-unittest", 631 category="testing", 632 description="Run the wpt tools and wptrunner unit tests", 633 parser=create_parser_unittest, 634 virtualenv_name="wpt", 635 ) 636 def wpt_unittest(command_context, **params): 637 runner = command_context._spawn(WebPlatformTestsUnittestRunner) 638 return 0 if runner.run(**params) else 1 639 640 641 @Command( 642 "wpt-test-paths", 643 category="testing", 644 description="Get a mapping from test ids to files", 645 parser=create_parser_testpaths, 646 virtualenv_name="wpt", 647 ) 648 def wpt_test_paths(command_context, **params): 649 runner = command_context._spawn(WebPlatformTestsTestPathsRunner) 650 runner.run(**params) 651 return 0 652 653 654 @Command( 655 "wpt-fetch-logs", 656 category="testing", 657 description="Fetch wptreport.json logs from taskcluster", 658 parser=create_parser_fetch_logs, 659 virtualenv_name="wpt-interop", 660 ) 661 def wpt_fetch_logs(command_context, **params): 662 import interop 663 664 interop.fetch_logs(**params) 665 return 0 666 667 668 @Command( 669 "wpt-interop-score", 670 category="testing", 671 description="Score a run according to Interop 2023", 672 parser=create_parser_interop_score, 673 virtualenv_name="wpt-interop", 674 ) 675 def wpt_interop_score(command_context, **params): 676 import interop 677 678 interop.score_runs(**params) 679 return 0