reftestcommandline.py (20200B)
1 import argparse 2 import os 3 import sys 4 from collections import OrderedDict 5 from urllib.parse import urlparse 6 7 import mozinfo 8 import mozlog 9 10 here = os.path.abspath(os.path.dirname(__file__)) 11 12 13 class ReftestArgumentsParser(argparse.ArgumentParser): 14 def __init__(self, **kwargs): 15 super().__init__(**kwargs) 16 17 # Try to import a MozbuildObject. Success indicates that we are 18 # running from a source tree. This allows some defaults to be set 19 # from the source tree. 20 try: 21 from mozbuild.base import MozbuildObject 22 23 self.build_obj = MozbuildObject.from_environment(cwd=here) 24 except ImportError: 25 self.build_obj = None 26 27 self.add_argument( 28 "--xre-path", 29 action="store", 30 type=str, 31 dest="xrePath", 32 # individual scripts will set a sane default 33 default=None, 34 help="absolute path to directory containing XRE (probably xulrunner)", 35 ) 36 37 self.add_argument( 38 "--symbols-path", 39 action="store", 40 type=str, 41 dest="symbolsPath", 42 default=None, 43 help="absolute path to directory containing breakpad symbols, " 44 "or the URL of a zip file containing symbols", 45 ) 46 47 self.add_argument( 48 "--debugger", 49 action="store", 50 dest="debugger", 51 help="use the given debugger to launch the application", 52 ) 53 54 self.add_argument( 55 "--debugger-args", 56 action="store", 57 dest="debuggerArgs", 58 help="pass the given args to the debugger _before_ " 59 "the application on the command line", 60 ) 61 62 self.add_argument( 63 "--debugger-interactive", 64 action="store_true", 65 dest="debuggerInteractive", 66 help="prevents the test harness from redirecting " 67 "stdout and stderr for interactive debuggers", 68 ) 69 70 self.add_argument( 71 "--appname", 72 action="store", 73 type=str, 74 dest="app", 75 default=None, 76 help="absolute path to application, overriding default", 77 ) 78 79 self.add_argument( 80 "--extra-profile-file", 81 action="append", 82 dest="extraProfileFiles", 83 default=[], 84 help="copy specified files/dirs to testing profile", 85 ) 86 87 self.add_argument( 88 "--timeout", 89 action="store", 90 dest="timeout", 91 type=int, 92 default=300, # 5 minutes per bug 479518 93 help="reftest will timeout in specified number of seconds. " 94 "[default %(default)s].", 95 ) 96 97 self.add_argument( 98 "--leak-threshold", 99 action="store", 100 type=int, 101 dest="defaultLeakThreshold", 102 default=0, 103 help="fail if the number of bytes leaked in default " 104 "processes through refcounted objects (or bytes " 105 "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " 106 "is greater than the given number", 107 ) 108 109 self.add_argument( 110 "--utility-path", 111 action="store", 112 type=str, 113 dest="utilityPath", 114 default=self.build_obj.bindir if self.build_obj else None, 115 help="absolute path to directory containing utility " 116 "programs (xpcshell, ssltunnel, certutil)", 117 ) 118 119 self.add_argument( 120 "--total-chunks", 121 type=int, 122 dest="totalChunks", 123 help="how many chunks to split the tests up into", 124 ) 125 126 self.add_argument( 127 "--this-chunk", 128 type=int, 129 dest="thisChunk", 130 help="which chunk to run between 1 and --total-chunks", 131 ) 132 133 self.add_argument( 134 "--log-file", 135 action="store", 136 type=str, 137 dest="logFile", 138 default=None, 139 help="file to log output to in addition to stdout", 140 ) 141 142 self.add_argument( 143 "--skip-slow-tests", 144 dest="skipSlowTests", 145 action="store_true", 146 default=False, 147 help="skip tests marked as slow when running", 148 ) 149 150 self.add_argument( 151 "--ignore-window-size", 152 dest="ignoreWindowSize", 153 action="store_true", 154 default=False, 155 help="ignore the window size, which may cause spurious failures and passes", 156 ) 157 158 self.add_argument( 159 "--install-extension", 160 action="append", 161 dest="extensionsToInstall", 162 default=[], 163 help="install the specified extension in the testing profile. " 164 "The extension file's name should be <id>.xpi where <id> is " 165 "the extension's id as indicated in its install.rdf. " 166 "An optional path can be specified too.", 167 ) 168 169 self.add_argument( 170 "--marionette", 171 default=None, 172 help="host:port to use when connecting to Marionette", 173 ) 174 175 self.add_argument( 176 "--marionette-socket-timeout", default=None, help=argparse.SUPPRESS 177 ) 178 179 self.add_argument( 180 "--marionette-startup-timeout", default=None, help=argparse.SUPPRESS 181 ) 182 183 self.add_argument( 184 "--setenv", 185 action="append", 186 type=str, 187 default=[], 188 dest="environment", 189 metavar="NAME=VALUE", 190 help="sets the given variable in the application's environment", 191 ) 192 193 self.add_argument( 194 "--filter", 195 action="store", 196 type=str, 197 dest="filter", 198 help="specifies a regular expression (as could be passed to the JS " 199 "RegExp constructor) to test against URLs in the reftest manifest; " 200 "only test items that have a matching test URL will be run.", 201 ) 202 203 self.add_argument( 204 "--shuffle", 205 action="store_true", 206 default=False, 207 dest="shuffle", 208 help="run reftests in random order", 209 ) 210 211 self.add_argument( 212 "--run-until-failure", 213 action="store_true", 214 default=False, 215 dest="runUntilFailure", 216 help="stop running on the first failure. Useful for RR recordings.", 217 ) 218 219 self.add_argument( 220 "--repeat", 221 action="store", 222 type=int, 223 default=0, 224 dest="repeat", 225 help="number of times the select test(s) will be executed. Useful for " 226 "finding intermittent failures.", 227 ) 228 229 self.add_argument( 230 "--focus-filter-mode", 231 action="store", 232 type=str, 233 dest="focusFilterMode", 234 default="all", 235 help="filters tests to run by whether they require focus. " 236 "Valid values are `all', `needs-focus', or `non-needs-focus'. " 237 "Defaults to `all'.", 238 ) 239 240 self.add_argument( 241 "--disable-e10s", 242 action="store_false", 243 default=True, 244 dest="e10s", 245 help="disables content processes", 246 ) 247 248 self.add_argument( 249 "--disable-fission", 250 action="store_true", 251 default=False, 252 dest="disableFission", 253 help="Run tests with fission (site isolation) disabled.", 254 ) 255 256 self.add_argument( 257 "--setpref", 258 action="append", 259 type=str, 260 default=[], 261 dest="extraPrefs", 262 metavar="PREF=VALUE", 263 help="defines an extra user preference", 264 ) 265 266 self.add_argument( 267 "--reftest-extension-path", 268 action="store", 269 dest="reftestExtensionPath", 270 help="Path to the reftest extension", 271 ) 272 273 self.add_argument( 274 "--special-powers-extension-path", 275 action="store", 276 dest="specialPowersExtensionPath", 277 help="Path to the special powers extension", 278 ) 279 280 self.add_argument( 281 "--suite", 282 choices=["reftest", "crashtest", "jstestbrowser"], 283 default=None, 284 help=argparse.SUPPRESS, 285 ) 286 287 self.add_argument( 288 "--cleanup-crashes", 289 action="store_true", 290 dest="cleanupCrashes", 291 default=False, 292 help="Delete pending crash reports before running tests.", 293 ) 294 295 self.add_argument( 296 "--max-retries", 297 type=int, 298 dest="maxRetries", 299 default=4, 300 help="The maximum number of attempts to try and recover from a " 301 "crash before aborting the test run [default 4].", 302 ) 303 304 self.add_argument( 305 "tests", 306 metavar="TEST_PATH", 307 nargs="*", 308 help="Path to test file, manifest file, or directory containing " 309 "tests. For jstestbrowser, the relative path can be either from " 310 "topsrcdir or the staged area " 311 "($OBJDIR/dist/test-stage/jsreftest/tests)", 312 ) 313 314 self.add_argument( 315 "--sandbox-read-whitelist", 316 action="append", 317 dest="sandboxReadWhitelist", 318 default=[], 319 help="Path to add to the sandbox whitelist.", 320 ) 321 322 self.add_argument( 323 "--verify", 324 action="store_true", 325 default=False, 326 help="Run tests in verification mode: Run many times in different " 327 "ways, to see if there are intermittent failures.", 328 ) 329 330 self.add_argument( 331 "--verify-max-time", 332 type=int, 333 default=3600, 334 help="Maximum time, in seconds, to run in --verify mode..", 335 ) 336 337 self.add_argument( 338 "--enable-webrender", 339 action="store_true", 340 dest="enable_webrender", 341 default=False, 342 help="Enable the WebRender compositor in Gecko.", 343 ) 344 345 self.add_argument( 346 "--headless", 347 action="store_true", 348 dest="headless", 349 default=False, 350 help="Run tests in headless mode.", 351 ) 352 353 self.add_argument( 354 "--topsrcdir", 355 action="store", 356 type=str, 357 dest="topsrcdir", 358 default=None, 359 help="Path to source directory", 360 ) 361 362 mozlog.commandline.add_logging_group(self) 363 364 def get_ip(self): 365 import moznetwork 366 367 return moznetwork.get_ip() 368 369 def set_default_suite(self, options): 370 manifests = OrderedDict([ 371 ("reftest.list", "reftest"), 372 ("crashtests.list", "crashtest"), 373 ("jstests.list", "jstestbrowser"), 374 ]) 375 376 for test_path in options.tests: 377 file_name = os.path.basename(test_path) 378 if file_name in manifests: 379 options.suite = manifests[file_name] 380 return 381 382 for test_path in options.tests: 383 for manifest_file, suite in manifests.items(): 384 if os.path.exists(os.path.join(test_path, manifest_file)): 385 options.suite = suite 386 return 387 388 self.error( 389 "Failed to determine test suite; supply --suite to set this explicitly" 390 ) 391 392 def validate(self, options, reftest): 393 if not options.tests: 394 # Can't just set this in the argument parser because mach will set a default 395 self.error( 396 "Must supply at least one path to a manifest file, " 397 "test directory, or test file to run." 398 ) 399 400 if options.suite is None: 401 self.set_default_suite(options) 402 403 if options.totalChunks is not None and options.thisChunk is None: 404 self.error("thisChunk must be specified when totalChunks is specified") 405 406 if options.totalChunks: 407 if not 1 <= options.thisChunk <= options.totalChunks: 408 self.error("thisChunk must be between 1 and totalChunks") 409 410 if not options.disableFission and not options.e10s: 411 self.error("Fission is not supported without e10s.") 412 413 if options.logFile: 414 options.logFile = reftest.getFullPath(options.logFile) 415 416 if options.xrePath is not None: 417 if not os.access(options.xrePath, os.F_OK): 418 self.error("--xre-path '%s' not found" % options.xrePath) 419 if not os.path.isdir(options.xrePath): 420 self.error("--xre-path '%s' is not a directory" % options.xrePath) 421 options.xrePath = reftest.getFullPath(options.xrePath) 422 423 if options.reftestExtensionPath is None: 424 if self.build_obj is not None: 425 reftestExtensionPath = os.path.join( 426 self.build_obj.distdir, "xpi-stage", "reftest" 427 ) 428 else: 429 reftestExtensionPath = os.path.join(here, "reftest") 430 options.reftestExtensionPath = os.path.normpath(reftestExtensionPath) 431 432 if options.specialPowersExtensionPath is None: 433 if self.build_obj is not None: 434 specialPowersExtensionPath = os.path.join( 435 self.build_obj.distdir, "xpi-stage", "specialpowers" 436 ) 437 else: 438 specialPowersExtensionPath = os.path.join(here, "specialpowers") 439 options.specialPowersExtensionPath = os.path.normpath( 440 specialPowersExtensionPath 441 ) 442 443 options.leakThresholds = { 444 "default": options.defaultLeakThreshold, 445 "tab": options.defaultLeakThreshold, 446 } 447 448 if mozinfo.isWin: 449 if mozinfo.info["bits"] == 32: 450 # See bug 1408554. 451 options.leakThresholds["tab"] = 3000 452 else: 453 # See bug 1404482. 454 options.leakThresholds["tab"] = 100 455 456 if options.topsrcdir is None: 457 if self.build_obj: 458 options.topsrcdir = self.build_obj.topsrcdir 459 else: 460 options.topsrcdir = os.getcwd() 461 462 463 class DesktopArgumentsParser(ReftestArgumentsParser): 464 def __init__(self, **kwargs): 465 super().__init__(**kwargs) 466 467 self.add_argument( 468 "--run-tests-in-parallel", 469 action="store_true", 470 default=False, 471 dest="runTestsInParallel", 472 help="run tests in parallel if possible", 473 ) 474 475 def validate(self, options, reftest): 476 super().validate(options, reftest) 477 478 if options.runTestsInParallel: 479 if options.logFile is not None: 480 self.error("cannot specify logfile with parallel tests") 481 if options.totalChunks is not None or options.thisChunk is not None: 482 self.error( 483 "cannot specify thisChunk or totalChunks with parallel tests" 484 ) 485 if options.focusFilterMode != "all": 486 self.error("cannot specify focusFilterMode with parallel tests") 487 if options.debugger is not None: 488 self.error("cannot specify a debugger with parallel tests") 489 490 if options.debugger: 491 # valgrind and some debuggers may cause Gecko to start slowly. Make sure 492 # marionette waits long enough to connect. 493 options.marionette_startup_timeout = 900 494 options.marionette_socket_timeout = 540 495 496 if not options.tests: 497 self.error("No test files specified.") 498 499 if options.app is None: 500 if ( 501 self.build_obj 502 and self.build_obj.substs["MOZ_BUILD_APP"] != "mobile/android" 503 ): 504 from mozbuild.base import BinaryNotFoundException 505 506 try: 507 bin_dir = self.build_obj.get_binary_path() 508 except BinaryNotFoundException as e: 509 print(f"{e}\n\n{e.help()}\n", file=sys.stderr) 510 sys.exit(1) 511 else: 512 bin_dir = None 513 514 if bin_dir: 515 options.app = bin_dir 516 517 if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: 518 options.symbolsPath = reftest.getFullPath(options.symbolsPath) 519 520 options.utilityPath = reftest.getFullPath(options.utilityPath) 521 522 523 class RemoteArgumentsParser(ReftestArgumentsParser): 524 def __init__(self, **kwargs): 525 super().__init__() 526 527 # app, xrePath and utilityPath variables are set in main function 528 self.set_defaults( 529 logFile="reftest.log", app="", xrePath="", utilityPath="", localLogName=None 530 ) 531 532 self.add_argument( 533 "--adbpath", 534 action="store", 535 type=str, 536 dest="adb_path", 537 default=None, 538 help="Path to adb binary.", 539 ) 540 541 self.add_argument( 542 "--deviceSerial", 543 action="store", 544 type=str, 545 dest="deviceSerial", 546 help="adb serial number of remote device. This is required " 547 "when more than one device is connected to the host. " 548 "Use 'adb devices' to see connected devices.", 549 ) 550 551 self.add_argument( 552 "--remote-webserver", 553 action="store", 554 type=str, 555 dest="remoteWebServer", 556 help="IP address of the remote web server.", 557 ) 558 559 self.add_argument( 560 "--http-port", 561 action="store", 562 type=str, 563 dest="httpPort", 564 help="http port of the remote web server.", 565 ) 566 567 self.add_argument( 568 "--ssl-port", 569 action="store", 570 type=str, 571 dest="sslPort", 572 help="ssl port of the remote web server.", 573 ) 574 575 self.add_argument( 576 "--remoteTestRoot", 577 action="store", 578 type=str, 579 dest="remoteTestRoot", 580 help="Remote directory to use as test root " 581 "(eg. /data/local/tmp/test_root).", 582 ) 583 584 self.add_argument( 585 "--httpd-path", 586 action="store", 587 type=str, 588 dest="httpdPath", 589 help="Path to the httpd.js file.", 590 ) 591 592 self.add_argument( 593 "--no-install", 594 action="store_true", 595 default=False, 596 help="Skip the installation of the APK.", 597 ) 598 599 def validate_remote(self, options): 600 DEFAULT_HTTP_PORT = 8888 601 DEFAULT_SSL_PORT = 4443 602 603 if options.remoteWebServer is None: 604 options.remoteWebServer = self.get_ip() 605 606 if options.remoteWebServer == "127.0.0.1": 607 self.error( 608 "ERROR: Either you specified the loopback for the remote webserver or ", 609 "your local IP cannot be detected. " 610 "Please provide the local ip in --remote-webserver", 611 ) 612 613 if not options.httpPort: 614 options.httpPort = DEFAULT_HTTP_PORT 615 616 if not options.sslPort: 617 options.sslPort = DEFAULT_SSL_PORT 618 619 if options.xrePath is None: 620 self.error( 621 "ERROR: You must specify the path to the controller xre directory" 622 ) 623 else: 624 # Ensure xrepath is a full path 625 options.xrePath = os.path.abspath(options.xrePath) 626 627 # httpd-path is specified by standard makefile targets and may be specified 628 # on the command line to select a particular version of httpd.js. If not 629 # specified, try to select the one from hostutils.zip, as required in 630 # bug 882932. 631 if not options.httpdPath: 632 options.httpdPath = os.path.join(options.utilityPath, "components") 633 634 return options