runjunit.py (27022B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import os 7 import posixpath 8 import re 9 import shutil 10 import sys 11 import tempfile 12 import traceback 13 14 import mozcrash 15 import mozinfo 16 import mozlog 17 import moznetwork 18 from mozdevice import ADBDeviceFactory, ADBError, ADBTimeoutError 19 from mozprofile import DEFAULT_PORTS, Profile 20 from mozprofile.cli import parse_preferences 21 from mozprofile.permissions import ServerLocations 22 from runtests import MochitestDesktop, update_mozinfo 23 24 here = os.path.abspath(os.path.dirname(__file__)) 25 26 try: 27 from mach.util import UserError 28 from mozbuild.base import MachCommandConditions as conditions 29 from mozbuild.base import MozbuildObject 30 31 build_obj = MozbuildObject.from_environment(cwd=here) 32 except ImportError: 33 build_obj = None 34 conditions = None 35 UserError = Exception 36 37 38 class JavaTestHarnessException(Exception): 39 pass 40 41 42 class JUnitTestRunner(MochitestDesktop): 43 """ 44 A test harness to run geckoview junit tests on a remote device. 45 """ 46 47 def __init__(self, log, options): 48 self.log = log 49 self.verbose = False 50 self.http3Server = None 51 self.http2Server = None 52 self.dohServer = None 53 if ( 54 options.log_tbpl_level == "debug" 55 or options.log_mach_level == "debug" 56 or options.verbose 57 ): 58 self.verbose = True 59 self.device = ADBDeviceFactory( 60 adb=options.adbPath or "adb", 61 device=options.deviceSerial, 62 test_root=options.remoteTestRoot, 63 verbose=self.verbose, 64 run_as_package=options.app, 65 ) 66 self.options = options 67 self.log.debug("options=%s" % vars(options)) 68 update_mozinfo() 69 self.remote_profile = posixpath.join(self.device.test_root, "junit-profile") 70 self.remote_filter_list = posixpath.join( 71 self.device.test_root, "junit-filters.list" 72 ) 73 74 if self.options.coverage and not self.options.coverage_output_dir: 75 raise UserError( 76 "--coverage-output-dir is required when using --enable-coverage" 77 ) 78 if self.options.coverage: 79 self.remote_coverage_output_file = posixpath.join( 80 self.device.test_root, "junit-coverage.ec" 81 ) 82 self.coverage_output_file = os.path.join( 83 self.options.coverage_output_dir, "junit-coverage.ec" 84 ) 85 86 self.server_init() 87 88 self.cleanup() 89 self.device.clear_logcat() 90 self.build_profile() 91 self.startServers(self.options, debuggerInfo=None, public=True) 92 self.log.debug("Servers started") 93 94 def collectLogcatForCurrentTest(self): 95 # These are unique start and end markers logged by GeckoSessionTestRule.java 96 START_MARKER = "1f0befec-3ff2-40ff-89cf-b127eb38b1ec" 97 END_MARKER = "c5ee677f-bc83-49bd-9e28-2d35f3d0f059" 98 logcat = self.device.get_logcat() 99 test_logcat = "" 100 started = False 101 for l in logcat: 102 if START_MARKER in l and self.test_name in l: 103 started = True 104 if started: 105 test_logcat += l + "\n" 106 if started and END_MARKER in l: 107 return test_logcat 108 109 def needsWebsocketProcessBridge(self, options): 110 """ 111 Overrides MochitestDesktop.needsWebsocketProcessBridge and always 112 returns False as the junit tests do not use the websocket process 113 bridge. This is needed to satisfy MochitestDesktop.startServers. 114 """ 115 return False 116 117 def server_init(self): 118 """ 119 Additional initialization required to satisfy MochitestDesktop.startServers 120 """ 121 self._locations = None 122 self.server = None 123 self.wsserver = None 124 self.websocketProcessBridge = None 125 self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get("debug") else 90 126 if self.options.remoteWebServer is None: 127 self.options.remoteWebServer = moznetwork.get_ip() 128 self.options.webServer = self.options.remoteWebServer 129 self.options.webSocketPort = "9988" 130 self.options.httpdPath = None 131 self.options.http3ServerPath = None 132 self.options.http2ServerPath = None 133 self.options.keep_open = False 134 self.options.pidFile = "" 135 self.options.subsuite = None 136 self.options.xrePath = None 137 self.options.useHttp3Server = False 138 self.options.useHttp2Server = False 139 if build_obj and "MOZ_HOST_BIN" in os.environ: 140 self.options.xrePath = os.environ["MOZ_HOST_BIN"] 141 if not self.options.utilityPath: 142 self.options.utilityPath = self.options.xrePath 143 if not self.options.xrePath: 144 self.options.xrePath = self.options.utilityPath 145 if build_obj: 146 self.options.certPath = os.path.join( 147 build_obj.topsrcdir, "build", "pgo", "certs" 148 ) 149 150 def build_profile(self): 151 """ 152 Create a local profile with test prefs and proxy definitions and 153 push it to the remote device. 154 """ 155 156 self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options)) 157 self.options.profilePath = self.profile.profile 158 159 # Set preferences 160 self.merge_base_profiles(self.options, "geckoview-junit") 161 162 if self.options.web_content_isolation_strategy is not None: 163 self.options.extraPrefs.append( 164 "fission.webContentIsolationStrategy=%s" 165 % self.options.web_content_isolation_strategy 166 ) 167 self.options.extraPrefs.append("fission.autostart=true") 168 if self.options.disable_fission: 169 self.options.extraPrefs.pop() 170 self.options.extraPrefs.append("fission.autostart=false") 171 prefs = parse_preferences(self.options.extraPrefs) 172 self.profile.set_preferences(prefs) 173 174 if self.fillCertificateDB(self.options): 175 self.log.error("Certificate integration failed") 176 177 self.device.push(self.profile.profile, self.remote_profile) 178 self.log.debug( 179 "profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile)) 180 ) 181 182 def cleanup(self): 183 try: 184 self.stopServers() 185 self.log.debug("Servers stopped") 186 self.device.stop_application(self.options.app) 187 self.device.rm(self.remote_profile, force=True, recursive=True) 188 if hasattr(self, "profile"): 189 del self.profile 190 self.device.rm(self.remote_filter_list, force=True) 191 except Exception: 192 traceback.print_exc() 193 self.log.info("Caught and ignored an exception during cleanup") 194 195 def build_command_line(self, test_filters_file, test_filters): 196 """ 197 Construct and return the 'am instrument' command line. 198 """ 199 cmd = "am instrument -w -r" 200 # profile location 201 cmd = cmd + " -e args '-profile %s'" % self.remote_profile 202 # chunks (shards) 203 shards = self.options.totalChunks 204 shard = self.options.thisChunk 205 if shards is not None and shard is not None: 206 shard -= 1 # shard index is 0 based 207 cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard) 208 209 # test filters: limit run to specific test(s) 210 # filter can be class-name or 'class-name#method-name' (single test) 211 # Multiple filters must be specified as a line-separated text file 212 # and then pushed to the device. 213 filter_list_name = None 214 215 if test_filters_file: 216 # We specified a pre-existing file, so use that 217 filter_list_name = test_filters_file 218 elif test_filters: 219 if len(test_filters) > 1: 220 # Generate the list file from test_filters 221 with tempfile.NamedTemporaryFile(delete=False, mode="w") as filter_list: 222 for f in test_filters: 223 print(f, file=filter_list) 224 filter_list_name = filter_list.name 225 else: 226 # A single filter may be directly appended to the command line 227 cmd = cmd + " -e class %s" % test_filters[0] 228 229 if filter_list_name: 230 self.device.push(filter_list_name, self.remote_filter_list) 231 232 if test_filters: 233 # We only remove the filter list if we generated it as a 234 # temporary file. 235 os.remove(filter_list_name) 236 237 cmd = cmd + " -e testFile %s" % self.remote_filter_list 238 239 # enable code coverage reports 240 if self.options.coverage: 241 cmd = cmd + " -e coverage true" 242 cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_file 243 # environment 244 env = {} 245 env["MOZ_CRASHREPORTER"] = "1" 246 env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" 247 env["XPCOM_DEBUG_BREAK"] = "stack" 248 env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" 249 env["MOZ_IN_AUTOMATION"] = "1" 250 env["R_LOG_VERBOSE"] = "1" 251 env["R_LOG_LEVEL"] = "6" 252 env["R_LOG_DESTINATION"] = "stderr" 253 # webrender needs gfx.webrender.all=true, gtest doesn't use prefs 254 env["MOZ_WEBRENDER"] = "1" 255 # FIXME: When android switches to using Fission by default, 256 # MOZ_FORCE_DISABLE_FISSION will need to be configured correctly. 257 if self.options.disable_fission: 258 env["MOZ_FORCE_DISABLE_FISSION"] = "1" 259 else: 260 env["MOZ_FORCE_ENABLE_FISSION"] = "1" 261 if self.options.enable_isolated_zygote_process: 262 env["MOZ_ANDROID_CONTENT_SERVICE_ISOLATED_WITH_ZYGOTE"] = "1" 263 else: 264 env["MOZ_ANDROID_CONTENT_SERVICE_ISOLATED_WITH_ZYGOTE"] = "0" 265 266 # Add additional env variables 267 for [key, value] in [p.split("=", 1) for p in self.options.add_env]: 268 env[key] = value 269 270 for env_count, (env_key, env_val) in enumerate(env.items()): 271 cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val) 272 # runner 273 cmd = cmd + " %s/%s" % (self.options.app, self.options.runner) 274 return cmd 275 276 @property 277 def locations(self): 278 if self._locations is not None: 279 return self._locations 280 locations_file = os.path.join(here, "server-locations.txt") 281 self._locations = ServerLocations(locations_file) 282 return self._locations 283 284 def need_more_runs(self): 285 if self.options.run_until_failure and (self.fail_count == 0): 286 return True 287 if self.runs <= self.options.repeat: 288 return True 289 return False 290 291 def run_tests(self, test_filters_file=None, test_filters=None): 292 """ 293 Run the tests. 294 """ 295 if not self.device.is_app_installed(self.options.app): 296 raise UserError("%s is not installed" % self.options.app) 297 if self.device.process_exist(self.options.app): 298 raise UserError( 299 "%s already running before starting tests" % self.options.app 300 ) 301 # test_filters_file and test_filters must be mutually-exclusive 302 if test_filters_file and test_filters: 303 raise UserError( 304 "Test filters may not be specified when test-filters-file is provided" 305 ) 306 307 self.test_started = False 308 self.pass_count = 0 309 self.fail_count = 0 310 self.todo_count = 0 311 self.total_count = 0 312 self.runs = 0 313 self.seen_last_test = False 314 315 def callback(line): 316 # Output callback: Parse the raw junit log messages, translating into 317 # treeherder-friendly test start/pass/fail messages. 318 319 line = line.decode("utf-8") 320 self.log.process_output(self.options.app, line) 321 # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something" 322 match = re.match(r"INSTRUMENTATION_STATUS:\s*class=(.*)", line) 323 if match: 324 self.class_name = match.group(1) 325 # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something" 326 match = re.match(r"INSTRUMENTATION_STATUS:\s*test=(.*)", line) 327 if match: 328 self.test_name = match.group(1) 329 match = re.match(r"INSTRUMENTATION_STATUS:\s*numtests=(.*)", line) 330 if match: 331 self.total_count = int(match.group(1)) 332 match = re.match(r"INSTRUMENTATION_STATUS:\s*current=(.*)", line) 333 if match: 334 self.current_test_id = int(match.group(1)) 335 match = re.match(r"INSTRUMENTATION_STATUS:\s*stack=(.*)", line) 336 if match: 337 self.exception_message = match.group(1) 338 if ( 339 "org.mozilla.geckoview.test.rule.TestHarnessException" 340 in self.exception_message 341 ): 342 # This is actually a problem in the test harness itself 343 raise JavaTestHarnessException(self.exception_message) 344 345 # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..." 346 match = re.match(r"INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)", line) 347 if match: 348 status = match.group(1) 349 full_name = "%s#%s" % (self.class_name, self.test_name) 350 if full_name == self.current_full_name: 351 # A crash in the test harness might cause us to ignore tests, 352 # so we double check that we've actually ran all the tests 353 if self.total_count == self.current_test_id: 354 self.seen_last_test = True 355 356 if status == "0": 357 message = "" 358 status = "PASS" 359 expected = "PASS" 360 self.pass_count += 1 361 if self.verbose: 362 self.log.info("Printing logcat for test:") 363 print(self.collectLogcatForCurrentTest()) 364 elif status == "-3": # ignored (skipped) 365 message = "" 366 status = "SKIP" 367 expected = "SKIP" 368 self.todo_count += 1 369 elif status == "-4": # known fail 370 message = "" 371 status = "FAIL" 372 expected = "FAIL" 373 self.todo_count += 1 374 else: 375 if self.exception_message: 376 message = self.exception_message 377 else: 378 message = "status %s" % status 379 status = "FAIL" 380 expected = "PASS" 381 self.fail_count += 1 382 self.log.info("Printing logcat for test:") 383 print(self.collectLogcatForCurrentTest()) 384 self.log.test_end(full_name, status, expected, message) 385 self.test_started = False 386 else: 387 if self.test_started: 388 # next test started without reporting previous status 389 self.fail_count += 1 390 status = "FAIL" 391 expected = "PASS" 392 self.log.test_end( 393 self.current_full_name, 394 status, 395 expected, 396 "missing test completion status", 397 ) 398 self.log.test_start(full_name) 399 self.test_started = True 400 self.current_full_name = full_name 401 402 # Ideally all test names should be reported to suite_start, but these test 403 # names are not known in advance. 404 self.log.suite_start(["geckoview-junit"]) 405 try: 406 self.device.grant_runtime_permissions(self.options.app) 407 self.device.add_change_device_settings(self.options.app) 408 self.device.add_mock_location(self.options.app) 409 cmd = self.build_command_line( 410 test_filters_file=test_filters_file, test_filters=test_filters 411 ) 412 while self.need_more_runs(): 413 self.class_name = "" 414 self.exception_message = "" 415 self.test_name = "" 416 self.current_full_name = "" 417 self.current_test_id = 0 418 self.runs += 1 419 self.log.info("launching %s" % cmd) 420 p = self.device.shell( 421 cmd, timeout=self.options.max_time, stdout_callback=callback 422 ) 423 if p.timedout: 424 self.log.error( 425 "TEST-UNEXPECTED-TIMEOUT | runjunit.py | " 426 "Timed out after %d seconds" % self.options.max_time 427 ) 428 self.log.info("Passed: %d" % self.pass_count) 429 self.log.info("Failed: %d" % self.fail_count) 430 self.log.info("Todo: %d" % self.todo_count) 431 if not self.seen_last_test: 432 self.log.error( 433 "TEST-UNEXPECTED-FAIL | runjunit.py | " 434 "Some tests did not run (probably due to a crash in the harness)" 435 ) 436 finally: 437 self.log.suite_end() 438 439 if self.check_for_crashes(): 440 self.fail_count = 1 441 442 if self.options.coverage: 443 try: 444 self.device.pull( 445 self.remote_coverage_output_file, self.coverage_output_file 446 ) 447 except ADBError: 448 # Avoid a task retry in case the code coverage file is not found. 449 self.log.error( 450 "No code coverage file (%s) found on remote device" 451 % self.remote_coverage_output_file 452 ) 453 return -1 454 455 return 1 if self.fail_count else 0 456 457 def check_for_crashes(self): 458 symbols_path = self.options.symbolsPath 459 try: 460 dump_dir = tempfile.mkdtemp() 461 remote_dir = posixpath.join(self.remote_profile, "minidumps") 462 if not self.device.is_dir(remote_dir): 463 return False 464 self.device.pull(remote_dir, dump_dir) 465 crashed = mozcrash.log_crashes( 466 self.log, dump_dir, symbols_path, test=self.current_full_name 467 ) 468 finally: 469 try: 470 shutil.rmtree(dump_dir) 471 except Exception: 472 self.log.warning("unable to remove directory: %s" % dump_dir) 473 return crashed 474 475 476 class JunitArgumentParser(argparse.ArgumentParser): 477 """ 478 An argument parser for geckoview-junit. 479 """ 480 481 def __init__(self, **kwargs): 482 super().__init__(**kwargs) 483 484 self.add_argument( 485 "--appname", 486 action="store", 487 type=str, 488 dest="app", 489 default="org.mozilla.geckoview.test", 490 help="Test package name.", 491 ) 492 self.add_argument( 493 "--adbpath", 494 action="store", 495 type=str, 496 dest="adbPath", 497 default=None, 498 help="Path to adb binary.", 499 ) 500 self.add_argument( 501 "--deviceSerial", 502 action="store", 503 type=str, 504 dest="deviceSerial", 505 help="adb serial number of remote device. This is required " 506 "when more than one device is connected to the host. " 507 "Use 'adb devices' to see connected devices. ", 508 ) 509 self.add_argument( 510 "--setenv", 511 dest="add_env", 512 action="append", 513 default=[], 514 help="Set target environment variable, like FOO=BAR", 515 ) 516 self.add_argument( 517 "--remoteTestRoot", 518 action="store", 519 type=str, 520 dest="remoteTestRoot", 521 help="Remote directory to use as test root " 522 "(eg. /data/local/tmp/test_root).", 523 ) 524 self.add_argument( 525 "--max-time", 526 action="store", 527 type=int, 528 dest="max_time", 529 default="3000", 530 help="Max time in seconds to wait for tests (default 3000s).", 531 ) 532 self.add_argument( 533 "--runner", 534 action="store", 535 type=str, 536 dest="runner", 537 default="androidx.test.runner.AndroidJUnitRunner", 538 help="Test runner name.", 539 ) 540 self.add_argument( 541 "--symbols-path", 542 action="store", 543 type=str, 544 dest="symbolsPath", 545 default=None, 546 help="Path to directory containing breakpad symbols, " 547 "or the URL of a zip file containing symbols.", 548 ) 549 self.add_argument( 550 "--utility-path", 551 action="store", 552 type=str, 553 dest="utilityPath", 554 default=None, 555 help="Path to directory containing host utility programs.", 556 ) 557 self.add_argument( 558 "--total-chunks", 559 action="store", 560 type=int, 561 dest="totalChunks", 562 default=None, 563 help="Total number of chunks to split tests into.", 564 ) 565 self.add_argument( 566 "--this-chunk", 567 action="store", 568 type=int, 569 dest="thisChunk", 570 default=None, 571 help="If running tests by chunks, the chunk number to run.", 572 ) 573 self.add_argument( 574 "--verbose", 575 "-v", 576 action="store_true", 577 dest="verbose", 578 default=False, 579 help="Verbose output - enable debug log messages", 580 ) 581 self.add_argument( 582 "--enable-coverage", 583 action="store_true", 584 dest="coverage", 585 default=False, 586 help="Enable code coverage collection.", 587 ) 588 self.add_argument( 589 "--coverage-output-dir", 590 action="store", 591 type=str, 592 dest="coverage_output_dir", 593 default=None, 594 help="If collecting code coverage, save the report file in this dir.", 595 ) 596 self.add_argument( 597 "--disable-fission", 598 action="store_true", 599 dest="disable_fission", 600 default=False, 601 help="Run the tests without Fission (site isolation) enabled.", 602 ) 603 self.add_argument( 604 "--enable-isolated-zygote-process", 605 action="store_true", 606 dest="enable_isolated_zygote_process", 607 default=False, 608 help="Run with app Zygote preloading enabled.", 609 ) 610 self.add_argument( 611 "--web-content-isolation-strategy", 612 type=int, 613 dest="web_content_isolation_strategy", 614 help="Strategy used to determine whether or not a particular site should load into " 615 "a webIsolated content process, see fission.webContentIsolationStrategy.", 616 ) 617 self.add_argument( 618 "--repeat", 619 type=int, 620 default=0, 621 help="Repeat the tests the given number of times.", 622 ) 623 self.add_argument( 624 "--run-until-failure", 625 action="store_true", 626 dest="run_until_failure", 627 default=False, 628 help="Run tests repeatedly but stop the first time a test fails.", 629 ) 630 self.add_argument( 631 "--setpref", 632 action="append", 633 dest="extraPrefs", 634 default=[], 635 metavar="PREF=VALUE", 636 help="Defines an extra user preference.", 637 ) 638 # Additional options for server. 639 ( 640 self.add_argument( 641 "--certificate-path", 642 action="store", 643 type=str, 644 dest="certPath", 645 default=None, 646 help="Path to directory containing certificate store.", 647 ), 648 ) 649 self.add_argument( 650 "--http-port", 651 action="store", 652 type=str, 653 dest="httpPort", 654 default=DEFAULT_PORTS["http"], 655 help="http port of the remote web server.", 656 ) 657 self.add_argument( 658 "--remote-webserver", 659 action="store", 660 type=str, 661 dest="remoteWebServer", 662 help="IP address of the remote web server.", 663 ) 664 self.add_argument( 665 "--ssl-port", 666 action="store", 667 type=str, 668 dest="sslPort", 669 default=DEFAULT_PORTS["https"], 670 help="ssl port of the remote web server.", 671 ) 672 self.add_argument( 673 "--test-filters-file", 674 action="store", 675 type=str, 676 dest="test_filters_file", 677 default=None, 678 help="Line-delimited file containing test filter(s)", 679 ) 680 # Remaining arguments are test filters. 681 self.add_argument( 682 "test_filters", 683 nargs="*", 684 help="Test filter(s): class and/or method names of test(s) to run.", 685 ) 686 687 mozlog.commandline.add_logging_group(self) 688 689 690 def run_test_harness(parser, options): 691 if hasattr(options, "log"): 692 log = options.log 693 else: 694 log = mozlog.commandline.setup_logging( 695 "runjunit", options, {"tbpl": sys.stdout} 696 ) 697 runner = JUnitTestRunner(log, options) 698 result = -1 699 try: 700 device_exception = False 701 result = runner.run_tests( 702 test_filters_file=options.test_filters_file, 703 test_filters=options.test_filters, 704 ) 705 except KeyboardInterrupt: 706 log.info("runjunit.py | Received keyboard interrupt") 707 result = -1 708 except JavaTestHarnessException as e: 709 log.error( 710 "TEST-UNEXPECTED-FAIL | runjunit.py | The previous test failed because " 711 "of an error in the test harness | %s" % (str(e)) 712 ) 713 except Exception as e: 714 traceback.print_exc() 715 log.error("runjunit.py | Received unexpected exception while running tests") 716 result = 1 717 if isinstance(e, ADBTimeoutError): 718 device_exception = True 719 finally: 720 if not device_exception: 721 runner.cleanup() 722 return result 723 724 725 def main(args=sys.argv[1:]): 726 parser = JunitArgumentParser() 727 options = parser.parse_args() 728 return run_test_harness(parser, options) 729 730 731 if __name__ == "__main__": 732 sys.exit(main())