remotegtests.py (18820B)
1 #!/usr/bin/env python 2 # 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 import argparse 8 import datetime 9 import glob 10 import json 11 import os 12 import pathlib 13 import posixpath 14 import re 15 import shutil 16 import sys 17 import tempfile 18 import time 19 import traceback 20 21 import mozcrash 22 import mozdevice 23 import mozinfo 24 import mozlog 25 import six 26 27 LOGGER_NAME = "gtest" 28 log = mozlog.unstructured.getLogger(LOGGER_NAME) 29 PERFHERDER_MATCHER = re.compile(r"PERFHERDER_DATA:\s*(\{.*\})\s*$") 30 31 32 class RemoteGTests: 33 """ 34 A test harness to run gtest on Android. 35 """ 36 37 def __init__(self): 38 self.device = None 39 self.perfherder_data = [] 40 41 def build_environment(self, shuffle, test_filter): 42 """ 43 Create and return a dictionary of all the appropriate env variables 44 and values. 45 """ 46 env = {} 47 env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" 48 env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" 49 env["MOZ_CRASHREPORTER"] = "1" 50 env["MOZ_RUN_GTEST"] = "1" 51 # custom output parser is mandatory on Android 52 env["MOZ_TBPL_PARSER"] = "1" 53 env["MOZ_GTEST_LOG_PATH"] = self.remote_log 54 env["MOZ_GTEST_CWD"] = self.remote_profile 55 env["MOZ_GTEST_MINIDUMPS_PATH"] = self.remote_minidumps 56 env["MOZ_IN_AUTOMATION"] = "1" 57 env["MOZ_ANDROID_LIBDIR_OVERRIDE"] = posixpath.join( 58 self.remote_libdir, "libxul.so" 59 ) 60 if shuffle: 61 env["GTEST_SHUFFLE"] = "True" 62 if test_filter: 63 env["GTEST_FILTER"] = test_filter 64 65 # webrender needs gfx.webrender.all=true, gtest doesn't use prefs 66 env["MOZ_WEBRENDER"] = "1" 67 68 return env 69 70 def run_gtest( 71 self, 72 test_dir, 73 shuffle, 74 test_filter, 75 package, 76 adb_path, 77 device_serial, 78 remote_test_root, 79 libxul_path, 80 symbols_path, 81 ): 82 """ 83 Launch the test app, run gtest, collect test results and wait for completion. 84 Return False if a crash or other failure is detected, else True. 85 """ 86 update_mozinfo() 87 self.device = mozdevice.ADBDeviceFactory( 88 adb=adb_path, 89 device=device_serial, 90 test_root=remote_test_root, 91 logger_name=LOGGER_NAME, 92 verbose=False, 93 run_as_package=package, 94 ) 95 root = self.device.test_root 96 self.remote_profile = posixpath.join(root, "gtest-profile") 97 self.remote_minidumps = posixpath.join(root, "gtest-minidumps") 98 self.remote_log = posixpath.join(root, "gtest.log") 99 self.remote_libdir = posixpath.join(root, "gtest") 100 101 self.package = package 102 self.cleanup() 103 self.device.mkdir(self.remote_profile) 104 self.device.mkdir(self.remote_minidumps) 105 self.device.mkdir(self.remote_libdir) 106 107 log.info("Running Android gtest") 108 if not self.device.is_app_installed(self.package): 109 raise Exception("%s is not installed on this device" % self.package) 110 111 # Push the gtest libxul.so to the device. The harness assumes an architecture- 112 # appropriate library is specified and pushes it to the arch-agnostic remote 113 # directory. 114 # TODO -- consider packaging the gtest libxul.so in an apk 115 self.device.push(libxul_path, self.remote_libdir) 116 117 for buildid in ["correct", "broken", "missing"]: 118 libxul_buildid_name = f"libxul_{buildid}_buildid.so" 119 libxul_buildid_path = os.path.join( 120 os.path.dirname(libxul_path), libxul_buildid_name 121 ) 122 if os.path.isfile(libxul_buildid_path): 123 self.device.push(libxul_buildid_path, self.remote_libdir) 124 125 # Push support files to device. Avoid gtest_bin so that libxul.so 126 # is not included. 127 for f in glob.glob(os.path.join(test_dir, "**"), recursive=True): 128 if not "gtest_bin" in os.path.abspath(f): 129 self.device.push(f, self.remote_profile) 130 131 if test_filter is not None: 132 test_filter = six.ensure_text(test_filter) 133 env = self.build_environment(shuffle, test_filter) 134 args = [ 135 "-unittest", 136 "--gtest_death_test_style=threadsafe", 137 "-profile %s" % self.remote_profile, 138 ] 139 if "geckoview" in self.package: 140 activity = "TestRunnerActivity" 141 self.device.launch_activity( 142 self.package, 143 activity_name=activity, 144 moz_env=env, 145 extra_args=args, 146 wait=False, 147 ) 148 else: 149 self.device.launch_fennec(self.package, moz_env=env, extra_args=args) 150 waiter = AppWaiter( 151 self.device, self.remote_log, on_perfherder=self.perfherder_data.append 152 ) 153 timed_out = waiter.wait(self.package) 154 self.shutdown(use_kill=True if timed_out else False) 155 if self.perfherder_data and "MOZ_AUTOMATION" in os.environ: 156 upload_dir = pathlib.Path(os.getenv("MOZ_UPLOAD_DIR")) 157 upload_dir.mkdir(parents=True, exist_ok=True) 158 merged_perfherder_data = self.merge_perfherder_data(self.perfherder_data) 159 for framework_name, data in merged_perfherder_data.items(): 160 file_name = ( 161 "perfherder-data-gtest.json" 162 if len(merged_perfherder_data) == 1 163 else f"perfherder-data-gtest-{framework_name}.json" 164 ) 165 out_path = upload_dir / file_name 166 with out_path.open("w", encoding="utf-8") as f: 167 json.dump(data, f) 168 if self.check_for_crashes(symbols_path): 169 return False 170 return True 171 172 def shutdown(self, use_kill): 173 """ 174 Stop the remote application. 175 If use_kill is specified, a multi-stage kill procedure is used, 176 attempting to trigger ANR and minidump reports before ending 177 the process. 178 """ 179 if not use_kill: 180 self.device.stop_application(self.package) 181 else: 182 # Trigger an ANR report with "kill -3" (SIGQUIT) 183 try: 184 self.device.pkill(self.package, sig=3, attempts=1) 185 except mozdevice.ADBTimeoutError: 186 raise 187 except: # NOQA: E722 188 pass 189 time.sleep(3) 190 # Trigger a breakpad dump with "kill -6" (SIGABRT) 191 try: 192 self.device.pkill(self.package, sig=6, attempts=1) 193 except mozdevice.ADBTimeoutError: 194 raise 195 except: # NOQA: E722 196 pass 197 # Wait for process to end 198 retries = 0 199 while retries < 3: 200 if self.device.process_exist(self.package): 201 log.info("%s still alive after SIGABRT: waiting..." % self.package) 202 time.sleep(5) 203 else: 204 break 205 retries += 1 206 if self.device.process_exist(self.package): 207 try: 208 self.device.pkill(self.package, sig=9, attempts=1) 209 except mozdevice.ADBTimeoutError: 210 raise 211 except: # NOQA: E722 212 log.warning("%s still alive after SIGKILL!" % self.package) 213 if self.device.process_exist(self.package): 214 self.device.stop_application(self.package) 215 # Test harnesses use the MOZ_CRASHREPORTER environment variables to suppress 216 # the interactive crash reporter, but that may not always be effective; 217 # check for and cleanup errant crashreporters. 218 crashreporter = "%s.CrashReporter" % self.package 219 if self.device.process_exist(crashreporter): 220 log.warning("%s unexpectedly found running. Killing..." % crashreporter) 221 try: 222 self.device.pkill(crashreporter) 223 except mozdevice.ADBTimeoutError: 224 raise 225 except: # NOQA: E722 226 pass 227 if self.device.process_exist(crashreporter): 228 log.error("%s still running!!" % crashreporter) 229 230 def check_for_crashes(self, symbols_path): 231 """ 232 Pull minidumps from the remote device and generate crash reports. 233 Returns True if a crash was detected, or suspected. 234 """ 235 try: 236 dump_dir = tempfile.mkdtemp() 237 remote_dir = self.remote_minidumps 238 if not self.device.is_dir(remote_dir): 239 return False 240 self.device.pull(remote_dir, dump_dir) 241 crashed = mozcrash.check_for_crashes( 242 dump_dir, symbols_path, test_name="gtest" 243 ) 244 except Exception as e: 245 log.error("unable to check for crashes: %s" % str(e)) 246 crashed = True 247 finally: 248 try: 249 shutil.rmtree(dump_dir) 250 except Exception: 251 log.warning("unable to remove directory: %s" % dump_dir) 252 return crashed 253 254 def cleanup(self): 255 if self.device: 256 self.device.stop_application(self.package) 257 self.device.rm(self.remote_log, force=True) 258 self.device.rm(self.remote_profile, recursive=True, force=True) 259 self.device.rm(self.remote_minidumps, recursive=True, force=True) 260 self.device.rm(self.remote_libdir, recursive=True, force=True) 261 262 def merge_perfherder_data(self, perfherder_data): 263 grouped = {} 264 265 for data in perfherder_data: 266 framework_name = data.get("framework", {}).get("name") 267 suites_by_name = grouped.setdefault(framework_name, {}) 268 for suite in data.get("suites", []): 269 suite_name = suite.get("name") 270 suite_data = suites_by_name.setdefault( 271 suite_name, {"name": suite_name, "subtests": []} 272 ) 273 suite_data["subtests"].extend(suite.get("subtests", [])) 274 275 results = {} 276 for framework_name, suites in grouped.items(): 277 results[framework_name] = { 278 "framework": {"name": framework_name}, 279 "suites": list(suites.values()), 280 } 281 282 return results 283 284 285 class AppWaiter: 286 def __init__( 287 self, 288 device, 289 remote_log, 290 test_proc_timeout=1200, 291 test_proc_no_output_timeout=300, 292 test_proc_start_timeout=60, 293 output_poll_interval=10, 294 on_perfherder=None, 295 ): 296 self.device = device 297 self.remote_log = remote_log 298 self.start_time = datetime.datetime.now() 299 self.timeout_delta = datetime.timedelta(seconds=test_proc_timeout) 300 self.output_timeout_delta = datetime.timedelta( 301 seconds=test_proc_no_output_timeout 302 ) 303 self.start_timeout_delta = datetime.timedelta(seconds=test_proc_start_timeout) 304 self.output_poll_interval = output_poll_interval 305 self.last_output_time = datetime.datetime.now() 306 self.remote_log_len = 0 307 self.on_perfherder = on_perfherder or (lambda _data: None) 308 309 def start_timed_out(self): 310 if datetime.datetime.now() - self.start_time > self.start_timeout_delta: 311 return True 312 return False 313 314 def timed_out(self): 315 if datetime.datetime.now() - self.start_time > self.timeout_delta: 316 return True 317 return False 318 319 def output_timed_out(self): 320 if datetime.datetime.now() - self.last_output_time > self.output_timeout_delta: 321 return True 322 return False 323 324 def get_top(self): 325 top = self.device.get_top_activity(timeout=60) 326 if top is None: 327 log.info("Failed to get top activity, retrying, once...") 328 top = self.device.get_top_activity(timeout=60) 329 return top 330 331 def wait_for_start(self, package): 332 top = None 333 while top != package and not self.start_timed_out(): 334 if self.update_log(): 335 # if log content is available, assume the app started; otherwise, 336 # a short run (few tests) might complete without ever being detected 337 # in the foreground 338 return package 339 time.sleep(1) 340 top = self.get_top() 341 return top 342 343 def wait(self, package): 344 """ 345 Wait until: 346 - the app loses foreground, or 347 - no new output is observed for the output timeout, or 348 - the timeout is exceeded. 349 While waiting, update the log every periodically: pull the gtest log from 350 device and log any new content. 351 """ 352 top = self.wait_for_start(package) 353 if top != package: 354 log.testFail("gtest | %s failed to start" % package) 355 return 356 while not self.timed_out(): 357 if not self.update_log(): 358 top = self.get_top() 359 if top != package or self.output_timed_out(): 360 time.sleep(self.output_poll_interval) 361 break 362 time.sleep(self.output_poll_interval) 363 self.update_log() 364 if self.timed_out(): 365 log.testFail( 366 "gtest | timed out after %d seconds", self.timeout_delta.seconds 367 ) 368 elif self.output_timed_out(): 369 log.testFail( 370 "gtest | timed out after %d seconds without output", 371 self.output_timeout_delta.seconds, 372 ) 373 else: 374 log.info("gtest | wait for %s complete; top activity=%s" % (package, top)) 375 return True if top == package else False 376 377 def update_log(self): 378 """ 379 Pull the test log from the remote device and display new content. 380 """ 381 if not self.device.is_file(self.remote_log): 382 log.info("gtest | update_log %s is not a file." % self.remote_log) 383 return False 384 try: 385 new_content = self.device.get_file( 386 self.remote_log, offset=self.remote_log_len 387 ) 388 except mozdevice.ADBTimeoutError: 389 raise 390 except Exception as e: 391 log.info("gtest | update_log : exception reading log: %s" % str(e)) 392 return False 393 if not new_content: 394 log.info("gtest | update_log : no new content") 395 return False 396 new_content = six.ensure_text(new_content) 397 last_full_line_pos = new_content.rfind("\n") 398 if last_full_line_pos <= 0: 399 # wait for a full line 400 return False 401 # trim partial line 402 new_content = new_content[:last_full_line_pos] 403 self.remote_log_len += len(new_content) 404 for line in new_content.lstrip("\n").split("\n"): 405 print(line) 406 match = PERFHERDER_MATCHER.search(line) 407 if match: 408 data = json.loads(match.group(1)) 409 self.on_perfherder(data) 410 self.last_output_time = datetime.datetime.now() 411 return True 412 413 414 class remoteGtestOptions(argparse.ArgumentParser): 415 def __init__(self): 416 super().__init__(usage="usage: %prog [options] test_filter") 417 self.add_argument( 418 "--package", 419 dest="package", 420 default="org.mozilla.geckoview.test_runner", 421 help="Package name of test app.", 422 ) 423 self.add_argument( 424 "--adbpath", 425 action="store", 426 type=str, 427 dest="adb_path", 428 default="adb", 429 help="Path to adb binary.", 430 ) 431 self.add_argument( 432 "--deviceSerial", 433 action="store", 434 type=str, 435 dest="device_serial", 436 help="adb serial number of remote device. This is required " 437 "when more than one device is connected to the host. " 438 "Use 'adb devices' to see connected devices. ", 439 ) 440 self.add_argument( 441 "--remoteTestRoot", 442 action="store", 443 type=str, 444 dest="remote_test_root", 445 help="Remote directory to use as test root " 446 "(eg. /data/local/tmp/test_root).", 447 ) 448 self.add_argument( 449 "--libxul", 450 action="store", 451 type=str, 452 dest="libxul_path", 453 default=None, 454 help="Path to gtest libxul.so.", 455 ) 456 self.add_argument( 457 "--symbols-path", 458 dest="symbols_path", 459 default=None, 460 help="absolute path to directory containing breakpad " 461 "symbols, or the URL of a zip file containing symbols", 462 ) 463 self.add_argument( 464 "--shuffle", 465 action="store_true", 466 default=False, 467 help="Randomize the execution order of tests.", 468 ) 469 self.add_argument( 470 "--tests-path", 471 default=None, 472 help="Path to gtest directory containing test support files.", 473 ) 474 self.add_argument("args", nargs=argparse.REMAINDER) 475 476 477 def update_mozinfo(): 478 """ 479 Walk up directories to find mozinfo.json and update the info. 480 """ 481 path = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) 482 dirs = set() 483 while path != os.path.expanduser("~"): 484 if path in dirs: 485 break 486 dirs.add(path) 487 path = os.path.split(path)[0] 488 mozinfo.find_and_update_from_json(*dirs) 489 490 491 def main(): 492 parser = remoteGtestOptions() 493 options = parser.parse_args() 494 args = options.args 495 if not options.libxul_path: 496 parser.error("--libxul is required") 497 sys.exit(1) 498 if len(args) > 1: 499 parser.error("only one test_filter is allowed") 500 sys.exit(1) 501 test_filter = args[0] if args else None 502 tester = RemoteGTests() 503 result = False 504 try: 505 device_exception = False 506 result = tester.run_gtest( 507 options.tests_path, 508 options.shuffle, 509 test_filter, 510 options.package, 511 options.adb_path, 512 options.device_serial, 513 options.remote_test_root, 514 options.libxul_path, 515 options.symbols_path, 516 ) 517 except KeyboardInterrupt: 518 log.info("gtest | Received keyboard interrupt") 519 except Exception as e: 520 log.error(str(e)) 521 traceback.print_exc() 522 if isinstance(e, mozdevice.ADBTimeoutError): 523 device_exception = True 524 finally: 525 if not device_exception: 526 tester.cleanup() 527 sys.exit(0 if result else 1) 528 529 530 if __name__ == "__main__": 531 main()