android_hardware_unittest.py (19832B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import copy 7 import datetime 8 import json 9 import os 10 import subprocess 11 import sys 12 13 # load modules from parent dir 14 sys.path.insert(1, os.path.dirname(sys.path[0])) 15 16 from mozharness.base.log import WARNING 17 from mozharness.base.script import BaseScript, PreScriptAction 18 from mozharness.mozilla.automation import TBPL_RETRY 19 from mozharness.mozilla.mozbase import MozbaseMixin 20 from mozharness.mozilla.testing.android import AndroidMixin 21 from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin 22 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 23 24 SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"] 25 SUITE_NO_E10S = ["cppunittest", "gtest", "jittest"] 26 SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"] 27 28 29 class AndroidHardwareTest( 30 TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin 31 ): 32 config_options = [ 33 [["--test-suite"], {"action": "store", "dest": "test_suite", "default": None}], 34 [ 35 ["--adb-path"], 36 { 37 "action": "store", 38 "dest": "adb_path", 39 "default": None, 40 "help": "Path to adb", 41 }, 42 ], 43 [ 44 ["--total-chunk"], 45 { 46 "action": "store", 47 "dest": "total_chunks", 48 "default": None, 49 "help": "Number of total chunks", 50 }, 51 ], 52 [ 53 ["--this-chunk"], 54 { 55 "action": "store", 56 "dest": "this_chunk", 57 "default": None, 58 "help": "Number of this chunk", 59 }, 60 ], 61 [ 62 ["--log-raw-level"], 63 { 64 "action": "store", 65 "dest": "log_raw_level", 66 "default": "info", 67 "help": "Set log level (debug|info|warning|error|critical|fatal)", 68 }, 69 ], 70 [ 71 ["--log-tbpl-level"], 72 { 73 "action": "store", 74 "dest": "log_tbpl_level", 75 "default": "info", 76 "help": "Set log level (debug|info|warning|error|critical|fatal)", 77 }, 78 ], 79 [ 80 ["--disable-e10s"], 81 { 82 "action": "store_false", 83 "dest": "e10s", 84 "default": True, 85 "help": "Run tests without multiple processes (e10s).", 86 }, 87 ], 88 [ 89 ["--disable-fission"], 90 { 91 "action": "store_true", 92 "dest": "disable_fission", 93 "default": False, 94 "help": "Run with Fission disabled.", 95 }, 96 ], 97 [ 98 ["--repeat"], 99 { 100 "action": "store", 101 "type": "int", 102 "dest": "repeat", 103 "default": 0, 104 "help": "Repeat the tests the given number of times. Supported " 105 "by mochitest, reftest, crashtest, ignored otherwise.", 106 }, 107 ], 108 [ 109 [ 110 "--setpref", 111 ], 112 { 113 "action": "append", 114 "dest": "extra_prefs", 115 "default": [], 116 "help": "Extra user prefs.", 117 }, 118 ], 119 [ 120 ["--jittest-flags"], 121 { 122 "action": "store", 123 "dest": "jittest_flags", 124 "default": "debug", 125 "help": "Flags to run with jittest (all, debug, etc.).", 126 }, 127 ], 128 [ 129 ["--tag"], 130 { 131 "action": "append", 132 "default": [], 133 "dest": "test_tags", 134 "help": "Filter out tests that don't have the given tag. Can be used multiple " 135 "times in which case the test must contain at least one of the given tags.", 136 }, 137 ], 138 ] + copy.deepcopy(testing_config_options) 139 140 def __init__(self, require_config_file=False): 141 super().__init__( 142 config_options=self.config_options, 143 all_actions=[ 144 "clobber", 145 "download-and-extract", 146 "create-virtualenv", 147 "verify-device", 148 "install", 149 "run-tests", 150 ], 151 require_config_file=require_config_file, 152 config={ 153 "virtualenv_modules": [], 154 "virtualenv_requirements": [], 155 "require_test_zip": True, 156 # IP address of the host as seen from the device. 157 "remote_webserver": os.environ["HOST_IP"], 158 }, 159 ) 160 161 # these are necessary since self.config is read only 162 c = self.config 163 self.installer_url = c.get("installer_url") 164 self.installer_path = c.get("installer_path") 165 self.test_url = c.get("test_url") 166 self.test_packages_url = c.get("test_packages_url") 167 self.test_manifest = c.get("test_manifest") 168 suite = c.get("test_suite") 169 self.test_suite = suite 170 self.this_chunk = c.get("this_chunk") 171 self.total_chunks = c.get("total_chunks") 172 self.xre_path = None 173 self.log_raw_level = c.get("log_raw_level") 174 self.log_tbpl_level = c.get("log_tbpl_level") 175 self.disable_e10s = c.get("disable_e10s") 176 self.disable_fission = c.get("disable_fission") 177 self.extra_prefs = c.get("extra_prefs") 178 self.jittest_flags = c.get("jittest_flags") 179 self.test_tags = c.get("test_tags") 180 181 def query_abs_dirs(self): 182 if self.abs_dirs: 183 return self.abs_dirs 184 abs_dirs = super().query_abs_dirs() 185 dirs = {} 186 dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") 187 dirs["abs_test_bin_dir"] = os.path.join( 188 abs_dirs["abs_work_dir"], "tests", "bin" 189 ) 190 dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules") 191 dirs["abs_blob_upload_dir"] = os.path.join( 192 abs_dirs["abs_work_dir"], "blobber_upload_dir" 193 ) 194 dirs["abs_mochitest_dir"] = os.path.join( 195 dirs["abs_test_install_dir"], "mochitest" 196 ) 197 dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest") 198 dirs["abs_xpcshell_dir"] = os.path.join( 199 dirs["abs_test_install_dir"], "xpcshell" 200 ) 201 work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"] 202 dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils") 203 204 for key in dirs.keys(): 205 if key not in abs_dirs: 206 abs_dirs[key] = dirs[key] 207 self.abs_dirs = abs_dirs 208 return self.abs_dirs 209 210 def _query_tests_dir(self): 211 dirs = self.query_abs_dirs() 212 try: 213 test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"] 214 except Exception: 215 test_dir = self.test_suite 216 return os.path.join(dirs["abs_test_install_dir"], test_dir) 217 218 def _build_command(self): 219 c = self.config 220 dirs = self.query_abs_dirs() 221 222 if self.test_suite not in self.config["suite_definitions"]: 223 self.fatal("Key '%s' not defined in the config!" % self.test_suite) 224 225 cmd = [ 226 self.query_python_path("python"), 227 "-u", 228 os.path.join( 229 self._query_tests_dir(), 230 self.config["suite_definitions"][self.test_suite]["run_filename"], 231 ), 232 ] 233 234 raw_log_file, error_summary_file = self.get_indexed_logs( 235 dirs["abs_blob_upload_dir"], self.test_suite 236 ) 237 238 # LambdaTest provides a list of recommended ports via env var: 239 # UserPorts=port/27045,port/27046,port/27047,port/27048,port/27049 240 # These are only for android, so no need to put in mozprofile 241 # NOTE: mozprofile.DEFAULT_PORTS has http:8888. 242 DEFAULT_PORTS = {"http": 8854, "https": 4454, "ws": 9988, "wss": 4454} 243 ports = [p.split("/")[-1] for p in os.environ.get("UserPorts", "").split(",")] 244 if len(ports) > 3: 245 DEFAULT_PORTS = { 246 "http": ports[0], 247 "https": ports[1], 248 "ws": ports[2], 249 "wss": ports[3], 250 } 251 252 str_format_values = { 253 "device_serial": self.device_serial, 254 "remote_webserver": c["remote_webserver"], 255 "xre_path": self.xre_path, 256 "utility_path": self.xre_path, 257 "http_port": DEFAULT_PORTS[ 258 "http" 259 ], # starting http port to use for the mochitest server 260 "ssl_port": DEFAULT_PORTS[ 261 "https" 262 ], # starting ssl port to use for the server 263 "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"), 264 # TestingMixin._download_and_extract_symbols() will set 265 # self.symbols_path when downloading/extracting. 266 "symbols_path": self.symbols_path, 267 "modules_dir": dirs["abs_modules_dir"], 268 "installer_path": self.installer_path, 269 "raw_log_file": raw_log_file, 270 "log_tbpl_level": self.log_tbpl_level, 271 "log_raw_level": self.log_raw_level, 272 "error_summary_file": error_summary_file, 273 "xpcshell_extra": c.get("xpcshell_extra", ""), 274 "jittest_flags": self.jittest_flags, 275 "test_tags": self.test_tags, 276 } 277 278 user_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) 279 confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) 280 281 for option in self.config["suite_definitions"][self.test_suite]["options"]: 282 opt = option.split("=")[0] 283 # override configured chunk options with script args, if specified 284 if opt in ("--this-chunk", "--total-chunks"): 285 if ( 286 user_paths 287 or getattr(self, opt.replace("-", "_").strip("_"), None) is not None 288 ): 289 continue 290 291 if "%(app)" in option: 292 # only query package name if requested 293 cmd.extend([option % {"app": self.query_package_name()}]) 294 else: 295 option = option % str_format_values 296 if option: 297 cmd.extend([option]) 298 299 if not self.verify_enabled and not user_paths: 300 if self.this_chunk is not None: 301 cmd.extend(["--this-chunk", self.this_chunk]) 302 if self.total_chunks is not None: 303 cmd.extend(["--total-chunks", self.total_chunks]) 304 305 if "mochitest" in self.test_suite: 306 category = "mochitest" 307 elif "reftest" in self.test_suite or "crashtest" in self.test_suite: 308 category = "reftest" 309 else: 310 category = self.test_suite 311 if c.get("repeat"): 312 if category in SUITE_REPEATABLE: 313 cmd.extend(["--repeat=%s" % c.get("repeat")]) 314 else: 315 self.log(f"--repeat not supported in {category}", level=WARNING) 316 317 if category not in SUITE_NO_E10S: 318 if category in SUITE_DEFAULT_E10S and not c["e10s"]: 319 cmd.append("--disable-e10s") 320 elif category not in SUITE_DEFAULT_E10S and c["e10s"]: 321 cmd.append("--e10s") 322 323 if self.disable_fission and category not in SUITE_NO_E10S: 324 cmd.append("--disable-fission") 325 326 cmd.extend([f"--setpref={p}" for p in self.extra_prefs]) 327 328 cmd.extend([f"--tag={t}" for t in self.test_tags]) 329 330 try_options, try_tests = self.try_args(self.test_suite) 331 if try_options: 332 cmd.extend(try_options) 333 334 if user_paths: 335 # reftest on android-hw uses a subset (reftest-qr) of tests, 336 # but scheduling only knows about 'reftest' 337 suite = self.test_suite 338 if suite == "reftest-qr": 339 suite = "reftest" 340 341 if user_paths.get(suite, []): 342 suite_test_paths = user_paths.get(suite, []) 343 # NOTE: we do not want to prepend 'tests' if a single path 344 if confirm_paths and confirm_paths.get(suite, []): 345 suite_test_paths = confirm_paths.get(suite, []) 346 suite_test_paths = [os.path.join("tests", p) for p in suite_test_paths] 347 cmd.extend(suite_test_paths) 348 349 elif not self.verify_enabled and not self.per_test_coverage: 350 cmd.extend( 351 self.query_tests_args( 352 self.config["suite_definitions"][self.test_suite].get("tests"), 353 None, 354 try_tests, 355 ) 356 ) 357 358 if self.config.get("restartAfterFailure", False): 359 cmd.append("--restartAfterFailure") 360 361 return cmd 362 363 def _query_suites(self): 364 if self.test_suite: 365 return [(self.test_suite, self.test_suite)] 366 # per-test mode: determine test suites to run 367 all = [ 368 ( 369 "mochitest", 370 { 371 "mochitest-plain": "mochitest-plain", 372 "mochitest-plain-gpu": "mochitest-plain-gpu", 373 }, 374 ), 375 ("reftest", {"reftest": "reftest", "crashtest": "crashtest"}), 376 ("xpcshell", {"xpcshell": "xpcshell"}), 377 ] 378 suites = [] 379 for category, all_suites in all: 380 cat_suites = self.query_per_test_category_suites(category, all_suites) 381 for k in cat_suites.keys(): 382 suites.append((k, cat_suites[k])) 383 return suites 384 385 def _query_suite_categories(self): 386 if self.test_suite: 387 categories = [self.test_suite] 388 else: 389 # per-test mode 390 categories = ["mochitest", "reftest", "xpcshell"] 391 return categories 392 393 ########################################## 394 # Actions for AndroidHardwareTest # 395 ########################################## 396 397 def preflight_install(self): 398 # in the base class, this checks for mozinstall, but we don't use it 399 pass 400 401 @PreScriptAction("create-virtualenv") 402 def pre_create_virtualenv(self, action): 403 dirs = self.query_abs_dirs() 404 requirements = None 405 suites = self._query_suites() 406 if ("mochitest-media", "mochitest-media") in suites: 407 # mochitest-media is the only thing that needs this 408 requirements = os.path.join( 409 dirs["abs_mochitest_dir"], 410 "websocketprocessbridge", 411 "websocketprocessbridge_requirements_3.txt", 412 ) 413 if requirements: 414 self.register_virtualenv_module(requirements=[requirements]) 415 416 def download_and_extract(self): 417 """ 418 Download and extract product APK, tests.zip, and host utils. 419 """ 420 super().download_and_extract(suite_categories=self._query_suite_categories()) 421 dirs = self.query_abs_dirs() 422 self.xre_path = dirs["abs_xre_dir"] 423 424 def install(self): 425 """ 426 Install APKs on the device. 427 """ 428 install_needed = (not self.test_suite) or self.config["suite_definitions"][ 429 self.test_suite 430 ].get("install") 431 if install_needed is False: 432 self.info("Skipping apk installation for %s" % self.test_suite) 433 return 434 assert self.installer_path is not None, ( 435 "Either add installer_path to the config or use --installer-path." 436 ) 437 self.uninstall_android_app() 438 self.install_android_app(self.installer_path) 439 self.info("Finished installing apps for %s" % self.device_name) 440 441 def run_tests(self): 442 """ 443 Run the tests 444 """ 445 self.start_time = datetime.datetime.now() 446 max_per_test_time = datetime.timedelta(minutes=60) 447 448 per_test_args = [] 449 suites = self._query_suites() 450 minidump = self.query_minidump_stackwalk() 451 for per_test_suite, suite in suites: 452 self.test_suite = suite 453 454 try: 455 cwd = self._query_tests_dir() 456 except Exception: 457 self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite) 458 env = self.query_env() 459 if minidump: 460 env["MINIDUMP_STACKWALK"] = minidump 461 env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] 462 env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] 463 env["RUST_BACKTRACE"] = "full" 464 465 summary = None 466 for per_test_args in self.query_args(per_test_suite): 467 if (datetime.datetime.now() - self.start_time) > max_per_test_time: 468 # Running tests has run out of time. That is okay! Stop running 469 # them so that a task timeout is not triggered, and so that 470 # (partial) results are made available in a timely manner. 471 self.info( 472 "TinderboxPrint: Running tests took too long: " 473 "Not all tests were executed.<br/>" 474 ) 475 # Signal per-test time exceeded, to break out of suites and 476 # suite categories loops also. 477 return 478 479 cmd = self._build_command() 480 final_cmd = copy.copy(cmd) 481 if len(per_test_args) > 0: 482 # in per-test mode, remove any chunk arguments from command 483 for arg in final_cmd: 484 if "total-chunk" in arg or "this-chunk" in arg: 485 final_cmd.remove(arg) 486 final_cmd.extend(per_test_args) 487 488 self.info( 489 "Running on %s the command %s" 490 % (self.device_name, subprocess.list2cmdline(final_cmd)) 491 ) 492 self.info("##### %s log begins" % self.test_suite) 493 494 suite_category = self.test_suite 495 parser = self.get_test_output_parser( 496 suite_category, 497 config=self.config, 498 log_obj=self.log_obj, 499 error_list=[], 500 ) 501 self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser) 502 tbpl_status, log_level, summary = parser.evaluate_parser(0, summary) 503 parser.append_tinderboxprint_line(self.test_suite) 504 505 self.info("##### %s log ends" % self.test_suite) 506 507 if len(per_test_args) > 0: 508 self.record_status(tbpl_status, level=log_level) 509 self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) 510 if tbpl_status == TBPL_RETRY: 511 self.info("Per-test run abandoned due to RETRY status") 512 return 513 else: 514 self.record_status(tbpl_status, level=log_level) 515 # report as INFO instead of log_level to avoid extra Treeherder lines 516 self.info( 517 "The %s suite: %s ran with return status: %s" 518 % (suite_category, suite, tbpl_status), 519 ) 520 521 522 if __name__ == "__main__": 523 test = AndroidHardwareTest() 524 test.run_and_exit()