android_emulator_unittest.py (23182B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import copy 7 import datetime 8 import json 9 import os 10 import subprocess 11 import sys 12 13 # load modules from parent dir 14 here = os.path.abspath(os.path.dirname(__file__)) 15 sys.path.insert(1, os.path.dirname(here)) 16 17 from mozharness.base.log import WARNING 18 from mozharness.base.script import BaseScript, PreScriptAction 19 from mozharness.mozilla.automation import TBPL_RETRY 20 from mozharness.mozilla.mozbase import MozbaseMixin 21 from mozharness.mozilla.testing.android import AndroidMixin 22 from mozharness.mozilla.testing.codecoverage import ( 23 CodeCoverageMixin, 24 code_coverage_config_options, 25 ) 26 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 27 28 SUITE_DEFAULT_E10S = ["geckoview-junit", "mochitest", "reftest"] 29 SUITE_NO_E10S = ["cppunittest", "gtest", "jittest", "xpcshell"] 30 SUITE_REPEATABLE = ["mochitest", "reftest", "xpcshell"] 31 32 33 class AndroidEmulatorTest( 34 TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin, AndroidMixin 35 ): 36 """ 37 A mozharness script for Android functional tests (like mochitests and reftests) 38 run on an Android emulator. This script starts and manages an Android emulator 39 for the duration of the required tests. This is like desktop_unittest.py, but 40 for Android emulator test platforms. 41 """ 42 43 config_options = ( 44 [ 45 [ 46 ["--test-suite"], 47 {"action": "store", "dest": "test_suite", "default": None}, 48 ], 49 [ 50 ["--total-chunk"], 51 { 52 "action": "store", 53 "dest": "total_chunks", 54 "default": None, 55 "help": "Number of total chunks", 56 }, 57 ], 58 [ 59 ["--this-chunk"], 60 { 61 "action": "store", 62 "dest": "this_chunk", 63 "default": None, 64 "help": "Number of this chunk", 65 }, 66 ], 67 [ 68 ["--timeout-factor"], 69 { 70 "action": "store", 71 "dest": "timeout_factor", 72 "default": None, 73 "help": "Multiplier for test timeout values", 74 }, 75 ], 76 [ 77 ["--enable-xorigin-tests"], 78 { 79 "action": "store_true", 80 "dest": "enable_xorigin_tests", 81 "default": False, 82 "help": "Run tests in a cross origin iframe.", 83 }, 84 ], 85 [ 86 ["--gpu-required"], 87 { 88 "action": "store_true", 89 "dest": "gpu_required", 90 "default": False, 91 "help": "Run additional verification on modified tests using gpu instances.", 92 }, 93 ], 94 [ 95 ["--log-raw-level"], 96 { 97 "action": "store", 98 "dest": "log_raw_level", 99 "default": "info", 100 "help": "Set log level (debug|info|warning|error|critical|fatal)", 101 }, 102 ], 103 [ 104 ["--log-tbpl-level"], 105 { 106 "action": "store", 107 "dest": "log_tbpl_level", 108 "default": "info", 109 "help": "Set log level (debug|info|warning|error|critical|fatal)", 110 }, 111 ], 112 [ 113 ["--disable-e10s"], 114 { 115 "action": "store_false", 116 "dest": "e10s", 117 "default": True, 118 "help": "Run tests without multiple processes (e10s).", 119 }, 120 ], 121 [ 122 ["--disable-fission"], 123 { 124 "action": "store_true", 125 "dest": "disable_fission", 126 "default": False, 127 "help": "Run without Fission enabled.", 128 }, 129 ], 130 [ 131 ["--web-content-isolation-strategy"], 132 { 133 "action": "store", 134 "type": "int", 135 "dest": "web_content_isolation_strategy", 136 "help": "Strategy used to determine whether or not a particular site should" 137 "load into a webIsolated content process, see " 138 "fission.webContentIsolationStrategy.", 139 }, 140 ], 141 [ 142 ["--enable-isolated-zygote-process"], 143 { 144 "action": "store_true", 145 "dest": "enable_isolated_zygote_process", 146 "default": False, 147 "help": "Run with app Zygote preloading enabled.", 148 }, 149 ], 150 [ 151 ["--repeat"], 152 { 153 "action": "store", 154 "type": "int", 155 "dest": "repeat", 156 "default": 0, 157 "help": "Repeat the tests the given number of times. Supported " 158 "by mochitest, reftest, crashtest, ignored otherwise.", 159 }, 160 ], 161 [ 162 ["--setpref"], 163 { 164 "action": "append", 165 "metavar": "PREF=VALUE", 166 "dest": "extra_prefs", 167 "default": [], 168 "help": "Extra user prefs.", 169 }, 170 ], 171 [ 172 ["--tag"], 173 { 174 "action": "append", 175 "default": [], 176 "dest": "test_tags", 177 "help": "Filter out tests that don't have the given tag. Can be used multiple " 178 "times in which case the test must contain at least one of the given tags.", 179 }, 180 ], 181 ] 182 + copy.deepcopy(testing_config_options) 183 + copy.deepcopy(code_coverage_config_options) 184 ) 185 186 def __init__(self, require_config_file=False): 187 super().__init__( 188 config_options=self.config_options, 189 all_actions=[ 190 "clobber", 191 "download-and-extract", 192 "create-virtualenv", 193 "start-emulator", 194 "verify-device", 195 "install", 196 "run-tests", 197 ], 198 require_config_file=require_config_file, 199 config={ 200 "virtualenv_modules": [], 201 "virtualenv_requirements": [], 202 "require_test_zip": True, 203 }, 204 ) 205 206 # these are necessary since self.config is read only 207 c = self.config 208 self.installer_url = c.get("installer_url") 209 self.installer_path = c.get("installer_path") 210 self.test_url = c.get("test_url") 211 self.test_packages_url = c.get("test_packages_url") 212 self.test_manifest = c.get("test_manifest") 213 suite = c.get("test_suite") 214 self.test_suite = suite 215 self.this_chunk = c.get("this_chunk") 216 self.total_chunks = c.get("total_chunks") 217 self.timeout_factor = c.get("timeout_factor") 218 self.xre_path = None 219 self.device_serial = "emulator-5554" 220 self.log_raw_level = c.get("log_raw_level") 221 self.log_tbpl_level = c.get("log_tbpl_level") 222 # AndroidMixin uses this when launching the emulator. We only want 223 # GLES3 if we're running WebRender (default) 224 self.use_gles3 = True 225 self.disable_e10s = c.get("disable_e10s") 226 self.disable_fission = c.get("disable_fission") 227 self.web_content_isolation_strategy = c.get("web_content_isolation_strategy") 228 self.enable_isolated_zygote_process = c.get("enable_isolated_zygote_process") 229 self.extra_prefs = c.get("extra_prefs") 230 self.test_tags = c.get("test_tags") 231 232 def query_abs_dirs(self): 233 if self.abs_dirs: 234 return self.abs_dirs 235 abs_dirs = super().query_abs_dirs() 236 dirs = {} 237 dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") 238 dirs["abs_test_bin_dir"] = os.path.join( 239 abs_dirs["abs_work_dir"], "tests", "bin" 240 ) 241 dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules") 242 dirs["abs_blob_upload_dir"] = os.path.join( 243 abs_dirs["abs_work_dir"], "blobber_upload_dir" 244 ) 245 dirs["abs_mochitest_dir"] = os.path.join( 246 dirs["abs_test_install_dir"], "mochitest" 247 ) 248 dirs["abs_reftest_dir"] = os.path.join(dirs["abs_test_install_dir"], "reftest") 249 dirs["abs_xpcshell_dir"] = os.path.join( 250 dirs["abs_test_install_dir"], "xpcshell" 251 ) 252 work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"] 253 dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils") 254 dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux") 255 dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device") 256 dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar") 257 258 for key in dirs.keys(): 259 if key not in abs_dirs: 260 abs_dirs[key] = dirs[key] 261 self.abs_dirs = abs_dirs 262 return self.abs_dirs 263 264 def _query_tests_dir(self, test_suite): 265 dirs = self.query_abs_dirs() 266 try: 267 test_dir = self.config["suite_definitions"][test_suite]["testsdir"] 268 except Exception: 269 test_dir = test_suite 270 return os.path.join(dirs["abs_test_install_dir"], test_dir) 271 272 def _get_mozharness_test_paths(self, suite): 273 test_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) 274 confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) 275 276 if not test_paths or not test_paths.get(suite, []): 277 return None 278 279 suite_test_paths = test_paths.get(suite, []) 280 if confirm_paths and confirm_paths.get(suite, []): 281 suite_test_paths = confirm_paths.get(suite, []) 282 283 if suite in ("reftest", "crashtest"): 284 dirs = self.query_abs_dirs() 285 suite_test_paths = [ 286 os.path.join(dirs["abs_reftest_dir"], "tests", p) 287 for p in suite_test_paths 288 ] 289 290 return suite_test_paths 291 292 def _build_command(self): 293 c = self.config 294 dirs = self.query_abs_dirs() 295 296 if self.test_suite not in self.config["suite_definitions"]: 297 self.fatal("Key '%s' not defined in the config!" % self.test_suite) 298 299 cmd = [ 300 self.query_python_path("python"), 301 "-u", 302 os.path.join( 303 self._query_tests_dir(self.test_suite), 304 self.config["suite_definitions"][self.test_suite]["run_filename"], 305 ), 306 ] 307 308 raw_log_file, error_summary_file = self.get_indexed_logs( 309 dirs["abs_blob_upload_dir"], self.test_suite 310 ) 311 312 str_format_values = { 313 "device_serial": self.device_serial, 314 # IP address of the host as seen from the emulator 315 "remote_webserver": "10.0.2.2", 316 "xre_path": self.xre_path, 317 "utility_path": self.xre_path, 318 "http_port": "8854", # starting http port to use for the mochitest server 319 "ssl_port": "4454", # starting ssl port to use for the server 320 "certs_path": os.path.join(dirs["abs_work_dir"], "tests/certs"), 321 # TestingMixin._download_and_extract_symbols() will set 322 # self.symbols_path when downloading/extracting. 323 "symbols_path": self.symbols_path, 324 "modules_dir": dirs["abs_modules_dir"], 325 "installer_path": self.installer_path, 326 "raw_log_file": raw_log_file, 327 "log_tbpl_level": self.log_tbpl_level, 328 "log_raw_level": self.log_raw_level, 329 "error_summary_file": error_summary_file, 330 "xpcshell_extra": c.get("xpcshell_extra", ""), 331 "gtest_dir": os.path.join(dirs["abs_test_install_dir"], "gtest"), 332 } 333 334 user_paths = self._get_mozharness_test_paths(self.test_suite) 335 336 for option in self.config["suite_definitions"][self.test_suite]["options"]: 337 opt = option.split("=")[0] 338 # override configured chunk options with script args, if specified 339 if opt in ("--this-chunk", "--total-chunks"): 340 if ( 341 user_paths 342 or getattr(self, opt.replace("-", "_").strip("_"), None) is not None 343 ): 344 continue 345 346 if "%(app)" in option: 347 # only query package name if requested 348 cmd.extend([option % {"app": self.query_package_name()}]) 349 else: 350 option = option % str_format_values 351 if option: 352 cmd.extend([option]) 353 354 if "mochitest" in self.test_suite: 355 category = "mochitest" 356 elif "reftest" in self.test_suite or "crashtest" in self.test_suite: 357 category = "reftest" 358 else: 359 category = self.test_suite 360 if c.get("repeat"): 361 if category in SUITE_REPEATABLE: 362 cmd.extend(["--repeat=%s" % c.get("repeat")]) 363 else: 364 self.log(f"--repeat not supported in {category}", level=WARNING) 365 366 # do not add --disable fission if we don't have --disable-e10s 367 if c["disable_fission"] and category not in ["gtest", "cppunittest"]: 368 cmd.append("--disable-fission") 369 370 if c["enable_isolated_zygote_process"]: 371 cmd.append("--enable-isolated-zygote-process") 372 373 if "web_content_isolation_strategy" in c: 374 cmd.append( 375 "--web-content-isolation-strategy=%s" 376 % c["web_content_isolation_strategy"] 377 ) 378 cmd.extend([f"--setpref={p}" for p in self.extra_prefs]) 379 380 if not (self.verify_enabled or self.per_test_coverage): 381 if user_paths or self.test_tags: 382 if user_paths: 383 cmd.extend(user_paths) 384 if self.test_tags: 385 cmd.extend([f"--tag={t}" for t in self.test_tags]) 386 else: 387 if self.this_chunk is not None: 388 cmd.extend(["--this-chunk", self.this_chunk]) 389 if self.total_chunks is not None: 390 cmd.extend(["--total-chunks", self.total_chunks]) 391 392 if self.timeout_factor is not None: 393 cmd.extend(["--timeout-factor", self.timeout_factor]) 394 395 if category not in SUITE_NO_E10S: 396 if category in SUITE_DEFAULT_E10S and not c["e10s"]: 397 cmd.append("--disable-e10s") 398 elif category not in SUITE_DEFAULT_E10S and c["e10s"]: 399 cmd.append("--e10s") 400 401 if c.get("enable_xorigin_tests"): 402 cmd.extend(["--enable-xorigin-tests"]) 403 404 try_options, try_tests = self.try_args(self.test_suite) 405 cmd.extend(try_options) 406 if not self.verify_enabled and not self.per_test_coverage and not user_paths: 407 cmd.extend( 408 self.query_tests_args( 409 self.config["suite_definitions"][self.test_suite].get("tests"), 410 None, 411 try_tests, 412 ) 413 ) 414 415 if self.java_code_coverage_enabled: 416 cmd.extend([ 417 "--enable-coverage", 418 "--coverage-output-dir", 419 self.java_coverage_output_dir, 420 ]) 421 422 if self.config.get("restartAfterFailure", False): 423 cmd.append("--restartAfterFailure") 424 425 return cmd 426 427 def _query_suites(self): 428 if self.test_suite: 429 return [(self.test_suite, self.test_suite)] 430 # per-test mode: determine test suites to run 431 432 # For each test category, provide a list of supported sub-suites and a mapping 433 # between the per_test_base suite name and the android suite name. 434 all = [ 435 ( 436 "mochitest", 437 { 438 "mochitest-plain": "mochitest-plain", 439 "mochitest-media": "mochitest-media", 440 "mochitest-plain-gpu": "mochitest-plain-gpu", 441 }, 442 ), 443 ( 444 "reftest", 445 { 446 "reftest": "reftest", 447 "crashtest": "crashtest", 448 "jsreftest": "jsreftest", 449 }, 450 ), 451 ("xpcshell", {"xpcshell": "xpcshell"}), 452 ] 453 suites = [] 454 for category, all_suites in all: 455 cat_suites = self.query_per_test_category_suites(category, all_suites) 456 for k in cat_suites.keys(): 457 suites.append((k, cat_suites[k])) 458 return suites 459 460 def _query_suite_categories(self): 461 if self.test_suite: 462 categories = [self.test_suite] 463 else: 464 # per-test mode 465 categories = ["mochitest", "reftest", "xpcshell"] 466 return categories 467 468 ########################################## 469 # Actions for AndroidEmulatorTest # 470 ########################################## 471 472 def preflight_install(self): 473 # in the base class, this checks for mozinstall, but we don't use it 474 pass 475 476 @PreScriptAction("create-virtualenv") 477 def pre_create_virtualenv(self, action): 478 dirs = self.query_abs_dirs() 479 requirements = None 480 suites = self._query_suites() 481 if ("mochitest-media", "mochitest-media") in suites: 482 # mochitest-media is the only thing that needs this 483 requirements = os.path.join( 484 dirs["abs_mochitest_dir"], 485 "websocketprocessbridge", 486 "websocketprocessbridge_requirements_3.txt", 487 ) 488 if requirements: 489 self.register_virtualenv_module(requirements=[requirements]) 490 491 def download_and_extract(self): 492 """ 493 Download and extract product APK, tests.zip, and host utils. 494 """ 495 super().download_and_extract(suite_categories=self._query_suite_categories()) 496 dirs = self.query_abs_dirs() 497 self.xre_path = dirs["abs_xre_dir"] 498 499 def install(self): 500 """ 501 Install APKs on the device. 502 """ 503 install_needed = (not self.test_suite) or self.config["suite_definitions"][ 504 self.test_suite 505 ].get("install") 506 if install_needed is False: 507 self.info("Skipping apk installation for %s" % self.test_suite) 508 return 509 assert self.installer_path is not None, ( 510 "Either add installer_path to the config or use --installer-path." 511 ) 512 self.install_android_app(self.installer_path) 513 self.info("Finished installing apps for %s" % self.device_serial) 514 515 def run_tests(self): 516 """ 517 Run the tests 518 """ 519 self.start_time = datetime.datetime.now() 520 max_per_test_time = datetime.timedelta(minutes=60) 521 522 per_test_args = [] 523 suites = self._query_suites() 524 minidump = self.query_minidump_stackwalk() 525 for per_test_suite, suite in suites: 526 self.test_suite = suite 527 528 try: 529 cwd = self._query_tests_dir(self.test_suite) 530 except Exception: 531 self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite) 532 533 env = self.query_env() 534 if minidump: 535 env["MINIDUMP_STACKWALK"] = minidump 536 env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] 537 env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] 538 env["RUST_BACKTRACE"] = "full" 539 if self.config["nodejs_path"]: 540 env["MOZ_NODE_PATH"] = self.config["nodejs_path"] 541 542 summary = {} 543 for per_test_args in self.query_args(per_test_suite): 544 if (datetime.datetime.now() - self.start_time) > max_per_test_time: 545 # Running tests has run out of time. That is okay! Stop running 546 # them so that a task timeout is not triggered, and so that 547 # (partial) results are made available in a timely manner. 548 self.info( 549 "TinderboxPrint: Running tests took too long: " 550 "Not all tests were executed.<br/>" 551 ) 552 # Signal per-test time exceeded, to break out of suites and 553 # suite categories loops also. 554 return 555 556 cmd = self._build_command() 557 final_cmd = copy.copy(cmd) 558 if len(per_test_args) > 0: 559 # in per-test mode, remove any chunk arguments from command 560 for arg in final_cmd: 561 if "total-chunk" in arg or "this-chunk" in arg: 562 final_cmd.remove(arg) 563 final_cmd.extend(per_test_args) 564 565 self.info("Running the command %s" % subprocess.list2cmdline(final_cmd)) 566 self.info("##### %s log begins" % self.test_suite) 567 568 suite_category = self.test_suite 569 parser = self.get_test_output_parser( 570 suite_category, 571 config=self.config, 572 log_obj=self.log_obj, 573 error_list=[], 574 ) 575 self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser) 576 tbpl_status, log_level, summary = parser.evaluate_parser( 577 0, previous_summary=summary 578 ) 579 parser.append_tinderboxprint_line(self.test_suite) 580 581 self.info("##### %s log ends" % self.test_suite) 582 583 if len(per_test_args) > 0: 584 self.record_status(tbpl_status, level=log_level) 585 self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) 586 if tbpl_status == TBPL_RETRY: 587 self.info("Per-test run abandoned due to RETRY status") 588 return 589 else: 590 self.record_status(tbpl_status, level=log_level) 591 # report as INFO instead of log_level to avoid extra Treeherder lines 592 self.info( 593 "The %s suite: %s ran with return status: %s" 594 % (suite_category, suite, tbpl_status), 595 ) 596 597 598 if __name__ == "__main__": 599 test = AndroidEmulatorTest() 600 test.run_and_exit()