marionette.py (17494B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import copy 7 import json 8 import os 9 import sys 10 11 # load modules from parent dir 12 sys.path.insert(1, os.path.dirname(sys.path[0])) 13 14 from mozharness.base.errors import BaseErrorList, TarErrorList 15 from mozharness.base.log import INFO 16 from mozharness.base.script import PreScriptAction 17 from mozharness.base.transfer import TransferMixin 18 from mozharness.base.vcs.vcsbase import MercurialScript 19 from mozharness.mozilla.structuredlog import StructuredOutputParser 20 from mozharness.mozilla.testing.codecoverage import ( 21 CodeCoverageMixin, 22 code_coverage_config_options, 23 ) 24 from mozharness.mozilla.testing.errors import HarnessErrorList, LogcatErrorList 25 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 26 from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper 27 28 29 class MarionetteTest(TestingMixin, MercurialScript, TransferMixin, CodeCoverageMixin): 30 config_options = ( 31 [ 32 [ 33 ["--application"], 34 { 35 "action": "store", 36 "dest": "application", 37 "default": None, 38 "help": "application name of binary", 39 }, 40 ], 41 [ 42 ["--app-arg"], 43 { 44 "action": "store", 45 "dest": "app_arg", 46 "default": None, 47 "help": "Optional command-line argument to pass to the browser", 48 }, 49 ], 50 [ 51 ["--marionette-address"], 52 { 53 "action": "store", 54 "dest": "marionette_address", 55 "default": None, 56 "help": "The host:port of the Marionette server running inside Gecko. " 57 "Unused for emulator testing", 58 }, 59 ], 60 [ 61 ["--emulator"], 62 { 63 "action": "store", 64 "type": "choice", 65 "choices": ["arm", "x86"], 66 "dest": "emulator", 67 "default": None, 68 "help": "Use an emulator for testing", 69 }, 70 ], 71 [ 72 ["--test-manifest"], 73 { 74 "action": "store", 75 "dest": "test_manifest", 76 "default": "unit-tests.toml", 77 "help": "Path to test manifest to run relative to the Marionette " 78 "tests directory", 79 }, 80 ], 81 [ 82 ["--tag"], 83 { 84 "action": "store", 85 "dest": "test_tag", 86 "default": "", 87 "help": "Tag that identifies how to filter which tests to run.", 88 }, 89 ], 90 [ 91 ["--total-chunks"], 92 { 93 "action": "store", 94 "dest": "total_chunks", 95 "help": "Number of total chunks", 96 }, 97 ], 98 [ 99 ["--this-chunk"], 100 { 101 "action": "store", 102 "dest": "this_chunk", 103 "help": "Number of this chunk", 104 }, 105 ], 106 [ 107 ["--setpref"], 108 { 109 "action": "append", 110 "metavar": "PREF=VALUE", 111 "dest": "extra_prefs", 112 "default": [], 113 "help": "Extra user prefs.", 114 }, 115 ], 116 [ 117 ["--headless"], 118 { 119 "action": "store_true", 120 "dest": "headless", 121 "default": False, 122 "help": "Run tests in headless mode.", 123 }, 124 ], 125 [ 126 ["--headless-width"], 127 { 128 "action": "store", 129 "dest": "headless_width", 130 "default": "1600", 131 "help": "Specify headless virtual screen width (default: 1600).", 132 }, 133 ], 134 [ 135 ["--headless-height"], 136 { 137 "action": "store", 138 "dest": "headless_height", 139 "default": "1200", 140 "help": "Specify headless virtual screen height (default: 1200).", 141 }, 142 ], 143 [ 144 ["--allow-software-gl-layers"], 145 { 146 "action": "store_true", 147 "dest": "allow_software_gl_layers", 148 "default": False, 149 "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor.", # NOQA: E501 150 }, 151 ], 152 [ 153 ["--disable-fission"], 154 { 155 "action": "store_true", 156 "dest": "disable_fission", 157 "default": False, 158 "help": "Run the browser without fission enabled", 159 }, 160 ], 161 [ 162 ["--subsuite"], 163 { 164 "action": "store", 165 "dest": "subsuite", 166 "default": "marionette-integration", 167 "help": "Selects test paths from test-manifests.active", 168 }, 169 ], 170 ] 171 + copy.deepcopy(testing_config_options) 172 + copy.deepcopy(code_coverage_config_options) 173 ) 174 175 repos = [] 176 177 def __init__(self, require_config_file=False): 178 super().__init__( 179 config_options=self.config_options, 180 all_actions=[ 181 "clobber", 182 "pull", 183 "download-and-extract", 184 "create-virtualenv", 185 "install", 186 "run-tests", 187 ], 188 default_actions=[ 189 "clobber", 190 "pull", 191 "download-and-extract", 192 "create-virtualenv", 193 "install", 194 "run-tests", 195 ], 196 require_config_file=require_config_file, 197 config={"require_test_zip": True}, 198 ) 199 200 # these are necessary since self.config is read only 201 c = self.config 202 self.installer_url = c.get("installer_url") 203 self.installer_path = c.get("installer_path") 204 self.binary_path = c.get("binary_path") 205 self.test_url = c.get("test_url") 206 self.test_packages_url = c.get("test_packages_url") 207 self.subsuite = c.get("subsuite") 208 209 self.test_suite = self._get_test_suite(c.get("emulator")) 210 if self.test_suite not in self.config["suite_definitions"]: 211 self.fatal(f"{self.test_suite} is not defined in the config!") 212 213 if c.get("structured_output"): 214 self.parser_class = StructuredOutputParser 215 else: 216 self.parser_class = TestSummaryOutputParserHelper 217 218 def _pre_config_lock(self, rw_config): 219 super()._pre_config_lock(rw_config) 220 if not self.config.get("emulator") and not self.config.get( 221 "marionette_address" 222 ): 223 self.fatal( 224 "You need to specify a --marionette-address for non-emulator tests! " 225 "(Try --marionette-address localhost:2828 )" 226 ) 227 228 def _query_tests_dir(self): 229 dirs = self.query_abs_dirs() 230 test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"] 231 232 return os.path.join(dirs["abs_test_install_dir"], test_dir) 233 234 def query_abs_dirs(self): 235 if self.abs_dirs: 236 return self.abs_dirs 237 abs_dirs = super().query_abs_dirs() 238 dirs = {} 239 dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") 240 dirs["abs_marionette_dir"] = os.path.join( 241 dirs["abs_test_install_dir"], "marionette", "harness", "marionette_harness" 242 ) 243 dirs["abs_marionette_tests_dir"] = os.path.join( 244 dirs["abs_test_install_dir"], 245 "marionette", 246 "tests", 247 "testing", 248 "marionette", 249 "harness", 250 "marionette_harness", 251 "tests", 252 ) 253 dirs["abs_gecko_dir"] = os.path.join(abs_dirs["abs_work_dir"], "gecko") 254 dirs["abs_emulator_dir"] = os.path.join(abs_dirs["abs_work_dir"], "emulator") 255 256 dirs["abs_blob_upload_dir"] = os.path.join( 257 abs_dirs["abs_work_dir"], "blobber_upload_dir" 258 ) 259 260 for key in dirs.keys(): 261 if key not in abs_dirs: 262 abs_dirs[key] = dirs[key] 263 self.abs_dirs = abs_dirs 264 return self.abs_dirs 265 266 @PreScriptAction("create-virtualenv") 267 def _configure_marionette_virtualenv(self, action): 268 dirs = self.query_abs_dirs() 269 requirements = os.path.join( 270 dirs["abs_test_install_dir"], "config", "marionette_requirements.txt" 271 ) 272 if not os.path.isfile(requirements): 273 self.fatal(f"Could not find marionette requirements file: {requirements}") 274 275 self.register_virtualenv_module(requirements=[requirements]) 276 277 def _get_test_suite(self, is_emulator): 278 """ 279 Determine which in tree options group to use and return the 280 appropriate key. 281 """ 282 platform = "emulator" if is_emulator else "desktop" 283 # Currently running marionette on an emulator means webapi 284 # tests. This method will need to change if this does. 285 testsuite = "webapi" if is_emulator else "marionette" 286 return f"{testsuite}_{platform}" 287 288 def download_and_extract(self): 289 super().download_and_extract() 290 291 if self.config.get("emulator"): 292 dirs = self.query_abs_dirs() 293 294 self.mkdir_p(dirs["abs_emulator_dir"]) 295 tar = self.query_exe("tar", return_type="list") 296 self.run_command( 297 tar + ["zxf", self.installer_path], 298 cwd=dirs["abs_emulator_dir"], 299 error_list=TarErrorList, 300 halt_on_failure=True, 301 fatal_exit_code=3, 302 ) 303 304 def install(self): 305 if self.config.get("emulator"): 306 self.info("Emulator tests; skipping.") 307 else: 308 super().install() 309 310 def run_tests(self): 311 """ 312 Run the Marionette tests 313 """ 314 dirs = self.query_abs_dirs() 315 316 raw_log_file = os.path.join(dirs["abs_blob_upload_dir"], "marionette_raw.log") 317 error_summary_file = os.path.join( 318 dirs["abs_blob_upload_dir"], "marionette_errorsummary.log" 319 ) 320 html_report_file = os.path.join(dirs["abs_blob_upload_dir"], "report.html") 321 322 config_fmt_args = { 323 # emulator builds require a longer timeout 324 "timeout": 60000 if self.config.get("emulator") else 10000, 325 "profile": os.path.join(dirs["abs_work_dir"], "profile"), 326 "xml_output": os.path.join(dirs["abs_work_dir"], "output.xml"), 327 "html_output": os.path.join(dirs["abs_blob_upload_dir"], "output.html"), 328 "logcat_dir": dirs["abs_work_dir"], 329 "emulator": "arm", 330 "symbols_path": self.symbols_path, 331 "binary": self.binary_path, 332 "address": self.config.get("marionette_address"), 333 "raw_log_file": raw_log_file, 334 "error_summary_file": error_summary_file, 335 "html_report_file": html_report_file, 336 "gecko_log": dirs["abs_blob_upload_dir"], 337 "this_chunk": self.config.get("this_chunk", 1), 338 "total_chunks": self.config.get("total_chunks", 1), 339 } 340 341 self.info("The emulator type: %s" % config_fmt_args["emulator"]) 342 # build the marionette command arguments 343 python = self.query_python_path("python") 344 345 cmd = [python, "-u", os.path.join(dirs["abs_marionette_dir"], "runtests.py")] 346 347 if self.config.get("test_tag", ""): 348 cmd.extend(["--tag", self.config["test_tag"]]) 349 350 manifest = os.path.join( 351 dirs["abs_marionette_tests_dir"], self.config["test_manifest"] 352 ) 353 354 if self.config.get("app_arg"): 355 config_fmt_args["app_arg"] = self.config["app_arg"] 356 357 cmd.extend([f"--setpref={p}" for p in self.config["extra_prefs"]]) 358 359 cmd.append("--gecko-log=-") 360 361 if self.config.get("structured_output"): 362 cmd.append("--log-raw=-") 363 364 if self.config["disable_fission"]: 365 cmd.append("--disable-fission") 366 cmd.extend(["--setpref=fission.autostart=false"]) 367 368 for arg in self.config["suite_definitions"][self.test_suite]["options"]: 369 cmd.append(arg % config_fmt_args) 370 371 if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1: 372 # Make sure that the logging directory exists 373 self.fatal("Could not create blobber upload directory") 374 375 test_paths = json.loads(os.environ.get("MOZHARNESS_TEST_PATHS", '""')) 376 confirm_paths = json.loads(os.environ.get("MOZHARNESS_CONFIRM_PATHS", '""')) 377 378 suite = self.subsuite 379 if test_paths and suite in test_paths: 380 suite_test_paths = test_paths[suite] 381 if confirm_paths and suite in confirm_paths and confirm_paths[suite]: 382 suite_test_paths = confirm_paths[suite] 383 384 paths = [ 385 os.path.join(dirs["abs_test_install_dir"], "marionette", "tests", p) 386 for p in suite_test_paths 387 ] 388 cmd.extend(paths) 389 else: 390 cmd.append(manifest) 391 392 try_options, try_tests = self.try_args("marionette") 393 cmd.extend(self.query_tests_args(try_tests, str_format_values=config_fmt_args)) 394 395 env = {} 396 if self.query_minidump_stackwalk(): 397 env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path 398 env["MOZ_UPLOAD_DIR"] = self.query_abs_dirs()["abs_blob_upload_dir"] 399 env["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] 400 env["RUST_BACKTRACE"] = "full" 401 402 if self.config["allow_software_gl_layers"]: 403 env["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1" 404 405 if self.config["headless"]: 406 env["MOZ_HEADLESS"] = "1" 407 env["MOZ_HEADLESS_WIDTH"] = self.config["headless_width"] 408 env["MOZ_HEADLESS_HEIGHT"] = self.config["headless_height"] 409 410 if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): 411 self.mkdir_p(env["MOZ_UPLOAD_DIR"]) 412 413 # Causes Firefox to crash when using non-local connections. 414 env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" 415 416 # Avoid issues when printing messages containing unicode characters on 417 # windows (Bug 1800035). 418 if self._is_windows(): 419 env["PYTHONIOENCODING"] = "utf-8" 420 421 env = self.query_env(partial_env=env) 422 423 try: 424 cwd = self._query_tests_dir() 425 except Exception as e: 426 self.fatal(f"Don't know how to run --test-suite '{self.test_suite}': {e}!") 427 428 marionette_parser = self.parser_class( 429 config=self.config, 430 log_obj=self.log_obj, 431 error_list=BaseErrorList + HarnessErrorList, 432 strict=False, 433 ) 434 return_code = self.run_command( 435 cmd, cwd=cwd, output_timeout=1000, output_parser=marionette_parser, env=env 436 ) 437 level = INFO 438 tbpl_status, log_level, summary = marionette_parser.evaluate_parser( 439 return_code=return_code 440 ) 441 marionette_parser.append_tinderboxprint_line("marionette") 442 443 qemu = os.path.join(dirs["abs_work_dir"], "qemu.log") 444 if os.path.isfile(qemu): 445 self.copyfile(qemu, os.path.join(dirs["abs_blob_upload_dir"], "qemu.log")) 446 447 # dump logcat output if there were failures 448 if self.config.get("emulator"): 449 if ( 450 marionette_parser.failed != "0" 451 or "T-FAIL" in marionette_parser.tsummary 452 ): 453 logcat = os.path.join(dirs["abs_work_dir"], "emulator-5554.log") 454 if os.access(logcat, os.F_OK): 455 self.info("dumping logcat") 456 self.run_command(["cat", logcat], error_list=LogcatErrorList) 457 else: 458 self.info("no logcat file found") 459 else: 460 # .. or gecko.log if it exists 461 gecko_log = os.path.join(self.config["base_work_dir"], "gecko.log") 462 if os.access(gecko_log, os.F_OK): 463 self.info("dumping gecko.log") 464 self.run_command(["cat", gecko_log]) 465 self.rmtree(gecko_log) 466 else: 467 self.info("gecko.log not found") 468 469 marionette_parser.print_summary("marionette") 470 471 self.log( 472 "Marionette exited with return code %s: %s" % (return_code, tbpl_status), 473 level=level, 474 ) 475 self.record_status(tbpl_status) 476 477 478 if __name__ == "__main__": 479 marionetteTest = MarionetteTest() 480 marionetteTest.run_and_exit()