firefox_ui_tests.py (9611B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 7 import copy 8 import os 9 import sys 10 11 # load modules from parent dir 12 sys.path.insert(1, os.path.dirname(sys.path[0])) 13 14 from mozharness.base.python import PreScriptAction 15 from mozharness.mozilla.structuredlog import StructuredOutputParser 16 from mozharness.mozilla.testing.codecoverage import ( 17 CodeCoverageMixin, 18 code_coverage_config_options, 19 ) 20 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 21 from mozharness.mozilla.vcstools import VCSToolsScript 22 23 # General command line arguments for Firefox ui tests 24 firefox_ui_tests_config_options = ( 25 [ 26 [ 27 ["--allow-software-gl-layers"], 28 { 29 "action": "store_true", 30 "dest": "allow_software_gl_layers", 31 "default": False, 32 "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL " 33 "compositor.", 34 }, 35 ], 36 [ 37 ["--dry-run"], 38 { 39 "dest": "dry_run", 40 "default": False, 41 "help": "Only show what was going to be tested.", 42 }, 43 ], 44 [ 45 ["--disable-e10s"], 46 { 47 "dest": "e10s", 48 "action": "store_false", 49 "default": True, 50 "help": "Disable multi-process (e10s) mode when running tests.", 51 }, 52 ], 53 [ 54 ["--disable-fission"], 55 { 56 "dest": "disable_fission", 57 "action": "store_true", 58 "default": False, 59 "help": "Disable fission mode when running tests.", 60 }, 61 ], 62 [ 63 ["--setpref"], 64 { 65 "dest": "extra_prefs", 66 "action": "append", 67 "default": [], 68 "help": "Extra user prefs.", 69 }, 70 ], 71 [ 72 ["--symbols-path=SYMBOLS_PATH"], 73 { 74 "dest": "symbols_path", 75 "help": "absolute path to directory containing breakpad " 76 "symbols, or the url of a zip file containing symbols.", 77 }, 78 ], 79 ] 80 + copy.deepcopy(testing_config_options) 81 + copy.deepcopy(code_coverage_config_options) 82 ) 83 84 85 class FirefoxUIFunctionalTests(TestingMixin, VCSToolsScript, CodeCoverageMixin): 86 def __init__( 87 self, 88 config_options=None, 89 all_actions=None, 90 default_actions=None, 91 *args, 92 **kwargs, 93 ): 94 config_options = config_options or firefox_ui_tests_config_options 95 actions = [ 96 "clobber", 97 "download-and-extract", 98 "create-virtualenv", 99 "install", 100 "run-tests", 101 "uninstall", 102 ] 103 104 super().__init__( 105 config_options=config_options, 106 all_actions=all_actions or actions, 107 default_actions=default_actions or actions, 108 *args, 109 **kwargs, 110 ) 111 112 # Code which runs in automation has to include the following properties 113 self.binary_path = self.config.get("binary_path") 114 self.installer_path = self.config.get("installer_path") 115 self.installer_url = self.config.get("installer_url") 116 self.test_packages_url = self.config.get("test_packages_url") 117 self.test_url = self.config.get("test_url") 118 119 if not self.test_url and not self.test_packages_url: 120 self.fatal("You must use --test-url, or --test-packages-url") 121 122 @PreScriptAction("create-virtualenv") 123 def _pre_create_virtualenv(self, action): 124 dirs = self.query_abs_dirs() 125 126 requirements = os.path.join( 127 dirs["abs_test_install_dir"], "config", "firefox_ui_requirements.txt" 128 ) 129 self.register_virtualenv_module(requirements=[requirements]) 130 131 def download_and_extract(self): 132 """Override method from TestingMixin for more specific behavior.""" 133 extract_dirs = [ 134 "config/*", 135 "firefox-ui/*", 136 "marionette/*", 137 "mozbase/*", 138 "tools/mozterm/*", 139 "tools/wptserve/*", 140 "tools/wpt_third_party/*", 141 "mozpack/*", 142 "mozbuild/*", 143 ] 144 super().download_and_extract(extract_dirs=extract_dirs) 145 146 def query_abs_dirs(self): 147 if self.abs_dirs: 148 return self.abs_dirs 149 150 abs_dirs = super().query_abs_dirs() 151 abs_tests_install_dir = os.path.join(abs_dirs["abs_work_dir"], "tests") 152 153 dirs = { 154 "abs_blob_upload_dir": os.path.join( 155 abs_dirs["abs_work_dir"], "blobber_upload_dir" 156 ), 157 "abs_fxui_dir": os.path.join(abs_tests_install_dir, "firefox-ui"), 158 "abs_fxui_manifest_dir": os.path.join( 159 abs_tests_install_dir, 160 "firefox-ui", 161 "tests", 162 "testing", 163 "firefox-ui", 164 "tests", 165 ), 166 "abs_test_install_dir": abs_tests_install_dir, 167 } 168 169 for key in dirs: 170 if key not in abs_dirs: 171 abs_dirs[key] = dirs[key] 172 self.abs_dirs = abs_dirs 173 174 return self.abs_dirs 175 176 def query_harness_args(self, extra_harness_config_options=None): 177 """Collects specific test related command line arguments. 178 179 Sub classes should override this method for their own specific arguments. 180 """ 181 config_options = extra_harness_config_options or [] 182 183 args = [] 184 for option in config_options: 185 dest = option[1]["dest"] 186 name = self.config.get(dest) 187 188 if name: 189 if type(name) is bool: 190 args.append(option[0][0]) 191 else: 192 args.extend([option[0][0], self.config[dest]]) 193 194 return args 195 196 def run_test(self, binary_path, env=None, marionette_port=2828): 197 """All required steps for running the tests against an installer.""" 198 dirs = self.query_abs_dirs() 199 200 # Import the harness to retrieve the location of the cli scripts 201 import firefox_ui_harness 202 203 cmd = [ 204 self.query_python_path(), 205 os.path.join( 206 os.path.dirname(firefox_ui_harness.__file__), "cli_functional.py" 207 ), 208 "--binary", 209 binary_path, 210 "--address", 211 f"localhost:{marionette_port}", 212 # Resource files to serve via local webserver 213 "--server-root", 214 os.path.join(dirs["abs_fxui_dir"], "resources"), 215 # Use the work dir to get temporary data stored 216 "--workspace", 217 dirs["abs_work_dir"], 218 # logging options 219 "--gecko-log=-", # output from the gecko process redirected to stdout 220 "--log-raw=-", # structured log for output parser redirected to stdout 221 # Enable tracing output to log transmission protocol 222 "-vv", 223 ] 224 225 # Collect all pass-through harness options to the script 226 cmd.extend(self.query_harness_args()) 227 228 if not self.config.get("e10s"): 229 cmd.append("--disable-e10s") 230 231 if self.config.get("disable_fission"): 232 cmd.append("--disable-fission") 233 234 cmd.extend([f"--setpref={p}" for p in self.config.get("extra_prefs")]) 235 236 if self.symbols_url: 237 cmd.extend(["--symbols-path", self.symbols_url]) 238 239 parser = StructuredOutputParser( 240 config=self.config, log_obj=self.log_obj, strict=False 241 ) 242 243 # Add the tests to run 244 cmd.append( 245 os.path.join(dirs["abs_fxui_manifest_dir"], "functional", "manifest.toml") 246 ) 247 248 # Set further environment settings 249 env = env or self.query_env() 250 env.update({"MINIDUMP_SAVE_PATH": dirs["abs_blob_upload_dir"]}) 251 if self.query_minidump_stackwalk(): 252 env.update({"MINIDUMP_STACKWALK": self.minidump_stackwalk_path}) 253 env["RUST_BACKTRACE"] = "full" 254 255 # If code coverage is enabled, set GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR 256 # env variables 257 if self.config.get("code_coverage"): 258 env["GCOV_PREFIX"] = self.gcov_dir 259 env["JS_CODE_COVERAGE_OUTPUT_DIR"] = self.jsvm_dir 260 261 if self.config["allow_software_gl_layers"]: 262 env["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1" 263 264 return_code = self.run_command( 265 cmd, 266 cwd=dirs["abs_fxui_dir"], 267 output_timeout=1000, 268 output_parser=parser, 269 env=env, 270 ) 271 272 tbpl_status, log_level, summary = parser.evaluate_parser(return_code) 273 self.record_status(tbpl_status, level=log_level) 274 275 return return_code 276 277 @PreScriptAction("run-tests") 278 def _pre_run_tests(self, action): 279 if not self.installer_path and not self.installer_url: 280 self.critical( 281 "Please specify an installer via --installer-path or --installer-url." 282 ) 283 sys.exit(1) 284 285 def run_tests(self): 286 """Run all the tests""" 287 return self.run_test( 288 binary_path=self.binary_path, 289 env=self.query_env(), 290 ) 291 292 293 if __name__ == "__main__": 294 myScript = FirefoxUIFunctionalTests() 295 myScript.run_and_exit()