telemetry_client.py (8942B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 7 import copy 8 import os 9 import sys 10 11 # load modules from parent dir 12 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) 13 14 from mozharness.base.python import PreScriptAction 15 from mozharness.mozilla.structuredlog import StructuredOutputParser 16 from mozharness.mozilla.testing.codecoverage import ( 17 CodeCoverageMixin, 18 code_coverage_config_options, 19 ) 20 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 21 from mozharness.mozilla.vcstools import VCSToolsScript 22 23 # General command line arguments for Firefox ui tests 24 telemetry_tests_config_options = ( 25 [ 26 [ 27 ["--allow-software-gl-layers"], 28 { 29 "action": "store_true", 30 "dest": "allow_software_gl_layers", 31 "default": False, 32 "help": "Permits a software GL implementation (such as LLVMPipe) " 33 "to use the GL compositor.", 34 }, 35 ], 36 [ 37 ["--dry-run"], 38 { 39 "dest": "dry_run", 40 "default": False, 41 "help": "Only show what was going to be tested.", 42 }, 43 ], 44 [ 45 ["--disable-e10s"], 46 { 47 "dest": "e10s", 48 "action": "store_false", 49 "default": True, 50 "help": "Disable multi-process (e10s) mode when running tests.", 51 }, 52 ], 53 [ 54 ["--disable-fission"], 55 { 56 "dest": "disable_fission", 57 "action": "store_true", 58 "default": False, 59 "help": "Disable fission mode when running tests.", 60 }, 61 ], 62 [ 63 ["--setpref"], 64 { 65 "dest": "extra_prefs", 66 "action": "append", 67 "default": [], 68 "help": "Extra user prefs.", 69 }, 70 ], 71 [ 72 ["--symbols-path=SYMBOLS_PATH"], 73 { 74 "dest": "symbols_path", 75 "help": "absolute path to directory containing breakpad " 76 "symbols, or the url of a zip file containing symbols.", 77 }, 78 ], 79 [ 80 ["--tag=TAG"], 81 { 82 "dest": "tag", 83 "help": "Subset of tests to run (local, remote).", 84 }, 85 ], 86 ] 87 + copy.deepcopy(testing_config_options) 88 + copy.deepcopy(code_coverage_config_options) 89 ) 90 91 92 class TelemetryTests(TestingMixin, VCSToolsScript, CodeCoverageMixin): 93 def __init__( 94 self, 95 config_options=None, 96 all_actions=None, 97 default_actions=None, 98 *args, 99 **kwargs, 100 ): 101 config_options = config_options or telemetry_tests_config_options 102 actions = [ 103 "clobber", 104 "download-and-extract", 105 "create-virtualenv", 106 "install", 107 "run-tests", 108 "uninstall", 109 ] 110 111 super().__init__( 112 config_options=config_options, 113 all_actions=all_actions or actions, 114 default_actions=default_actions or actions, 115 *args, 116 **kwargs, 117 ) 118 119 # Code which runs in automation has to include the following properties 120 self.binary_path = self.config.get("binary_path") 121 self.installer_path = self.config.get("installer_path") 122 self.installer_url = self.config.get("installer_url") 123 self.test_packages_url = self.config.get("test_packages_url") 124 self.test_url = self.config.get("test_url") 125 self.disable_fission = self.config.get("disable_fission") 126 127 if not self.test_url and not self.test_packages_url: 128 self.fatal("You must use --test-url, or --test-packages-url") 129 130 @PreScriptAction("create-virtualenv") 131 def _pre_create_virtualenv(self, action): 132 abs_dirs = self.query_abs_dirs() 133 134 requirements = os.path.join( 135 abs_dirs["abs_test_install_dir"], 136 "config", 137 "telemetry_tests_requirements.txt", 138 ) 139 self.register_virtualenv_module(requirements=[requirements]) 140 141 def query_abs_dirs(self): 142 if self.abs_dirs: 143 return self.abs_dirs 144 145 abs_dirs = super().query_abs_dirs() 146 147 abs_test_install_dir = os.path.join(abs_dirs["abs_work_dir"], "tests") 148 149 dirs = { 150 "abs_test_install_dir": abs_test_install_dir, 151 "abs_telemetry_dir": os.path.join( 152 abs_test_install_dir, "telemetry", "marionette" 153 ), 154 "abs_blob_upload_dir": os.path.join( 155 abs_dirs["abs_work_dir"], "blobber_upload_dir" 156 ), 157 } 158 159 for key in dirs: 160 if key not in abs_dirs: 161 abs_dirs[key] = dirs[key] 162 163 self.abs_dirs = abs_dirs 164 165 return self.abs_dirs 166 167 def run_test(self, binary_path, env=None, marionette_port=2828): 168 """All required steps for running the tests against an installer.""" 169 dirs = self.query_abs_dirs() 170 171 # Import the harness to retrieve the location of the cli scripts 172 import telemetry_harness 173 174 cmd = [ 175 self.query_python_path(), 176 os.path.join(os.path.dirname(telemetry_harness.__file__), self.cli_script), 177 "--binary", 178 binary_path, 179 "--address", 180 f"localhost:{marionette_port}", 181 # Resource files to serve via local webserver 182 "--server-root", 183 os.path.join(dirs["abs_telemetry_dir"], "harness", "www"), 184 # Use the work dir to get temporary data stored 185 "--workspace", 186 dirs["abs_work_dir"], 187 # logging options 188 "--gecko-log=-", # output from the gecko process redirected to stdout 189 "--log-raw=-", # structured log for output parser redirected to stdout 190 # additional reports helpful for Jenkins and inpection via Treeherder 191 "--log-html", 192 os.path.join(dirs["abs_blob_upload_dir"], "report.html"), 193 "--log-xunit", 194 os.path.join(dirs["abs_blob_upload_dir"], "report.xml"), 195 # Enable tracing output to log transmission protocol 196 "-vv", 197 ] 198 199 # Symbols for crash reports 200 if self.symbols_path: 201 cmd.extend(["--symbols-path", self.symbols_path]) 202 203 if self.disable_fission: 204 cmd.append("--disable-fission") 205 cmd.extend([f"--setpref={p}" for p in self.config["extra_prefs"]]) 206 207 if not self.config["e10s"]: 208 cmd.append("--disable-e10s") 209 210 parser = StructuredOutputParser( 211 config=self.config, log_obj=self.log_obj, strict=False 212 ) 213 214 # Add the default tests to run 215 tests = [ 216 os.path.join(dirs["abs_telemetry_dir"], "tests", test) 217 for test in self.default_tests 218 ] 219 cmd.extend(tests) 220 221 # Set further environment settings 222 env = env or self.query_env() 223 env.update({"MINIDUMP_SAVE_PATH": dirs["abs_blob_upload_dir"]}) 224 if self.query_minidump_stackwalk(): 225 env.update({"MINIDUMP_STACKWALK": self.minidump_stackwalk_path}) 226 env["RUST_BACKTRACE"] = "1" 227 env["MOZ_IGNORE_NSS_SHUTDOWN_LEAKS"] = "1" 228 229 # Causes Firefox to crash when using non-local connections. 230 env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" 231 232 # If code coverage is enabled, set GCOV_PREFIX env variable 233 if self.config.get("code_coverage"): 234 env["GCOV_PREFIX"] = self.gcov_dir 235 236 return_code = self.run_command( 237 cmd, 238 cwd=dirs["abs_work_dir"], 239 output_timeout=1000, 240 output_parser=parser, 241 env=env, 242 ) 243 244 tbpl_status, log_level, _ = parser.evaluate_parser(return_code) 245 self.record_status(tbpl_status, level=log_level) 246 247 return return_code 248 249 @PreScriptAction("run-tests") 250 def _pre_run_tests(self, action): 251 if not self.installer_path and not self.installer_url: 252 self.critical( 253 "Please specify an installer via --installer-path or --installer-url." 254 ) 255 sys.exit(1) 256 257 def run_tests(self): 258 """Run all the tests""" 259 return self.run_test( 260 binary_path=self.binary_path, 261 env=self.query_env(), 262 ) 263 264 265 class TelemetryClientTests(TelemetryTests): 266 cli_script = "runtests.py" 267 default_tests = [ 268 os.path.join("client", "manifest.toml"), 269 os.path.join("unit", "manifest.toml"), 270 ] 271 272 273 if __name__ == "__main__": 274 myScript = TelemetryClientTests() 275 myScript.run_and_exit()