awsy_script.py (11080B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 """ 6 run awsy tests in a virtualenv 7 """ 8 9 import copy 10 import json 11 import os 12 import re 13 import sys 14 15 # load modules from parent dir 16 sys.path.insert(1, os.path.dirname(sys.path[0])) 17 18 import mozharness 19 import mozinfo 20 from mozharness.base.log import ERROR, INFO 21 from mozharness.base.script import PreScriptAction 22 from mozharness.base.vcs.vcsbase import MercurialScript 23 from mozharness.mozilla.structuredlog import StructuredOutputParser 24 from mozharness.mozilla.testing.codecoverage import ( 25 CodeCoverageMixin, 26 code_coverage_config_options, 27 ) 28 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 29 from mozharness.mozilla.tooltool import TooltoolMixin 30 31 PY2 = sys.version_info.major == 2 32 scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) 33 external_tools_path = os.path.join(scripts_path, "external_tools") 34 35 36 class AWSY(TestingMixin, MercurialScript, TooltoolMixin, CodeCoverageMixin): 37 config_options = ( 38 [ 39 [ 40 ["--disable-e10s"], 41 { 42 "action": "store_false", 43 "dest": "e10s", 44 "default": True, 45 "help": "Run tests without multiple processes (e10s). (Desktop builds only)", 46 }, 47 ], 48 [ 49 ["--setpref"], 50 { 51 "action": "append", 52 "dest": "extra_prefs", 53 "default": [], 54 "help": "Extra user prefs.", 55 }, 56 ], 57 [ 58 ["--base"], 59 { 60 "action": "store_true", 61 "dest": "test_about_blank", 62 "default": False, 63 "help": "Runs the about:blank base case memory test.", 64 }, 65 ], 66 [ 67 ["--dmd"], 68 { 69 "action": "store_true", 70 "dest": "dmd", 71 "default": False, 72 "help": "Runs tests with DMD enabled.", 73 }, 74 ], 75 [ 76 ["--tp5"], 77 { 78 "action": "store_true", 79 "dest": "tp5", 80 "default": False, 81 "help": "Runs tests with the tp5 pageset.", 82 }, 83 ], 84 ] 85 + testing_config_options 86 + copy.deepcopy(code_coverage_config_options) 87 ) 88 89 error_list = [ 90 {"regex": re.compile(r"""(TEST-UNEXPECTED|PROCESS-CRASH)"""), "level": ERROR}, 91 ] 92 93 def __init__(self, **kwargs): 94 kwargs.setdefault("config_options", self.config_options) 95 kwargs.setdefault( 96 "all_actions", 97 [ 98 "clobber", 99 "download-and-extract", 100 "populate-webroot", 101 "create-virtualenv", 102 "install", 103 "run-tests", 104 ], 105 ) 106 kwargs.setdefault( 107 "default_actions", 108 [ 109 "clobber", 110 "download-and-extract", 111 "populate-webroot", 112 "create-virtualenv", 113 "install", 114 "run-tests", 115 ], 116 ) 117 kwargs.setdefault("config", {}) 118 super().__init__(**kwargs) 119 self.installer_url = self.config.get("installer_url") 120 self.tests = None 121 122 self.testdir = self.query_abs_dirs()["abs_test_install_dir"] 123 self.awsy_path = os.path.join(self.testdir, "awsy") 124 self.awsy_libdir = os.path.join(self.awsy_path, "awsy") 125 self.webroot_dir = os.path.join(self.testdir, "html") 126 self.results_dir = os.path.join(self.testdir, "results") 127 self.binary_path = self.config.get("binary_path") 128 129 def query_abs_dirs(self): 130 if self.abs_dirs: 131 return self.abs_dirs 132 abs_dirs = super().query_abs_dirs() 133 134 dirs = {} 135 dirs["abs_blob_upload_dir"] = os.path.join( 136 abs_dirs["abs_work_dir"], "blobber_upload_dir" 137 ) 138 dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_work_dir"], "tests") 139 abs_dirs.update(dirs) 140 self.abs_dirs = abs_dirs 141 return self.abs_dirs 142 143 def download_and_extract(self, extract_dirs=None, suite_categories=None): 144 ret = super().download_and_extract(suite_categories=["common", "awsy"]) 145 return ret 146 147 @PreScriptAction("create-virtualenv") 148 def _pre_create_virtualenv(self, action): 149 requirements_files = [ 150 os.path.join(self.testdir, "config", "marionette_requirements.txt") 151 ] 152 153 for requirements_file in requirements_files: 154 self.register_virtualenv_module(requirements=[requirements_file]) 155 156 self.register_virtualenv_module("awsy", self.awsy_path) 157 158 def populate_webroot(self): 159 """Populate the production test machines' webroots""" 160 self.info("Downloading pageset with tooltool...") 161 manifest_file = os.path.join(self.awsy_path, "tp5n-pageset.manifest") 162 page_load_test_dir = os.path.join(self.webroot_dir, "page_load_test") 163 if not os.path.isdir(page_load_test_dir): 164 self.mkdir_p(page_load_test_dir) 165 self.tooltool_fetch( 166 manifest_file, 167 output_dir=page_load_test_dir, 168 cache=self.config.get("tooltool_cache"), 169 ) 170 archive = os.path.join(page_load_test_dir, "tp5n.zip") 171 unzip = self.query_exe("unzip") 172 unzip_cmd = [unzip, "-q", "-o", archive, "-d", page_load_test_dir] 173 self.run_command(unzip_cmd, halt_on_failure=False) 174 self.run_command("ls %s" % page_load_test_dir) 175 176 def run_tests(self, args=None, **kw): 177 """ 178 AWSY test should be implemented here 179 """ 180 dirs = self.abs_dirs 181 env = {} 182 error_summary_file = os.path.join( 183 dirs["abs_blob_upload_dir"], "marionette_errorsummary.log" 184 ) 185 186 runtime_testvars = { 187 "webRootDir": self.webroot_dir, 188 "resultsDir": self.results_dir, 189 "bin": self.binary_path, 190 } 191 192 # Check if this is a DMD build and if so enable it. 193 dmd_enabled = False 194 dmd_py_lib_dir = os.path.dirname(self.binary_path) 195 if mozinfo.os == "mac": 196 # On mac binary is in MacOS and dmd.py is in Resources, ie: 197 # Name.app/Contents/MacOS/libdmd.dylib 198 # Name.app/Contents/Resources/dmd.py 199 dmd_py_lib_dir = os.path.join(dmd_py_lib_dir, "../Resources/") 200 201 dmd_path = os.path.join(dmd_py_lib_dir, "dmd.py") 202 if self.config["dmd"] and os.path.isfile(dmd_path): 203 dmd_enabled = True 204 runtime_testvars["dmd"] = True 205 206 # Allow the child process to import dmd.py 207 python_path = os.environ.get("PYTHONPATH") 208 209 if python_path: 210 os.environ["PYTHONPATH"] = "%s%s%s" % ( 211 python_path, 212 os.pathsep, 213 dmd_py_lib_dir, 214 ) 215 else: 216 os.environ["PYTHONPATH"] = dmd_py_lib_dir 217 218 env["DMD"] = "--mode=dark-matter --stacks=full" 219 220 runtime_testvars["tp5"] = self.config["tp5"] 221 if not self.config["tp5"]: 222 # mitmproxy needs path to mozharness when installing the cert, and tooltool 223 env["SCRIPTSPATH"] = scripts_path 224 env["EXTERNALTOOLSPATH"] = external_tools_path 225 226 runtime_testvars_path = os.path.join(self.awsy_path, "runtime-testvars.json") 227 runtime_testvars_file = open(runtime_testvars_path, "wb" if PY2 else "w") 228 runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) 229 runtime_testvars_file.close() 230 231 cmd = ["marionette"] 232 233 test_vars_file = None 234 if self.config["test_about_blank"]: 235 test_vars_file = "base-testvars.json" 236 else: 237 test_vars_file = "tp6-testvars.json" 238 if self.config["tp5"]: 239 test_vars_file = "testvars.json" 240 241 cmd.append( 242 "--testvars=%s" % os.path.join(self.awsy_path, "conf", test_vars_file) 243 ) 244 cmd.append("--testvars=%s" % runtime_testvars_path) 245 cmd.append("--log-raw=-") 246 cmd.append("--log-errorsummary=%s" % error_summary_file) 247 cmd.append("--binary=%s" % self.binary_path) 248 cmd.append("--profile=%s" % (os.path.join(dirs["abs_work_dir"], "profile"))) 249 if not self.config["e10s"]: 250 cmd.append("--disable-e10s") 251 cmd.extend([f"--setpref={p}" for p in self.config["extra_prefs"]]) 252 cmd.append( 253 "--gecko-log=%s" % os.path.join(dirs["abs_blob_upload_dir"], "gecko.log") 254 ) 255 # TestingMixin._download_and_extract_symbols() should set 256 # self.symbols_path 257 cmd.append("--symbols-path=%s" % self.symbols_path) 258 259 if self.config["test_about_blank"]: 260 test_file = os.path.join(self.awsy_libdir, "test_base_memory_usage.py") 261 prefs_file = "base-prefs.json" 262 else: 263 test_file = os.path.join(self.awsy_libdir, "test_memory_usage.py") 264 prefs_file = "tp6-prefs.json" 265 if self.config["tp5"]: 266 prefs_file = "prefs.json" 267 268 cmd.append( 269 "--preferences=%s" % os.path.join(self.awsy_path, "conf", prefs_file) 270 ) 271 if dmd_enabled: 272 cmd.append("--setpref=security.sandbox.content.level=0") 273 cmd.append("--setpref=layout.css.stylo-threads=4") 274 275 cmd.append(test_file) 276 277 env["MOZ_UPLOAD_DIR"] = dirs["abs_blob_upload_dir"] 278 if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): 279 self.mkdir_p(env["MOZ_UPLOAD_DIR"]) 280 if self.query_minidump_stackwalk(): 281 env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path 282 env["MINIDUMP_SAVE_PATH"] = dirs["abs_blob_upload_dir"] 283 env["RUST_BACKTRACE"] = "1" 284 env = self.query_env(partial_env=env) 285 parser = StructuredOutputParser( 286 config=self.config, 287 log_obj=self.log_obj, 288 error_list=self.error_list, 289 strict=False, 290 ) 291 return_code = self.run_command( 292 command=cmd, 293 cwd=self.awsy_path, 294 output_timeout=self.config.get("cmd_timeout"), 295 env=env, 296 output_parser=parser, 297 ) 298 299 level = INFO 300 tbpl_status, log_level, summary = parser.evaluate_parser( 301 return_code=return_code 302 ) 303 304 self.log( 305 "AWSY exited with return code %s: %s" % (return_code, tbpl_status), 306 level=level, 307 ) 308 self.record_status(tbpl_status) 309 310 311 if __name__ == "__main__": 312 awsy_test = AWSY() 313 awsy_test.run_and_exit()