runner.py (9500B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import errno 6 import os 7 import shutil 8 import tempfile 9 10 import pytest 11 12 GVE = "org.mozilla.geckoview_example" 13 14 15 def run( 16 logger, 17 path, 18 webdriver_binary, 19 webdriver_port, 20 webdriver_ws_port, 21 browser_binary=None, 22 device_serial=None, 23 package_name=None, 24 environ=None, 25 bugs=None, 26 debug=False, 27 interventions=None, 28 shims=None, 29 config=None, 30 headless=False, 31 addon=None, 32 do2fa=False, 33 log_level="INFO", 34 failure_screenshots_dir=None, 35 no_failure_screenshots=None, 36 platform_override=None, 37 ): 38 """""" 39 old_environ = os.environ.copy() 40 try: 41 with TemporaryDirectory() as cache: 42 if environ: 43 os.environ.update(environ) 44 45 config_plugin = WDConfig() 46 result_recorder = ResultRecorder(logger) 47 48 args = [ 49 "--strict", # turn warnings into errors 50 "-vv", # show each individual subtest and full failure logs 51 "--capture", 52 "no", # enable stdout/stderr from tests 53 "--basetemp", 54 cache, # temporary directory 55 "--showlocals", # display contents of variables in local scope 56 "-p", 57 "no:mozlog", # use the WPT result recorder 58 "--disable-warnings", 59 "-rfEs", 60 "-p", 61 "no:cacheprovider", # disable state preservation across invocations 62 "-o=console_output_style=classic", # disable test progress bar 63 "--browser", 64 "firefox", 65 "--webdriver-binary", 66 webdriver_binary, 67 "--webdriver-port", 68 webdriver_port, 69 "--webdriver-ws-port", 70 webdriver_ws_port, 71 "--webdriver-log-level", 72 log_level, 73 "--capture=no", 74 "--show-capture=no", 75 ] 76 77 if debug: 78 args.append("--pdb") 79 80 if headless: 81 args.append("--headless") 82 83 if browser_binary: 84 args.append("--browser-binary") 85 args.append(browser_binary) 86 87 if device_serial: 88 args.append("--device-serial") 89 args.append(device_serial) 90 91 if package_name: 92 args.append("--package-name") 93 args.append(package_name) 94 95 if addon: 96 args.append("--addon") 97 args.append(addon) 98 99 if do2fa: 100 args.append("--do2fa") 101 102 if config: 103 args.append("--config") 104 args.append(config) 105 106 if failure_screenshots_dir: 107 args.append("--failure-screenshots-dir") 108 args.append(failure_screenshots_dir) 109 110 if platform_override: 111 args.append("--platform-override") 112 args.append(platform_override) 113 114 if no_failure_screenshots: 115 args.append("--no-failure-screenshots") 116 117 if interventions is not None and shims is not None: 118 raise ValueError( 119 "Must provide only one of interventions or shims argument" 120 ) 121 elif interventions is None and shims is None: 122 raise ValueError( 123 "Must provide either an interventions or shims argument" 124 ) 125 126 name = "webcompat-interventions" 127 if interventions == "enabled": 128 args.extend(["-m", "with_interventions"]) 129 elif interventions == "disabled": 130 args.extend(["-m", "without_interventions"]) 131 elif interventions is not None: 132 raise ValueError(f"Invalid value for interventions {interventions}") 133 if shims == "enabled": 134 args.extend(["-m", "with_shims"]) 135 name = "smartblock-shims" 136 elif shims == "disabled": 137 args.extend(["-m", "without_shims"]) 138 name = "smartblock-shims" 139 elif shims is not None: 140 raise ValueError(f"Invalid value for shims {shims}") 141 else: 142 name = "smartblock-shims" 143 144 if bugs is not None: 145 args.extend(["-k", " or ".join(bugs)]) 146 147 args.append(path) 148 try: 149 logger.suite_start([], name=name) 150 pytest.main(args, plugins=[config_plugin, result_recorder]) 151 except Exception as e: 152 logger.critical(str(e)) 153 finally: 154 logger.suite_end() 155 156 finally: 157 os.environ = old_environ 158 159 160 class WDConfig: 161 def pytest_addoption(self, parser): 162 parser.addoption( 163 "--browser-binary", action="store", help="Path to browser binary" 164 ) 165 parser.addoption( 166 "--webdriver-binary", action="store", help="Path to webdriver binary" 167 ) 168 parser.addoption( 169 "--webdriver-port", 170 action="store", 171 default="4444", 172 help="Port on which to run WebDriver", 173 ) 174 parser.addoption( 175 "--webdriver-ws-port", 176 action="store", 177 default="9222", 178 help="Port on which to run WebDriver BiDi websocket", 179 ) 180 parser.addoption( 181 "--webdriver-log-level", 182 action="store", 183 default="INFO", 184 help="Log level to use for WebDriver", 185 ) 186 parser.addoption( 187 "--browser", action="store", choices=["firefox"], help="Name of the browser" 188 ) 189 parser.addoption( 190 "--do2fa", 191 action="store_true", 192 default=False, 193 help="Do two-factor auth live in supporting tests", 194 ) 195 parser.addoption( 196 "--failure-screenshots-dir", 197 action="store", 198 help="Path to save failure screenshots", 199 ) 200 parser.addoption( 201 "--no-failure-screenshots", 202 action="store_true", 203 default=False, 204 help="Do not save screenshots on failure", 205 ) 206 parser.addoption( 207 "--config", 208 action="store", 209 help="Path to JSON file containing logins and other settings", 210 ) 211 parser.addoption( 212 "--addon", 213 action="store", 214 help="Path to the webcompat addon XPI to use", 215 ) 216 parser.addoption( 217 "--device-serial", 218 action="store", 219 help="Emulator device serial number", 220 ) 221 parser.addoption( 222 "--package-name", 223 action="store", 224 default=GVE, 225 help="Android package to run/connect to", 226 ) 227 parser.addoption( 228 "--headless", action="store_true", help="Run browser in headless mode" 229 ) 230 parser.addoption( 231 "--platform-override", 232 action="store", 233 choices=["android", "linux", "mac", "windows"], 234 help="Override key navigator properties to match the given platform and/or use responsive design mode to mimic the given platform", 235 ) 236 237 238 class ResultRecorder: 239 def __init__(self, logger): 240 self.logger = logger 241 242 def pytest_runtest_logreport(self, report): 243 if report.passed and report.when == "call": 244 self.record_pass(report) 245 elif report.failed: 246 if report.when != "call": 247 self.record_error(report) 248 else: 249 self.record_fail(report) 250 elif report.skipped: 251 self.record_skip(report) 252 253 def record_pass(self, report): 254 self.record(report.nodeid, "PASS") 255 256 def record_fail(self, report): 257 # pytest outputs the stacktrace followed by an error message prefixed 258 # with "E ", e.g. 259 # 260 # def test_example(): 261 # > assert "fuu" in "foobar" 262 # > E AssertionError: assert 'fuu' in 'foobar' 263 message = "" 264 for line in report.longreprtext.splitlines(): 265 if line.startswith("E "): 266 message = line[1:].strip() 267 break 268 269 self.record(report.nodeid, "FAIL", message=message, stack=report.longrepr) 270 271 def record_error(self, report): 272 # error in setup/teardown 273 if report.when != "call": 274 message = f"{report.when} error" 275 self.record(report.nodeid, "ERROR", message, report.longrepr) 276 277 def record_skip(self, report): 278 self.record(report.nodeid, "SKIP") 279 280 def record(self, test, status, message=None, stack=None): 281 if stack is not None: 282 stack = str(stack) 283 self.logger.test_start(test) 284 self.logger.test_end( 285 test=test, status=status, expected="PASS", message=message, stack=stack 286 ) 287 288 289 class TemporaryDirectory: 290 def __enter__(self): 291 self.path = tempfile.mkdtemp(prefix="pytest-") 292 return self.path 293 294 def __exit__(self, *args): 295 try: 296 shutil.rmtree(self.path) 297 except OSError as e: 298 # no such file or directory 299 if e.errno != errno.ENOENT: 300 raise