test_mochitest_integration.py (11864B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import os 6 from functools import partial 7 8 import mozunit 9 import pytest 10 from conftest import setup_args 11 from manifestparser import TestManifest 12 from mozharness.base.log import ERROR, INFO, WARNING 13 from mozharness.mozilla.automation import TBPL_FAILURE, TBPL_SUCCESS, TBPL_WARNING 14 from moztest.selftest.output import filter_action, get_mozharness_status 15 16 here = os.path.abspath(os.path.dirname(__file__)) 17 get_mozharness_status = partial(get_mozharness_status, "mochitest") 18 19 20 @pytest.fixture 21 def test_name(request): 22 flavor = request.getfixturevalue("flavor") 23 24 def inner(name): 25 if flavor == "plain": 26 return f"test_{name}.html" 27 elif flavor == "browser-chrome": 28 return f"browser_{name}.js" 29 30 return inner 31 32 33 @pytest.fixture 34 def test_manifest(setup_test_harness, request): 35 flavor = request.getfixturevalue("flavor") 36 test_root = setup_test_harness(*setup_args, flavor=flavor) 37 assert test_root 38 39 def inner(manifestFileNames): 40 return TestManifest( 41 manifests=[os.path.join(test_root, name) for name in manifestFileNames], 42 strict=False, 43 rootdir=test_root, 44 ) 45 46 return inner 47 48 49 @pytest.mark.parametrize( 50 "flavor,manifest", 51 [ 52 ("plain", "mochitest-args.ini"), 53 ("browser-chrome", "browser-args.toml"), 54 ], 55 ) 56 def test_output_extra_args(flavor, manifest, runtests, test_manifest, test_name): 57 # Explicitly provide a manifestFile property that includes the 58 # manifest file that contains command line arguments. 59 extra_opts = { 60 "manifestFile": test_manifest([manifest]), 61 "runByManifest": True, 62 } 63 64 results = { 65 "status": 0, 66 "tbpl_status": TBPL_SUCCESS, 67 "log_level": (INFO, WARNING), 68 } 69 70 status, lines = runtests(test_name("pass"), **extra_opts) 71 assert status == results["status"] 72 73 tbpl_status, log_level, _ = get_mozharness_status(lines, status) 74 assert tbpl_status == results["tbpl_status"] 75 assert log_level in results["log_level"] 76 77 # Filter log entries for the application command including the used 78 # command line arguments. 79 lines = filter_action("log", lines) 80 command = next( 81 l["message"] for l in lines if l["message"].startswith("Application command") 82 ) 83 assert "--headless --window-size 800,600 --new-tab http://example.org" in command 84 85 86 @pytest.mark.parametrize("runFailures", ["selftest", ""]) 87 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 88 def test_output_pass(flavor, runFailures, runtests, test_name): 89 extra_opts = {} 90 results = { 91 "status": 1 if runFailures else 0, 92 "tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS, 93 "log_level": (INFO, WARNING), 94 "lines": 2 if runFailures else 1, 95 "line_status": "PASS", 96 } 97 if runFailures: 98 extra_opts["runFailures"] = runFailures 99 extra_opts["crashAsPass"] = True 100 extra_opts["timeoutAsPass"] = True 101 102 status, lines = runtests(test_name("pass"), **extra_opts) 103 assert status == results["status"] 104 105 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 106 assert tbpl_status == results["tbpl_status"] 107 assert log_level in results["log_level"] 108 109 lines = filter_action("test_status", lines) 110 assert len(lines) == results["lines"] 111 assert lines[0]["status"] == results["line_status"] 112 113 114 @pytest.mark.parametrize("runFailures", ["selftest", ""]) 115 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 116 def test_output_fail(flavor, runFailures, runtests, test_name): 117 extra_opts = {} 118 results = { 119 "status": 0 if runFailures else 1, 120 "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING, 121 "log_level": (INFO, WARNING), 122 "lines": 1, 123 "line_status": "PASS" if runFailures else "FAIL", 124 } 125 if runFailures: 126 extra_opts["runFailures"] = runFailures 127 extra_opts["crashAsPass"] = True 128 extra_opts["timeoutAsPass"] = True 129 130 status, lines = runtests(test_name("fail"), **extra_opts) 131 assert status == results["status"] 132 133 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 134 assert tbpl_status == results["tbpl_status"] 135 assert log_level in results["log_level"] 136 137 lines = filter_action("test_status", lines) 138 assert len(lines) == results["lines"] 139 assert lines[0]["status"] == results["line_status"] 140 141 142 @pytest.mark.parametrize("runFailures", ["selftest", ""]) 143 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 144 def test_output_restart_after_failure(flavor, runFailures, runtests, test_name): 145 extra_opts = {} 146 results = { 147 "status": 0 if runFailures else 1, 148 "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING, 149 "log_level": (INFO, WARNING), 150 "lines": 2, 151 "line_status": "PASS" if runFailures else "FAIL", 152 } 153 extra_opts["restartAfterFailure"] = True 154 if runFailures: 155 extra_opts["runFailures"] = runFailures 156 extra_opts["crashAsPass"] = True 157 extra_opts["timeoutAsPass"] = True 158 159 tests = [test_name("fail"), test_name("fail2")] 160 status, lines = runtests(tests, **extra_opts) 161 assert status == results["status"] 162 163 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 164 assert tbpl_status == results["tbpl_status"] 165 assert log_level in results["log_level"] 166 167 # Ensure the harness cycled when failing (look for launching browser) 168 start_lines = [ 169 line for line in lines if "Application command:" in line.get("message", "") 170 ] 171 if not runFailures: 172 assert len(start_lines) == results["lines"] 173 else: 174 assert len(start_lines) == 1 175 176 177 @pytest.mark.skip_mozinfo("!crashreporter") 178 @pytest.mark.parametrize("runFailures", ["selftest", ""]) 179 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 180 def test_output_crash(flavor, runFailures, runtests, test_name): 181 extra_opts = {} 182 results = { 183 "status": 0 if runFailures else 1, 184 "tbpl_status": TBPL_FAILURE, 185 "log_level": ERROR, 186 "lines": 1, 187 } 188 if runFailures: 189 extra_opts["runFailures"] = runFailures 190 extra_opts["crashAsPass"] = True 191 extra_opts["timeoutAsPass"] = True 192 # bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome 193 # the error regex's don't pick this up as a failure 194 if flavor == "browser-chrome": 195 results["tbpl_status"] = TBPL_SUCCESS 196 results["log_level"] = (INFO, WARNING) 197 198 status, lines = runtests( 199 test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts 200 ) 201 assert status == results["status"] 202 203 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 204 assert tbpl_status == results["tbpl_status"] 205 assert log_level in results["log_level"] 206 207 if not runFailures: 208 crash = filter_action("crash", lines) 209 assert len(crash) == 1 210 assert crash[0]["action"] == "crash" 211 assert crash[0]["signature"] 212 assert crash[0]["minidump_path"] 213 214 lines = filter_action("test_end", lines) 215 assert len(lines) == results["lines"] 216 217 218 @pytest.mark.skip_mozinfo("!asan") 219 @pytest.mark.parametrize("runFailures", [""]) 220 @pytest.mark.parametrize("flavor", ["plain"]) 221 def test_output_asan(flavor, runFailures, runtests, test_name): 222 extra_opts = {} 223 results = { 224 "status": 1, 225 "tbpl_status": TBPL_FAILURE, 226 "log_level": ERROR, 227 "lines": 0, 228 } 229 230 status, lines = runtests( 231 test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts 232 ) 233 assert status == results["status"] 234 235 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 236 assert tbpl_status == results["tbpl_status"] 237 assert log_level == results["log_level"] 238 239 crash = filter_action("crash", lines) 240 assert len(crash) == results["lines"] 241 242 process_output = filter_action("process_output", lines) 243 assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output) 244 245 246 @pytest.mark.skip_mozinfo("!debug") 247 @pytest.mark.parametrize("runFailures", [""]) 248 @pytest.mark.parametrize("flavor", ["plain"]) 249 def test_output_assertion(flavor, runFailures, runtests, test_name): 250 extra_opts = {} 251 results = { 252 "status": 0, 253 "tbpl_status": TBPL_WARNING, 254 "log_level": WARNING, 255 "lines": 1, 256 "assertions": 1, 257 } 258 259 status, lines = runtests(test_name("assertion"), **extra_opts) 260 # TODO: mochitest should return non-zero here 261 assert status == results["status"] 262 263 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 264 assert tbpl_status == results["tbpl_status"] 265 assert log_level == results["log_level"] 266 267 # assertion failure prints error, but still emits test-ok 268 test_end = filter_action("test_end", lines) 269 assert len(test_end) == results["lines"] 270 assert test_end[0]["status"] == "OK" 271 272 assertions = filter_action("assertion_count", lines) 273 assert len(assertions) == results["assertions"] 274 assert assertions[0]["count"] == results["assertions"] 275 276 277 @pytest.mark.skip_mozinfo("!debug") 278 @pytest.mark.parametrize("runFailures", [""]) 279 @pytest.mark.parametrize("flavor", ["plain"]) 280 def test_output_leak(flavor, runFailures, runtests, test_name): 281 extra_opts = {} 282 results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING} 283 284 status, lines = runtests(test_name("leak"), **extra_opts) 285 # TODO: mochitest should return non-zero here 286 assert status == results["status"] 287 288 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 289 assert tbpl_status == results["tbpl_status"] 290 assert log_level == results["log_level"] 291 292 leak_totals = filter_action("mozleak_total", lines) 293 found_leaks = False 294 for lt in leak_totals: 295 if lt["bytes"] == 0: 296 # No leaks in this process. 297 assert len(lt["objects"]) == 0 298 continue 299 300 assert not found_leaks, "Only one process should have leaked" 301 found_leaks = True 302 assert lt["process"] == "tab" 303 assert lt["bytes"] == 1 304 assert lt["objects"] == ["IntentionallyLeakedObject"] 305 306 assert found_leaks, "At least one process should have leaked" 307 308 309 @pytest.mark.parametrize("flavor", ["plain"]) 310 def test_output_testfile_in_dupe_manifests(flavor, runtests, test_name, test_manifest): 311 results = { 312 "status": 0, 313 "tbpl_status": TBPL_SUCCESS, 314 "log_level": (INFO, WARNING), 315 "line_status": "PASS", 316 # We expect the test to be executed exactly 2 times, 317 # once for each manifest where the test file has been included. 318 "lines": 2, 319 } 320 321 # Explicitly provide a manifestFile property that includes the 322 # two manifest files that share the same test file. 323 extra_opts = { 324 "manifestFile": test_manifest([ 325 "mochitest-dupemanifest-1.ini", 326 "mochitest-dupemanifest-2.ini", 327 ]), 328 "runByManifest": True, 329 } 330 331 # Execute mochitest by explicitly request the test file listed 332 # in two manifest files to be executed. 333 status, lines = runtests(test_name("pass"), **extra_opts) 334 assert status == results["status"] 335 336 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 337 assert tbpl_status == results["tbpl_status"] 338 assert log_level in results["log_level"] 339 340 lines = filter_action("test_status", lines) 341 assert len(lines) == results["lines"] 342 assert lines[0]["status"] == results["line_status"] 343 344 345 if __name__ == "__main__": 346 mozunit.main()