test_errorsummary.py (11380B)
1 # -*- coding: utf-8 -*- 2 3 import json 4 import time 5 6 import mozunit 7 import pytest 8 9 # flake8: noqa 10 11 12 @pytest.mark.parametrize( 13 "logs,expected", 14 ( 15 pytest.param( 16 [ 17 ( 18 "suite_start", 19 { 20 "manifestA": ["test_foo", "test_bar", "test_baz"], 21 "manifestB": ["test_something"], 22 }, 23 ), 24 ("test_start", "test_foo"), 25 ("test_end", "test_foo", "SKIP"), 26 ("test_start", "test_bar"), 27 ("test_end", "test_bar", "OK"), 28 ("test_start", "test_something"), 29 ("test_end", "test_something", "OK"), 30 ("test_start", "test_baz"), 31 ("test_end", "test_baz", "PASS", "FAIL"), 32 ("suite_end",), 33 ], 34 """ 35 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 36 {"test": "test_baz", "subtest": null, "group": "manifestA", "status": "PASS", "expected": "FAIL", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 8} 37 {"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 9} 38 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9} 39 """.strip(), 40 id="basic", 41 ), 42 pytest.param( 43 [ 44 ("suite_start", {"manifest": ["test_foo"]}), 45 ("test_start", "test_foo"), 46 ("suite_end",), 47 ], 48 """ 49 {"groups": ["manifest"], "action": "test_groups", "line": 0} 50 {"group": "manifest", "status": null, "duration": 0, "action": "group_result", "line": 2} 51 """.strip(), 52 id="missing_test_end", 53 ), 54 pytest.param( 55 [ 56 ("suite_start", {"manifest": ["test_foo"]}), 57 ("test_start", "test_foo"), 58 ("test_status", "test_foo", "subtest", "PASS"), 59 ("suite_end",), 60 ], 61 """ 62 {"groups": ["manifest"], "action": "test_groups", "line": 0} 63 {"group": "manifest", "status": "ERROR", "duration": null, "action": "group_result", "line": 3} 64 """.strip(), 65 id="missing_test_end_with_test_status_ok", 66 marks=pytest.mark.xfail, # status is OK but should be ERROR 67 ), 68 pytest.param( 69 [ 70 ( 71 "suite_start", 72 { 73 "manifestA": ["test_foo", "test_bar", "test_baz"], 74 "manifestB": ["test_something"], 75 }, 76 ), 77 ("test_start", "test_foo"), 78 ("test_end", "test_foo", "SKIP"), 79 ("test_start", "test_bar"), 80 ("test_end", "test_bar", "CRASH"), 81 ("test_start", "test_something"), 82 ("test_end", "test_something", "OK"), 83 ("test_start", "test_baz"), 84 ("test_end", "test_baz", "FAIL", "FAIL"), 85 ("suite_end",), 86 ], 87 """ 88 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 89 {"test": "test_bar", "subtest": null, "group": "manifestA", "status": "CRASH", "expected": "OK", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 4} 90 {"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 9} 91 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9} 92 """.strip(), 93 id="crash_and_group_status", 94 ), 95 pytest.param( 96 [ 97 ( 98 "suite_start", 99 { 100 "manifestA": ["test_foo", "test_bar", "test_baz"], 101 "manifestB": ["test_something"], 102 }, 103 ), 104 ("test_start", "test_foo"), 105 ("test_end", "test_foo", "SKIP"), 106 ("test_start", "test_bar"), 107 ("test_end", "test_bar", "OK"), 108 ("test_start", "test_something"), 109 ("test_end", "test_something", "OK"), 110 ("test_start", "test_baz"), 111 ("test_status", "test_baz", "subtest", "FAIL", "FAIL"), 112 ("test_end", "test_baz", "OK"), 113 ("suite_end",), 114 ], 115 """ 116 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 117 {"group": "manifestA", "status": "OK", "duration": 29, "action": "group_result", "line": 10} 118 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 10} 119 """.strip(), 120 id="fail_expected_fail", 121 ), 122 pytest.param( 123 [ 124 ( 125 "suite_start", 126 { 127 "manifestA": ["test_foo", "test_bar", "test_baz"], 128 "manifestB": ["test_something"], 129 }, 130 ), 131 ("test_start", "test_foo"), 132 ("test_end", "test_foo", "SKIP"), 133 ("test_start", "test_bar"), 134 ("test_end", "test_bar", "OK"), 135 ("test_start", "test_something"), 136 ("test_end", "test_something", "OK"), 137 ("test_start", "test_baz"), 138 ("test_status", "test_baz", "Test timed out", "FAIL", "PASS"), 139 ("test_status", "test_baz", "", "TIMEOUT", "PASS"), 140 ("crash", "", "signature", "manifestA"), 141 ("test_end", "test_baz", "OK"), 142 ("suite_end",), 143 ], 144 """ 145 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 146 {"test": "test_baz", "subtest": "Test timed out", "group": "manifestA", "status": "FAIL", "expected": "PASS", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 8} 147 {"test": "test_baz", "subtest": "", "group": "manifestA", "status": "TIMEOUT", "expected": "PASS", "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 9} 148 {"test": "manifestA", "group": "manifestA", "signature": "signature", "stackwalk_stderr": null, "stackwalk_stdout": null, "action": "crash", "line": 10} 149 {"group": "manifestA", "status": "ERROR", "duration": 49, "action": "group_result", "line": 12} 150 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 12} 151 """.strip(), 152 id="timeout_and_crash", 153 ), 154 pytest.param( 155 [ 156 ( 157 "suite_start", 158 { 159 "manifestA": ["test_foo", "test_bar", "test_baz"], 160 "manifestB": ["test_something"], 161 }, 162 ), 163 ("test_start", "test_foo"), 164 ("test_end", "test_foo", "SKIP"), 165 ("test_start", "test_bar"), 166 ("test_end", "test_bar", "CRASH", "CRASH"), 167 ("test_start", "test_something"), 168 ("test_end", "test_something", "OK"), 169 ("test_start", "test_baz"), 170 ("test_end", "test_baz", "FAIL", "FAIL"), 171 ("suite_end",), 172 ], 173 """ 174 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 175 {"group": "manifestA", "status": "OK", "duration": 20, "action": "group_result", "line": 9} 176 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 9} 177 """.strip(), 178 id="crash_expected_crash", 179 ), 180 pytest.param( 181 [ 182 ( 183 "suite_start", 184 { 185 "manifestA": ["test_foo", "test_bar", "test_baz"], 186 "manifestB": ["test_something"], 187 }, 188 ), 189 ("test_start", "test_foo"), 190 ("test_end", "test_foo", "SKIP"), 191 ("test_start", "test_bar"), 192 ("test_end", "test_bar", "OK"), 193 ("test_start", "test_something"), 194 ("test_end", "test_something", "OK"), 195 ("test_start", "test_baz"), 196 ("test_end", "test_baz", "FAIL", "FAIL"), 197 ("crash", "", "", "manifestA"), 198 ("suite_end",), 199 ], 200 """ 201 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 202 {"test": "manifestA", "group": "manifestA", "signature": "", "stackwalk_stderr": null, "stackwalk_stdout": null, "action": "crash", "line": 9} 203 {"group": "manifestA", "status": "ERROR", "duration": 20, "action": "group_result", "line": 10} 204 {"group": "manifestB", "status": "OK", "duration": 10, "action": "group_result", "line": 10} 205 """.strip(), 206 id="assertion_crash_on_shutdown", 207 ), 208 pytest.param( 209 [ 210 ( 211 "suite_start", 212 { 213 "manifestA": ["test_foo", "test_bar", "test_baz"], 214 "manifestB": ["test_something"], 215 }, 216 ), 217 ("test_start", "test_foo"), 218 ("test_end", "test_foo", "SKIP"), 219 ("test_start", "test_bar"), 220 ("test_end", "test_bar", "OK"), 221 ("test_start", "test_something"), 222 ("test_end", "test_something", "OK"), 223 ("test_start", "test_baz"), 224 ("test_end", "test_baz", "FAIL"), 225 ], 226 """ 227 {"groups": ["manifestA", "manifestB"], "action": "test_groups", "line": 0} 228 {"test": "test_baz", "group": "manifestA", "status": "FAIL", "expected": "OK", "subtest": null, "message": null, "stack": null, "modifiers": "", "known_intermittent": [], "action": "test_result", "line": 8} 229 """.strip(), 230 id="timeout_no_group_status", 231 ), 232 ), 233 ) 234 def test_errorsummary(monkeypatch, get_logger, logs, expected): 235 ts = {"ts": 0.0} # need to use dict since 'nonlocal' doesn't exist on PY2 236 237 def fake_time(): 238 ts["ts"] += 0.01 239 return ts["ts"] 240 241 monkeypatch.setattr(time, "time", fake_time) 242 logger = get_logger("errorsummary") 243 244 for log in logs: 245 getattr(logger, log[0])(*log[1:]) 246 247 buf = logger.handlers[0].stream 248 result = buf.getvalue() 249 print("Dumping result for copy/paste:") 250 print(result) 251 252 expected = expected.split("\n") 253 for i, line in enumerate(result.split("\n")): 254 if not line: 255 continue 256 257 data = json.loads(line) 258 assert data == json.loads(expected[i]) 259 260 261 if __name__ == "__main__": 262 mozunit.main()