test_reftest_output.py (4926B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import os 6 7 try: 8 # Python2 9 from cStringIO import StringIO 10 except ImportError: 11 # Python3 12 from io import StringIO 13 14 from functools import partial 15 16 import mozunit 17 import pytest 18 from mozharness.base.log import ERROR, INFO, WARNING 19 from mozharness.mozilla.automation import TBPL_FAILURE, TBPL_SUCCESS, TBPL_WARNING 20 from moztest.selftest.output import filter_action, get_mozharness_status 21 22 here = os.path.abspath(os.path.dirname(__file__)) 23 get_mozharness_status = partial(get_mozharness_status, "reftest") 24 25 26 def test_output_pass(runtests): 27 status, lines = runtests("reftest-pass.list") 28 assert status == 0 29 30 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 31 assert tbpl_status == TBPL_SUCCESS 32 assert log_level in (INFO, WARNING) 33 34 test_status = filter_action("test_status", lines) 35 assert len(test_status) == 3 36 assert all(t["status"] == "PASS" for t in test_status) 37 38 test_end = filter_action("test_end", lines) 39 assert len(test_end) == 3 40 assert all(t["status"] == "OK" for t in test_end) 41 42 43 def test_output_fail(runtests): 44 formatter = pytest.importorskip("output").ReftestFormatter() 45 46 status, lines = runtests("reftest-fail.list") 47 assert status == 0 48 49 buf = StringIO() 50 tbpl_status, log_level, summary = get_mozharness_status( 51 lines, status, formatter=formatter, buf=buf 52 ) 53 54 assert tbpl_status == TBPL_WARNING 55 assert log_level == WARNING 56 57 test_status = filter_action("test_status", lines) 58 assert len(test_status) == 3 59 assert all(t["status"] == "FAIL" for t in test_status) 60 assert all("reftest_screenshots" in t["extra"] for t in test_status) 61 62 test_end = filter_action("test_end", lines) 63 assert len(test_end) == 3 64 assert all(t["status"] == "OK" for t in test_end) 65 66 # ensure screenshots were printed 67 formatted = buf.getvalue() 68 assert "REFTEST IMAGE 1" in formatted 69 assert "REFTEST IMAGE 2" in formatted 70 71 72 @pytest.mark.skip_mozinfo("!crashreporter") 73 def test_output_crash(runtests): 74 status, lines = runtests( 75 "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] 76 ) 77 assert status == 245 78 79 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 80 assert tbpl_status == TBPL_FAILURE 81 assert log_level == ERROR 82 83 crash = filter_action("crash", lines) 84 assert len(crash) == 1 85 assert crash[0]["action"] == "crash" 86 assert crash[0]["signature"] 87 assert crash[0]["minidump_path"] 88 89 lines = filter_action("test_end", lines) 90 assert len(lines) == 0 91 92 93 @pytest.mark.skip_mozinfo("!asan") 94 def test_output_asan(runtests): 95 status, lines = runtests( 96 "reftest-crash.list", environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] 97 ) 98 assert status == 245 99 100 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 101 assert tbpl_status == TBPL_FAILURE 102 assert log_level == ERROR 103 104 crash = filter_action("crash", lines) 105 assert len(crash) == 0 106 107 process_output = filter_action("process_output", lines) 108 assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output) 109 110 111 @pytest.mark.skip_mozinfo("!debug") 112 def test_output_assertion(runtests): 113 status, lines = runtests("reftest-assert.list") 114 assert status == 0 115 116 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 117 assert tbpl_status == TBPL_WARNING 118 assert log_level == WARNING 119 120 test_status = filter_action("test_status", lines) 121 assert len(test_status) == 1 122 assert test_status[0]["status"] == "PASS" 123 124 test_end = filter_action("test_end", lines) 125 assert len(test_end) == 1 126 assert test_end[0]["status"] == "OK" 127 128 assertions = filter_action("assertion_count", lines) 129 assert len(assertions) == 1 130 assert assertions[0]["count"] == 1 131 132 133 @pytest.mark.skip_mozinfo("!debug") 134 def test_output_leak(monkeypatch, runtests): 135 # Monkeypatch mozleak so we always process a failing leak log 136 # instead of the actual one. 137 import mozleak 138 139 old_process_leak_log = mozleak.process_leak_log 140 141 def process_leak_log(*args, **kwargs): 142 return old_process_leak_log( 143 os.path.join(here, "files", "leaks.log"), *args[1:], **kwargs 144 ) 145 146 monkeypatch.setattr("mozleak.process_leak_log", process_leak_log) 147 148 status, lines = runtests("reftest-pass.list") 149 assert status == 0 150 151 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 152 assert tbpl_status == TBPL_WARNING 153 assert log_level == WARNING 154 155 leaks = filter_action("mozleak_total", lines) 156 assert len(leaks) == 1 157 assert leaks[0]["process"] == "default" 158 assert leaks[0]["bytes"] == 19915 159 160 161 if __name__ == "__main__": 162 mozunit.main()