test_modes.py (7630B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import os 6 from pathlib import Path 7 8 import mozunit 9 import pytest 10 from manifestparser.toml import Mode 11 from skipfails import FAILURE_RATIO, Skipfails, read_json, write_json 12 13 DATA_PATH = Path(__file__).with_name("data") 14 REVISION: str = "7cf7a9720f4ead03213f1799f3dcc00a413c7a02" 15 RETRIEVING = "Retrieving" 16 TRY_URL: str = f"https://treeherder.mozilla.org/jobs?repo=try&revision={REVISION}" 17 META_BUG_ID: str = "1991977" 18 TEXT_FILE = 0 19 JSON_FILE = 1 20 TASKS_FILE = 1 21 22 23 def clear_cache(sf: Skipfails): 24 sf.clear_cache = META_BUG_ID 25 sf.check_cache() 26 sf.clear_cache = REVISION 27 sf.check_cache() 28 29 30 def copy_to_cache(skipfails: Skipfails, filename: str, kind: int = JSON_FILE): 31 from_path = DATA_PATH.joinpath(filename) 32 to_path = skipfails.cached_path(REVISION, filename) 33 if kind == TEXT_FILE: 34 with open(from_path) as from_fp, open(to_path, "w") as to_fp: 35 to_fp.write(from_fp.read()) 36 elif kind == JSON_FILE: 37 data = read_json(from_path) 38 write_json(to_path, data) 39 else: # TASKS_FILE 40 data = skipfails.read_tasks(from_path) 41 skipfails.write_tasks(to_path, data) 42 43 44 def cache_vs_expected( 45 skipfails: Skipfails, 46 to_filename: str, 47 from_filename: str = "", 48 cache_dir: str = REVISION, 49 ): 50 from_path = DATA_PATH.joinpath(from_filename if from_filename else to_filename) 51 to_path = skipfails.cached_path(cache_dir, to_filename) 52 with open(from_path) as from_fp, open(to_path) as to_fp: 53 from_data = from_fp.read() 54 to_data = to_fp.read() 55 return (to_data, from_data) 56 57 58 @pytest.fixture(scope="session") 59 def skipfails(): 60 sf = Skipfails( 61 None, # command_context 62 TRY_URL, 63 True, # verbose 64 "disable", # bugzilla 65 False, # dry_run 66 False, # turbo 67 False, # implicit_vars 68 None, # new_version 69 None, # task_id 70 None, # user_agent 71 None, # clear_cache 72 ) 73 clear_cache(sf) 74 yield sf 75 76 77 def test_carryover_mode(skipfails: Skipfails, capsys): 78 "Test --carryover" 79 80 copy_to_cache(skipfails, "tasks.json", TASKS_FILE) 81 copy_to_cache(skipfails, "job_ids.json") 82 copy_to_cache(skipfails, "browser.toml", TEXT_FILE) 83 copy_to_cache(skipfails, "suggest-531522970.json") 84 copy_to_cache(skipfails, "suggest-531522979.json") 85 copy_to_cache(skipfails, "suggest-531523119.json") 86 copy_to_cache(skipfails, "context-O304PG2lSOuef7JzFoSaow-12581.txt", TEXT_FILE) 87 copy_to_cache(skipfails, "context-BarnNoFwSCGQtnNGH-o08w-4028.txt", TEXT_FILE) 88 89 mode: int = Mode.CARRYOVER 90 skipfails.run( 91 META_BUG_ID, 92 None, # save_tasks 93 None, # use_tasks 94 None, # save_failures 95 None, # use_failures 96 -1, # max_failures 97 FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO, 98 mode, 99 ) 100 101 out, err = capsys.readouterr() 102 # save STDERR for debugging (don't clear cache at the end) 103 err_path = skipfails.cached_path(REVISION, "err-carryover.log") 104 with open(err_path, "w") as fp: 105 fp.write(err) 106 mode_string = "Carryover mode: only platform match conditions considered, no bugs created or updated" 107 assert mode_string in err 108 assert RETRIEVING not in err 109 110 failures, failures_expected = cache_vs_expected(skipfails, "failures.json") 111 assert failures == failures_expected 112 113 manifest, manifest_expected = cache_vs_expected( 114 skipfails, "browser.toml", "browser-carryover.toml" 115 ) 116 assert manifest == manifest_expected 117 118 actions, actions_expected = cache_vs_expected( 119 skipfails, "actions.json", "actions-carryover.json", META_BUG_ID 120 ) 121 assert actions == actions_expected 122 123 124 def test_known_intermittents_mode(skipfails: Skipfails, capsys): 125 "Test --known-intermittents" 126 127 mode: int = Mode.KNOWN_INTERMITTENT 128 skipfails.run( 129 META_BUG_ID, 130 None, # save_tasks 131 None, # use_tasks 132 None, # save_failures 133 None, # use_failures 134 -1, # max_failures 135 FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO, 136 mode, 137 ) 138 139 out, err = capsys.readouterr() 140 # save STDERR for debugging (don't clear cache at the end) 141 err_path = skipfails.cached_path(REVISION, "err-known.log") 142 with open(err_path, "w") as fp: 143 fp.write(err) 144 mode_string = "Known Intermittents mode: only failures with known intermittents considered, no bugs created or updated" 145 assert mode_string in err 146 assert RETRIEVING not in err 147 148 manifest, manifest_expected = cache_vs_expected( 149 skipfails, "browser.toml", "browser-known.toml" 150 ) 151 assert manifest == manifest_expected 152 153 actions, actions_expected = cache_vs_expected( 154 skipfails, "actions.json", "actions-known.json", META_BUG_ID 155 ) 156 assert actions == actions_expected 157 158 159 def test_new_failures_mode(skipfails: Skipfails, capsys): 160 "Test --new-failures" 161 162 mode: int = Mode.NEW_FAILURE 163 skipfails.run( 164 META_BUG_ID, 165 None, # save_tasks 166 None, # use_tasks 167 None, # save_failures 168 None, # use_failures 169 -1, # max_failures 170 FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO, 171 mode, 172 ) 173 174 out, err = capsys.readouterr() 175 # save STDERR for debugging (don't clear cache at the end) 176 err_path = skipfails.cached_path(REVISION, "err-new.log") 177 with open(err_path, "w") as fp: 178 fp.write(err) 179 mode_string = "New failures mode: Will only edit manifest skip-if conditions for new failures (i.e. not carryover nor known intermittents)" 180 assert mode_string in err 181 assert RETRIEVING not in err 182 183 manifest, manifest_expected = cache_vs_expected( 184 skipfails, "browser.toml", "browser-new.toml" 185 ) 186 assert manifest == manifest_expected 187 188 actions, actions_expected = cache_vs_expected( 189 skipfails, "actions.json", "actions-new.json", META_BUG_ID 190 ) 191 assert actions == actions_expected 192 193 194 def test_replace_tbd_mode(skipfails: Skipfails, capsys): 195 "Test --replace-tbd" 196 197 mode: int = Mode.REPLACE_TBD 198 skipfails.run( 199 META_BUG_ID, 200 None, # save_tasks 201 None, # use_tasks 202 None, # save_failures 203 None, # use_failures 204 -1, # max_failures 205 FAILURE_RATIO, # failure_ratio: float = FAILURE_RATIO, 206 mode, 207 ) 208 209 out, err = capsys.readouterr() 210 # save STDERR for debugging (don't clear cache at the end) 211 err_path = skipfails.cached_path(REVISION, "err-replace.log") 212 with open(err_path, "w") as fp: 213 fp.write(err) 214 mode_string = "Replace TBD mode: Will only edit manifest skip-if conditions for new failures by filing new bugs and replacing TBD with actual bug number." 215 assert mode_string in err 216 assert RETRIEVING not in err 217 218 carryover = "Bugzilla has been disabled: comment not added to Bug 1111111" 219 assert carryover in err 220 221 intermittent = "Error log line 4028: https://treeherder.mozilla.org/logviewer?repo=try&job_id=531522970&lineNumber=4028" 222 assert intermittent in err 223 224 new = "Error log line 12581: https://treeherder.mozilla.org/logviewer?repo=try&job_id=531523119&lineNumber=12581" 225 assert new in err 226 227 228 def test_cleanup(skipfails: Skipfails): 229 clear_cache(skipfails) 230 tasks_cached = skipfails.cached_path(REVISION, "tasks.json") 231 assert not os.path.exists(tasks_cached) 232 233 234 if __name__ == "__main__": 235 mozunit.main()