high_freq_skipfails.py (7267B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import datetime 6 import logging 7 import os 8 import sys 9 from pathlib import Path 10 from typing import Optional, TypedDict 11 12 import requests 13 from intermittent_failures import IntermittentFailuresFetcher 14 from mozci.util.taskcluster import get_task 15 from mozinfo.platforminfo import PlatformInfo 16 from skipfails import Skipfails 17 18 ERROR = "error" 19 USER_AGENT = "mach-manifest-high-freq-skipfails/1.0" 20 21 22 class FailureByBug(TypedDict): 23 task_id: str 24 bug_id: int 25 job_id: int 26 tree: str 27 28 29 class BugSuggestion(TypedDict): 30 path_end: Optional[str] 31 32 33 class TestInfoAllTestsItem(TypedDict): 34 manifest: list[str] 35 test: str 36 37 38 class TestInfoAllTests(TypedDict): 39 tests: dict[str, list[TestInfoAllTestsItem]] 40 41 42 class HighFreqSkipfails: 43 "mach manifest high-freq-skip-fails implementation: Update manifests to skip failing tests by looking at recent failures" 44 45 def __init__(self, command_context=None, failures: int = 30, days: int = 7) -> None: 46 self.command_context = command_context 47 if self.command_context is not None: 48 self.topsrcdir = self.command_context.topsrcdir 49 else: 50 self.topsrcdir = Path(__file__).parent.parent 51 self.topsrcdir = os.path.normpath(self.topsrcdir) 52 self.component = "high-freq-skip-fails" 53 54 self.failures = failures 55 self.days = days 56 57 self.fetcher = IntermittentFailuresFetcher( 58 days=days, threshold=failures, verbose=False 59 ) 60 61 self.start_date = datetime.datetime.now() 62 self.start_date = self.start_date - datetime.timedelta(days=self.days) 63 self.end_date = datetime.datetime.now() 64 self.test_info_all_tests: Optional[TestInfoAllTests] = None 65 66 def error(self, e): 67 if self.command_context is not None: 68 self.command_context.log( 69 logging.ERROR, self.component, {ERROR: str(e)}, "ERROR: {error}" 70 ) 71 else: 72 print(f"ERROR: {e}", file=sys.stderr, flush=True) 73 74 def info(self, e): 75 if self.command_context is not None: 76 self.command_context.log( 77 logging.INFO, self.component, {ERROR: str(e)}, "INFO: {error}" 78 ) 79 else: 80 print(f"INFO: {e}", file=sys.stderr, flush=True) 81 82 def run(self): 83 self.info( 84 f"Fetching bugs with failure count above {self.failures} in the last {self.days} days..." 85 ) 86 bug_list = self.fetcher.get_single_tracking_bugs_with_paths() 87 if len(bug_list) == 0: 88 self.info( 89 f"Could not find bugs wih at least {self.failures} failures in the last {self.days}" 90 ) 91 return 92 self.info(f"Found {len(bug_list)} bugs to inspect") 93 94 self.info("Fetching test_info_all_tests and caching it...") 95 self.test_info_all_tests = self.get_test_info_all_tests() 96 97 manifest_errors: set[tuple[int, str]] = set() 98 99 task_data: dict[str, tuple[int, str, str]] = {} 100 for bug_id, test_path in bug_list: 101 self.info(f"Getting failures for bug '{bug_id}'...") 102 failures_by_bug = self.get_failures_by_bug(bug_id) 103 self.info(f"Found {len(failures_by_bug)} failures") 104 manifest = self.get_manifest_from_path(test_path) 105 if manifest: 106 self.info(f"Found manifest '{manifest}' for path '{test_path}'") 107 for failure in failures_by_bug: 108 task_data[failure["task_id"]] = (bug_id, test_path, manifest) 109 else: 110 manifest_errors.add((bug_id, test_path)) 111 self.error(f"Could not find manifest for path '{test_path}'") 112 113 skipfails = Skipfails(self.command_context, "", True, "disable", True) 114 115 task_list = self.get_task_list([task_id for task_id in task_data]) 116 for task_id, task in task_list: 117 test_setting = task.get("extra", {}).get("test-setting", {}) 118 if not test_setting: 119 continue 120 platform_info = PlatformInfo(test_setting) 121 (bug_id, test_path, raw_manifest) = task_data[task_id] 122 123 kind, manifest = skipfails.get_kind_manifest(raw_manifest) 124 if kind is None or manifest is None: 125 self.error(f"Could not resolve kind for manifest {raw_manifest}") 126 continue 127 skipfails.skip_failure( 128 manifest, 129 kind, 130 test_path, 131 task_id, 132 platform_info, 133 str(bug_id), 134 high_freq=True, 135 ) 136 137 if len(manifest_errors) > 0: 138 self.info("\nExecution complete\n") 139 self.info("Script encountered errors while fetching manifests:") 140 for bug_id, test_path in manifest_errors: 141 self.info( 142 f"Bug {bug_id}: Could not find manifest for path '{test_path}'" 143 ) 144 145 def get_manifest_from_path(self, path: Optional[str]) -> Optional[str]: 146 manifest: Optional[str] = None 147 if path is not None and self.test_info_all_tests is not None: 148 for test_list in self.test_info_all_tests["tests"].values(): 149 for test in test_list: 150 # FIXME 151 # in case of wpt, we have an incoming path that is a subset of the full test["test"], for example, path could be: 152 # /navigation-api/ordering-and-transition/location-href-canceled.html 153 # but full path as found in test_info_all_tests is: 154 # testing/web-platform/tests/navigation-api/ordering-and-transition/location-href-canceled.html 155 # unfortunately in this case manifest ends up being: /navigation-api/ordering-and-transition 156 if test["test"] == path: 157 manifest = test["manifest"][0] 158 break 159 if manifest is not None: 160 break 161 return manifest 162 163 ################# 164 # API Calls # 165 ################# 166 167 def get_failures_by_bug(self, bug: int, branch="trunk") -> list[FailureByBug]: 168 url = f"https://treeherder.mozilla.org/api/failuresbybug/?startday={self.start_date.date()}&endday={self.end_date.date()}&tree={branch}&bug={bug}" 169 response = requests.get(url, headers={"User-agent": USER_AGENT}) 170 json_data = response.json() 171 return json_data 172 173 def get_task_list( 174 self, task_id_list: list[str], branch="trunk" 175 ) -> list[tuple[str, object]]: 176 retVal = [] 177 for tid in task_id_list: 178 task = get_task(tid) 179 retVal.append((tid, task)) 180 return retVal 181 182 def get_test_info_all_tests(self) -> TestInfoAllTests: 183 url = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.source.test-info-all/artifacts/public/test-info-all-tests.json" 184 response = requests.get(url, headers={"User-agent": USER_AGENT}) 185 json_data = response.json() 186 return json_data