interop.py (11095B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import csv 7 import math 8 import os 9 import re 10 import shutil 11 import sys 12 import tempfile 13 from collections.abc import Iterable, Mapping 14 from datetime import datetime 15 from typing import Callable, Optional 16 17 repos = ["autoland", "mozilla-central", "try", "mozilla-central", "mozilla-beta", "wpt"] 18 19 default_fetch_task_filters = { 20 "wpt": ["-firefox"], 21 None: ["-web-platform-tests-|-spidermonkey-"], 22 } 23 default_interop_task_filters = { 24 "wpt": ["-firefox-"], 25 None: [ 26 "web-platform-tests", 27 "linux.*-64", 28 "/opt", 29 "!-nofis|-headless|-asan|-tsan|-ccov|wayland", 30 ], 31 } 32 33 34 def get_parser_fetch_logs() -> argparse.Namespace: 35 parser = argparse.ArgumentParser() 36 parser.register("type", "list", lambda s: s.split(",")) 37 38 parser.add_argument( 39 "--log-dir", action="store", help="Directory into which to download logs" 40 ) 41 parser.add_argument( 42 "--task-filter", 43 dest="task_filters", 44 action="append", 45 help="Regex filter applied to task names. Filters starting ! must not match. Filters starting ^ (after any !) match the entire task name, otherwise any substring can match. Multiple filters must all match", 46 ) 47 parser.add_argument( 48 "--check-complete", 49 action="store_true", 50 help="Only download logs if the task is complete", 51 ) 52 group = parser.add_mutually_exclusive_group(required=True) 53 group.add_argument( 54 "commits", 55 nargs="*", 56 help="repo:commit e.g. mozilla-central:fae24810aef1 for the runs to include", 57 ) 58 group.add_argument( 59 "--local-logs", 60 action="store", 61 type="list", 62 help="Comma separated list of local log files to use", 63 ) 64 return parser 65 66 67 def get_default_year() -> int: 68 # Simple guess at current Interop year, based on switchover in Feburary 69 now = datetime.now() 70 year = now.year 71 if now.month < 2: 72 year -= 1 73 return year 74 75 76 def get_parser_interop_score() -> argparse.Namespace: 77 parser = get_parser_fetch_logs() 78 parser.add_argument( 79 "--year", 80 action="store", 81 default=get_default_year(), 82 type=int, 83 help="Interop year to score against", 84 ) 85 parser.add_argument( 86 "--category-filter", 87 action="append", 88 dest="category_filters", 89 help="Regex filter applied to category names. Filters starting ! must not match. Filters starting ^ (after any !) match the entire task name, otherwise any substring can match. Multiple filters must all match", 90 ) 91 parser.add_argument( 92 "--expected-failures", 93 help="Path to a file containing a list of tests which are not expected to pass", 94 ) 95 return parser 96 97 98 def print_scores( 99 runs: Iterable[tuple[str, str]], 100 results_by_category: Mapping[str, list[int]], 101 expected_failures_by_category: Optional[Mapping[str, list[tuple[int, int]]]], 102 include_total: bool, 103 ): 104 include_expected_failures = expected_failures_by_category is not None 105 106 writer = csv.writer(sys.stdout, delimiter="\t") 107 108 headers = ["Category"] 109 for repo, commit in runs: 110 prefix = f"{repo}:{commit}" 111 headers.append(f"{prefix}-score") 112 if include_expected_failures: 113 headers.append(f"{prefix}-expected-failures") 114 headers.append(f"{prefix}-adjusted-score") 115 116 writer.writerow(headers) 117 118 totals = {"score": [0.0] * len(runs)} 119 if include_expected_failures: 120 totals["expected_failures"] = [0.0] * len(runs) 121 totals["adjusted_score"] = [0.0] * len(runs) 122 123 for category, category_results in results_by_category.items(): 124 category_row = [] 125 category_row.append(category) 126 for category_index, result in enumerate(category_results): 127 for run_index, run_score in enumerate(category_results): 128 category_row.append(f"{run_score / 10:.1f}") 129 totals["score"][run_index] += run_score 130 if include_expected_failures: 131 expected_failures, adjusted_score = expected_failures_by_category[ 132 category 133 ][run_index] 134 category_row.append(f"{expected_failures / 10:.1f}") 135 category_row.append(f"{adjusted_score / 10:.1f}") 136 totals["expected_failures"][run_index] += expected_failures 137 totals["adjusted_score"][run_index] += adjusted_score 138 writer.writerow(category_row) 139 140 if include_total: 141 142 def get_total(score, floor=True): 143 total = float(score) / (len(results_by_category)) 144 if floor: 145 total = math.floor(total) 146 total /= 10.0 147 return total 148 149 totals_row = ["Total"] 150 for i in range(len(runs)): 151 totals_row.append(f"{get_total(totals['score'][i]):.1f}") 152 if include_expected_failures: 153 totals_row.append( 154 f"{get_total(totals['expected_failures'][i], floor=False):.1f}" 155 ) 156 totals_row.append(f"{get_total(totals['adjusted_score'][i]):.1f}") 157 writer.writerow(totals_row) 158 159 160 def get_wptreports( 161 repo: str, commit: str, task_filters: list[str], log_dir: str, check_complete: bool 162 ) -> list[str]: 163 import tcfetch 164 165 return tcfetch.download_artifacts( 166 repo, 167 commit, 168 task_filters=task_filters, 169 check_complete=check_complete, 170 out_dir=log_dir, 171 ) 172 173 174 def get_runs(commits: list[str]) -> list[tuple[str, str]]: 175 runs = [] 176 for item in commits: 177 if ":" not in item: 178 raise ValueError(f"Expected commits of the form repo:commit, got {item}") 179 repo, commit = item.split(":", 1) 180 if repo not in repos: 181 raise ValueError(f"Unsupported repo {repo}") 182 runs.append((repo, commit)) 183 return runs 184 185 186 def get_category_filter( 187 category_filters: Optional[list[str]], 188 ) -> Optional[Callable[[str], bool]]: 189 if category_filters is None: 190 return None 191 192 filters = [] 193 for item in category_filters: 194 if not item: 195 continue 196 invert = item[0] == "!" 197 if invert: 198 item = item[1:] 199 if item[0] == "^": 200 regex = re.compile(item) 201 else: 202 regex = re.compile(f"^(.*)(?:{item})") 203 filters.append((regex, invert)) 204 205 def match_filters(category): 206 for regex, invert in filters: 207 matches = regex.match(category) is not None 208 if invert: 209 matches = not matches 210 if not matches: 211 return False 212 return True 213 214 return match_filters 215 216 217 def fetch_logs( 218 commits: list[str], 219 task_filters: list[str], 220 log_dir: Optional[str], 221 check_complete: bool, 222 **kwargs, 223 ): 224 runs = get_runs(commits) 225 226 if not task_filters: 227 repos = {item[0] for item in runs} 228 task_filters = [] 229 need_default_filter = False 230 for repo in repos: 231 if repo in default_fetch_task_filters: 232 task_filters.extend(default_fetch_task_filters[repo]) 233 else: 234 need_default_filter = True 235 if need_default_filter: 236 task_filters.extend(default_fetch_task_filters[None]) 237 238 if log_dir is None: 239 log_dir = os.path.abspath(os.curdir) 240 241 for repo, commit in runs: 242 task_data = get_wptreports(repo, commit, task_filters, log_dir, check_complete) 243 print(f"Downloaded {len(task_data)} log files") 244 245 246 def get_expected_failures(path: str) -> Mapping[str, set[Optional[str]]]: 247 expected_failures = {} 248 with open(path) as f: 249 for i, entry in enumerate(csv.reader(f)): 250 entry = [item.strip() for item in entry] 251 if not any(item for item in entry) or entry[0][0] == "#": 252 continue 253 if len(entry) > 2: 254 raise ValueError( 255 f"{path}:{i + 1} expected at most two columns, got {len(entry)}" 256 ) 257 if entry[0][0] != "/": 258 raise ValueError( 259 f'{path}:{i + 1} "{entry[0]}" is not a valid test id (must start with "/")' 260 ) 261 test_id = entry[0] 262 if test_id not in expected_failures: 263 expected_failures[test_id] = set() 264 if len(entry) == 2: 265 subtest_id = entry[1] 266 if subtest_id == "": 267 print( 268 f"Warning: {path}:{i + 1} got empty string subtest id, remove the trailing comma to make this apply to the full test" 269 ) 270 expected_failures[test_id].add(subtest_id) 271 else: 272 expected_failures[test_id].add(None) 273 return expected_failures 274 275 276 def score_runs( 277 commits: list[str], 278 local_logs: list[str], 279 task_filters: list[str], 280 log_dir: Optional[str], 281 year: int, 282 check_complete: bool, 283 category_filters: Optional[list[str]], 284 expected_failures: Optional[str], 285 **kwargs, 286 ): 287 from wpt_interop import score 288 289 runs = get_runs(commits) 290 291 temp_dir = None 292 if log_dir is None: 293 temp_dir = tempfile.mkdtemp() 294 log_dir = temp_dir 295 296 try: 297 if expected_failures is not None: 298 expected_failures_data = get_expected_failures(expected_failures) 299 else: 300 expected_failures_data = None 301 302 run_logs = [] 303 for repo, commit in runs: 304 if not task_filters: 305 if repo in default_interop_task_filters: 306 filters = default_interop_task_filters[repo] 307 else: 308 filters = default_interop_task_filters[None] 309 else: 310 filters = task_filters 311 312 task_data = get_wptreports(repo, commit, filters, log_dir, check_complete) 313 if not task_data: 314 print(f"Failed to get any logs for {repo}:{commit}", file=sys.stderr) 315 else: 316 run_logs.append([item.path for item in task_data]) 317 318 if not run_logs and local_logs: 319 runs = [] 320 for log in local_logs: 321 run_logs.append([log]) 322 runs.append(("local", log)) 323 324 if not run_logs: 325 print("No logs to process", file=sys.stderr) 326 327 include_total = category_filters is None 328 329 category_filter = ( 330 get_category_filter(category_filters) if category_filters else None 331 ) 332 333 scores, expected_failure_scores = score.score_wptreports( 334 run_logs, 335 year=year, 336 category_filter=category_filter, 337 expected_failures=expected_failures_data, 338 ) 339 print_scores(runs, scores, expected_failure_scores, include_total) 340 finally: 341 if temp_dir is not None: 342 shutil.rmtree(temp_dir, True)