skipfails.py (116011B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import gzip 6 import json 7 import logging 8 import os 9 import os.path 10 import pprint 11 import re 12 import shutil 13 import sys 14 import tempfile 15 import time 16 import urllib.parse 17 from copy import deepcopy 18 from pathlib import Path 19 from statistics import median 20 21 # ruff linter deprecates Dict, List, Tuple required for Python 3.8 compatibility 22 from typing import Any, Callable, Dict, List, Literal, Tuple, Union, cast # noqa UP035 23 from xmlrpc.client import Fault 24 25 import bugzilla 26 import mozci.data 27 import requests 28 from bugzilla.bug import Bug 29 from failedplatform import FailedPlatform 30 from manifestparser import ManifestParser 31 from manifestparser.toml import ( 32 Mode, 33 add_skip_if, 34 alphabetize_toml_str, 35 replace_tbd_skip_if, 36 sort_paths, 37 ) 38 from mozci.push import Push 39 from mozci.task import Optional, TestTask 40 from mozci.util.taskcluster import get_task 41 from mozinfo.platforminfo import PlatformInfo 42 from taskcluster.exceptions import TaskclusterRestFailure 43 from wpt_path_utils import ( 44 WPT_META0, 45 WPT_META0_CLASSIC, 46 parse_wpt_path, 47 ) 48 from yaml import load 49 50 # Use faster LibYAML, if installed: https://pyyaml.org/wiki/PyYAMLDocumentation 51 try: 52 from yaml import CLoader as Loader 53 except ImportError: 54 from yaml import Loader 55 56 ArtifactList = List[Dict[Literal["name"], str]] # noqa UP006 57 CreateBug = Optional[Callable[[], Bug]] 58 DictStrList = Dict[str, List] # noqa UP006 59 Extras = Dict[str, PlatformInfo] # noqa UP006 60 FailedPlatforms = Dict[str, FailedPlatform] # noqa UP006 61 GenBugComment = Tuple[ # noqa UP006 62 CreateBug, # noqa UP006 63 str, # bugid 64 bool, # meta_bug_blocked 65 dict, # attachments 66 str, # comment 67 int, # line_number 68 str, # summary 69 str, # description 70 str, # product 71 str, # component 72 ] # noqa UP006 73 JSONType = Union[ 74 None, 75 bool, 76 int, 77 float, 78 str, 79 List["JSONType"], # noqa UP006 80 Dict[str, "JSONType"], # noqa UP006 81 ] 82 DictJSON = Dict[str, JSONType] # noqa UP006 83 ListBug = List[Bug] # noqa UP006 84 ListInt = List[int] # noqa UP006 85 ListStr = List[str] # noqa UP006 86 ManifestPaths = Dict[str, Dict[str, List[str]]] # noqa UP006 87 OptBug = Optional[Bug] 88 BugsBySummary = Dict[str, OptBug] # noqa UP006 89 OptDifferences = Optional[List[int]] # noqa UP006 90 OptInt = Optional[int] 91 OptJs = Optional[Dict[str, bool]] # noqa UP006 92 OptPlatformInfo = Optional[PlatformInfo] 93 OptStr = Optional[str] 94 OptTaskResult = Optional[Dict[str, Any]] # noqa UP006 95 PlatformPermutations = Dict[ # noqa UP006 96 str, # Manifest 97 Dict[ # noqa UP006 98 str, # OS 99 Dict[ # noqa UP006 100 str, # OS Version 101 Dict[ # noqa UP006 102 str, # Processor 103 Dict[ # noqa UP006 104 str, # Build type 105 Dict[ # noqa UP006 106 str, # Test Variant 107 Dict[str, int], # noqa UP006 {'pass': x, 'fail': y} 108 ], 109 ], 110 ], 111 ], 112 ], 113 ] 114 Runs = Dict[str, Dict[str, Any]] # noqa UP006 115 Suggestion = Tuple[OptInt, OptStr, OptStr] # noqa UP006 116 TaskIdOrPlatformInfo = Union[str, PlatformInfo] 117 Tasks = List[TestTask] # noqa UP006 118 TupleOptIntStrOptInt = Tuple[OptInt, str, OptInt] # noqa UP006 119 WptPaths = Tuple[OptStr, OptStr, OptStr, OptStr] # noqa UP006 120 121 BUGREF_REGEX = r"[Bb][Uu][Gg] ?([0-9]+|TBD)" 122 TASK_LOG = "live_backing.log" 123 TASK_ARTIFACT = "public/logs/" + TASK_LOG 124 ATTACHMENT_DESCRIPTION = "Compressed " + TASK_ARTIFACT + " for task " 125 ATTACHMENT_REGEX = ( 126 r".*Created attachment ([0-9]+)\n.*" 127 + ATTACHMENT_DESCRIPTION 128 + "([A-Za-z0-9_-]+)\n.*" 129 ) 130 131 BUGZILLA_AUTHENTICATION_HELP = "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py" 132 CACHE_EXPIRY = 45 # days to expire entries in .skip_fails_cache 133 CACHE_DIR = ".skip_fails_cache" 134 135 MS_PER_MINUTE = 60 * 1000 # ms per minute 136 DEBUG_THRESHOLD = 40 * MS_PER_MINUTE # 40 minutes in ms 137 OPT_THRESHOLD = 20 * MS_PER_MINUTE # 20 minutes in ms 138 139 ANYJS = "anyjs" 140 CC = "classification" 141 DEF = "DEFAULT" 142 DIFFERENCE = "difference" 143 DURATIONS = "durations" 144 EQEQ = "==" 145 ERROR = "error" 146 FAIL = "FAIL" 147 FAILED_RUNS = "runs_failed" 148 FAILURE_RATIO = 0.4 # more than this fraction of failures will disable 149 INTERMITTENT_RATIO_REFTEST = 0.4 # reftest low frequency intermittent 150 FAILURE_RATIO_REFTEST = 0.8 # disable ratio for reftest (high freq intermittent) 151 GROUP = "group" 152 KIND = "kind" 153 LINENO = "lineno" 154 LL = "label" 155 MEDIAN_DURATION = "duration_median" 156 MINIMUM_RUNS = 3 # mininum number of runs to consider success/failure 157 MOCK_BUG_DEFAULTS = {"blocks": [], "comments": []} 158 MOCK_TASK_DEFAULTS = {"extra": {}, "failure_types": {}, "results": []} 159 MOCK_TASK_INITS = ["results"] 160 MODIFIERS = "modifiers" 161 NOTEQ = "!=" 162 OPT = "opt" 163 PASS = "PASS" 164 PIXELS = "pixels" 165 PP = "path" 166 QUERY = "query" 167 RR = "result" 168 RUNS = "runs" 169 STATUS = "status" 170 SUBTEST = "subtest" 171 SUBTEST_REGEX = ( 172 r"image comparison, max difference: ([0-9]+), number of differing pixels: ([0-9]+)" 173 ) 174 SUM_BY_LABEL = "sum_by_label" 175 TEST = "test" 176 TEST_TYPES = [EQEQ, NOTEQ] 177 TOTAL_DURATION = "duration_total" 178 TOTAL_RUNS = "runs_total" 179 180 181 def read_json(filename: str): 182 """read data as JSON from filename""" 183 with open(filename, encoding="utf-8") as fp: 184 data = json.load(fp) 185 return data 186 187 188 def default_serializer(obj): 189 if hasattr(obj, "to_dict"): 190 return obj.to_dict() 191 return str(obj) 192 193 194 def write_json(filename: str, data): 195 """saves data as JSON to filename""" 196 # ensure that (at most ONE) parent dir exists 197 parent = os.path.dirname(filename) 198 grandparent = os.path.dirname(parent) 199 if not os.path.isdir(grandparent): 200 raise NotADirectoryError( 201 f"write_json: grand parent directory does not exist for: {filename}" 202 ) 203 if not os.path.isdir(parent): 204 os.mkdir(parent) 205 with open(filename, "w", encoding="utf-8") as fp: 206 s: str = json.dumps(data, indent=2, sort_keys=True, default=default_serializer) 207 if s[-1] != "\n": 208 s += "\n" # end with newline to match JSON linter 209 fp.write(s) 210 211 212 class Mock: 213 def __init__(self, data, defaults={}, inits=[]): 214 self._data = data 215 self._defaults = defaults 216 for name in inits: 217 values = self._data.get(name, []) # assume type is an array 218 values = [Mock(value, defaults, inits) for value in values] 219 self._data[name] = values 220 221 def __getattr__(self, name): 222 if name in self._data: 223 return self._data[name] 224 if name in self._defaults: 225 return self._defaults[name] 226 return "" 227 228 229 class Classification: 230 "Classification of the failure (not the task result)" 231 232 DISABLE_INTERMITTENT = "disable_intermittent" # reftest [40%, 80%) 233 DISABLE_FAILURE = "disable_failure" # reftest (80%,100%] failure 234 DISABLE_MANIFEST = "disable_manifest" # crash found 235 DISABLE_RECOMMENDED = "disable_recommended" # disable first failing path 236 DISABLE_TOO_LONG = "disable_too_long" # runtime threshold exceeded 237 INTERMITTENT = "intermittent" 238 SECONDARY = "secondary" # secondary failing path 239 SUCCESS = "success" # path always succeeds 240 UNKNOWN = "unknown" 241 242 243 class Kind: 244 "Kind of manifest" 245 246 LIST = "list" 247 TOML = "toml" 248 UNKNOWN = "unknown" 249 WPT = "wpt" 250 251 252 class SkipfailsMode(Mode): 253 "Skipfails mode of operation" 254 255 @classmethod 256 def from_flags( 257 cls, 258 carryover_mode: bool, 259 known_intermittents_mode: bool, 260 new_failures_mode: bool, 261 replace_tbd_mode: bool, 262 ) -> int: 263 if ( 264 sum([ 265 carryover_mode, 266 known_intermittents_mode, 267 new_failures_mode, 268 replace_tbd_mode, 269 ]) 270 > 1 271 ): 272 raise Exception( 273 "may not specifiy more than one mode: --carryover --known-intermittents --new-failures --replace-tbd" 274 ) 275 if carryover_mode: 276 return cls.CARRYOVER 277 elif known_intermittents_mode: 278 return cls.KNOWN_INTERMITTENT 279 elif new_failures_mode: 280 return cls.NEW_FAILURE 281 elif replace_tbd_mode: 282 return cls.REPLACE_TBD 283 return cls.NORMAL 284 285 @classmethod 286 def edits_bugzilla(cls, mode: int) -> bool: 287 if mode in [cls.NORMAL, cls.REPLACE_TBD]: 288 return True 289 return False 290 291 @classmethod 292 def description(cls, mode: int) -> str: 293 if mode == cls.CARRYOVER: 294 return "Carryover mode: only platform match conditions considered, no bugs created or updated" 295 elif mode == cls.KNOWN_INTERMITTENT: 296 return "Known Intermittents mode: only failures with known intermittents considered, no bugs created or updated" 297 elif mode == cls.NEW_FAILURE: 298 return "New failures mode: Will only edit manifest skip-if conditions for new failures (i.e. not carryover nor known intermittents)" 299 elif mode == cls.REPLACE_TBD: 300 return "Replace TBD mode: Will only edit manifest skip-if conditions for new failures by filing new bugs and replacing TBD with actual bug number." 301 return "Normal mode" 302 303 @classmethod 304 def name(cls, mode: int) -> str: 305 if mode == cls.NORMAL: 306 return "NORMAL" 307 if mode == cls.CARRYOVER: 308 return "CARRYOVER" 309 elif mode == cls.KNOWN_INTERMITTENT: 310 return "KNOWN_INTERMITTENT" 311 elif mode == cls.NEW_FAILURE: 312 return "NEW_FAILURE" 313 elif mode == cls.REPLACE_TBD: 314 return "REPLACE_TBD" 315 if mode == cls.CARRYOVER_FILED: 316 return "CARRYOVER_FILED" 317 elif mode == cls.KNOWN_INTERMITTENT_FILED: 318 return "KNOWN_INTERMITTENT_FILED" 319 elif mode == cls.NEW_FAILURE_FILED: 320 return "NEW_FAILURE_FILED" 321 return "" 322 323 @classmethod 324 def bug_filed(cls, mode: int) -> int: 325 if mode == cls.CARRYOVER: 326 return cls.CARRYOVER_FILED 327 elif mode == cls.KNOWN_INTERMITTENT: 328 return cls.KNOWN_INTERMITTENT_FILED 329 elif mode == cls.NEW_FAILURE: 330 return cls.NEW_FAILURE_FILED 331 else: 332 raise Exception( 333 f"Skipfails mode {cls.name(mode)} cannot be promoted to a _FILED mode" 334 ) 335 return mode 336 337 338 class Action: 339 """ 340 A defferred action to take for a failure as a result 341 of running in --carryover, --known-intermittents or --new-failures mode 342 to be acted upon in --replace-tbd mode 343 """ 344 345 SENTINEL = "|" 346 347 def __init__(self, **kwargs): 348 self.bugid: str = str(kwargs.get("bugid", "")) 349 self.comment: str = kwargs.get("comment", "") 350 self.component: str = kwargs.get("component", "") 351 self.description: str = kwargs.get("description", "") 352 self.disposition: str = kwargs.get("disposition", Mode.NORMAL) 353 self.label: str = kwargs.get("label", "") 354 self.manifest: str = kwargs.get("manifest", "") 355 self.path: str = kwargs.get("path", "") 356 self.product: str = kwargs.get("product", "") 357 self.revision: str = kwargs.get("revision", "") 358 self.skip_if: str = kwargs.get("skip_if", "") 359 self.summary: str = kwargs.get("summary", "") 360 self.task_id: str = kwargs.get("task_id", "") 361 362 @classmethod 363 def make_key(cls, manifest: str, path: str, label: str) -> str: 364 if not manifest: 365 raise Exception( 366 "cannot create a key for an Action if the manifest is not specified" 367 ) 368 if not path: 369 raise Exception( 370 "cannot create a key for an Action if the path is not specified" 371 ) 372 if not label: 373 raise Exception( 374 "cannot create a key for an Action if the label is not specified" 375 ) 376 return manifest + Action.SENTINEL + path + Action.SENTINEL + label 377 378 def key(self) -> str: 379 return Action.make_key(self.manifest, self.path, self.label) 380 381 def to_dict(self) -> Dict: # noqa UP006 382 return self.__dict__ 383 384 385 DictAction = Dict[str, Action] # noqa UP006 386 OptAction = Optional[Action] # noqa UP006 387 388 389 class Skipfails: 390 "mach manifest skip-fails implementation: Update manifests to skip failing tests" 391 392 REPO = "repo" 393 REVISION = "revision" 394 TREEHERDER = "treeherder.mozilla.org" 395 BUGZILLA_DISABLE = "disable" 396 BUGZILLA_SERVER = "bugzilla.allizom.org" 397 BUGZILLA_SERVER_DEFAULT = BUGZILLA_DISABLE 398 399 def __init__( 400 self, 401 command_context=None, 402 try_url="", 403 verbose=True, 404 bugzilla=None, 405 dry_run=False, 406 turbo=False, 407 implicit_vars=False, 408 new_version=None, 409 task_id=None, 410 user_agent=None, 411 clear_cache=None, 412 ): 413 self.command_context = command_context 414 if self.command_context is not None: 415 self.topsrcdir = self.command_context.topsrcdir 416 else: 417 self.topsrcdir = Path(__file__).parent.parent.parent 418 self.topsrcdir = os.path.normpath(self.topsrcdir) 419 if isinstance(try_url, list) and len(try_url) == 1: 420 self.try_url = try_url[0] 421 else: 422 self.try_url = try_url 423 self.implicit_vars = implicit_vars 424 self.new_version = new_version 425 self.verbose = verbose 426 self.turbo = turbo 427 self.edit_bugzilla = True 428 if bugzilla is None: 429 if "BUGZILLA" in os.environ: 430 self.bugzilla = os.environ["BUGZILLA"] 431 else: 432 self.bugzilla = Skipfails.BUGZILLA_SERVER_DEFAULT 433 else: 434 self.bugzilla = bugzilla 435 if self.bugzilla == Skipfails.BUGZILLA_DISABLE: 436 self.bugzilla = None # Bug filing disabled 437 self.edit_bugzilla = False 438 self.dry_run = dry_run 439 if self.dry_run: 440 self.edit_bugzilla = False 441 self.component = "skip-fails" 442 self._bzapi = None 443 self._attach_rx = None 444 self.variants = {} 445 self.tasks = {} 446 self.pp = None 447 self.headers = {} # for Treeherder requests 448 self.headers["Accept"] = "application/json" 449 self.headers["User-Agent"] = "treeherder-pyclient" 450 self.jobs_url = "https://treeherder.mozilla.org/api/jobs/" 451 self.push_ids = {} 452 self.job_ids = {} 453 self.extras: Extras = {} 454 self.bugs = [] # preloaded bugs, currently not an updated cache 455 self.error_summary = {} 456 self._subtest_rx = None 457 self.lmp = None 458 self.failure_types = None 459 self.task_id: OptStr = task_id 460 self.failed_platforms: FailedPlatforms = {} 461 self.platform_permutations: PlatformPermutations = {} 462 self.user_agent: OptStr = user_agent 463 self.suggestions: DictJSON = {} 464 self.clear_cache: OptStr = clear_cache 465 self.mode: int = Mode.NORMAL 466 self.bugs_by_summary: BugsBySummary = {} 467 self.actions: DictAction = {} 468 self._bugref_rx = None 469 self.check_cache() 470 471 def check_cache(self) -> None: 472 """ 473 Will ensure that the cache directory is present 474 And will clear any entries based upon --clear-cache 475 Or revisions older than 45 days 476 """ 477 cache_dir = self.full_path(CACHE_DIR) 478 if not os.path.exists(cache_dir): 479 self.vinfo(f"creating cache directory: {cache_dir}") 480 os.mkdir(cache_dir) 481 return 482 self.vinfo(f"clearing cache for revisions older than {CACHE_EXPIRY} days") 483 expiry: float = CACHE_EXPIRY * 24 * 3600 484 for rev_dir in [e.name for e in os.scandir(cache_dir) if e.is_dir()]: 485 rev_path = os.path.join(cache_dir, rev_dir) 486 if ( 487 self.clear_cache is not None and self.clear_cache in (rev_dir, "all") 488 ) or self.file_age(rev_path) > expiry: 489 self.vinfo(f"clearing revision: {rev_path}") 490 self.delete_dir(rev_path) 491 492 def cached_path(self, revision: str, filename: str) -> str: 493 """Return full path for a cached filename for revision""" 494 cache_dir = self.full_path(CACHE_DIR) 495 return os.path.join(cache_dir, revision, filename) 496 497 def record_new_bug(self, revision: OptStr, bugid: int) -> None: 498 """Records this bug id in the cache of created bugs for this revision""" 499 if revision is not None: 500 new_bugs_path = self.cached_path(revision, "new-bugs.json") 501 new_bugs: ListInt = [] 502 if os.path.exists(new_bugs_path): 503 self.vinfo(f"Reading cached new bugs for revision: {revision}") 504 new_bugs = read_json(new_bugs_path) 505 if bugid not in new_bugs: 506 new_bugs.append(bugid) 507 new_bugs.sort() 508 write_json(new_bugs_path, new_bugs) 509 510 def _initialize_bzapi(self): 511 """Lazily initializes the Bugzilla API (returns True on success)""" 512 if self._bzapi is None and self.bugzilla is not None: 513 self._bzapi = bugzilla.Bugzilla(self.bugzilla) 514 self._attach_rx = re.compile(ATTACHMENT_REGEX, flags=re.M) 515 return self._bzapi is not None 516 517 def pprint(self, obj): 518 if self.pp is None: 519 self.pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr) 520 self.pp.pprint(obj) 521 sys.stderr.flush() 522 523 def error(self, e): 524 if self.command_context is not None: 525 self.command_context.log( 526 logging.ERROR, self.component, {ERROR: str(e)}, "ERROR: {error}" 527 ) 528 else: 529 print(f"ERROR: {e}", file=sys.stderr, flush=True) 530 531 def warning(self, e): 532 if self.command_context is not None: 533 self.command_context.log( 534 logging.WARNING, self.component, {ERROR: str(e)}, "WARNING: {error}" 535 ) 536 else: 537 print(f"WARNING: {e}", file=sys.stderr, flush=True) 538 539 def info(self, e): 540 if self.command_context is not None: 541 self.command_context.log( 542 logging.INFO, self.component, {ERROR: str(e)}, "INFO: {error}" 543 ) 544 else: 545 print(f"INFO: {e}", file=sys.stderr, flush=True) 546 547 def vinfo(self, e): 548 if self.verbose: 549 self.info(e) 550 551 def full_path(self, filename): 552 """Returns full path for the relative filename""" 553 554 return os.path.join(self.topsrcdir, os.path.normpath(filename)) 555 556 def isdir(self, filename): 557 """Returns True if filename is a directory""" 558 559 return os.path.isdir(self.full_path(filename)) 560 561 def exists(self, filename): 562 """Returns True if filename exists""" 563 564 return os.path.exists(self.full_path(filename)) 565 566 def file_age(self, path: str) -> float: 567 """Returns age of filename in seconds""" 568 569 age: float = 0.0 570 if os.path.exists(path): 571 stat: os.stat_result = os.stat(path) 572 mtime: float = stat.st_mtime 573 now: float = time.time() 574 age = now - mtime 575 return age 576 577 def delete_dir(self, path: str) -> None: 578 """Recursively deletes dir at path""" 579 abs_path: str = os.path.abspath(path) 580 # Safety: prevent root or empty deletions 581 if abs_path in ("", os.path.sep): 582 raise ValueError(f"Refusing to delete unsafe path: {path}") 583 # Ensure path is inside topsrcdir 584 if not abs_path.startswith(self.topsrcdir + os.path.sep): 585 raise ValueError( 586 f"Refusing to delete: {path} is not inside {self.topsrcdir}" 587 ) 588 if not os.path.exists(abs_path): 589 raise FileNotFoundError(f"Path does not exist: {abs_path}") 590 if not os.path.isdir(abs_path): 591 raise NotADirectoryError(f"Not a directory: {abs_path}") 592 shutil.rmtree(abs_path) 593 594 def run( 595 self, 596 meta_bug_id: OptInt = None, 597 save_tasks: OptStr = None, 598 use_tasks: OptStr = None, 599 save_failures: OptStr = None, 600 use_failures: OptStr = None, 601 max_failures: int = -1, 602 failure_ratio: float = FAILURE_RATIO, 603 mode: int = Mode.NORMAL, 604 ): 605 "Run skip-fails on try_url, return True on success" 606 607 self.mode = mode 608 if self.mode != Mode.NORMAL and meta_bug_id is None: 609 raise Exception( 610 "must specifiy --meta-bug-id when using one of: --carryover --known-intermittents --new-failures --replace-tbd" 611 ) 612 if self.mode != Mode.NORMAL: 613 self.read_actions(meta_bug_id) 614 self.vinfo(SkipfailsMode.description(self.mode)) 615 if not SkipfailsMode.edits_bugzilla(self.mode): 616 self.edit_bugzilla = False 617 if self.bugzilla is None: 618 self.vinfo("Bugzilla has been disabled: bugs not created or updated.") 619 elif self.dry_run: 620 self.vinfo("Flag --dry-run: bugs not created or updated.") 621 if meta_bug_id is None: 622 self.edit_bugzilla = False 623 self.vinfo("No --meta-bug-id specified: bugs not created or updated.") 624 else: 625 self.vinfo(f"meta-bug-id: {meta_bug_id}") 626 if self.new_version is not None: 627 self.vinfo( 628 f"All skip-if conditions will use the --new-version for os_version: {self.new_version}" 629 ) 630 if failure_ratio != FAILURE_RATIO: 631 self.vinfo(f"Failure ratio for this run: {failure_ratio}") 632 self.vinfo( 633 f"skip-fails assumes implicit-vars for reftest: {self.implicit_vars}" 634 ) 635 try_url = self.try_url 636 revision, repo = self.get_revision(try_url) 637 self.cached_job_ids(revision) 638 use_tasks_cached = self.cached_path(revision, "tasks.json") 639 use_failures_cached = self.cached_path(revision, "failures.json") 640 if use_tasks is None and os.path.exists(use_tasks_cached): 641 use_tasks = use_tasks_cached 642 if save_tasks is None and use_tasks != use_tasks_cached: 643 save_tasks = use_tasks_cached 644 if use_failures is None and os.path.exists(use_failures_cached): 645 use_failures = use_failures_cached 646 if save_failures is None and use_failures != use_failures_cached: 647 save_failures = use_failures_cached 648 if use_tasks is not None: 649 tasks = self.read_tasks(use_tasks) 650 self.vinfo(f"use tasks: {use_tasks}") 651 self.failure_types = None # do NOT cache failure_types 652 else: 653 tasks = self.get_tasks(revision, repo) 654 if len(tasks) > 0 and self.task_id is not None: 655 tasks = [t for t in tasks if t.id == self.task_id] 656 self.failure_types = {} # cache failure_types 657 if use_failures is not None: 658 failures = self.read_failures(use_failures) 659 self.vinfo(f"use failures: {use_failures}") 660 else: 661 failures = self.get_failures(tasks, failure_ratio) 662 if save_failures is not None: 663 write_json(save_failures, failures) 664 self.vinfo(f"save failures: {save_failures}") 665 if save_tasks is not None: 666 self.write_tasks(save_tasks, tasks) 667 self.vinfo(f"save tasks: {save_tasks}") 668 num_failures = 0 669 if self.mode == Mode.REPLACE_TBD: 670 self.replace_tbd(meta_bug_id) 671 failures = [] 672 for manifest in failures: 673 kind = failures[manifest][KIND] 674 for label in failures[manifest][LL]: 675 for path in failures[manifest][LL][label][PP]: 676 classification = failures[manifest][LL][label][PP][path][CC] 677 if classification.startswith("disable_") or ( 678 self.turbo and classification == Classification.SECONDARY 679 ): 680 anyjs = {} # anyjs alternate basename = False 681 differences = [] 682 pixels = [] 683 status = FAIL 684 lineno = failures[manifest][LL][label][PP][path].get(LINENO, 0) 685 runs: Runs = failures[manifest][LL][label][PP][path][RUNS] 686 k = Action.make_key(manifest, path, label) 687 if ( 688 self.mode in [Mode.KNOWN_INTERMITTENT, Mode.NEW_FAILURE] 689 and k in self.actions 690 ): 691 self.info( 692 f"\n\n===== Previously handled {SkipfailsMode.name(self.actions[k].disposition)} in manifest: {manifest} =====" 693 ) 694 self.info(f" path: {path}") 695 self.info(f" label: {label}") 696 continue 697 698 # skip_failure only needs to run against one failing task for each path: first_task_id 699 first_task_id: OptStr = None 700 for task_id in runs: 701 if first_task_id is None and not runs[task_id].get( 702 RR, False 703 ): 704 first_task_id = task_id 705 if kind == Kind.TOML: 706 continue 707 elif kind == Kind.LIST: 708 difference = runs[task_id].get(DIFFERENCE, 0) 709 if difference > 0: 710 differences.append(difference) 711 pixel = runs[task_id].get(PIXELS, 0) 712 if pixel > 0: 713 pixels.append(pixel) 714 status = runs[task_id].get(STATUS, FAIL) 715 elif kind == Kind.WPT: 716 filename = os.path.basename(path) 717 anyjs[filename] = False 718 if QUERY in runs[task_id]: 719 query = runs[task_id][QUERY] 720 anyjs[filename + query] = False 721 else: 722 query = None 723 if ANYJS in runs[task_id]: 724 any_filename = os.path.basename( 725 runs[task_id][ANYJS] 726 ) 727 anyjs[any_filename] = False 728 if query is not None: 729 anyjs[any_filename + query] = False 730 731 self.skip_failure( 732 manifest, 733 kind, 734 path, 735 first_task_id, 736 None, # platform_info 737 None, # bug_id 738 False, # high_freq 739 anyjs, 740 differences, 741 pixels, 742 lineno, 743 status, 744 label, 745 classification, 746 try_url, 747 revision, 748 repo, 749 meta_bug_id, 750 ) 751 num_failures += 1 752 if max_failures >= 0 and num_failures >= max_failures: 753 self.warning( 754 f"max_failures={max_failures} threshold reached: stopping." 755 ) 756 return True 757 self.cache_job_ids(revision) 758 if self.mode != Mode.NORMAL: 759 self.write_actions(meta_bug_id) 760 return True 761 762 def get_revision(self, url): 763 parsed = urllib.parse.urlparse(url) 764 if parsed.scheme != "https": 765 raise ValueError("try_url scheme not https") 766 if parsed.netloc != Skipfails.TREEHERDER: 767 raise ValueError(f"try_url server not {Skipfails.TREEHERDER}") 768 if len(parsed.query) == 0: 769 raise ValueError("try_url query missing") 770 query = urllib.parse.parse_qs(parsed.query) 771 if Skipfails.REVISION not in query: 772 raise ValueError("try_url query missing revision") 773 revision = query[Skipfails.REVISION][0] 774 if Skipfails.REPO in query: 775 repo = query[Skipfails.REPO][0] 776 else: 777 repo = "try" 778 self.vinfo(f"considering {repo} revision={revision}") 779 return revision, repo 780 781 def get_tasks(self, revision, repo): 782 self.vinfo(f"Retrieving tasks for revision: {revision} ...") 783 push = Push(revision, repo) 784 tasks = None 785 try: 786 tasks = push.tasks 787 except requests.exceptions.HTTPError: 788 n = len(mozci.data.handler.sources) 789 self.error("Error querying mozci, sources are:") 790 tcs = -1 791 for i in range(n): 792 source = mozci.data.handler.sources[i] 793 self.error(f" sources[{i}] is type {source.__class__.__name__}") 794 if source.__class__.__name__ == "TreeherderClientSource": 795 tcs = i 796 if tcs < 0: 797 raise PermissionError("Error querying mozci with default User-Agent") 798 msg = f"Error querying mozci with User-Agent: {mozci.data.handler.sources[tcs].session.headers['User-Agent']}" 799 if self.user_agent is None: 800 raise PermissionError(msg) 801 else: 802 self.error(msg) 803 self.error(f"Re-try with User-Agent: {self.user_agent}") 804 mozci.data.handler.sources[tcs].session.headers = { 805 "User-Agent": self.user_agent 806 } 807 tasks = push.tasks 808 return tasks 809 810 def get_kind_manifest(self, manifest: str): 811 kind = Kind.UNKNOWN 812 if manifest.endswith(".ini"): 813 self.warning(f"cannot analyze skip-fails on INI manifests: {manifest}") 814 return (None, None) 815 elif manifest.endswith(".list"): 816 kind = Kind.LIST 817 elif manifest.endswith(".toml"): 818 # NOTE: manifest may be a compound of manifest1:manifest2 819 # where manifest1 includes manifest2 820 # The skip-condition will need to be associated with manifest2 821 includes = manifest.split(":") 822 if len(includes) > 1: 823 manifest = includes[-1] 824 self.warning( 825 f"manifest '{manifest}' is included from {':'.join(includes[0:-1])}" 826 ) 827 kind = Kind.TOML 828 else: 829 kind = Kind.WPT 830 path, wpt_manifest, _query, _anyjs = self.wpt_paths(manifest) 831 if path is None or wpt_manifest is None: # not WPT 832 self.warning( 833 f"cannot analyze skip-fails on unknown manifest type: {manifest}" 834 ) 835 return (None, None) 836 manifest = wpt_manifest 837 return (kind, manifest) 838 839 def get_task_config(self, task: TestTask): 840 if task.label is None: 841 self.warning(f"Cannot find task label for task: {task.id}") 842 return None 843 # strip chunk number - this finds failures across different chunks 844 try: 845 parts = task.label.split("-") 846 int(parts[-1]) 847 return "-".join(parts[:-1]) 848 except ValueError: 849 return task.label 850 851 def get_failures( 852 self, 853 tasks: Tasks, 854 failure_ratio: float = FAILURE_RATIO, 855 ): 856 """ 857 find failures and create structure comprised of runs by path: 858 result: 859 * False (failed) 860 * True (passed) 861 classification: Classification 862 * unknown (default) < 3 runs 863 * intermittent (not enough failures) 864 * disable_recommended (enough repeated failures) >3 runs >= 4 865 * disable_manifest (disable DEFAULT if no other failures) 866 * secondary (not first failure in group) 867 * success 868 """ 869 870 failures = {} 871 manifest_paths: ManifestPaths = {} 872 manifest_ = { 873 KIND: Kind.UNKNOWN, 874 LL: {}, 875 } 876 label_ = { 877 DURATIONS: {}, 878 MEDIAN_DURATION: 0, 879 OPT: False, 880 PP: {}, 881 SUM_BY_LABEL: {}, # All sums implicitly zero 882 TOTAL_DURATION: 0, 883 } 884 path_ = { 885 CC: Classification.UNKNOWN, 886 FAILED_RUNS: 0, 887 RUNS: {}, 888 TOTAL_RUNS: 0, 889 } 890 run_ = { 891 RR: False, 892 } 893 894 for task in tasks: # add explicit failures 895 config = self.get_task_config(task) 896 if config is None: 897 continue 898 899 try: 900 if len(task.results) == 0: 901 continue # ignore aborted tasks 902 _fail_types = task.failure_types # call magic property once 903 if _fail_types is None: 904 continue 905 if self.failure_types is None: 906 self.failure_types = {} 907 908 # This remove known failures 909 failure_types = {} 910 failing_groups = [r.group for r in task.results if not r.ok] 911 for ft in _fail_types: 912 failtype = ft 913 kind, manifest = self.get_kind_manifest(ft) 914 if kind == Kind.WPT: 915 failtype = parse_wpt_path(ft)[0] 916 917 if [fg for fg in failing_groups if fg.endswith(failtype)]: 918 failure_types[ft] = _fail_types[ft] 919 920 self.failure_types[task.id] = failure_types 921 self.vinfo(f"Getting failure_types from task: {task.id}") 922 for raw_manifest in failure_types: 923 kind, manifest = self.get_kind_manifest(raw_manifest) 924 if kind is None or manifest is None: 925 continue 926 927 if kind != Kind.WPT: 928 if manifest not in failures: 929 failures[manifest] = deepcopy(manifest_) 930 failures[manifest][KIND] = kind 931 if task.label not in failures[manifest][LL]: 932 failures[manifest][LL][task.label] = deepcopy(label_) 933 934 if manifest not in manifest_paths: 935 manifest_paths[manifest] = {} 936 if config not in manifest_paths[manifest]: 937 manifest_paths[manifest][config] = [] 938 939 for included_path_type in failure_types[raw_manifest]: 940 included_path, _type = included_path_type 941 path = included_path.split(":")[ 942 -1 943 ] # strip 'included_from.toml:' prefix 944 query = None 945 anyjs = {} 946 allpaths = [] 947 if kind == Kind.WPT: 948 path, mmpath, query, anyjs = self.wpt_paths(path) 949 if path is None or mmpath is None: 950 self.warning( 951 f"non existant failure path: {included_path_type[0]}" 952 ) 953 break 954 allpaths = [path] 955 manifest = os.path.dirname(mmpath) 956 if manifest not in manifest_paths: 957 # this can be a subdir, or translated path 958 manifest_paths[manifest] = {} 959 if config not in manifest_paths[manifest]: 960 manifest_paths[manifest][config] = [] 961 if manifest not in failures: 962 failures[manifest] = deepcopy(manifest_) 963 failures[manifest][KIND] = kind 964 if task.label not in failures[manifest][LL]: 965 failures[manifest][LL][task.label] = deepcopy(label_) 966 elif kind == Kind.LIST: 967 words = path.split() 968 if len(words) != 3 or words[1] not in TEST_TYPES: 969 self.warning(f"reftest type not supported: {path}") 970 continue 971 allpaths = self.get_allpaths(task.id, manifest, path) 972 elif kind == Kind.TOML: 973 if path == manifest: 974 path = DEF # refers to the manifest itself 975 allpaths = [path] 976 for path in allpaths: 977 if path not in manifest_paths[manifest].get(config, []): 978 manifest_paths[manifest][config].append(path) 979 self.vinfo( 980 f"Getting failure info in manifest: {manifest}, path: {path}" 981 ) 982 task_path_object = failures[manifest][LL][task.label][PP] 983 if path not in task_path_object: 984 task_path_object[path] = deepcopy(path_) 985 task_path = task_path_object[path] 986 if task.id not in task_path[RUNS]: 987 task_path[RUNS][task.id] = deepcopy(run_) 988 else: 989 continue 990 result = task.result == "passed" 991 task_path[RUNS][task.id][RR] = result 992 if query is not None: 993 task_path[RUNS][task.id][QUERY] = query 994 if anyjs is not None and len(anyjs) > 0: 995 task_path[RUNS][task.id][ANYJS] = anyjs 996 task_path[TOTAL_RUNS] += 1 997 if not result: 998 task_path[FAILED_RUNS] += 1 999 if kind == Kind.LIST: 1000 ( 1001 lineno, 1002 difference, 1003 pixels, 1004 status, 1005 ) = self.get_lineno_difference_pixels_status( 1006 task.id, manifest, path 1007 ) 1008 if lineno > 0: 1009 task_path[LINENO] = lineno 1010 else: 1011 self.vinfo(f"ERROR no lineno for {path}") 1012 if status != FAIL: 1013 task_path[RUNS][task.id][STATUS] = status 1014 if status == FAIL and difference == 0 and pixels == 0: 1015 # intermittent, not error 1016 task_path[RUNS][task.id][RR] = True 1017 task_path[FAILED_RUNS] -= 1 1018 elif difference > 0: 1019 task_path[RUNS][task.id][DIFFERENCE] = difference 1020 if pixels > 0: 1021 task_path[RUNS][task.id][PIXELS] = pixels 1022 except AttributeError: 1023 pass # self.warning(f"unknown attribute in task (#1): {ae}") 1024 1025 for task in tasks: # add results 1026 config = self.get_task_config(task) 1027 if config is None: 1028 continue 1029 1030 try: 1031 if len(task.results) == 0: 1032 continue # ignore aborted tasks 1033 if self.failure_types is None: 1034 continue 1035 self.vinfo(f"Getting results from task: {task.id}") 1036 for result in task.results: 1037 kind, manifest = self.get_kind_manifest(result.group) 1038 if kind is None or manifest is None: 1039 continue 1040 1041 if manifest not in manifest_paths: 1042 continue 1043 if config not in manifest_paths[manifest]: 1044 continue 1045 if manifest not in failures: 1046 failures[manifest] = deepcopy(manifest_) 1047 task_label_object = failures[manifest][LL] 1048 if task.label not in task_label_object: 1049 task_label_object[task.label] = deepcopy(label_) 1050 task_label = task_label_object[task.label] 1051 if task.id not in task_label[DURATIONS]: 1052 # duration may be None !!! 1053 task_label[DURATIONS][task.id] = result.duration or 0 1054 if task_label[OPT] is None: 1055 task_label[OPT] = self.get_opt_for_task(task.id) 1056 for path in manifest_paths[manifest][config]: # all known paths 1057 # path can be one of any paths that have failed for the manifest/config 1058 # ensure the path is in the specific task failure data 1059 if path not in [ 1060 path 1061 for path, type in self.failure_types.get(task.id, {}).get( 1062 manifest, [("", "")] 1063 ) 1064 ]: 1065 result.ok = True 1066 1067 task_path_object = task_label[PP] 1068 if path not in task_path_object: 1069 task_path_object[path] = deepcopy(path_) 1070 task_path = task_path_object[path] 1071 if task.id not in task_path[RUNS]: 1072 task_path[RUNS][task.id] = deepcopy(run_) 1073 task_path[RUNS][task.id][RR] = result.ok 1074 task_path[TOTAL_RUNS] += 1 1075 if not result.ok: 1076 task_path[FAILED_RUNS] += 1 1077 if kind == Kind.LIST: 1078 ( 1079 lineno, 1080 difference, 1081 pixels, 1082 status, 1083 ) = self.get_lineno_difference_pixels_status( 1084 task.id, manifest, path 1085 ) 1086 if lineno > 0: 1087 task_path[LINENO] = lineno 1088 else: 1089 self.vinfo(f"ERROR no lineno for {path}") 1090 if status != FAIL: 1091 task_path[RUNS][task.id][STATUS] = status 1092 if ( 1093 status == FAIL 1094 and difference == 0 1095 and pixels == 0 1096 and not result.ok 1097 ): 1098 # intermittent, not error 1099 task_path[RUNS][task.id][RR] = True 1100 task_path[FAILED_RUNS] -= 1 1101 if difference > 0: 1102 task_path[RUNS][task.id][DIFFERENCE] = difference 1103 if pixels > 0: 1104 task_path[RUNS][task.id][PIXELS] = pixels 1105 except AttributeError: 1106 pass # self.warning(f"unknown attribute in task (#2): {ae}") 1107 1108 for manifest in failures: # determine classifications 1109 kind = failures[manifest][KIND] 1110 for label in failures[manifest][LL]: 1111 task_label = failures[manifest][LL][label] 1112 opt = task_label[OPT] 1113 durations = [] # summarize durations 1114 try: 1115 first_task_id: str = next(iter(task_label.get(DURATIONS, {}))) 1116 except StopIteration: 1117 continue 1118 for task_id in task_label.get(DURATIONS, {}): 1119 duration = task_label[DURATIONS][task_id] 1120 durations.append(duration) 1121 if len(durations) > 0: 1122 total_duration = sum(durations) 1123 median_duration = median(durations) 1124 task_label[TOTAL_DURATION] = total_duration 1125 task_label[MEDIAN_DURATION] = median_duration 1126 if (opt and median_duration > OPT_THRESHOLD) or ( 1127 (not opt) and median_duration > DEBUG_THRESHOLD 1128 ): 1129 if kind == Kind.TOML: 1130 paths = [DEF] 1131 else: 1132 paths = task_label[PP].keys() 1133 for path in paths: 1134 task_path_object = task_label[PP] 1135 if path not in task_path_object: 1136 task_path_object[path] = deepcopy(path_) 1137 task_path = task_path_object[path] 1138 if first_task_id not in task_path[RUNS]: 1139 task_path[RUNS][first_task_id] = deepcopy(run_) 1140 task_path[RUNS][first_task_id][RR] = False 1141 task_path[TOTAL_RUNS] += 1 1142 task_path[FAILED_RUNS] += 1 1143 task_path[CC] = Classification.DISABLE_TOO_LONG 1144 primary = True # we have not seen the first failure 1145 for path in sort_paths(task_label[PP]): 1146 task_path = task_label[PP][path] 1147 classification = task_path[CC] 1148 if classification == Classification.UNKNOWN: 1149 failed_runs = task_path[FAILED_RUNS] 1150 total_runs = task_path[TOTAL_RUNS] 1151 status = FAIL # default status, only one run could be PASS 1152 for first_task_id in task_path[RUNS]: 1153 status = task_path[RUNS][first_task_id].get(STATUS, status) 1154 if kind == Kind.LIST: 1155 ratio = INTERMITTENT_RATIO_REFTEST 1156 else: 1157 ratio = failure_ratio 1158 if total_runs >= MINIMUM_RUNS: 1159 if failed_runs / total_runs < ratio: 1160 if failed_runs == 0: 1161 classification = Classification.SUCCESS 1162 else: 1163 classification = Classification.INTERMITTENT 1164 elif kind == Kind.LIST: 1165 if failed_runs / total_runs < FAILURE_RATIO_REFTEST: 1166 classification = Classification.DISABLE_INTERMITTENT 1167 else: 1168 classification = Classification.DISABLE_FAILURE 1169 elif primary: 1170 if path == DEF: 1171 classification = Classification.DISABLE_MANIFEST 1172 else: 1173 classification = Classification.DISABLE_RECOMMENDED 1174 primary = False 1175 else: 1176 classification = Classification.SECONDARY 1177 elif self.task_id is not None: 1178 classification = Classification.DISABLE_RECOMMENDED 1179 task_path[CC] = classification 1180 if classification not in task_label[SUM_BY_LABEL]: 1181 task_label[SUM_BY_LABEL][classification] = 0 1182 task_label[SUM_BY_LABEL][classification] += 1 1183 1184 return failures 1185 1186 def bugid_from_reference(self, bug_reference) -> str: 1187 if self._bugref_rx is None: 1188 self._bugref_rx = re.compile(BUGREF_REGEX) 1189 m = self._bugref_rx.findall(bug_reference) 1190 if len(m) == 1: 1191 bugid = m[0] 1192 else: 1193 raise Exception("Carryover bug reference does not include a bug id") 1194 return bugid 1195 1196 def get_bug_by_id(self, id) -> OptBug: 1197 """Get bug by bug id""" 1198 1199 bug: OptBug = None 1200 for b in self.bugs: 1201 if b.id == id: 1202 bug = b 1203 break 1204 if bug is None and self._initialize_bzapi(): 1205 self.vinfo(f"Retrieving bug id: {id} ...") 1206 bug = self._bzapi.getbug(id) 1207 return bug 1208 1209 def get_bugs_by_summary(self, summary, meta_bug_id: OptInt = None) -> ListBug: 1210 """Get bug by bug summary""" 1211 1212 bugs: ListBug = [] 1213 for b in self.bugs: 1214 if b.summary == summary: 1215 bugs.append(b) 1216 if ( 1217 not self.edit_bugzilla 1218 and len(bugs) == 0 1219 and summary in self.bugs_by_summary 1220 ): 1221 bug = self.bugs_by_summary[summary] 1222 if bug is not None: 1223 bugs.append(bug) 1224 return bugs 1225 if len(bugs) == 0 and self.bugzilla is not None and self._initialize_bzapi(): 1226 self.vinfo(f"Retrieving bugs by summary: {summary} ...") 1227 query = self._bzapi.build_query(short_desc=summary) 1228 query["include_fields"] = [ 1229 "id", 1230 "product", 1231 "component", 1232 "status", 1233 "resolution", 1234 "summary", 1235 "blocks", 1236 ] 1237 try: 1238 bugs = self._bzapi.query(query) 1239 except requests.exceptions.HTTPError: 1240 raise 1241 if len(bugs) > 0 and meta_bug_id is not None: 1242 # narrow results to those blocking meta_bug_id 1243 i = 0 1244 while i < len(bugs): 1245 if meta_bug_id not in bugs[i].blocks: 1246 del bugs[i] 1247 else: 1248 i += 1 1249 if not self.edit_bugzilla: 1250 self.bugs_by_summary[summary] = bugs[0] if len(bugs) > 0 else None 1251 return bugs 1252 1253 def create_bug( 1254 self, 1255 summary: str = "Bug short description", 1256 description: str = "Bug description", 1257 product: str = "Testing", 1258 component: str = "General", 1259 version: str = "unspecified", 1260 bugtype: str = "task", 1261 revision: OptStr = None, 1262 ) -> OptBug: 1263 """Create a bug""" 1264 1265 bug: OptBug = None 1266 if self._initialize_bzapi(): 1267 if not self._bzapi.logged_in: 1268 self.error( 1269 "Must create a Bugzilla API key per https://github.com/mozilla/mozci-tools/blob/main/citools/test_triage_bug_filer.py" 1270 ) 1271 raise PermissionError(f"Not authenticated for Bugzilla {self.bugzilla}") 1272 createinfo = self._bzapi.build_createbug( 1273 product=product, 1274 component=component, 1275 summary=summary, 1276 version=version, 1277 description=description, 1278 ) 1279 createinfo["type"] = bugtype 1280 bug = self._bzapi.createbug(createinfo) 1281 if bug is not None: 1282 self.vinfo(f'Created Bug {bug.id} {product}::{component} : "{summary}"') 1283 self.record_new_bug(revision, bug.id) 1284 return bug 1285 1286 def add_bug_comment(self, id: int, comment: str, meta_bug_id: OptInt = None): 1287 """Add a comment to an existing bug""" 1288 1289 if self._initialize_bzapi(): 1290 if not self._bzapi.logged_in: 1291 self.error(BUGZILLA_AUTHENTICATION_HELP) 1292 raise PermissionError("Not authenticated for Bugzilla") 1293 if meta_bug_id is not None: 1294 blocks_add = [meta_bug_id] 1295 else: 1296 blocks_add = None 1297 updateinfo = self._bzapi.build_update( 1298 comment=comment, blocks_add=blocks_add 1299 ) 1300 self._bzapi.update_bugs([id], updateinfo) 1301 1302 def generate_bugzilla_comment( 1303 self, 1304 manifest: str, 1305 kind: str, 1306 path: str, 1307 skip_if: str, 1308 filename: str, 1309 anyjs: OptJs, 1310 lineno: OptInt, 1311 label: OptStr, 1312 classification: OptStr, 1313 task_id: OptStr, 1314 try_url: OptStr, 1315 revision: OptStr, 1316 repo: OptStr, 1317 meta_bug_id: OptInt = None, 1318 ) -> GenBugComment: 1319 """ 1320 Will create a comment for the failure details provided as arguments. 1321 Will determine if a bug exists already for this manifest and meta-bug-id. 1322 If exactly one bug is found, then set 1323 bugid 1324 meta_bug_blocked 1325 attachments (to determine if the task log is in the attachments) 1326 else 1327 set bugid to TBD 1328 if we should edit_bugzilla (not --dry-run) then we return a lambda function 1329 to lazily create a bug later 1330 Returns a tuple with 1331 create_bug_lambda -- lambda function to create a bug (or None) 1332 bugid -- id of bug found, or TBD 1333 meta_bug_blocked -- True if bug found and meta_bug_id is already blocked 1334 attachments -- dictionary of attachments if bug found 1335 comment -- comment to add to the bug (if we should edit_bugzilla) 1336 """ 1337 1338 create_bug_lambda: CreateBug = None 1339 bugid: str = "TBD" 1340 meta_bug_blocked: bool = False 1341 attachments: dict = {} 1342 comment: str = "" 1343 line_number: OptInt = None 1344 if classification == Classification.DISABLE_MANIFEST: 1345 comment = "Disabled entire manifest due to crash result" 1346 elif classification == Classification.DISABLE_TOO_LONG: 1347 comment = "Disabled entire manifest due to excessive run time" 1348 else: 1349 comment = f'Disabled test due to failures in test file: "{filename}"' 1350 if classification == Classification.SECONDARY: 1351 comment += " (secondary)" 1352 if kind == Kind.WPT and anyjs is not None and len(anyjs) > 1: 1353 comment += "\nAdditional WPT wildcard paths:" 1354 for p in sorted(anyjs.keys()): 1355 if p != filename: 1356 comment += f'\n "{p}"' 1357 if task_id is not None: 1358 if kind != Kind.LIST: 1359 if revision is not None and repo is not None: 1360 push_id = self.get_push_id(revision, repo) 1361 if push_id is not None: 1362 job_id = self.get_job_id(push_id, task_id) 1363 if job_id is not None: 1364 ( 1365 line_number, 1366 line, 1367 log_url, 1368 ) = self.get_bug_suggestions( 1369 repo, revision, job_id, path, anyjs 1370 ) 1371 if log_url is not None: 1372 comment += f"\nError log line {line_number}: {log_url}" 1373 summary: str = f"MANIFEST {manifest}" 1374 bugs: ListBug = [] # performance optimization when not editing bugzilla 1375 if self.edit_bugzilla: 1376 bugs = self.get_bugs_by_summary(summary, meta_bug_id) 1377 if len(bugs) == 0: 1378 description = ( 1379 f"This bug covers excluded failing tests in the MANIFEST {manifest}" 1380 ) 1381 description += "\n(generated by `mach manifest skip-fails`)" 1382 product, component = self.get_file_info(path) 1383 if self.edit_bugzilla: 1384 create_bug_lambda = cast( 1385 CreateBug, 1386 lambda: self.create_bug( 1387 summary, 1388 description, 1389 product, 1390 component, 1391 "unspecified", 1392 "task", 1393 revision, 1394 ), 1395 ) 1396 elif len(bugs) == 1: 1397 bugid = str(bugs[0].id) 1398 product = bugs[0].product 1399 component = bugs[0].component 1400 self.vinfo(f'Found Bug {bugid} {product}::{component} "{summary}"') 1401 if meta_bug_id is not None: 1402 if meta_bug_id in bugs[0].blocks: 1403 self.vinfo(f" Bug {bugid} already blocks meta bug {meta_bug_id}") 1404 meta_bug_blocked = True 1405 comments = bugs[0].getcomments() 1406 for i in range(len(comments)): 1407 text = comments[i]["text"] 1408 attach_rx = self._attach_rx 1409 if attach_rx is not None: 1410 m = attach_rx.findall(text) 1411 if len(m) == 1: 1412 a_task_id = m[0][1] 1413 attachments[a_task_id] = m[0][0] 1414 if a_task_id == task_id: 1415 self.vinfo( 1416 f" Bug {bugid} already has the compressed log attached for this task" 1417 ) 1418 elif meta_bug_id is None: 1419 raise Exception(f'More than one bug found for summary: "{summary}"') 1420 else: 1421 raise Exception( 1422 f'More than one bug found for summary: "{summary}" for meta-bug-id: {meta_bug_id}' 1423 ) 1424 if kind == Kind.LIST: 1425 comment += f"\nfuzzy-if condition on line {lineno}: {skip_if}" 1426 return ( 1427 create_bug_lambda, 1428 bugid, 1429 meta_bug_blocked, 1430 attachments, 1431 comment, 1432 line_number, 1433 summary, 1434 description, 1435 product, 1436 component, 1437 ) 1438 1439 def resolve_failure_filename(self, path: str, kind: str, manifest: str) -> str: 1440 filename = DEF 1441 if kind == Kind.TOML: 1442 filename = self.get_filename_in_manifest(manifest.split(":")[-1], path) 1443 elif kind == Kind.WPT: 1444 filename = os.path.basename(path) 1445 elif kind == Kind.LIST: 1446 filename = [ 1447 am 1448 for am in self.error_summary.get(manifest, "") 1449 if self.error_summary[manifest][am]["test"].endswith(path) 1450 ] 1451 if filename: 1452 filename = filename[0] 1453 else: 1454 filename = path 1455 return filename 1456 1457 def resolve_failure_manifest(self, path: str, kind: str, manifest: str) -> str: 1458 if kind == Kind.WPT: 1459 _path, resolved_manifest, _query, _anyjs = self.wpt_paths(path) 1460 if resolved_manifest: 1461 return resolved_manifest 1462 raise Exception(f"Could not resolve WPT manifest for path {path}") 1463 return manifest 1464 1465 def skip_failure( 1466 self, 1467 manifest: str, 1468 kind: str, 1469 path: str, 1470 task_id: OptStr, 1471 platform_info: OptPlatformInfo = None, 1472 bug_id: OptStr = None, 1473 high_freq: bool = False, 1474 anyjs: OptJs = None, 1475 differences: OptDifferences = None, 1476 pixels: OptDifferences = None, 1477 lineno: OptInt = None, 1478 status: OptStr = None, 1479 label: OptStr = None, 1480 classification: OptStr = None, 1481 try_url: OptStr = None, 1482 revision: OptStr = None, 1483 repo: OptStr = None, 1484 meta_bug_id: OptInt = None, 1485 ): 1486 """ 1487 Skip a failure (for TOML, WPT and REFTEST manifests) 1488 For wpt anyjs is a dictionary mapping from alternate basename to 1489 a boolean (indicating if the basename has been handled in the manifest) 1490 """ 1491 1492 path: str = path.split(":")[-1] 1493 self.info(f"\n\n===== Skip failure in manifest: {manifest} =====") 1494 self.info(f" path: {path}") 1495 self.info(f" label: {label}") 1496 action: OptAction = None 1497 1498 if self.mode != Mode.NORMAL: 1499 if bug_id is not None: 1500 self.warning( 1501 f"skip_failure with bug_id specified not supported in {Mode.name(self.mode)} mode" 1502 ) 1503 return 1504 if kind != Kind.TOML: 1505 self.warning( 1506 f"skip_failure in {SkipfailsMode.name(self.mode)} mode only supported for TOML manifests" 1507 ) 1508 return 1509 1510 skip_if: OptStr 1511 if task_id is None: 1512 skip_if = "true" 1513 else: 1514 skip_if = self.task_to_skip_if( 1515 manifest, 1516 platform_info if platform_info is not None else task_id, 1517 kind, 1518 path, 1519 high_freq, 1520 ) 1521 if skip_if is None: 1522 self.info("Not adding skip-if condition") 1523 return 1524 self.vinfo(f"proposed skip-if: {skip_if}") 1525 1526 filename: str = self.resolve_failure_filename(path, kind, manifest) 1527 manifest: str = self.resolve_failure_manifest(path, kind, manifest) 1528 manifest_path: str = self.full_path(manifest) 1529 manifest_str: str = "" 1530 comment: str = "" 1531 line_number: OptInt = None 1532 additional_comment: str = "" 1533 meta_bug_blocked: bool = False 1534 create_bug_lambda: CreateBug = None 1535 bugid: OptInt 1536 1537 if bug_id is None: 1538 if self.mode == Mode.KNOWN_INTERMITTENT and kind == Kind.TOML: 1539 (bugid, comment, line_number) = self.find_known_intermittent( 1540 repo, revision, task_id, manifest, filename, skip_if 1541 ) 1542 if bugid is None: 1543 self.info("Ignoring failure as it is not a known intermittent") 1544 return 1545 self.vinfo(f"Found known intermittent: {bugid}") 1546 action = Action( 1547 manifest=manifest, 1548 path=path, 1549 label=label, 1550 revision=revision, 1551 disposition=self.mode, 1552 bugid=bugid, 1553 task_id=task_id, 1554 skip_if=skip_if, 1555 ) 1556 else: 1557 ( 1558 create_bug_lambda, 1559 bugid, 1560 meta_bug_blocked, 1561 attachments, 1562 comment, 1563 line_number, 1564 summary, 1565 description, 1566 product, 1567 component, 1568 ) = self.generate_bugzilla_comment( 1569 manifest, 1570 kind, 1571 path, 1572 skip_if, 1573 filename, 1574 anyjs, 1575 lineno, 1576 label, 1577 classification, 1578 task_id, 1579 try_url, 1580 revision, 1581 repo, 1582 meta_bug_id, 1583 ) 1584 bug_reference: str = f"Bug {bugid}" 1585 if classification == Classification.SECONDARY and kind != Kind.WPT: 1586 bug_reference += " (secondary)" 1587 if self.mode == Mode.NEW_FAILURE: 1588 action = Action( 1589 manifest=manifest, 1590 path=path, 1591 label=label, 1592 revision=revision, 1593 disposition=self.mode, 1594 bugid=bugid, 1595 task_id=task_id, 1596 skip_if=skip_if, 1597 summary=summary, 1598 description=description, 1599 product=product, 1600 component=component, 1601 ) 1602 else: 1603 bug_reference = f"Bug {bug_id}" 1604 1605 if kind == Kind.WPT: 1606 if os.path.exists(manifest_path): 1607 manifest_str = open(manifest_path, encoding="utf-8").read() 1608 else: 1609 # ensure parent directories exist 1610 os.makedirs(os.path.dirname(manifest_path), exist_ok=True) 1611 if bug_id is None and create_bug_lambda is not None: 1612 bug = create_bug_lambda() 1613 if bug is not None: 1614 bug_reference = f"Bug {bug.id}" 1615 manifest_str, additional_comment = self.wpt_add_skip_if( 1616 manifest_str, anyjs, skip_if, bug_reference 1617 ) 1618 elif kind == Kind.TOML: 1619 mp = ManifestParser(use_toml=True, document=True) 1620 try: 1621 mp.read(manifest_path) 1622 except OSError: 1623 raise Exception(f"Unable to find path: {manifest_path}") 1624 1625 document = mp.source_documents[manifest_path] 1626 try: 1627 additional_comment, carryover, bug_reference = add_skip_if( 1628 document, 1629 filename, 1630 skip_if, 1631 bug_reference, 1632 create_bug_lambda, 1633 self.mode, 1634 ) 1635 except Exception: 1636 # Note: this fails to find a comment at the desired index 1637 # Note: manifestparser len(skip_if) yields: TypeError: object of type 'bool' has no len() 1638 additional_comment = "" 1639 carryover = False 1640 if bug_reference is None: # skip-if was ignored 1641 self.warning( 1642 f'Did NOT add redundant skip-if to: ["{filename}"] in manifest: "{manifest}"' 1643 ) 1644 return 1645 if self.mode == Mode.CARRYOVER: 1646 if not carryover: 1647 self.vinfo( 1648 f'No --carryover in: ["{filename}"] in manifest: "{manifest}"' 1649 ) 1650 return 1651 bugid = self.bugid_from_reference(bug_reference) 1652 action = Action( 1653 manifest=manifest, 1654 path=path, 1655 label=label, 1656 revision=revision, 1657 disposition=self.mode, 1658 bugid=bugid, 1659 task_id=task_id, 1660 skip_if=skip_if, 1661 ) 1662 manifest_str = alphabetize_toml_str(document) 1663 elif kind == Kind.LIST: 1664 if lineno == 0: 1665 self.error( 1666 f"cannot determine line to edit in manifest: {manifest_path}" 1667 ) 1668 elif not os.path.exists(manifest_path): 1669 self.error(f"manifest does not exist: {manifest_path}") 1670 else: 1671 manifest_str = open(manifest_path, encoding="utf-8").read() 1672 if status == PASS: 1673 self.info(f"Unexpected status: {status}") 1674 if ( 1675 status == PASS 1676 or classification == Classification.DISABLE_INTERMITTENT 1677 ): 1678 zero = True # refest lower ranges should include zero 1679 else: 1680 zero = False 1681 if bug_id is None and create_bug_lambda is not None: 1682 bug = create_bug_lambda() 1683 if bug is not None: 1684 bug_reference = f"Bug {bug.id}" 1685 manifest_str, additional_comment = self.reftest_add_fuzzy_if( 1686 manifest_str, 1687 filename, 1688 skip_if, 1689 differences, 1690 pixels, 1691 lineno, 1692 zero, 1693 bug_reference, 1694 ) 1695 if not manifest_str and additional_comment: 1696 self.warning(additional_comment) 1697 if manifest_str: 1698 if line_number is not None: 1699 comment += "\n" + self.error_log_context(revision, task_id, line_number) 1700 if additional_comment: 1701 comment += "\n" + additional_comment 1702 if action is not None: 1703 action.comment = comment 1704 self.actions[action.key()] = action 1705 if self.dry_run: 1706 prefix = "Would have (--dry-run): " 1707 else: 1708 prefix = "" 1709 with open(manifest_path, "w", encoding="utf-8", newline="\n") as fp: 1710 fp.write(manifest_str) 1711 self.info(f'{prefix}Edited ["{filename}"] in manifest: "{manifest}"') 1712 if kind != Kind.LIST: 1713 self.info( 1714 f'{prefix}Added {SkipfailsMode.name(self.mode)} skip-if condition: "{skip_if} # {bug_reference}"' 1715 ) 1716 if bug_id is None: 1717 return 1718 if self.mode == Mode.NORMAL: 1719 if self.bugzilla is None: 1720 self.vinfo( 1721 f"Bugzilla has been disabled: comment not added to Bug {bugid}:\n{comment}" 1722 ) 1723 elif meta_bug_id is None: 1724 self.vinfo( 1725 f"No --meta-bug-id specified: comment not added to Bug {bugid}:\n{comment}" 1726 ) 1727 elif self.dry_run: 1728 self.vinfo( 1729 f"Flag --dry-run: comment not added to Bug {bugid}:\n{comment}" 1730 ) 1731 else: 1732 self.add_bug_comment( 1733 bugid, comment, None if meta_bug_blocked else meta_bug_id 1734 ) 1735 if meta_bug_id is not None: 1736 self.info(f" Bug {bugid} blocks meta Bug: {meta_bug_id}") 1737 self.info(f"Added comment to Bug {bugid}:\n{comment}") 1738 else: 1739 self.vinfo(f"New comment for Bug {bugid}:\n{comment}") 1740 else: 1741 self.error(f'Error editing ["{filename}"] in manifest: "{manifest}"') 1742 1743 def replace_tbd(self, meta_bug_id: int): 1744 # First pass: file new bugs for TBD, collect comments by bugid 1745 comments_by_bugid: DictStrList = {} 1746 for k in self.actions: 1747 action: Action = self.actions[k] 1748 self.info(f"\n\n===== Action in manifest: {action.manifest} =====") 1749 self.info(f" path: {action.path}") 1750 self.info(f" label: {action.label}") 1751 self.info(f" skip_if: {action.skip_if}") 1752 self.info(f" disposition: {SkipfailsMode.name(action.disposition)}") 1753 self.info(f" bug_id: {action.bugid}") 1754 1755 kind: Kind = Kind.TOML 1756 if not action.manifest.endswith(".toml"): 1757 raise Exception( 1758 f'Only TOML manifests supported for --replace-tbd: "{action.manifest}"' 1759 ) 1760 if action.disposition == Mode.NEW_FAILURE: 1761 if self.bugzilla is None: 1762 self.vinfo( 1763 f"Bugzilla has been disabled: new bug not created for Bug {action.bugid}" 1764 ) 1765 elif self.dry_run: 1766 self.vinfo( 1767 f"Flag --dry-run: new bug not created for Bug {action.bugid}" 1768 ) 1769 else: 1770 # Check for existing bug 1771 bugs = self.get_bugs_by_summary(action.summary, meta_bug_id) 1772 if len(bugs) == 0: 1773 bug = self.create_bug( 1774 action.summary, 1775 action.description, 1776 action.product, 1777 action.component, 1778 ) 1779 if bug is not None: 1780 action.bugid = str(bug.id) 1781 elif len(bugs) == 1: 1782 action.bugid = str(bugs[0].id) 1783 self.vinfo(f'Found Bug {action.bugid} "{action.summary}"') 1784 else: 1785 raise Exception( 1786 f'More than one bug found for summary: "{action.summary}"' 1787 ) 1788 manifest_path: str = self.full_path(action.manifest) 1789 filename: str = self.resolve_failure_filename( 1790 action.path, kind, action.manifest 1791 ) 1792 1793 mp = ManifestParser(use_toml=True, document=True) 1794 mp.read(manifest_path) 1795 document = mp.source_documents[manifest_path] 1796 updated = replace_tbd_skip_if( 1797 document, filename, action.skip_if, action.bugid 1798 ) 1799 if updated: 1800 manifest_str = alphabetize_toml_str(document) 1801 with open( 1802 manifest_path, "w", encoding="utf-8", newline="\n" 1803 ) as fp: 1804 fp.write(manifest_str) 1805 self.info( 1806 f'Edited ["{filename}"] in manifest: "{action.manifest}"' 1807 ) 1808 else: 1809 self.error( 1810 f'Error editing ["{filename}"] in manifest: "{action.manifest}"' 1811 ) 1812 self.actions[k] = action 1813 comments: ListStr = comments_by_bugid.get(action.bugid, []) 1814 comments.append(action.comment) 1815 comments_by_bugid[action.bugid] = comments 1816 # Second pass: Add combined comment for each bugid 1817 for k in self.actions: 1818 action: Action = self.actions[k] 1819 comments: ListStr = comments_by_bugid.get(action.bugid, []) 1820 if self.bugzilla is not None and not self.dry_run: 1821 action.disposition = SkipfailsMode.bug_filed(action.disposition) 1822 self.actions[k] = action 1823 if len(comments) > 0: # comments not yet added 1824 self.info( 1825 f"\n\n===== Filing Combined Comment for Bug {action.bugid} =====" 1826 ) 1827 comment: str = "" 1828 for c in comments: 1829 comment += c + "\n" 1830 if self.bugzilla is None: 1831 self.vinfo( 1832 f"Bugzilla has been disabled: comment not added to Bug {action.bugid}:\n{comment}" 1833 ) 1834 elif self.dry_run: 1835 self.vinfo( 1836 f"Flag --dry-run: comment not added to Bug {action.bugid}:\n{comment}" 1837 ) 1838 else: 1839 self.add_bug_comment(int(action.bugid), comment) 1840 self.info(f"Added comment to Bug {action.bugid}:\n{comment}") 1841 comments_by_bugid[action.bugid] = [] 1842 1843 def get_variants(self): 1844 """Get mozinfo for each test variants""" 1845 1846 if len(self.variants) == 0: 1847 variants_file = "taskcluster/test_configs/variants.yml" 1848 variants_path = self.full_path(variants_file) 1849 with open(variants_path, encoding="utf-8") as fp: 1850 raw_variants = load(fp, Loader=Loader) 1851 for k, v in raw_variants.items(): 1852 mozinfo = k 1853 if "mozinfo" in v: 1854 mozinfo = v["mozinfo"] 1855 self.variants[k] = mozinfo 1856 return self.variants 1857 1858 def get_task_details(self, task_id): 1859 """Download details for task task_id""" 1860 1861 if task_id in self.tasks: # if cached 1862 task = self.tasks[task_id] 1863 else: 1864 self.vinfo(f"get_task_details for task: {task_id}") 1865 try: 1866 task = get_task(task_id) 1867 except TaskclusterRestFailure: 1868 self.warning(f"Task {task_id} no longer exists.") 1869 return None 1870 self.tasks[task_id] = task 1871 return task 1872 1873 def get_extra(self, task_id): 1874 """Calculate extra for task task_id""" 1875 1876 if task_id in self.extras: # if cached 1877 platform_info = self.extras[task_id] 1878 else: 1879 self.get_variants() 1880 task = self.get_task_details(task_id) or {} 1881 test_setting = task.get("extra", {}).get("test-setting", {}) 1882 platform = test_setting.get("platform", {}) 1883 platform_os = platform.get("os", {}) 1884 if self.new_version: 1885 platform_os["version"] = self.new_version 1886 if not test_setting: 1887 return None 1888 platform_info = PlatformInfo(test_setting) 1889 self.extras[task_id] = platform_info 1890 return platform_info 1891 1892 def get_opt_for_task(self, task_id): 1893 extra = self.get_extra(task_id) 1894 return extra.opt 1895 1896 def _fetch_platform_permutations(self): 1897 self.info("Fetching platform permutations...") 1898 import taskcluster 1899 1900 url: OptStr = None 1901 index = taskcluster.Index({ 1902 "rootUrl": "https://firefox-ci-tc.services.mozilla.com", 1903 }) 1904 route = "gecko.v2.mozilla-central.latest.source.test-info-all" 1905 queue = taskcluster.Queue({ 1906 "rootUrl": "https://firefox-ci-tc.services.mozilla.com", 1907 }) 1908 1909 # Typing from findTask is wrong, so we need to convert to Any 1910 result: OptTaskResult = index.findTask(route) 1911 if result is not None: 1912 task_id: str = result["taskId"] 1913 result = queue.listLatestArtifacts(task_id) 1914 if result is not None and task_id is not None: 1915 artifact_list: ArtifactList = result["artifacts"] 1916 for artifact in artifact_list: 1917 artifact_name = artifact["name"] 1918 if artifact_name.endswith("test-info-testrun-matrix.json"): 1919 url = queue.buildUrl( 1920 "getLatestArtifact", task_id, artifact_name 1921 ) 1922 break 1923 1924 if url is not None: 1925 self.vinfo("Retrieving platform permutations ...") 1926 response = requests.get(url, headers={"User-agent": "mach-test-info/1.0"}) 1927 self.platform_permutations = response.json() 1928 else: 1929 self.info("Failed fetching platform permutations...") 1930 1931 def _get_list_skip_if(self, platform_info: PlatformInfo): 1932 aa = "&&" 1933 nn = "!" 1934 1935 os = platform_info.os 1936 build_type = platform_info.build_type 1937 runtimes = platform_info.test_variant.split("+") 1938 1939 skip_if = None 1940 if os == "linux": 1941 skip_if = "gtkWidget" 1942 elif os == "win": 1943 skip_if = "winWidget" 1944 elif os == "mac": 1945 skip_if = "cocoaWidget" 1946 elif os == "android": 1947 skip_if = "Android" 1948 else: 1949 self.error(f"cannot calculate skip-if for unknown OS: '{os}'") 1950 if skip_if is not None: 1951 ccov = "ccov" in build_type 1952 asan = "asan" in build_type 1953 tsan = "tsan" in build_type 1954 optimized = ( 1955 (not platform_info.debug) and (not ccov) and (not asan) and (not tsan) 1956 ) 1957 skip_if += aa 1958 if optimized: 1959 skip_if += "optimized" 1960 elif platform_info.debug: 1961 skip_if += "isDebugBuild" 1962 elif ccov: 1963 skip_if += "isCoverageBuild" 1964 elif asan: 1965 skip_if += "AddressSanitizer" 1966 elif tsan: 1967 skip_if += "ThreadSanitizer" 1968 # See implicit VARIANT_DEFAULTS in 1969 # https://searchfox.org/mozilla-central/source/layout/tools/reftest/manifest.sys.mjs#30 1970 no_fission = "!fission" not in runtimes 1971 snapshot = "snapshot" in runtimes 1972 swgl = "swgl" in runtimes 1973 nogpu = "nogpu" in runtimes 1974 if not self.implicit_vars and no_fission: 1975 skip_if += aa + "fission" 1976 elif not no_fission: # implicit default: fission 1977 skip_if += aa + nn + "fission" 1978 if platform_info.bits is not None: 1979 if platform_info.bits == "32": 1980 skip_if += aa + nn + "is64Bit" # override implicit is64Bit 1981 elif not self.implicit_vars and os == "winWidget": 1982 skip_if += aa + "is64Bit" 1983 if not self.implicit_vars and not swgl: 1984 skip_if += aa + nn + "swgl" 1985 elif swgl: # implicit default: !swgl 1986 skip_if += aa + "swgl" 1987 1988 if not self.implicit_vars and not nogpu: 1989 skip_if += aa + nn + "nogpu" 1990 elif nogpu: # implicit default: !swgl 1991 skip_if += aa + "nogpu" 1992 1993 if os == "gtkWidget": 1994 if not self.implicit_vars and not snapshot: 1995 skip_if += aa + nn + "useDrawSnapshot" 1996 elif snapshot: # implicit default: !useDrawSnapshot 1997 skip_if += aa + "useDrawSnapshot" 1998 return skip_if 1999 2000 def task_to_skip_if( 2001 self, 2002 manifest: str, 2003 task: TaskIdOrPlatformInfo, 2004 kind: str, 2005 file_path: str, 2006 high_freq: bool, 2007 ) -> OptStr: 2008 """Calculate the skip-if condition for failing task task_id""" 2009 if isinstance(task, str): 2010 extra = self.get_extra(task) 2011 else: 2012 extra = task 2013 if kind == Kind.WPT: 2014 qq = '"' 2015 aa = " and " 2016 else: 2017 qq = "'" 2018 aa = " && " 2019 eq = " == " 2020 skip_if = None 2021 os = extra.os 2022 os_version = extra.os_version 2023 if os is not None: 2024 if kind == Kind.LIST: 2025 skip_if = self._get_list_skip_if(extra) 2026 else: 2027 skip_if = "os" + eq + qq + os + qq 2028 if os_version is not None: 2029 skip_if += aa + "os_version" + eq + qq + os_version + qq 2030 arch = extra.arch 2031 if arch is not None and skip_if is not None and kind != Kind.LIST: 2032 skip_if += aa + "arch" + eq + qq + arch + qq 2033 if high_freq: 2034 failure_key = os + os_version + arch + manifest + file_path 2035 if self.failed_platforms.get(failure_key) is None: 2036 if not self.platform_permutations: 2037 self._fetch_platform_permutations() 2038 permutations = ( 2039 self.platform_permutations.get(manifest, {}) 2040 .get(os, {}) 2041 .get(os_version, {}) 2042 .get(arch, None) 2043 ) 2044 self.failed_platforms[failure_key] = FailedPlatform( 2045 permutations, high_freq 2046 ) 2047 skip_cond = self.failed_platforms[failure_key].get_skip_string( 2048 aa, extra.build_type, extra.test_variant 2049 ) 2050 if skip_cond is not None: 2051 skip_if += skip_cond 2052 else: 2053 skip_if = None 2054 else: # not high_freq 2055 skip_if += aa + extra.build_type 2056 variants = extra.test_variant.split("+") 2057 if len(variants) >= 3: 2058 self.warning( 2059 f'Removing all variants "{" ".join(variants)}" from skip-if condition in manifest={manifest} and file={file_path}' 2060 ) 2061 else: 2062 for tv in variants: 2063 if tv != "no_variant": 2064 skip_if += aa + tv 2065 elif skip_if is None: 2066 raise Exception( 2067 f"Unable to calculate skip-if condition from manifest={manifest} and file={file_path}" 2068 ) 2069 if skip_if is not None and kind == Kind.WPT: 2070 # ensure ! -> 'not', primarily fission and e10s 2071 skip_if = skip_if.replace("!", "not ") 2072 return skip_if 2073 2074 def get_file_info(self, path, product="Testing", component="General"): 2075 """ 2076 Get bugzilla product and component for the path. 2077 Provide defaults (in case command_context is not defined 2078 or there isn't file info available). 2079 """ 2080 if path != DEF and self.command_context is not None: 2081 reader = self.command_context.mozbuild_reader(config_mode="empty") 2082 info = reader.files_info([path]) 2083 try: 2084 cp = info[path]["BUG_COMPONENT"] 2085 except TypeError: 2086 # TypeError: BugzillaComponent.__new__() missing 2 required positional arguments: 'product' and 'component' 2087 pass 2088 else: 2089 product = cp.product 2090 component = cp.component 2091 return product, component 2092 2093 def get_filename_in_manifest(self, manifest: str, path: str) -> str: 2094 """return relative filename for path in manifest""" 2095 2096 filename = os.path.basename(path) 2097 if filename == DEF: 2098 return filename 2099 manifest_dir = os.path.dirname(manifest) 2100 i = 0 2101 j = min(len(manifest_dir), len(path)) 2102 while i < j and manifest_dir[i] == path[i]: 2103 i += 1 2104 if i < len(manifest_dir): 2105 for _ in range(manifest_dir.count("/", i) + 1): 2106 filename = "../" + filename 2107 elif i < len(path): 2108 filename = path[i + 1 :] 2109 return filename 2110 2111 def get_push_id(self, revision: str, repo: str): 2112 """Return the push_id for revision and repo (or None)""" 2113 2114 if revision in self.push_ids: # if cached 2115 self.vinfo(f"Getting push_id for {repo} revision: {revision} ...") 2116 push_id = self.push_ids[revision] 2117 else: 2118 push_id = None 2119 push_url = f"https://treeherder.mozilla.org/api/project/{repo}/push/" 2120 params = {} 2121 params["full"] = "true" 2122 params["count"] = 10 2123 params["revision"] = revision 2124 self.vinfo(f"Retrieving push_id for {repo} revision: {revision} ...") 2125 r = requests.get(push_url, headers=self.headers, params=params) 2126 if r.status_code != 200: 2127 self.warning(f"FAILED to query Treeherder = {r} for {r.url}") 2128 else: 2129 response = r.json() 2130 if "results" in response: 2131 results = response["results"] 2132 if len(results) > 0: 2133 r0 = results[0] 2134 if "id" in r0: 2135 push_id = r0["id"] 2136 self.push_ids[revision] = push_id 2137 return push_id 2138 2139 def cached_job_ids(self, revision): 2140 if len(self.push_ids) == 0 and len(self.job_ids) == 0: 2141 # no pre-caching for tests 2142 job_ids_cached = self.cached_path(revision, "job_ids.json") 2143 if os.path.exists(job_ids_cached): 2144 self.job_ids = read_json(job_ids_cached) 2145 for k in self.job_ids: 2146 push_id, _task_id = k.split(":") 2147 self.push_ids[revision] = push_id 2148 break 2149 2150 def cache_job_ids(self, revision): 2151 job_ids_cached = self.cached_path(revision, "job_ids.json") 2152 if not os.path.exists(job_ids_cached): 2153 write_json(job_ids_cached, self.job_ids) 2154 2155 def get_job_id(self, push_id, task_id): 2156 """Return the job_id for push_id, task_id (or None)""" 2157 2158 k = f"{push_id}:{task_id}" 2159 if k in self.job_ids: # if cached 2160 self.vinfo(f"Getting job_id for push_id: {push_id}, task_id: {task_id} ...") 2161 job_id = self.job_ids[k] 2162 else: 2163 job_id = None 2164 params = {} 2165 params["push_id"] = push_id 2166 self.vinfo( 2167 f"Retrieving job_id for push_id: {push_id}, task_id: {task_id} ..." 2168 ) 2169 r = requests.get(self.jobs_url, headers=self.headers, params=params) 2170 if r.status_code != 200: 2171 self.warning(f"FAILED to query Treeherder = {r} for {r.url}") 2172 else: 2173 response = r.json() 2174 if "results" in response: 2175 results = response["results"] 2176 if len(results) > 0: 2177 for result in results: 2178 if len(result) > 14: 2179 if result[14] == task_id: 2180 job_id = result[1] 2181 break 2182 self.job_ids[k] = job_id 2183 return job_id 2184 2185 def cached_bug_suggestions(self, repo, revision, job_id) -> JSONType: 2186 """ 2187 Return the bug_suggestions JSON for the job_id 2188 Use the cache, if present, else download from treeherder 2189 """ 2190 2191 if job_id in self.suggestions: 2192 self.vinfo( 2193 f"Getting bug_suggestions for {repo} revision: {revision} job_id: {job_id}" 2194 ) 2195 suggestions = self.suggestions[job_id] 2196 else: 2197 suggestions_path = self.cached_path(revision, f"suggest-{job_id}.json") 2198 if os.path.exists(suggestions_path): 2199 self.vinfo( 2200 f"Reading cached bug_suggestions for {repo} revision: {revision} job_id: {job_id}" 2201 ) 2202 suggestions = read_json(suggestions_path) 2203 else: 2204 suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/" 2205 self.vinfo( 2206 f"Retrieving bug_suggestions for {repo} revision: {revision} job_id: {job_id}" 2207 ) 2208 r = requests.get(suggestions_url, headers=self.headers) 2209 if r.status_code != 200: 2210 self.warning(f"FAILED to query Treeherder = {r} for {r.url}") 2211 return None 2212 suggestions = r.json() 2213 write_json(suggestions_path, suggestions) 2214 self.suggestions[job_id] = suggestions 2215 return suggestions 2216 2217 def get_bug_suggestions( 2218 self, repo, revision, job_id, path, anyjs=None 2219 ) -> Suggestion: 2220 """ 2221 Return the (line_number, line, log_url) 2222 for the given repo and job_id 2223 """ 2224 line_number: int = None 2225 line: str = None 2226 log_url: str = None 2227 suggestions: JSONType = self.cached_bug_suggestions(repo, revision, job_id) 2228 if suggestions is not None: 2229 paths: ListStr 2230 if anyjs is not None and len(anyjs) > 0: 2231 pathdir: str = os.path.dirname(path) + "/" 2232 paths = [pathdir + f for f in anyjs.keys()] 2233 else: 2234 paths = [path] 2235 if len(suggestions) > 0: 2236 for suggestion in suggestions: 2237 for p in paths: 2238 path_end = suggestion.get("path_end", None) 2239 # handles WPT short paths 2240 if path_end is not None and p.endswith(path_end): 2241 line_number = suggestion["line_number"] + 1 2242 line = suggestion["search"] 2243 log_url = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}" 2244 break 2245 return (line_number, line, log_url) 2246 2247 def read_tasks(self, filename): 2248 """read tasks as JSON from filename""" 2249 2250 if not os.path.exists(filename): 2251 msg = f"use-tasks JSON file does not exist: {filename}" 2252 raise OSError(2, msg, filename) 2253 tasks = read_json(filename) 2254 tasks = [Mock(task, MOCK_TASK_DEFAULTS, MOCK_TASK_INITS) for task in tasks] 2255 for task in tasks: 2256 if len(task.extra) > 0: # pre-warm cache for extra information 2257 platform_info = PlatformInfo() 2258 extra: Any = task.extra 2259 platform_info.from_dict(extra) 2260 self.extras[task.id] = platform_info 2261 return tasks 2262 2263 def read_failures(self, filename): 2264 """read failures as JSON from filename""" 2265 2266 if not os.path.exists(filename): 2267 msg = f"use-failures JSON file does not exist: {filename}" 2268 raise OSError(2, msg, filename) 2269 failures = read_json(filename) 2270 return failures 2271 2272 def read_bugs(self, filename): 2273 """read bugs as JSON from filename""" 2274 2275 if not os.path.exists(filename): 2276 msg = f"bugs JSON file does not exist: {filename}" 2277 raise OSError(2, msg, filename) 2278 bugs = read_json(filename) 2279 bugs = [Mock(bug, MOCK_BUG_DEFAULTS) for bug in bugs] 2280 return bugs 2281 2282 def write_tasks(self, save_tasks, tasks): 2283 """saves tasks as JSON to save_tasks""" 2284 jtasks = [] 2285 for task in tasks: 2286 extras = self.get_extra(task.id) 2287 if not extras: 2288 continue 2289 2290 jtask = {} 2291 jtask["id"] = task.id 2292 jtask["label"] = task.label 2293 jtask["duration"] = task.duration 2294 jtask["result"] = task.result 2295 jtask["state"] = task.state 2296 jtask["extra"] = extras.to_dict() 2297 jtags = {} 2298 for k, v in task.tags.items(): 2299 if k == "createdForUser": 2300 jtags[k] = "ci@mozilla.com" 2301 else: 2302 jtags[k] = v 2303 jtask["tags"] = jtags 2304 jtask["tier"] = task.tier 2305 jtask["results"] = [ 2306 {"group": r.group, "ok": r.ok, "duration": r.duration} 2307 for r in task.results 2308 ] 2309 jtask["errors"] = None # Bug with task.errors property?? 2310 jft = {} 2311 if self.failure_types is not None and task.id in self.failure_types: 2312 failure_types = self.failure_types[task.id] # use cache 2313 else: 2314 try: 2315 failure_types = task.failure_types 2316 except requests.exceptions.HTTPError: 2317 continue 2318 except TaskclusterRestFailure: 2319 continue 2320 for k in failure_types: 2321 if isinstance(task, TestTask): 2322 jft[k] = [[f[0], f[1].value] for f in task.failure_types[k]] 2323 else: 2324 jft[k] = [[f[0], f[1]] for f in task.failure_types[k]] 2325 jtask["failure_types"] = jft 2326 jtasks.append(jtask) 2327 write_json(save_tasks, jtasks) 2328 2329 def add_attachment_log_for_task(self, bugid: str, task_id: str): 2330 """Adds compressed log for this task to bugid""" 2331 2332 log_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/logs/live_backing.log" 2333 self.vinfo(f"Retrieving full log for task: {task_id}") 2334 r = requests.get(log_url, headers=self.headers) 2335 if r.status_code != 200: 2336 self.error(f"Unable to get log for task: {task_id}") 2337 return 2338 attach_fp = tempfile.NamedTemporaryFile() 2339 with gzip.open(attach_fp, "wb") as fp: 2340 fp.write(r.text.encode("utf-8")) 2341 if self._initialize_bzapi(): 2342 description = ATTACHMENT_DESCRIPTION + task_id 2343 file_name = TASK_LOG + ".gz" 2344 comment = "Added compressed log" 2345 content_type = "application/gzip" 2346 try: 2347 self._bzapi.attachfile( 2348 [int(bugid)], 2349 attach_fp.name, 2350 description, 2351 file_name=file_name, 2352 comment=comment, 2353 content_type=content_type, 2354 is_private=False, 2355 ) 2356 except Fault: 2357 pass # Fault expected: Failed to fetch key 9372091 from network storage: The specified key does not exist. 2358 2359 def wpt_paths(self, shortpath: str) -> WptPaths: 2360 """ 2361 Analyzes the WPT short path for a test and returns 2362 (path, manifest, query, anyjs) where 2363 path is the relative path to the test file 2364 manifest is the relative path to the file metadata 2365 query is the test file query paramters (or None) 2366 anyjs is the html test file as reported by mozci (or None) 2367 """ 2368 path, manifest, query, anyjs = parse_wpt_path(shortpath, self.isdir) 2369 if manifest and manifest.startswith(WPT_META0): 2370 manifest_classic = manifest.replace(WPT_META0, WPT_META0_CLASSIC, 1) 2371 if self.exists(manifest_classic): 2372 if self.exists(manifest): 2373 self.warning( 2374 f"Both classic {manifest_classic} and metadata {manifest} manifests exist" 2375 ) 2376 else: 2377 self.warning( 2378 f"Using the classic {manifest_classic} manifest as the metadata manifest {manifest} does not exist" 2379 ) 2380 manifest = manifest_classic 2381 2382 self.vinfo(f"wpt_paths::{path},{manifest}") 2383 2384 if path and not self.exists(path): 2385 return (None, None, None, None) 2386 return (path, manifest, query, anyjs) 2387 2388 def wpt_add_skip_if(self, manifest_str, anyjs, skip_if, bug_reference): 2389 """ 2390 Edits a WPT manifest string to add disabled condition 2391 anyjs is a dictionary mapping from filename and any alternate basenames to 2392 a boolean (indicating if the file has been handled in the manifest). 2393 Returns additional_comment (if any) 2394 """ 2395 2396 additional_comment = "" 2397 disabled_key = False 2398 disabled = " disabled:" 2399 condition_start = " if " 2400 condition = condition_start + skip_if + ": " + bug_reference 2401 lines = manifest_str.splitlines() 2402 section = None # name of the section 2403 i = 0 2404 n = len(lines) 2405 while i < n: 2406 line = lines[i] 2407 if line.startswith("["): 2408 if section is not None and not anyjs[section]: # not yet handled 2409 if not disabled_key: 2410 lines.insert(i, disabled) 2411 i += 1 2412 lines.insert(i, condition) 2413 lines.insert(i + 1, "") # blank line after condition 2414 i += 2 2415 n += 2 2416 anyjs[section] = True 2417 section = line[1:-1] 2418 if section and anyjs and section in anyjs and not anyjs[section]: 2419 disabled_key = False 2420 else: 2421 section = None # ignore section we are not interested in 2422 elif section is not None: 2423 if line == disabled: 2424 disabled_key = True 2425 elif line.startswith(" ["): 2426 if i > 0 and i - 1 < n and lines[i - 1] == "": 2427 del lines[i - 1] 2428 i -= 1 2429 n -= 1 2430 if not disabled_key: 2431 lines.insert(i, disabled) 2432 i += 1 2433 n += 1 2434 lines.insert(i, condition) 2435 lines.insert(i + 1, "") # blank line after condition 2436 i += 2 2437 n += 2 2438 anyjs[section] = True 2439 section = None 2440 elif line.startswith(" ") and not line.startswith(" "): 2441 if disabled_key: # insert condition above new key 2442 lines.insert(i, condition) 2443 i += 1 2444 n += 1 2445 anyjs[section] = True 2446 section = None 2447 disabled_key = False 2448 elif line.startswith(" "): 2449 if disabled_key and line == condition: 2450 anyjs[section] = True # condition already present 2451 section = None 2452 i += 1 2453 if section is not None and not anyjs[section]: # not yet handled 2454 if i > 0 and i - 1 < n and lines[i - 1] == "": 2455 del lines[i - 1] 2456 if not disabled_key: 2457 lines.append(disabled) 2458 i += 1 2459 n += 1 2460 lines.append(condition) 2461 lines.append("") # blank line after condition 2462 i += 2 2463 n += 2 2464 anyjs[section] = True 2465 if len(anyjs) > 0: 2466 for section in anyjs: 2467 if not anyjs[section]: 2468 if i > 0 and i - 1 < n and lines[i - 1] != "": 2469 lines.append("") # blank line before condition 2470 i += 1 2471 n += 1 2472 lines.append("[" + section + "]") 2473 lines.append(disabled) 2474 lines.append(condition) 2475 lines.append("") # blank line after condition 2476 i += 4 2477 n += 4 2478 manifest_str = "\n".join(lines) + "\n" 2479 return manifest_str, additional_comment 2480 2481 def reftest_add_fuzzy_if( 2482 self, 2483 manifest_str, 2484 filename, 2485 fuzzy_if, 2486 differences, 2487 pixels, 2488 lineno, 2489 zero, 2490 bug_reference, 2491 ): 2492 """ 2493 Edits a reftest manifest string to add disabled condition 2494 """ 2495 2496 if self.lmp is None: 2497 from parse_reftest import ListManifestParser 2498 2499 self.lmp = ListManifestParser( 2500 self.implicit_vars, self.verbose, self.error, self.warning, self.info 2501 ) 2502 manifest_str, additional_comment = self.lmp.reftest_add_fuzzy_if( 2503 manifest_str, 2504 filename, 2505 fuzzy_if, 2506 differences, 2507 pixels, 2508 lineno, 2509 zero, 2510 bug_reference, 2511 ) 2512 return manifest_str, additional_comment 2513 2514 def get_lineno_difference_pixels_status(self, task_id, manifest, allmods): 2515 """ 2516 Returns 2517 - lineno in manifest 2518 - image comparison, max *difference* 2519 - number of differing *pixels* 2520 - status (PASS or FAIL) 2521 as cached from reftest_errorsummary.log for a task 2522 """ 2523 manifest_obj = self.error_summary.get(manifest, {}) 2524 # allmods_obj: manifest_test_name: {test: allmods, error: ...} 2525 allmods_obj = manifest_obj[ 2526 [am for am in manifest_obj if manifest_obj[am]["test"].endswith(allmods)][0] 2527 ] 2528 lineno = allmods_obj.get(LINENO, 0) 2529 runs_obj = allmods_obj.get(RUNS, {}) 2530 task_obj = runs_obj.get(task_id, {}) 2531 difference = task_obj.get(DIFFERENCE, 0) 2532 pixels = task_obj.get(PIXELS, 0) 2533 status = task_obj.get(STATUS, FAIL) 2534 return lineno, difference, pixels, status 2535 2536 def reftest_find_lineno(self, manifest, modifiers, allmods): 2537 """ 2538 Return the line number with modifiers in manifest (else 0) 2539 """ 2540 2541 lineno = 0 2542 mods = [] 2543 prefs = [] 2544 for i in range(len(modifiers)): 2545 if modifiers[i].find("pref(") >= 0 or modifiers[i].find("skip-if(") >= 0: 2546 prefs.append(modifiers[i]) 2547 else: 2548 mods.append(modifiers[i]) 2549 m = len(mods) 2550 manifest_str = open(manifest, encoding="utf-8").read() 2551 lines = manifest_str.splitlines() 2552 defaults = [] 2553 found = False 2554 alt_lineno = 0 2555 for linenum in range(len(lines)): 2556 line = lines[linenum] 2557 if len(line) > 0 and line[0] == "#": 2558 continue 2559 comment_start = line.find(" #") # MUST NOT match anchors! 2560 if comment_start > 0: 2561 line = line[0:comment_start].strip() 2562 words = line.split() 2563 n = len(words) 2564 if n > 1 and words[0] == "defaults": 2565 defaults = words[1:].copy() 2566 continue 2567 line_defaults = defaults.copy() 2568 i = 0 2569 while i < n: 2570 if words[i].find("pref(") >= 0 or words[i].find("skip-if(") >= 0: 2571 line_defaults.append(words[i]) 2572 del words[i] 2573 n -= 1 2574 else: 2575 i += 1 2576 if (len(prefs) == 0 or prefs == line_defaults) and words == mods: 2577 found = True 2578 lineno = linenum + 1 2579 break 2580 elif m > 2 and n > 2: 2581 if words[-3:] == mods[-3:]: 2582 alt_lineno = linenum + 1 2583 else: 2584 bwords = [os.path.basename(f) for f in words[-2:]] 2585 bmods = [os.path.basename(f) for f in mods[-2:]] 2586 if bwords == bmods: 2587 alt_lineno = linenum + 1 2588 if not found: 2589 if alt_lineno > 0: 2590 lineno = alt_lineno 2591 self.warning( 2592 f"manifest '{manifest}' found lineno: {lineno}, but it does not contain all the prefs from modifiers,\nSEARCH: {allmods}\nFOUND : {lines[alt_lineno - 1]}" 2593 ) 2594 else: 2595 lineno = 0 2596 self.error( 2597 f"manifest '{manifest}' does not contain line with modifiers: {allmods}" 2598 ) 2599 return lineno 2600 2601 def get_allpaths(self, task_id, manifest, path): 2602 """ 2603 Looks up the reftest_errorsummary.log for a task 2604 and caches the details in self.error_summary by 2605 task_id, manifest, allmods 2606 where allmods is the concatenation of all modifiers 2607 and the details include 2608 - image comparison, max *difference* 2609 - number of differing *pixels* 2610 - status: unexpected PASS or FAIL 2611 2612 The list iof unique modifiers (allmods) for the given path are returned 2613 """ 2614 2615 allpaths = [] 2616 words = path.split() 2617 if len(words) != 3 or words[1] not in TEST_TYPES: 2618 self.warning( 2619 f"reftest_errorsummary.log for task: {task_id} has unsupported test type '{path}'" 2620 ) 2621 return allpaths 2622 if manifest in self.error_summary: 2623 for allmods in self.error_summary[manifest]: 2624 if self.error_summary[manifest][allmods][ 2625 TEST 2626 ] == path and task_id in self.error_summary[manifest][allmods].get( 2627 RUNS, {} 2628 ): 2629 allpaths.append(path) 2630 if len(allpaths) > 0: 2631 return allpaths # cached (including self tests) 2632 error_url = f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}/artifacts/public/test_info/reftest_errorsummary.log" 2633 self.vinfo(f"Retrieving reftest_errorsummary.log for task: {task_id}") 2634 r = requests.get(error_url, headers=self.headers) 2635 if r.status_code != 200: 2636 self.error(f"Unable to get reftest_errorsummary.log for task: {task_id}") 2637 return allpaths 2638 for line in r.text.splitlines(): 2639 summary = json.loads(line) 2640 group = summary.get(GROUP, "") 2641 if not group or not os.path.exists(group): # not error line 2642 continue 2643 test = summary.get(TEST, None) 2644 if test is None: 2645 continue 2646 if not MODIFIERS in summary: 2647 self.warning( 2648 f"reftest_errorsummary.log for task: {task_id} does not have modifiers for '{test}'" 2649 ) 2650 continue 2651 words = test.split() 2652 if len(words) != 3 or words[1] not in TEST_TYPES: 2653 self.warning( 2654 f"reftest_errorsummary.log for task: {task_id} has unsupported test '{test}'" 2655 ) 2656 continue 2657 status = summary.get(STATUS, "") 2658 if status not in [FAIL, PASS]: 2659 self.warning( 2660 f"reftest_errorsummary.log for task: {task_id} has unknown status: {status} for '{test}'" 2661 ) 2662 continue 2663 error = summary.get(SUBTEST, "") 2664 mods = summary[MODIFIERS] 2665 allmods = " ".join(mods) 2666 if group not in self.error_summary: 2667 self.error_summary[group] = {} 2668 if allmods not in self.error_summary[group]: 2669 self.error_summary[group][allmods] = {} 2670 self.error_summary[group][allmods][TEST] = test 2671 lineno = self.error_summary[group][allmods].get(LINENO, 0) 2672 if lineno == 0: 2673 lineno = self.reftest_find_lineno(group, mods, allmods) 2674 if lineno > 0: 2675 self.error_summary[group][allmods][LINENO] = lineno 2676 if RUNS not in self.error_summary[group][allmods]: 2677 self.error_summary[group][allmods][RUNS] = {} 2678 if task_id not in self.error_summary[group][allmods][RUNS]: 2679 self.error_summary[group][allmods][RUNS][task_id] = {} 2680 self.error_summary[group][allmods][RUNS][task_id][ERROR] = error 2681 if self._subtest_rx is None: 2682 self._subtest_rx = re.compile(SUBTEST_REGEX) 2683 m = self._subtest_rx.findall(error) 2684 if len(m) == 1: 2685 difference = int(m[0][0]) 2686 pixels = int(m[0][1]) 2687 else: 2688 difference = 0 2689 pixels = 0 2690 if difference > 0: 2691 self.error_summary[group][allmods][RUNS][task_id][DIFFERENCE] = ( 2692 difference 2693 ) 2694 if pixels > 0: 2695 self.error_summary[group][allmods][RUNS][task_id][PIXELS] = pixels 2696 if status != FAIL: 2697 self.error_summary[group][allmods][RUNS][task_id][STATUS] = status 2698 if test == path: 2699 allpaths.append(test) 2700 return allpaths 2701 2702 def find_known_intermittent( 2703 self, 2704 repo: str, 2705 revision: str, 2706 task_id: str, 2707 manifest: str, 2708 filename: str, 2709 skip_if: str, 2710 ) -> TupleOptIntStrOptInt: 2711 """ 2712 Returns bugid if a known intermittent is found. 2713 Also returns a suggested comment to be added to the known intermittent 2714 bug... (currently not implemented). The args 2715 manifest, filename, skip_if 2716 are only used to create the comment 2717 """ 2718 bugid = None 2719 suggestions: JSONType = None 2720 line_number: OptInt = None 2721 comment: str = f'Intermittent failure in manifest: "{manifest}"' 2722 comment += f'\n in test: "[{filename}]"' 2723 comment += f'\n added skip-if: "{skip_if}"' 2724 if revision is not None and repo is not None: 2725 push_id = self.get_push_id(revision, repo) 2726 if push_id is not None: 2727 job_id = self.get_job_id(push_id, task_id) 2728 if job_id is not None: 2729 suggestions: JSONType = self.cached_bug_suggestions( 2730 repo, revision, job_id 2731 ) 2732 if suggestions is not None: 2733 top: JSONType = None 2734 for suggestion in suggestions: 2735 path_end = suggestion.get("path_end", None) 2736 search: str = suggestion.get("search", "") 2737 if ( 2738 path_end is not None 2739 and path_end.endswith(filename) 2740 and ( 2741 search.startswith("PROCESS-CRASH") 2742 or (search.startswith("TEST-UNEXPECTED") and top is None) 2743 ) 2744 ): 2745 top = suggestion 2746 if top is not None: 2747 recent_bugs = top.get("bugs", {}).get("open_recent", []) 2748 for bug in recent_bugs: 2749 summary: str = bug.get("summary", "") 2750 if summary.endswith("single tracking bug"): 2751 bugid: int = bug.get("id", None) 2752 line_number = top["line_number"] + 1 2753 log_url: str = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}" 2754 comment += f"\nError log line {line_number}: {log_url}" 2755 return (bugid, comment, line_number) 2756 2757 def error_log_context(self, revision: str, task_id: str, line_number: int) -> str: 2758 context: str = "" 2759 context_path: str = self.cached_path( 2760 revision, f"context-{task_id}-{line_number}.txt" 2761 ) 2762 path = Path(context_path) 2763 if path.exists(): 2764 self.vinfo( 2765 f"Reading cached error log context for revision: {revision} task-id: {task_id} line: {line_number}" 2766 ) 2767 context = path.read_text(encoding="utf-8") 2768 else: 2769 delta: int = 10 2770 log_url = f"https://firefoxci.taskcluster-artifacts.net/{task_id}/0/public/logs/live_backing.log" 2771 self.vinfo( 2772 f"Retrieving error log context for revision: {revision} task-id: {task_id} line: {line_number}" 2773 ) 2774 r = requests.get(log_url, headers=self.headers) 2775 if r.status_code != 200: 2776 self.warning(f"Unable to get log for task: {task_id}") 2777 return context 2778 log: str = r.text 2779 n: int = len(log) 2780 i: int = 0 2781 j: int = log.find("\n", i) 2782 if j < 0: 2783 j = n 2784 line: int = 1 2785 prefix: str 2786 while i < n: 2787 if line >= line_number - delta and line <= line_number + delta: 2788 prefix = f"{line:6d}" 2789 if line == line_number: 2790 prefix = prefix.replace(" ", ">") 2791 context += f"{prefix}: {log[i:j]}\n" 2792 i = j + 1 2793 j = log.find("\n", i) 2794 if j < 0: 2795 j = n 2796 line += 1 2797 path.write_text(context, encoding="utf-8") 2798 return context 2799 2800 def read_actions(self, meta_bug_id: int): 2801 cache_dir = self.full_path(CACHE_DIR) 2802 meta_dir = os.path.join(cache_dir, str(meta_bug_id)) 2803 actions_path = os.path.join(meta_dir, "actions.json") 2804 if not os.path.exists(meta_dir): 2805 self.vinfo(f"creating meta_bug_id cache dir: {meta_dir}") 2806 os.mkdir(meta_dir) 2807 actions: DictAction = {} 2808 if os.path.exists(actions_path): 2809 actions = read_json(actions_path) 2810 for k in actions: 2811 if k not in self.actions: # do not supercede newly created actions 2812 self.actions[k] = Action(**actions[k]) 2813 2814 def write_actions(self, meta_bug_id: int): 2815 cache_dir = self.full_path(CACHE_DIR) 2816 meta_dir = os.path.join(cache_dir, str(meta_bug_id)) 2817 actions_path = os.path.join(meta_dir, "actions.json") 2818 if not os.path.exists(meta_dir): 2819 self.vinfo(f"creating meta_bug_id cache dir: {meta_dir}") 2820 os.mkdir(meta_dir) 2821 write_json(actions_path, self.actions)