testinfo.py (48493B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import datetime 6 import errno 7 import json 8 import os 9 import posixpath 10 import re 11 import subprocess 12 from collections import defaultdict 13 14 import mozpack.path as mozpath 15 import requests 16 import six.moves.urllib_parse as urlparse 17 from mozbuild.base import MachCommandConditions as conditions 18 from mozbuild.base import MozbuildObject 19 from mozfile import which 20 from mozinfo.platforminfo import PlatformInfo 21 from moztest.resolve import TestManifestLoader, TestResolver 22 from redo import retriable 23 24 REFERER = "https://wiki.developer.mozilla.org/en-US/docs/Mozilla/Test-Info" 25 MAX_DAYS = 30 26 27 28 class SetEncoder(json.JSONEncoder): 29 def default(self, obj): 30 if isinstance(obj, set): 31 return list(obj) 32 return super().default(obj) 33 34 35 class TestInfo: 36 """ 37 Support 'mach test-info'. 38 """ 39 40 def __init__(self, verbose): 41 self.verbose = verbose 42 here = os.path.abspath(os.path.dirname(__file__)) 43 self.build_obj = MozbuildObject.from_environment(cwd=here) 44 45 def log_verbose(self, what): 46 if self.verbose: 47 print(what) 48 49 50 class TestInfoTests(TestInfo): 51 """ 52 Support 'mach test-info tests': Detailed report of specified tests. 53 """ 54 55 def __init__(self, verbose): 56 TestInfo.__init__(self, verbose) 57 58 self._hg = None 59 if conditions.is_hg(self.build_obj): 60 self._hg = which("hg") 61 if not self._hg: 62 raise OSError(errno.ENOENT, "Could not find 'hg' on PATH.") 63 64 self._git = None 65 if conditions.is_git(self.build_obj): 66 self._git = which("git") 67 if not self._git: 68 raise OSError(errno.ENOENT, "Could not find 'git' on PATH.") 69 70 def find_in_hg_or_git(self, test_name): 71 if self._hg: 72 cmd = [self._hg, "files", "-I", test_name] 73 elif self._git: 74 cmd = [self._git, "ls-files", test_name] 75 else: 76 return None 77 try: 78 out = subprocess.check_output(cmd, universal_newlines=True).splitlines() 79 except subprocess.CalledProcessError: 80 out = None 81 return out 82 83 def set_test_name(self): 84 # Generating a unified report for a specific test is complicated 85 # by differences in the test name used in various data sources. 86 # Consider: 87 # - It is often convenient to request a report based only on 88 # a short file name, rather than the full path; 89 # - Bugs may be filed in bugzilla against a simple, short test 90 # name or the full path to the test; 91 # This function attempts to find appropriate names for different 92 # queries based on the specified test name. 93 94 # full_test_name is full path to file in hg (or git) 95 self.full_test_name = None 96 out = self.find_in_hg_or_git(self.test_name) 97 if out and len(out) == 1: 98 self.full_test_name = out[0] 99 elif out and len(out) > 1: 100 print("Ambiguous test name specified. Found:") 101 for line in out: 102 print(line) 103 else: 104 out = self.find_in_hg_or_git("**/%s*" % self.test_name) 105 if out and len(out) == 1: 106 self.full_test_name = out[0] 107 elif out and len(out) > 1: 108 print("Ambiguous test name. Found:") 109 for line in out: 110 print(line) 111 if self.full_test_name: 112 self.full_test_name.replace(os.sep, posixpath.sep) 113 print("Found %s in source control." % self.full_test_name) 114 else: 115 print("Unable to validate test name '%s'!" % self.test_name) 116 self.full_test_name = self.test_name 117 118 # search for full_test_name in test manifests 119 here = os.path.abspath(os.path.dirname(__file__)) 120 resolver = TestResolver.from_environment( 121 cwd=here, loader_cls=TestManifestLoader 122 ) 123 relpath = self.build_obj._wrap_path_argument(self.full_test_name).relpath() 124 tests = list(resolver.resolve_tests(paths=[relpath])) 125 if len(tests) == 1: 126 relpath = self.build_obj._wrap_path_argument(tests[0]["manifest"]).relpath() 127 print("%s found in manifest %s" % (self.full_test_name, relpath)) 128 if tests[0].get("flavor"): 129 print(" flavor: %s" % tests[0]["flavor"]) 130 if tests[0].get("skip-if"): 131 print(" skip-if: %s" % tests[0]["skip-if"]) 132 if tests[0].get("fail-if"): 133 print(" fail-if: %s" % tests[0]["fail-if"]) 134 elif len(tests) == 0: 135 print("%s not found in any test manifest!" % self.full_test_name) 136 else: 137 print("%s found in more than one manifest!" % self.full_test_name) 138 139 # short_name is full_test_name without path 140 self.short_name = None 141 name_idx = self.full_test_name.rfind("/") 142 if name_idx > 0: 143 self.short_name = self.full_test_name[name_idx + 1 :] 144 if self.short_name and self.short_name == self.test_name: 145 self.short_name = None 146 147 def get_platform(self, record): 148 if "platform" in record["build"]: 149 platform = record["build"]["platform"] 150 else: 151 platform = "-" 152 platform_words = platform.split("-") 153 types_label = "" 154 # combine run and build types and eliminate duplicates 155 run_types = [] 156 if "run" in record and "type" in record["run"]: 157 run_types = record["run"]["type"] 158 run_types = run_types if isinstance(run_types, list) else [run_types] 159 build_types = [] 160 if "build" in record and "type" in record["build"]: 161 build_types = record["build"]["type"] 162 build_types = ( 163 build_types if isinstance(build_types, list) else [build_types] 164 ) 165 run_types = list(set(run_types + build_types)) 166 # '1proc' is used as a treeherder label but does not appear in run types 167 if "e10s" not in run_types: 168 run_types = run_types + ["1proc"] 169 for run_type in run_types: 170 # chunked is not interesting 171 if run_type == "chunked": 172 continue 173 # e10s is the default: implied 174 if run_type == "e10s": 175 continue 176 # sometimes a build/run type is already present in the build platform 177 if run_type in platform_words: 178 continue 179 if types_label: 180 types_label += "-" 181 types_label += run_type 182 return "%s/%s:" % (platform, types_label) 183 184 def report_bugs(self): 185 # Report open bugs matching test name 186 search = self.full_test_name 187 if self.test_name: 188 search = "%s,%s" % (search, self.test_name) 189 if self.short_name: 190 search = "%s,%s" % (search, self.short_name) 191 payload = {"quicksearch": search, "include_fields": "id,summary"} 192 response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload) 193 response.raise_for_status() 194 json_response = response.json() 195 print("\nBugzilla quick search for '%s':" % search) 196 if "bugs" in json_response: 197 for bug in json_response["bugs"]: 198 print("Bug %s: %s" % (bug["id"], bug["summary"])) 199 else: 200 print("No bugs found.") 201 202 def report( 203 self, 204 test_names, 205 start, 206 end, 207 show_info, 208 show_bugs, 209 ): 210 self.start = start 211 self.end = end 212 self.show_info = show_info 213 214 if not self.show_info and not show_bugs: 215 # by default, show everything 216 self.show_info = True 217 show_bugs = True 218 219 for test_name in test_names: 220 print("===== %s =====" % test_name) 221 self.test_name = test_name 222 if len(self.test_name) < 6: 223 print("'%s' is too short for a test name!" % self.test_name) 224 continue 225 self.set_test_name() 226 if show_bugs: 227 self.report_bugs() 228 229 230 class TestInfoReport(TestInfo): 231 """ 232 Support 'mach test-info report': Report of test runs summarized by 233 manifest and component. 234 """ 235 236 def __init__(self, verbose): 237 TestInfo.__init__(self, verbose) 238 self.threads = [] 239 240 @retriable(attempts=3, sleeptime=5, sleepscale=2) 241 def get_url(self, target_url): 242 # if we fail to get valid json (i.e. end point has malformed data), return {} 243 retVal = {} 244 try: 245 self.log_verbose("getting url: %s" % target_url) 246 r = requests.get(target_url, headers={"User-agent": "mach-test-info/1.0"}) 247 self.log_verbose("got status: %s" % r.status_code) 248 r.raise_for_status() 249 retVal = r.json() 250 except json.decoder.JSONDecodeError: 251 self.log_verbose("Error retrieving data from %s" % target_url) 252 253 return retVal 254 255 def update_report(self, by_component, result, path_mod): 256 def update_item(item, label, value): 257 # It is important to include any existing item value in case ActiveData 258 # returns multiple records for the same test; that can happen if the report 259 # sometimes maps more than one ActiveData record to the same path. 260 new_value = item.get(label, 0) + value 261 if type(new_value) is int: 262 item[label] = new_value 263 else: 264 item[label] = float(round(new_value, 2)) # pylint: disable=W1633 265 266 if "test" in result and "tests" in by_component: 267 test = result["test"] 268 if path_mod: 269 test = path_mod(test) 270 for bc in by_component["tests"]: 271 for item in by_component["tests"][bc]: 272 if test == item["test"]: 273 # pylint: disable=W1633 274 seconds = float(round(result.get("duration", 0), 2)) 275 update_item(item, "total run time, seconds", seconds) 276 update_item(item, "total runs", result.get("count", 0)) 277 update_item(item, "skipped runs", result.get("skips", 0)) 278 update_item(item, "failed runs", result.get("failures", 0)) 279 return True 280 return False 281 282 def path_mod_reftest(self, path): 283 # "<path1> == <path2>" -> "<path1>" 284 path = path.split(" ")[0] 285 # "<path>?<params>" -> "<path>" 286 path = path.split("?")[0] 287 # "<path>#<fragment>" -> "<path>" 288 path = path.split("#")[0] 289 return path 290 291 def path_mod_jsreftest(self, path): 292 # "<path>;assert" -> "<path>" 293 path = path.split(";")[0] 294 return path 295 296 def path_mod_marionette(self, path): 297 # "<path> <test-name>" -> "<path>" 298 path = path.split(" ")[0] 299 # "part1\part2" -> "part1/part2" 300 path = path.replace("\\", os.path.sep) 301 return path 302 303 def path_mod_wpt(self, path): 304 if path[0] == os.path.sep: 305 # "/<path>" -> "<path>" 306 path = path[1:] 307 # "<path>" -> "testing/web-platform/tests/<path>" 308 path = os.path.join("testing", "web-platform", "tests", path) 309 # "<path>?<params>" -> "<path>" 310 path = path.split("?")[0] 311 return path 312 313 def path_mod_jittest(self, path): 314 # "part1\part2" -> "part1/part2" 315 path = path.replace("\\", os.path.sep) 316 # "<path>" -> "js/src/jit-test/tests/<path>" 317 return os.path.join("js", "src", "jit-test", "tests", path) 318 319 def path_mod_xpcshell(self, path): 320 # <manifest>.{ini|toml}:<path> -> "<path>" 321 path = path.split(":")[-1] 322 return path 323 324 def description( 325 self, 326 components, 327 flavor, 328 subsuite, 329 paths, 330 show_manifests, 331 show_tests, 332 show_summary, 333 show_annotations, 334 filter_values, 335 filter_keys, 336 start_date, 337 end_date, 338 ): 339 # provide a natural language description of the report options 340 what = [] 341 if show_manifests: 342 what.append("test manifests") 343 if show_tests: 344 what.append("tests") 345 if show_annotations: 346 what.append("test manifest annotations") 347 if show_summary and len(what) == 0: 348 what.append("summary of tests only") 349 if len(what) > 1: 350 what[-1] = "and " + what[-1] 351 what = ", ".join(what) 352 d = "Test summary report for " + what 353 if components: 354 d += ", in specified components (%s)" % components 355 else: 356 d += ", in all components" 357 if flavor: 358 d += ", in specified flavor (%s)" % flavor 359 if subsuite: 360 d += ", in specified subsuite (%s)" % subsuite 361 if paths: 362 d += ", on specified paths (%s)" % paths 363 if filter_values: 364 d += ", containing '%s'" % filter_values 365 if filter_keys: 366 d += " in manifest keys '%s'" % filter_keys 367 else: 368 d += " in any part of manifest entry" 369 d += ", including historical run-time data for the last " 370 371 start = datetime.datetime.strptime(start_date, "%Y-%m-%d") 372 end = datetime.datetime.strptime(end_date, "%Y-%m-%d") 373 d += "%s days on trunk (autoland/m-c)" % ((end - start).days) 374 d += " as of %s." % end_date 375 return d 376 377 # TODO: this is hacked for now and very limited 378 def parse_test(self, summary): 379 if summary.endswith("single tracking bug"): 380 name_part = summary.split("|")[0] # remove 'single tracking bug' 381 name_part.strip() 382 return name_part.split()[-1] # get just the test name, not extra words 383 return None 384 385 def get_runcount_data(self, runcounts_input_file, start, end): 386 # TODO: use start/end properly 387 if runcounts_input_file: 388 try: 389 with open(runcounts_input_file) as f: 390 runcounts = json.load(f) 391 except: 392 print("Unable to load runcounts from path: %s" % runcounts_input_file) 393 raise 394 else: 395 runcounts = self.get_runcounts(days=MAX_DAYS) 396 runcounts = self.squash_runcounts(runcounts, days=MAX_DAYS) 397 return runcounts 398 399 def get_testinfoall_index_url(self): 400 import taskcluster 401 402 index = taskcluster.Index({ 403 "rootUrl": "https://firefox-ci-tc.services.mozilla.com", 404 }) 405 route = "gecko.v2.mozilla-central.latest.source.test-info-all" 406 queue = taskcluster.Queue({ 407 "rootUrl": "https://firefox-ci-tc.services.mozilla.com", 408 }) 409 410 task_id = index.findTask(route)["taskId"] 411 artifacts = queue.listLatestArtifacts(task_id)["artifacts"] 412 413 url = "" 414 for artifact in artifacts: 415 if artifact["name"].endswith("test-run-info.json"): 416 url = queue.buildUrl("getLatestArtifact", task_id, artifact["name"]) 417 break 418 return url 419 420 def get_runcounts(self, days=MAX_DAYS): 421 testrundata = {} 422 # get historical data from test-info job artifact; if missing get fresh 423 url = self.get_testinfoall_index_url() 424 print("INFO: requesting runcounts url: %s" % url) 425 olddata = self.get_url(url) 426 427 # fill in any holes we have 428 endday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( 429 days=1 430 ) 431 startday = endday - datetime.timedelta(days=days) 432 urls_to_fetch = [] 433 # build list of dates with missing data 434 while startday < endday: 435 nextday = startday + datetime.timedelta(days=1) 436 if not olddata.get(str(nextday.date()), {}): 437 url = "https://treeherder.mozilla.org/api/groupsummary/" 438 url += "?startdate=%s&enddate=%s" % ( 439 startday.date(), 440 nextday.date(), 441 ) 442 urls_to_fetch.append([str(nextday.date()), url]) 443 testrundata[str(nextday.date())] = olddata.get(str(nextday.date()), {}) 444 445 startday = nextday 446 447 # limit missing data collection to 5 most recent days days to reduce overall runtime 448 for date, url in urls_to_fetch[-5:]: 449 try: 450 testrundata[date] = self.get_url(url) 451 except requests.exceptions.HTTPError: 452 # We want to see other errors, but can accept HTTPError failures 453 print(f"Unable to retrieve results for url: {url}") 454 pass 455 456 return testrundata 457 458 def optimize_runcounts_data(self, runcounts, num_days): 459 yesterday = datetime.date.today() - datetime.timedelta(days=1) 460 if num_days > 1: 461 startday = yesterday - datetime.timedelta(days=num_days) 462 else: 463 startday = yesterday 464 465 days = [ 466 (startday + datetime.timedelta(days=i)).strftime("%Y-%m-%d") 467 for i in range(num_days) 468 ] 469 470 summary_groups = {key: runcounts[key] for key in days if key in runcounts} 471 tasks_and_count = {"manifests": []} 472 for day in days: 473 if day not in summary_groups or not summary_groups[day]: 474 continue 475 all_task_labels = summary_groups[day]["job_type_names"] 476 for tasks_by_manifest in summary_groups[day]["manifests"]: 477 for manifest in tasks_by_manifest: 478 tasks_and_count.setdefault(manifest, {}) 479 for task_index, _, _, count in tasks_by_manifest[manifest]: 480 task_label = all_task_labels[task_index] 481 if task_label not in tasks_and_count["manifests"]: 482 tasks_and_count["manifests"].append(task_label) 483 new_index = len(tasks_and_count["manifests"]) - 1 484 else: 485 new_index = tasks_and_count["manifests"].index(task_label) 486 487 if new_index not in tasks_and_count[manifest]: 488 tasks_and_count[manifest][new_index] = 0 489 tasks_and_count[manifest][new_index] += count 490 491 return tasks_and_count 492 493 def squash_runcounts(self, runcounts, days=MAX_DAYS): 494 # squash all testrundata together into 1 big happy family for the last X days 495 endday = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( 496 days=1 497 ) 498 oldest = endday - datetime.timedelta(days=days) 499 500 testgroup_runinfo = defaultdict(lambda: defaultdict(int)) 501 502 retVal = {} 503 for datekey in runcounts.keys(): 504 # strip out older days 505 if datetime.date.fromisoformat(datekey) < oldest.date(): 506 continue 507 508 jtn = runcounts[datekey].get("job_type_names", {}) 509 if not jtn: 510 print("Warning: Missing job type names from date: %s" % datekey) 511 continue 512 513 # TODO: changed this to include all manifests, not just first 514 for m in runcounts[datekey]["manifests"]: 515 for man_name in m.keys(): 516 for job_type_id, result, classification, count in m[man_name]: 517 # HACK: we treat shippable and opt the same for mozinfo and runcounts 518 job_name = jtn[job_type_id].replace("-shippable", "") 519 520 # format: job_type_name, result, classification, count 521 # find matching jtn, result, classification and increment 'count' 522 key = (job_name, result, classification) 523 524 # only keep the "parent" manifest 525 testgroup_runinfo[man_name.split(":")[0]][key] += count 526 527 for m in testgroup_runinfo: 528 retVal[m] = [ 529 list(x) + [testgroup_runinfo[m][x]] for x in testgroup_runinfo[m] 530 ] 531 return retVal 532 533 def get_intermittent_failure_data(self, start, end): 534 retVal = {} 535 536 # get IFV bug list 537 # i.e. https://th.m.o/api/failures/?startday=2022-06-22&endday=2022-06-29&tree=all 538 url = ( 539 "https://treeherder.mozilla.org/api/failures/?startday=%s&endday=%s&tree=trunk" 540 % (start, end) 541 ) 542 if_data = self.get_url(url) 543 buglist = [x["bug_id"] for x in if_data] 544 545 # get bug data for summary, 800 bugs at a time 546 # i.e. https://b.m.o/rest/bug?include_fields=id,product,component,summary&id=1,2,3... 547 max_bugs = 800 548 bug_data = [] 549 fields = ["id", "product", "component", "summary"] 550 for bug_index in range(0, len(buglist), max_bugs): 551 bugs = [str(x) for x in buglist[bug_index : bug_index + max_bugs]] 552 if not bugs: 553 print(f"warning: found no bugs in range {bug_index}, +{max_bugs}") 554 continue 555 556 url = "https://bugzilla.mozilla.org/rest/bug?include_fields=%s&id=%s" % ( 557 ",".join(fields), 558 ",".join(bugs), 559 ) 560 data = self.get_url(url) 561 if data and "bugs" in data.keys(): 562 bug_data.extend(data["bugs"]) 563 564 # for each summary, parse filename, store component 565 # IF we find >1 bug with same testname, for now summarize as one 566 for bug in bug_data: 567 test_name = self.parse_test(bug["summary"]) 568 if not test_name: 569 continue 570 571 c = int([x["bug_count"] for x in if_data if x["bug_id"] == bug["id"]][0]) 572 if test_name not in retVal.keys(): 573 retVal[test_name] = { 574 "id": bug["id"], 575 "count": 0, 576 "product": bug["product"], 577 "component": bug["component"], 578 } 579 retVal[test_name]["count"] += c 580 581 if bug["product"] != retVal[test_name]["product"]: 582 print( 583 "ERROR | %s | mismatched bugzilla product, bugzilla (%s) != repo (%s)" 584 % (bug["id"], bug["product"], retVal[test_name]["product"]) 585 ) 586 if bug["component"] != retVal[test_name]["component"]: 587 print( 588 "ERROR | %s | mismatched bugzilla component, bugzilla (%s) != repo (%s)" 589 % (bug["id"], bug["component"], retVal[test_name]["component"]) 590 ) 591 return retVal 592 593 def report( 594 self, 595 components, 596 flavor, 597 subsuite, 598 paths, 599 show_manifests, 600 show_tests, 601 show_summary, 602 show_annotations, 603 filter_values, 604 filter_keys, 605 show_components, 606 output_file, 607 start, 608 end, 609 show_testruns, 610 runcounts_input_file, 611 config_matrix_output_file, 612 ): 613 def matches_filters(test): 614 """ 615 Return True if all of the requested filter_values are found in this test; 616 if filter_keys are specified, restrict search to those test keys. 617 """ 618 for value in filter_values: 619 value_found = False 620 for key in test: 621 if not filter_keys or key in filter_keys: 622 if re.search(value, test[key]): 623 value_found = True 624 break 625 if not value_found: 626 return False 627 return True 628 629 start_time = datetime.datetime.now() 630 631 # Ensure useful report by default 632 if ( 633 not show_manifests 634 and not show_tests 635 and not show_summary 636 and not show_annotations 637 ): 638 show_manifests = True 639 show_summary = True 640 641 trunk = False 642 if os.environ.get("GECKO_HEAD_REPOSITORY", "") in [ 643 "https://hg.mozilla.org/mozilla-central", 644 "https://hg.mozilla.org/try", 645 ]: 646 trunk = True 647 else: 648 show_testruns = False 649 650 by_component = {} 651 if components: 652 components = components.split(",") 653 if filter_keys: 654 filter_keys = filter_keys.split(",") 655 if filter_values: 656 filter_values = filter_values.split(",") 657 else: 658 filter_values = [] 659 display_keys = (filter_keys or []) + [ 660 "run-if", 661 "skip-if", 662 "fail-if", 663 "fails-if", 664 ] 665 display_keys = set(display_keys) 666 ifd = self.get_intermittent_failure_data(start, end) 667 668 runcount = {} 669 if show_testruns and trunk: 670 runcount = self.get_runcount_data(runcounts_input_file, start, end) 671 672 print("Finding tests...") 673 here = os.path.abspath(os.path.dirname(__file__)) 674 resolver = TestResolver.from_environment( 675 cwd=here, loader_cls=TestManifestLoader 676 ) 677 tests = list( 678 resolver.resolve_tests(paths=paths, flavor=flavor, subsuite=subsuite) 679 ) 680 681 manifest_paths = set() 682 for t in tests: 683 if t.get("manifest", None): 684 manifest_path = t["manifest"] 685 if t.get("ancestor_manifest", None): 686 manifest_path = "%s:%s" % (t["ancestor_manifest"], t["manifest"]) 687 manifest_paths.add(manifest_path) 688 manifest_count = len(manifest_paths) 689 print(f"Resolver found {len(tests)} tests, {manifest_count} manifests") 690 691 if config_matrix_output_file and trunk: 692 topsrcdir = self.build_obj.topsrcdir 693 config_matrix = {} 694 for manifest in manifest_paths: 695 # we want the first part of the parent:child, as parent shows up in MHTP 696 # TODO: figure out a better solution for child manifests 697 if ".toml" in manifest: 698 relpath = mozpath.relpath( 699 f"{manifest.split('.toml')[0]}.toml", topsrcdir 700 ) 701 else: 702 relpath = mozpath.relpath(manifest, topsrcdir) 703 # hack for wpt manifests 704 if relpath.startswith(".."): 705 relpath = "/" + relpath.replace("../", "") 706 config_matrix[relpath] = self.create_matrix_from_task_graph( 707 relpath, runcount 708 ) 709 self.write_report(config_matrix, config_matrix_output_file) 710 711 if show_manifests: 712 topsrcdir = self.build_obj.topsrcdir 713 by_component["manifests"] = {} 714 manifest_paths = list(manifest_paths) 715 manifest_paths.sort() 716 relpaths = [] 717 for manifest_path in manifest_paths: 718 relpath = mozpath.relpath(manifest_path, topsrcdir) 719 if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir: 720 continue 721 relpaths.append(relpath) 722 reader = self.build_obj.mozbuild_reader(config_mode="empty") 723 files_info = reader.files_info(relpaths) 724 for manifest_path in manifest_paths: 725 relpath = mozpath.relpath(manifest_path, topsrcdir) 726 if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir: 727 continue 728 manifest_info = None 729 if relpath in files_info: 730 bug_component = files_info[relpath].get("BUG_COMPONENT") 731 if bug_component: 732 key = f"{bug_component.product}::{bug_component.component}" 733 else: 734 key = "<unknown bug component>" 735 if (not components) or (key in components): 736 manifest_info = {"manifest": relpath, "tests": 0, "skipped": 0} 737 rkey = key if show_components else "all" 738 if rkey in by_component["manifests"]: 739 by_component["manifests"][rkey].append(manifest_info) 740 else: 741 by_component["manifests"][rkey] = [manifest_info] 742 if manifest_info: 743 for t in tests: 744 if t["manifest"] == manifest_path: 745 manifest_info["tests"] += 1 746 if t.get("skip-if"): 747 manifest_info["skipped"] += 1 748 for key in by_component["manifests"]: 749 by_component["manifests"][key].sort(key=lambda k: k["manifest"]) 750 751 if show_tests: 752 by_component["tests"] = {} 753 754 if show_tests or show_summary or show_annotations: 755 test_count = 0 756 failed_count = 0 757 skipped_count = 0 758 annotation_count = 0 759 condition_count = 0 760 component_set = set() 761 relpaths = [] 762 conditions = {} 763 known_unconditional_annotations = ["skip", "fail", "asserts", "random"] 764 known_conditional_annotations = [ 765 "skip-if", 766 "fail-if", 767 "run-if", 768 "fails-if", 769 "fuzzy-if", 770 "random-if", 771 "asserts-if", 772 ] 773 for t in tests: 774 relpath = t.get("srcdir_relpath") 775 relpaths.append(relpath) 776 reader = self.build_obj.mozbuild_reader(config_mode="empty") 777 files_info = reader.files_info(relpaths) 778 for t in tests: 779 if not matches_filters(t): 780 continue 781 if "referenced-test" in t: 782 # Avoid double-counting reftests: disregard reference file entries 783 continue 784 if show_annotations: 785 for key in t: 786 if key in known_unconditional_annotations: 787 annotation_count += 1 788 if key in known_conditional_annotations: 789 annotation_count += 1 790 # Here 'key' is a manifest annotation type like 'skip-if' and t[key] 791 # is the associated condition. For example, the manifestparser 792 # manifest annotation, "skip-if = os == 'win'", is expected to be 793 # encoded as t['skip-if'] = "os == 'win'". 794 # To allow for reftest manifests, t[key] may have multiple entries 795 # separated by ';', each corresponding to a condition for that test 796 # and annotation type. For example, 797 # "skip-if(Android&&webrender) skip-if(OSX)", would be 798 # encoded as t['skip-if'] = "Android&&webrender;OSX". 799 annotation_conditions = t[key].split(";") 800 801 # if key has \n in it, we need to strip it. for manifestparser format 802 # 1) from the beginning of the line 803 # 2) different conditions if in the middle of the line 804 annotation_conditions = [ 805 x.strip("\n") for x in annotation_conditions 806 ] 807 temp = [] 808 for condition in annotation_conditions: 809 temp.extend(condition.split("\n")) 810 annotation_conditions = temp 811 812 for c in annotation_conditions: 813 condition_count += 1 814 # Trim reftest fuzzy-if ranges: everything after the first comma 815 # eg. "Android,0-2,1-3" -> "Android" 816 condition = c.split(",")[0] 817 if condition not in conditions: 818 conditions[condition] = 0 819 conditions[condition] += 1 820 test_count += 1 821 relpath = t.get("srcdir_relpath") 822 if relpath in files_info: 823 bug_component = files_info[relpath].get("BUG_COMPONENT") 824 if bug_component: 825 key = f"{bug_component.product}::{bug_component.component}" 826 else: 827 key = "<unknown bug component>" 828 if (not components) or (key in components): 829 component_set.add(key) 830 test_info = {"test": relpath} 831 for test_key in display_keys: 832 value = t.get(test_key) 833 if value: 834 test_info[test_key] = value 835 if t.get("fail-if"): 836 failed_count += 1 837 if t.get("fails-if"): 838 failed_count += 1 839 if t.get("skip-if"): 840 skipped_count += 1 841 842 if "manifest_relpath" in t and "manifest" in t: 843 if "web-platform" in t["manifest_relpath"]: 844 test_info["manifest"] = [t["manifest"]] 845 else: 846 test_info["manifest"] = [t["manifest_relpath"]] 847 848 # handle included manifests as ancestor:child 849 if t.get("ancestor_manifest", None): 850 test_info["manifest"] = [ 851 "%s:%s" 852 % (t["ancestor_manifest"], test_info["manifest"][0]) 853 ] 854 855 # add in intermittent failure data 856 if ifd.get(relpath): 857 if_data = ifd.get(relpath) 858 test_info["failure_count"] = if_data["count"] 859 if show_testruns: 860 total_runs = 0 861 for m in test_info["manifest"]: 862 if m in runcount.keys(): 863 for x in runcount.get(m, []): 864 if not x: 865 break 866 total_runs += x[3] 867 if total_runs > 0: 868 test_info["total_runs"] = total_runs 869 870 if show_tests: 871 rkey = key if show_components else "all" 872 if rkey in by_component["tests"]: 873 # Avoid duplicates: Some test paths have multiple TestResolver 874 # entries, as when a test is included by multiple manifests. 875 found = False 876 for ctest in by_component["tests"][rkey]: 877 if ctest["test"] == test_info["test"]: 878 found = True 879 break 880 if not found: 881 by_component["tests"][rkey].append(test_info) 882 else: 883 for ti in by_component["tests"][rkey]: 884 if ti["test"] == test_info["test"]: 885 if ( 886 test_info["manifest"][0] 887 not in ti["manifest"] 888 ): 889 ti_manifest = test_info["manifest"] 890 if test_info.get( 891 "ancestor_manifest", None 892 ): 893 ti_manifest = "%s:%s" % ( 894 test_info["ancestor_manifest"], 895 ti_manifest, 896 ) 897 ti["manifest"].extend(ti_manifest) 898 else: 899 by_component["tests"][rkey] = [test_info] 900 if show_tests: 901 for key in by_component["tests"]: 902 by_component["tests"][key].sort(key=lambda k: k["test"]) 903 904 by_component["description"] = self.description( 905 components, 906 flavor, 907 subsuite, 908 paths, 909 show_manifests, 910 show_tests, 911 show_summary, 912 show_annotations, 913 filter_values, 914 filter_keys, 915 start, 916 end, 917 ) 918 919 if show_summary: 920 by_component["summary"] = {} 921 by_component["summary"]["components"] = len(component_set) 922 by_component["summary"]["manifests"] = manifest_count 923 by_component["summary"]["tests"] = test_count 924 by_component["summary"]["failed tests"] = failed_count 925 by_component["summary"]["skipped tests"] = skipped_count 926 927 if show_annotations: 928 by_component["annotations"] = {} 929 by_component["annotations"]["total annotations"] = annotation_count 930 by_component["annotations"]["total conditions"] = condition_count 931 by_component["annotations"]["unique conditions"] = len(conditions) 932 by_component["annotations"]["conditions"] = conditions 933 934 self.write_report(by_component, output_file) 935 936 end_time = datetime.datetime.now() 937 self.log_verbose( 938 "%d seconds total to generate report" 939 % (end_time - start_time).total_seconds() 940 ) 941 942 def write_report(self, by_component, output_file): 943 json_report = json.dumps(by_component, indent=2, sort_keys=True, cls=SetEncoder) 944 if output_file: 945 output_file = os.path.abspath(output_file) 946 output_dir = os.path.dirname(output_file) 947 if not os.path.isdir(output_dir): 948 os.makedirs(output_dir) 949 950 with open(output_file, "w") as f: 951 f.write(json_report) 952 else: 953 print(json_report) 954 955 def report_diff(self, before, after, output_file): 956 """ 957 Support for 'mach test-info report-diff'. 958 """ 959 960 def get_file(path_or_url): 961 if urlparse.urlparse(path_or_url).scheme: 962 response = requests.get(path_or_url) 963 response.raise_for_status() 964 return json.loads(response.text) 965 with open(path_or_url) as f: 966 return json.load(f) 967 968 report1 = get_file(before) 969 report2 = get_file(after) 970 971 by_component = {"tests": {}, "summary": {}} 972 self.diff_summaries(by_component, report1["summary"], report2["summary"]) 973 self.diff_all_components(by_component, report1["tests"], report2["tests"]) 974 self.write_report(by_component, output_file) 975 976 def diff_summaries(self, by_component, summary1, summary2): 977 """ 978 Update by_component with comparison of summaries. 979 """ 980 all_keys = set(summary1.keys()) | set(summary2.keys()) 981 for key in all_keys: 982 delta = summary2.get(key, 0) - summary1.get(key, 0) 983 by_component["summary"]["%s delta" % key] = delta 984 985 def diff_all_components(self, by_component, tests1, tests2): 986 """ 987 Update by_component with any added/deleted tests, for all components. 988 """ 989 self.added_count = 0 990 self.deleted_count = 0 991 for component in tests1: 992 component1 = tests1[component] 993 component2 = [] if component not in tests2 else tests2[component] 994 self.diff_component(by_component, component, component1, component2) 995 for component in tests2: 996 if component not in tests1: 997 component2 = tests2[component] 998 self.diff_component(by_component, component, [], component2) 999 by_component["summary"]["added tests"] = self.added_count 1000 by_component["summary"]["deleted tests"] = self.deleted_count 1001 1002 def diff_component(self, by_component, component, component1, component2): 1003 """ 1004 Update by_component[component] with any added/deleted tests for the 1005 named component. 1006 "added": tests found in component2 but missing from component1. 1007 "deleted": tests found in component1 but missing from component2. 1008 """ 1009 tests1 = set([t["test"] for t in component1]) 1010 tests2 = set([t["test"] for t in component2]) 1011 deleted = tests1 - tests2 1012 added = tests2 - tests1 1013 if deleted or added: 1014 by_component["tests"][component] = {} 1015 if deleted: 1016 by_component["tests"][component]["deleted"] = sorted(list(deleted)) 1017 if added: 1018 by_component["tests"][component]["added"] = sorted(list(added)) 1019 self.added_count += len(added) 1020 self.deleted_count += len(deleted) 1021 common = len(tests1.intersection(tests2)) 1022 self.log_verbose( 1023 "%s: %d deleted, %d added, %d common" 1024 % (component, len(deleted), len(added), common) 1025 ) 1026 1027 ################################################################################ 1028 ### 1029 ### Below is code for creating a os/version/processor/config/variant matrix 1030 ### 1031 1032 def build_matrix_cache(self): 1033 # this is an attempt to cache the .json for the duration of the task 1034 filename = "task-graph.json" 1035 if os.path.exists(filename): 1036 with open(filename) as f: 1037 data = json.load(f) 1038 else: 1039 url = ( 1040 "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.taskgraph.decision/artifacts/public/" 1041 + filename 1042 ) 1043 1044 response = requests.get(url, headers={"User-agent": "mach-test-info/1.0"}) 1045 data = response.json() 1046 with open(filename, "w") as f: 1047 json.dump(data, f) 1048 1049 for task in data.values(): 1050 task_label: str = task["label"] 1051 # HACK: we treat shippable and opt the same from mozinfo, runtime counts 1052 task_label = task_label.replace("-shippable", "") 1053 1054 # we only want test tasks 1055 if not task_label.startswith("test-"): 1056 continue 1057 if task_label.endswith("-cf"): 1058 continue 1059 1060 # skip tier-3 1061 if ( 1062 task.get("task", {}) 1063 .get("extra", {}) 1064 .get("treeherder", {}) 1065 .get("tier", 3) 1066 == 3 1067 ): 1068 continue 1069 1070 try: 1071 parts = task_label.split("-") 1072 if int(parts[-1]): 1073 task_label = "-".join(parts[:-1]) 1074 except ValueError: 1075 pass 1076 1077 # TODO: this only works for tasks where we schedule by manifest 1078 env = task.get("task", {}).get("payload", {}).get("env", {}) 1079 1080 mhtp = json.loads(env.get("MOZHARNESS_TEST_PATHS", "{}")) 1081 if not mhtp: 1082 # mock up logic here if matching task 1083 suite = self.find_non_test_path_loader(task_label) 1084 if not suite: 1085 continue 1086 mhtp[suite] = [suite] 1087 1088 # TODO: figure out a better method for dealing with TEST_TAG 1089 # when we have a test_tag, all skipped manifests are added to chunk 1. 1090 # we are skipping real manifests, but avoiding many overreported manifests. 1091 # 1092 # NOTE: some variants only have a single chunk, so no numbers 1093 if json.loads(env.get("MOZHARNESS_TEST_TAG", "{}")): 1094 if not json.loads(env.get("MOZHARNESS_TEST_PATHS", "{}")): 1095 # mock up logic here if matching task 1096 suite = self.find_non_test_path_loader(task_label) 1097 if not suite: 1098 continue 1099 mhtp[suite] = [suite] 1100 1101 for suite in mhtp: 1102 for manifest in mhtp[suite]: 1103 self.matrix_map[manifest].append(task_label) 1104 1105 extra = task.get("task", {}).get("extra", {}).get("test-setting", {}) 1106 platform_info = PlatformInfo(extra) 1107 1108 self.task_tuples[task_label] = platform_info 1109 1110 matrix_map = defaultdict(list) 1111 task_tuples: dict[str, PlatformInfo] = {} 1112 1113 def find_non_test_path_loader(self, label): 1114 # TODO: how to keep this list synchronized? 1115 known_suites = [ 1116 "mochitest-browser-media", 1117 "telemetry-tests-client", 1118 "mochitest-webgl2-ext", 1119 "mochitest-webgl1-ext", 1120 "jittest-1proc", 1121 "mochitest-browser-translations", 1122 "jsreftest", 1123 "mochitest-browser-screenshots", 1124 "marionette-unittest", 1125 ] 1126 match = [x for x in known_suites if x in label] 1127 if match: 1128 return match[0] 1129 return "" 1130 1131 # find manifest in matrix_map and for all tasks that run this 1132 # pull the tuples out and create a definitive list 1133 def create_matrix_from_task_graph(self, target_manifest, runcount): 1134 results = {} 1135 1136 if not self.matrix_map: 1137 self.build_matrix_cache() 1138 1139 # for tasks with no MOZHARNESS_TEST_PATHS, provide basic data 1140 if target_manifest in runcount and self.find_non_test_path_loader( 1141 runcount[target_manifest][0][0] 1142 ): 1143 suite = self.find_non_test_path_loader(runcount[target_manifest][0][0]) 1144 self.matrix_map[target_manifest] = self.matrix_map[suite] 1145 1146 for tl in self.matrix_map.get(target_manifest, []): 1147 task_label = tl.replace("-shippable", "") 1148 platform_info = self.task_tuples[task_label] 1149 1150 # add in runcounts, we can find find the index of the given task_label in 'job_type_names', 1151 # use that to get specific runs 1152 passed = 0 1153 failed = 0 1154 if target_manifest in runcount: 1155 # data = [[job_name, result, classification, count], ...] 1156 for data in [ 1157 x for x in runcount[target_manifest] if task_label == x[0] 1158 ]: 1159 if data[1] == "passed": 1160 passed += data[-1] 1161 else: 1162 failed += data[-1] 1163 1164 # this helps avoid 'skipped' manifests 1165 if passed == 0 and failed == 0: 1166 continue 1167 1168 if platform_info.os not in results: 1169 results[platform_info.os] = {} 1170 os = results[platform_info.os] 1171 if platform_info.os_version not in os: 1172 os[platform_info.os_version] = {} 1173 os_version = os[platform_info.os_version] 1174 if platform_info.arch not in os_version: 1175 os_version[platform_info.arch] = {} 1176 arch = os_version[platform_info.arch] 1177 if platform_info.build_type not in arch: 1178 arch[platform_info.build_type] = {} 1179 1180 if platform_info.test_variant not in arch[platform_info.build_type]: 1181 arch[platform_info.build_type][platform_info.test_variant] = { 1182 "pass": 0, 1183 "fail": 0, 1184 } 1185 arch[platform_info.build_type][platform_info.test_variant]["pass"] += passed 1186 arch[platform_info.build_type][platform_info.test_variant]["fail"] += failed 1187 1188 return results