commit 4424263193cb9ba03d1bf8b07c222d7a2d80f987 parent ff19491773d070b3e9d8ab72d6b1949be1221a50 Author: Tom Marble <tmarble@info9.net> Date: Wed, 15 Oct 2025 20:21:41 +0000 Bug 1991971 - Implement skip-fails --known-intermittents r=jmaher Enhanced mach manifest skip-fails to support --known-intermittents where bugzilla won't be updated and skip-if conditions will only be added to manifests for failures that are known intermittents Documented --known-intermittents mode in SKIP-FAILS.txt Fixed handling of included manifests from mozci task info by making sure the _included_ manifest is the one listed in the failure and the test path is recorded without the include manifest prefix Fixed handling of a failure to ensure that the task_id used to create the skip-if condition is from one of the failing tasks Removed use of FailedPlatform in TOML (non high-freq) manifests as it unnecessarily included some unused, negated variants in skip-if conidtions Fixed handling of multiple runtimes in mozinfo.test_variant Signed-off-by: Tom Marble <tmarble@info9.net> Differential Revision: https://phabricator.services.mozilla.com/D268463 Diffstat:
15 files changed, 824 insertions(+), 180 deletions(-)
diff --git a/testing/failedplatform.py b/testing/failedplatform.py @@ -107,14 +107,20 @@ class FailedPlatform: This function gets all available test variants for the given build type and negates them to create a skip-if that handle tasks without test variants """ - variants = [ - tv - for tv in self.get_possible_test_variants(build_type) - if tv != "no_variant" - ] - return_str = "" - for tv in variants: - return_str += and_str + self.get_negated_variant(tv) + variants_to_negate = [] + for tv in self.get_possible_test_variants(build_type): + if tv != "no_variant": + variants_to_negate.extend(tv.split("+")) + negated = [] + for tv in variants_to_negate: + ntv = self.get_negated_variant(tv) + if not ntv in variants_to_negate: + negated.append(ntv) # ignore mutually exclusive variants + return_str = reduce( + (lambda rs, npv: rs + and_str + npv), + negated, + "", + ) return return_str def get_test_variant_condition( diff --git a/testing/mach_commands.py b/testing/mach_commands.py @@ -245,7 +245,7 @@ def addtest( if editor is not MISSING_ARG: if editor is not None: - editor = editor + pass elif "VISUAL" in os.environ: editor = os.environ["VISUAL"] elif "EDITOR" in os.environ: @@ -258,7 +258,7 @@ def addtest( if editor: import subprocess - proc = subprocess.Popen("%s %s" % (editor, " ".join(paths)), shell=True) + proc = subprocess.Popen(f"{editor} {' '.join(paths)}", shell=True) if proc: proc.wait() @@ -302,21 +302,18 @@ def guess_suite(abs_test): and guess_doc(abs_test) == "js" ): guessed_suite = "xpcshell" - else: - if filename.startswith("browser_") and (has_browser_ini or has_browser_toml): - guessed_suite = "mochitest-browser-chrome" - elif filename.startswith("test_"): - if (has_chrome_ini or has_chrome_toml) and ( - has_plain_ini or has_plain_toml - ): - err = ( - "Error: directory contains both a chrome.{ini|toml} and mochitest.{ini|toml}. " - "Please set --suite=mochitest-chrome or --suite=mochitest-plain." - ) - elif has_chrome_ini or has_chrome_toml: - guessed_suite = "mochitest-chrome" - elif has_plain_ini or has_plain_toml: - guessed_suite = "mochitest-plain" + elif filename.startswith("browser_") and (has_browser_ini or has_browser_toml): + guessed_suite = "mochitest-browser-chrome" + elif filename.startswith("test_"): + if (has_chrome_ini or has_chrome_toml) and (has_plain_ini or has_plain_toml): + err = ( + "Error: directory contains both a chrome.toml and mochitest.toml. " + "Please set --suite=mochitest-chrome or --suite=mochitest-plain." + ) + elif has_chrome_ini or has_chrome_toml: + guessed_suite = "mochitest-chrome" + elif has_plain_ini or has_plain_toml: + guessed_suite = "mochitest-plain" return guessed_suite, err @@ -533,7 +530,7 @@ def run_desktop_test( try: result = cppunittests.run_test_harness(options, tests) except Exception as e: - log.error("Caught exception running cpp unit tests: %s" % str(e)) + log.error(f"Caught exception running cpp unit tests: {str(e)}") result = False raise @@ -565,7 +562,7 @@ def run_android_test(command_context, tests, symbols_path, manifest_path, log): try: result = remotecppunittests.run_test_harness(options, tests) except Exception as e: - log.error("Caught exception running cpp unit tests: %s" % str(e)) + log.error(f"Caught exception running cpp unit tests: {str(e)}") result = False raise @@ -999,17 +996,14 @@ def test_info_failures( return # get bug info - url = ( - "https://bugzilla.mozilla.org/rest/bug?include_fields=summary,depends_on&id=%s" - % bugid - ) + url = f"https://bugzilla.mozilla.org/rest/bug?include_fields=summary,depends_on&id={bugid}" r = requests.get(url, headers={"User-agent": "mach-test-info/1.0"}) if r.status_code != 200: - print("%s error retrieving url: %s" % (r.status_code, url)) + print(f"{r.status_code} error retrieving url: {url}") data = r.json() if not data: - print("unable to get bugzilla information for %s" % bugid) + print(f"unable to get bugzilla information for {bugid}") return summary = data["bugs"][0]["summary"] @@ -1029,7 +1023,7 @@ def test_info_failures( data = [] for b in buglist: url = "https://treeherder.mozilla.org/api/failuresbybug/" - url += "?startday=%s&endday=%s&tree=trunk&bug=%s" % (start, end, b) + url += f"?startday={start}&endday={end}&tree=trunk&bug={b}" r = requests.get(url, headers={"User-agent": "mach-test-info/1.0"}) r.raise_for_status() @@ -1049,8 +1043,7 @@ def test_info_failures( variants = yaml.safe_load(r.text) print( - "\nQuerying data for bug %s annotated from %s to %s on trunk.\n\n" - % (buglist, start, end) + f"\nQuerying data for bug {buglist} annotated from {start} to {end} on trunk.\n\n" ) jobs = {} lines = {} @@ -1058,13 +1051,12 @@ def test_info_failures( # config = platform/buildtype # testsuite (<suite>[-variant][-<chunk>]) # lines - group by patterns that contain test name - config = "%s/%s" % (failure["platform"], failure["build_type"]) - + config = f"{failure['platform']}/{failure['build_type']}" variant = "" suite = "" varpos = len(failure["test_suite"]) for v in variants.keys(): - var = "-%s" % variants[v]["suffix"] + var = f"-{variants[v]['suffix']}" if var in failure["test_suite"]: if failure["test_suite"].find(var) < varpos: variant = var @@ -1080,9 +1072,9 @@ def test_info_failures( pass # if this works, then the last '-X' is a number :) if suite == "": - print("Error: failure to find variant in %s" % failure["test_suite"]) + print(f"Error: failure to find variant in {failure['test_suite']}") - job = "%s-%s%s" % (config, suite, variant) + job = f"{config}-{suite}{variant}" if job not in jobs.keys(): jobs[job] = 0 jobs[job] += 1 @@ -1094,11 +1086,11 @@ def test_info_failures( continue # strip off timestamp and mozharness status parts = line.split("TEST-UNEXPECTED") - l = "TEST-UNEXPECTED%s" % parts[-1] + l = f"TEST-UNEXPECTED{parts[-1]}" # only keep 25 characters of the failure, often longer is random numbers parts = l.split(testname) - l = "%s%s%s" % (parts[0], testname, parts[1][:25]) + l = parts[0] + testname + parts[1][:25] hvalue += hash(l) @@ -1113,9 +1105,11 @@ def test_info_failures( lines[hvalue]["config"].append(job) for h in lines.keys(): - print("%s errors with:" % (len(lines[h]["config"]))) - for l in lines[h]["lines"]: - print(l) + print(f"{len(lines[h]['config'])} errors with:") + failure_lines = lines[h]["lines"] + if len(failure_lines) > 0: + for l in failure_lines: + print(l) else: print( "... no failure lines recorded in" @@ -1125,7 +1119,7 @@ def test_info_failures( for job in jobs: count = len([x for x in lines[h]["config"] if x == job]) if count > 0: - print(" %s: %s" % (job, count)) + print(f" {job}: {count}") print("") @@ -1284,6 +1278,12 @@ def manifest(_command_context): help="Task id to write a condition for instead of all tasks from the push", ) @CommandArgument( + "-k", + "--known-intermittents", + action="store_true", + help="Set known intermittents mode (only skip failures known intermittents)", +) +@CommandArgument( "-M", "--max-failures", type=int, @@ -1339,6 +1339,7 @@ def skipfails( carryover=False, failure_ratio=0.4, clear_cache=None, + known_intermittents=False, ): from skipfails import Skipfails @@ -1363,6 +1364,7 @@ def skipfails( max_failures, carryover, failure_ratio, + known_intermittents, ) @@ -1432,7 +1434,7 @@ def high_freq_skipfails(command_context, failures: str, days: str): ) def clean_skipfails( command_context, - manifest_search_path: List[str], + manifest_search_path: List[str], # noqa UP006 os_name: Optional[str] = None, os_version: Optional[str] = None, processor: Optional[str] = None, diff --git a/testing/mozbase/manifestparser/manifestparser/toml.py b/testing/mozbase/manifestparser/manifestparser/toml.py @@ -16,12 +16,12 @@ from .ini import combine_fields __all__ = ["read_toml", "alphabetize_toml_str", "add_skip_if", "sort_paths"] CreateBug = Optional[Callable[[], object]] -ListStr = List[str] +ListStr = List[str] # noqa UP006 OptArray = Optional[Array] OptRegex = Optional[re.Pattern] OptStr = Optional[str] -OptConditions = List[Tuple[str, OptStr]] -TupleStrBool = Tuple[str, bool] +OptConditions = List[Tuple[str, OptStr]] # noqa UP006 +TupleStrBoolStr = Tuple[str, bool, str] # noqa UP006 FILENAME_REGEX = r"^([A-Za-z0-9_./-]*)([Bb][Uu][Gg])([-_]*)([0-9]+)([A-Za-z0-9_./-]*)$" DEFAULT_SECTION = "DEFAULT" @@ -409,7 +409,7 @@ def add_skip_if( bug_reference: OptStr = None, create_bug_lambda: CreateBug = None, carryover_mode: bool = False, -) -> TupleStrBool: +) -> TupleStrBoolStr: """ Will take a TOMLkit manifest document (i.e. from a previous invocation of read_toml(..., document=True) and accessing the document @@ -533,7 +533,7 @@ def add_skip_if( skip_if = {"skip-if": mp_array} del keyvals["skip-if"] keyvals.update(skip_if) - return (additional_comment, carryover) + return (additional_comment, carryover, bug_reference) def _should_remove_condition( diff --git a/testing/mozbase/manifestparser/tests/edit-manifest-after.toml b/testing/mozbase/manifestparser/tests/edit-manifest-after.toml @@ -19,8 +19,8 @@ skip-if = [ ["bug_100.js"] skip-if = [ "apple_catalina", # Bug 200 - "os == 'android' && debug", # Bug 100, will be carried over - "os == 'android' && os_version == '14' && debug", # Bug 100, will be carried over + "os == 'android' && asan", # Bug 100, will be carried over + "os == 'android' && os_version == '14' && ccov", # Bug 100, will be carried over ] ["test_bar.html"] diff --git a/testing/mozbase/manifestparser/tests/edit-manifest-before.toml b/testing/mozbase/manifestparser/tests/edit-manifest-before.toml @@ -2,7 +2,7 @@ ["bug_100.js"] skip-if = [ - "os == 'android' && debug", # Bug 100, will be carried over + "os == 'android' && asan", # Bug 100, will be carried over ] ["bug_3.js"] diff --git a/testing/mozbase/manifestparser/tests/test_manifestparser.py b/testing/mozbase/manifestparser/tests/test_manifestparser.py @@ -610,13 +610,16 @@ yellow = submarine condition = "apple_catalina" bug = "Bug 200" manifestparser.toml.add_skip_if(manifest, filename, condition, bug) - condition = "os == 'android' && os_version == '14' && debug" + condition = "os == 'android' && os_version == '14' && ccov" bug = "Bug 99999" - (additional_comment, carryover) = manifestparser.toml.add_skip_if( - manifest, filename, condition, bug, None, True + (additional_comment, carryover, bug_reference) = ( + manifestparser.toml.add_skip_if( + manifest, filename, condition, bug, None, True + ) ) assert not additional_comment assert carryover + assert bug_reference == "Bug 100, will be carried over" filename = "bug_3.js" assert filename in manifest diff --git a/testing/mozbase/mozinfo/mozinfo/platforminfo.py b/testing/mozbase/mozinfo/mozinfo/platforminfo.py @@ -9,8 +9,8 @@ from typing import Any, Dict, Optional # noqa UP035 import yaml -DictAny = Dict[str, Any] -DictStr = Dict[str, str] +DictAny = Dict[str, Any] # noqa UP006 +DictStr = Dict[str, str] # noqa UP006 OptTestSettings = Optional[DictAny] # From https://developer.android.com/tools/releases/platforms @@ -203,9 +203,9 @@ class PlatformInfo: def _clean_test_variant(self) -> str: # TODO: consider adding display here runtimes = list(self._runtime.keys()) - test_variant = "+".join( - [v for v in [self.get_variant_condition(x) for x in runtimes] if v] - ) + variants = [v for v in [self.get_variant_condition(x) for x in runtimes] if v] + variants.sort() # keep variants in consistent order + test_variant = "+".join(variants) if not runtimes or not test_variant: test_variant = "no_variant" return test_variant diff --git a/testing/mozbase/mozinfo/tests/test_platforminfo.py b/testing/mozbase/mozinfo/tests/test_platforminfo.py @@ -236,7 +236,20 @@ def test_runtimes(): # combines multiple runtimes test_settings["runtime"] = {"xorigin": True, "1proc": True} platform_info = PlatformInfo(test_settings) - assert platform_info.test_variant == "xorigin+!e10s" + assert platform_info.test_variant == "!e10s+xorigin" + + # combines multiple runtimes 2 + test_settings["runtime"] = {"no-fission": True, "socketprocess_networking": True} + platform_info = PlatformInfo(test_settings) + assert platform_info.test_variant == "!fission+socketprocess_networking" + + # combines multiple runtimes 3 + test_settings["runtime"] = { + "socketprocess_networking": True, + "no-fission": True, + } + platform_info = PlatformInfo(test_settings) + assert platform_info.test_variant == "!fission+socketprocess_networking" if __name__ == "__main__": diff --git a/testing/skipfails.py b/testing/skipfails.py @@ -51,55 +51,57 @@ try: except ImportError: from yaml import Loader -ArtifactList = List[Dict[Literal["name"], str]] +ArtifactList = List[Dict[Literal["name"], str]] # noqa UP006 CreateBug = Optional[Callable[[], Bug]] -Extras = Dict[str, PlatformInfo] -FailedPlatforms = Dict[str, FailedPlatform] -GenBugComment = Tuple[CreateBug, str, bool, dict, str] +Extras = Dict[str, PlatformInfo] # noqa UP006 +FailedPlatforms = Dict[str, FailedPlatform] # noqa UP006 +GenBugComment = Tuple[CreateBug, str, bool, dict, str] # noqa UP006 JSONType = Union[ None, bool, int, float, str, - List["JSONType"], - Dict[str, "JSONType"], + List["JSONType"], # noqa UP006 + Dict[str, "JSONType"], # noqa UP006 ] -ListBug = List[Bug] -ListInt = List[int] -ListStr = List[str] -ManifestPaths = Dict[str, Dict[str, List[str]]] +DictJSON = Dict[str, JSONType] # noqa UP006 +ListBug = List[Bug] # noqa UP006 +ListInt = List[int] # noqa UP006 +ListStr = List[str] # noqa UP006 +ManifestPaths = Dict[str, Dict[str, List[str]]] # noqa UP006 OptBug = Optional[Bug] -OptDifferences = Optional[List[int]] +OptDifferences = Optional[List[int]] # noqa UP006 OptInt = Optional[int] -OptJs = Optional[Dict[str, bool]] +OptJs = Optional[Dict[str, bool]] # noqa UP006 OptPlatformInfo = Optional[PlatformInfo] OptStr = Optional[str] -OptTaskResult = Optional[Dict[str, Any]] -PlatformPermutations = Dict[ +OptTaskResult = Optional[Dict[str, Any]] # noqa UP006 +PlatformPermutations = Dict[ # noqa UP006 str, # Manifest - Dict[ + Dict[ # noqa UP006 str, # OS - Dict[ + Dict[ # noqa UP006 str, # OS Version - Dict[ + Dict[ # noqa UP006 str, # Processor - Dict[ + Dict[ # noqa UP006 str, # Build type - Dict[ + Dict[ # noqa UP006 str, # Test Variant - Dict[str, int], # {'pass': x, 'fail': y} + Dict[str, int], # noqa UP006 {'pass': x, 'fail': y} ], ], ], ], ], ] -Runs = Dict[str, Dict[str, Any]] -Suggestion = Tuple[OptInt, OptStr, OptStr] +Runs = Dict[str, Dict[str, Any]] # noqa UP006 +Suggestion = Tuple[OptInt, OptStr, OptStr] # noqa UP006 TaskIdOrPlatformInfo = Union[str, PlatformInfo] -Tasks = List[TestTask] -WptPaths = Tuple[OptStr, OptStr, OptStr, OptStr] +Tasks = List[TestTask] # noqa UP006 +TupleOptIntStr = Tuple[OptInt, str] # noqa UP006 +WptPaths = Tuple[OptStr, OptStr, OptStr, OptStr] # noqa UP006 TASK_LOG = "live_backing.log" TASK_ARTIFACT = "public/logs/" + TASK_LOG @@ -274,6 +276,7 @@ class Skipfails: self.failed_platforms: FailedPlatforms = {} self.platform_permutations: PlatformPermutations = {} self.user_agent: OptStr = user_agent + self.suggestions: DictJSON = {} self.clear_cache: OptStr = clear_cache self.check_cache() @@ -293,8 +296,7 @@ class Skipfails: for rev_dir in [e.name for e in os.scandir(cache_dir) if e.is_dir()]: rev_path = os.path.join(cache_dir, rev_dir) if ( - self.clear_cache is not None - and (self.clear_cache == rev_dir or self.clear_cache == "all") + self.clear_cache is not None and self.clear_cache in (rev_dir, "all") ) or self.file_age(rev_path) > expiry: self.vinfo(f"clearing revision: {rev_path}") self.delete_dir(rev_path) @@ -408,6 +410,7 @@ class Skipfails: max_failures: int = -1, carryover_mode: bool = False, failure_ratio: float = FAILURE_RATIO, + known_intermittents_mode: bool = False, ): "Run skip-fails on try_url, return True on success" @@ -415,11 +418,22 @@ class Skipfails: self.vinfo( f"All skip-if conditions will use the --new-version {self.new_version}" ) + if failure_ratio != FAILURE_RATIO: + self.vinfo(f"Failure ratio for this run: {failure_ratio}") + if carryover_mode and known_intermittents_mode: + raise Exception( + "may not specifiy both --carryover and --known-intermittents" + ) if carryover_mode: self.edit_bugzilla = False self.vinfo( "Carryover mode ON: only platform match conditions considered, no bugs created or updated" ) + elif known_intermittents_mode: + self.edit_bugzilla = False + self.vinfo( + "Known Intermittents mode ON: only failures with known intermittents considered, no bugs created or updated" + ) if self.bugzilla is None: self.vinfo("Bugzilla has been disabled: bugs not created or updated.") elif self.dry_run: @@ -484,11 +498,15 @@ class Skipfails: status = FAIL lineno = failures[manifest][LL][label][PP][path].get(LINENO, 0) runs: Runs = failures[manifest][LL][label][PP][path][RUNS] - # skip_failure only needs to run against one task for each path - first_task_id = next(iter(runs)) + # skip_failure only needs to run against one failing task for each path: first_task_id + first_task_id: OptStr = None for task_id in runs: + if first_task_id is None and not runs[task_id].get( + RR, False + ): + first_task_id = task_id if kind == Kind.TOML: - break + continue elif kind == Kind.LIST: difference = runs[task_id].get(DIFFERENCE, 0) if difference > 0: @@ -518,9 +536,9 @@ class Skipfails: kind, path, first_task_id, - None, - None, - False, + None, # platform_info + None, # bug_id + False, # high_freq anyjs, differences, pixels, @@ -533,6 +551,7 @@ class Skipfails: repo, meta_bug_id, carryover_mode, + known_intermittents_mode, ) num_failures += 1 if max_failures >= 0 and num_failures >= max_failures: @@ -597,6 +616,15 @@ class Skipfails: elif manifest.endswith(".list"): kind = Kind.LIST elif manifest.endswith(".toml"): + # NOTE: manifest may be a compound of manifest1:manifest2 + # where manifest1 includes manifest2 + # The skip-condition will need to be associated with manifest2 + includes = manifest.split(":") + if len(includes) > 1: + manifest = includes[-1] + self.warning( + f"manifest '{manifest}' is included from {':'.join(includes[0:-1])}" + ) kind = Kind.TOML else: kind = Kind.WPT @@ -709,8 +737,11 @@ class Skipfails: if config not in manifest_paths[manifest]: manifest_paths[manifest][config] = [] - for path_type in failure_types[raw_manifest]: - path, _type = path_type + for included_path_type in failure_types[raw_manifest]: + included_path, _type = included_path_type + path = included_path.split(":")[ + -1 + ] # strip 'included_from.toml:' prefix query = None anyjs = None allpaths = [] @@ -718,7 +749,7 @@ class Skipfails: path, mmpath, query, anyjs = self.wpt_paths(path) if path is None or mmpath is None: self.warning( - f"non existant failure path: {path_type[0]}" + f"non existant failure path: {included_path_type[0]}" ) break allpaths = [path] @@ -1237,6 +1268,7 @@ class Skipfails: repo: OptStr = None, meta_bug_id: OptInt = None, carryover_mode: bool = False, + known_intermittents_mode: bool = False, ): """ Skip a failure (for TOML, WPT and REFTEST manifests) @@ -1266,29 +1298,45 @@ class Skipfails: manifest: str = self.resolve_failure_manifest(path, kind, manifest) manifest_path: str = self.full_path(manifest) manifest_str: str = "" + comment: str = "" additional_comment: str = "" meta_bug_blocked: bool = False create_bug_lambda: CreateBug = None - carryover = False # this failure is not carried over from a previous skip-if (Bug 1971610) + carryover: bool = ( + False # not carried over from a previous skip-if (Bug 1971610) + ) + known_intermittent: bool = False + bugid: OptInt + if bug_id is None: - create_bug_lambda, bugid, meta_bug_blocked, attachments, comment = ( - self.generate_bugzilla_comment( - manifest, - kind, - path, - skip_if, - filename, - anyjs, - lineno, - label, - classification, - task_id, - try_url, - revision, - repo, - meta_bug_id, + if known_intermittents_mode and kind == Kind.TOML: + (bugid, comment) = self.find_known_intermittent( + repo, revision, task_id, manifest, filename, skip_if + ) + if bugid is None: + self.info("Ignoring failure as it is not a known intermittent") + return + known_intermittent = True + + else: + create_bug_lambda, bugid, meta_bug_blocked, attachments, comment = ( + self.generate_bugzilla_comment( + manifest, + kind, + path, + skip_if, + filename, + anyjs, + lineno, + label, + classification, + task_id, + try_url, + revision, + repo, + meta_bug_id, + ) ) - ) bug_reference: str = f"Bug {bugid}" if classification == Classification.SECONDARY and kind != Kind.WPT: bug_reference += " (secondary)" @@ -1300,6 +1348,11 @@ class Skipfails: f'Not editing in --carryover mode: ["{filename}"] in manifest: "{manifest}"' ) return + elif known_intermittents_mode: # not yet supported for WPT + self.info( + f'Not editing in --known-intermittents mode: ["{filename}"] in manifest: "{manifest}"' + ) + return if os.path.exists(manifest_path): manifest_str = open(manifest_path, encoding="utf-8").read() else: @@ -1321,7 +1374,7 @@ class Skipfails: document = mp.source_documents[manifest_path] try: - additional_comment, carryover = add_skip_if( + additional_comment, carryover, bug_reference = add_skip_if( document, filename, skip_if, @@ -1335,7 +1388,7 @@ class Skipfails: additional_comment = "" carryover = False if carryover_mode and not carryover: - self.info( + self.vinfo( f'No --carryover in: ["{filename}"] in manifest: "{manifest}"' ) return @@ -1346,6 +1399,11 @@ class Skipfails: f'Not editing in --carryover mode: ["{filename}"] in manifest: "{manifest}"' ) return + elif known_intermittents_mode: # not yet supported for LIST + self.info( + f'Not editing in --known-intermittents mode: ["{filename}"] in manifest: "{manifest}"' + ) + return if lineno == 0: self.error( f"cannot determine line to edit in manifest: {manifest_path}" @@ -1391,8 +1449,13 @@ class Skipfails: fp.close() self.info(f'{prefix}Edited ["{filename}"] in manifest: "{manifest}"') if kind != Kind.LIST: + mode: str = "" + if carryover: + mode = " CARRYOVER" + elif known_intermittent: + mode = " KNOWN INTERMITTENT" self.info( - f'{prefix}Added{" CARRYOVER" if carryover else ""} skip-if condition: "{skip_if}"' + f'{prefix}Added{mode} skip-if condition: "{skip_if} # {bug_reference}"' ) if bug_id is None: if self.bugzilla is None: @@ -1407,7 +1470,7 @@ class Skipfails: self.vinfo( f"Mode --dry-run: comment not added to Bug {bugid}:\n{comment}" ) - elif not carryover: + elif not carryover_mode and not known_intermittents_mode: self.add_bug_comment( bugid, comment, None if meta_bug_blocked else meta_bug_id ) @@ -1518,7 +1581,7 @@ class Skipfails: os = platform_info.os build_type = platform_info.build_type - runtimes = platform_info.test_variant + runtimes = platform_info.test_variant.split("+") skip_if = None if os == "linux": @@ -1551,13 +1614,13 @@ class Skipfails: skip_if += "ThreadSanitizer" # See implicit VARIANT_DEFAULTS in # https://searchfox.org/mozilla-central/source/layout/tools/reftest/manifest.sys.mjs#30 - fission = "no-fission" not in runtimes + no_fission = "!fission" not in runtimes snapshot = "snapshot" in runtimes swgl = "swgl" in runtimes nogpu = "nogpu" in runtimes - if not self.implicit_vars and fission: + if not self.implicit_vars and no_fission: skip_if += aa + "fission" - elif not fission: # implicit default: fission + elif not no_fission: # implicit default: fission skip_if += aa + nn + "fission" if platform_info.bits is not None: if platform_info.bits == "32": @@ -1595,7 +1658,6 @@ class Skipfails: extra = self.get_extra(task) else: extra = task - if kind == Kind.WPT: qq = '"' aa = " and " @@ -1616,34 +1678,39 @@ class Skipfails: arch = extra.arch if arch is not None and skip_if is not None and kind != Kind.LIST: skip_if += aa + "arch" + eq + qq + arch + qq - failure_key = os + os_version + arch + manifest + file_path - if self.failed_platforms.get(failure_key) is None: - if not self.platform_permutations: - self._fetch_platform_permutations() - permutations = ( - self.platform_permutations.get(manifest, {}) - .get(os, {}) - .get(os_version, {}) - .get(arch, None) - ) - self.failed_platforms[failure_key] = FailedPlatform( - permutations, high_freq + if high_freq: + failure_key = os + os_version + arch + manifest + file_path + if self.failed_platforms.get(failure_key) is None: + if not self.platform_permutations: + self._fetch_platform_permutations() + permutations = ( + self.platform_permutations.get(manifest, {}) + .get(os, {}) + .get(os_version, {}) + .get(arch, None) + ) + self.failed_platforms[failure_key] = FailedPlatform( + permutations, high_freq + ) + skip_cond = self.failed_platforms[failure_key].get_skip_string( + aa, extra.build_type, extra.test_variant ) - build_types = extra.build_type - skip_cond = self.failed_platforms[failure_key].get_skip_string( - aa, build_types, extra.test_variant - ) - if skip_cond is not None: - if kind == Kind.WPT: - # ensure ! -> 'not', primarily fission and e10s - skip_cond = skip_cond.replace("!", "not ") - skip_if += skip_cond - else: - skip_if = None + if skip_cond is not None: + skip_if += skip_cond + else: + skip_if = None + else: # not high_freq + skip_if += aa + extra.build_type + for tv in extra.test_variant.split("+"): + if tv != "no_variant": + skip_if += aa + tv elif skip_if is None: raise Exception( f"Unable to calculate skip-if condition from manifest={manifest} and file={file_path}" ) + if skip_if is not None and kind == Kind.WPT: + # ensure ! -> 'not', primarily fission and e10s + skip_if = skip_if.replace("!", "not ") return skip_if def get_file_info(self, path, product="Testing", component="General"): @@ -1742,23 +1809,28 @@ class Skipfails: Return the bug_suggestions JSON for the job_id Use the cache, if present, else download from treeherder """ - suggestions_path = self.cached_path(revision, f"suggest-{job_id}.json") - if os.path.exists(suggestions_path): - self.vinfo( - f"Reading cached bug_suggestions for {repo} revision: {revision} job_id: {job_id}" - ) - suggestions = self.read_json(suggestions_path) + + if job_id in self.suggestions: + suggestions = self.suggestions[job_id] else: - suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/" - self.vinfo( - f"Retrieving bug_suggestions for {repo} revision: {revision} job_id: {job_id}" - ) - r = requests.get(suggestions_url, headers=self.headers) - if r.status_code != 200: - self.warning(f"FAILED to query Treeherder = {r} for {r.url}") - return None - suggestions = r.json() - self.write_json(suggestions_path, suggestions) + suggestions_path = self.cached_path(revision, f"suggest-{job_id}.json") + if os.path.exists(suggestions_path): + self.vinfo( + f"Reading cached bug_suggestions for {repo} revision: {revision} job_id: {job_id}" + ) + suggestions = self.read_json(suggestions_path) + else: + suggestions_url = f"https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/" + self.vinfo( + f"Retrieving bug_suggestions for {repo} revision: {revision} job_id: {job_id}" + ) + r = requests.get(suggestions_url, headers=self.headers) + if r.status_code != 200: + self.warning(f"FAILED to query Treeherder = {r} for {r.url}") + return None + suggestions = r.json() + self.write_json(suggestions_path, suggestions) + self.suggestions[job_id] = suggestions return suggestions def get_bug_suggestions( @@ -2290,3 +2362,46 @@ class Skipfails: if test == path: allpaths.append(test) return allpaths + + def find_known_intermittent( + self, + repo: str, + revision: str, + task_id: str, + manifest: str, + filename: str, + skip_if: str, + ) -> TupleOptIntStr: + """ + Returns bugid if a known intermittent is found. + Also returns a suggested comment to be added to the known intermittent + bug... (currently not implemented). The args + manifest, filename, skip_if + are only used to create the comment + """ + bugid = None + suggestions: JSONType = None + comment: str = f'Intermittent failure in manifest: "{manifest}"' + comment += f'\n in test: "[{filename}]"' + comment += f'\n added skip-if: "{skip_if}"' + if revision is not None and repo is not None: + push_id = self.get_push_id(revision, repo) + if push_id is not None: + job_id = self.get_job_id(push_id, task_id) + if job_id is not None: + suggestions = self.cached_bug_suggestions(repo, revision, job_id) + if suggestions is not None: + top: JSONType = None + for suggestion in suggestions: + search: str = suggestion.get("search", "") + if search.startswith("PROCESS-CRASH") or ( + search.startswith("TEST-UNEXPECTED") and top is None + ): + top = suggestion + if top is not None: + recent_bugs = top.get("bugs", {}).get("open_recent", []) + for bug in recent_bugs: + summary: str = bug.get("summary", "") + if summary.endswith("single tracking bug"): + bugid = bug.get("id", None) + return (bugid, comment) diff --git a/testing/test/SKIP-FAILS.txt b/testing/test/SKIP-FAILS.txt @@ -64,6 +64,9 @@ Sub Command Arguments: -i, --task-id TASK_ID Task id to write a condition for instead of all tasks from the push + -k, --known-intermittents + Set known intermittents mode (only skip failures known + intermittents) -M, --max-failures MAX_FAILURES Maximum number of failures to skip (-1 == no limit) -m, --meta-bug-id META_BUG_ID @@ -98,6 +101,17 @@ Design a. Variables used in Python are set: https://searchfox.org/firefox-main/source/testing/mozbase/mozinfo/mozinfo/platforminfo.py + NOTE: that the task metadata runtimes are turned in to test variants by looking + up the "mozinfo" field that corresponds to the "runtime" key in + taskcluster/test_configs/variants.yml + + NOTE: special excpetions are these runtimes are mapped to these variants: + - no-fission -> !fission + - 1proc -> !e10s + + NOTE: multiple variants are stored in the field test_variant concatenated by '+', + for example: "socketprocess_networking+!fission" + b. Variables used in JavaScript are set: https://searchfox.org/firefox-main/source/layout/tools/reftest/manifest.sys.mjs @@ -171,8 +185,23 @@ Design b. Carry-over mode Only consider adding skip-if conditions which match the - platform (see above) of previous conditions. Does not perform any other - manifest edits or bugzilla changes. + platform (see above) of previous conditions. Does not perform any + bugzilla changes. + + c. Known intermittents mode + Only consider adding skip-if conditions which have known + intermittent bugs (and are not carryover bugs). + form (see above) of previous conditions. Does not perform any + bugzilla changes. + + For each failure, if the job_id can be determined then bug_suggestions will + be retrieved from + https://treeherder.mozilla.org/api/project/{repo}/jobs/{job_id}/bug_suggestions/ + each object with "search" field of ^PROCESS-CRASH or ^TEST-UNEXPECTED + will be considered if one of the bugs in + bugs.open_recent.<i>.summary ends with 'single tracking bug$' + then + bugs.open_recent.<i>.id has the bug id 5. Cache for skip-fails At the top of the source tree is a cache directory for skip-fails: diff --git a/testing/test/data/reftest-reorder-after.list b/testing/test/data/reftest-reorder-after.list @@ -1,2 +1,2 @@ # demonstrate reordering conditions -skip-if(Android) fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) fuzzy-if(gtkWidget,0-70,0-2239) fuzzy-if(winWidget,0-63,0-76799) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html # Bug TBD +skip-if(Android) fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) fuzzy-if(gtkWidget&&optimized&&(fission||!fission),0-70,0-2239) fuzzy-if(winWidget,0-63,0-76799) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html # Bug TBD diff --git a/testing/test/data/reftest-reorder-before.list b/testing/test/data/reftest-reorder-before.list @@ -1,2 +1,2 @@ # demonstrate reordering conditions -fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) skip-if(Android) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html +fuzzy-if(cocoaWidget,0-80,0-76800) fuzzy-if(appleSilicon,0-80,0-76800) skip-if(Android) fuzzy-if(winWidget,0-63,0-76799) fuzzy-if(gtkWidget&&optimized,0-70,0-2032) HTTP(..) == short.mp4.firstframe.html short.mp4.firstframe-ref.html diff --git a/testing/test/data/suggest-530457469.json b/testing/test/data/suggest-530457469.json @@ -0,0 +1,401 @@ +[ + { + "bugs": { + "all_others": [], + "open_recent": [ + { + "crash_signature": "", + "dupe_of": null, + "id": 1814775, + "internal_id": 1814775, + "keywords": "intermittent-failure,intermittent-testcase", + "occurrences": null, + "resolution": "", + "status": "REOPENED", + "summary": "Intermittent dom/midi/tests/test_midi_device_sysex.html | single tracking bug", + "whiteboard": "" + } + ] + }, + "counter": 1, + "failure_new_in_rev": false, + "line_number": 2545, + "path_end": "dom/midi/tests/test_midi_device_sysex.html", + "search": "TEST-UNEXPECTED-FAIL | dom/midi/tests/test_midi_device_sysex.html | Test timed out. -", + "search_terms": ["test_midi_device_sysex.html"] + }, + { + "bugs": { + "all_others": [ + { + "crash_signature": "", + "dupe_of": 1800035, + "id": 1838357, + "internal_id": 1958303, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "DUPLICATE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1964589, + "internal_id": 1956877, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 137", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1931816, + "internal_id": 1931816, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent perftest [taskcluster:error] exit status 1", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": 1800035, + "id": 1839987, + "internal_id": 1958686, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "DUPLICATE", + "status": "RESOLVED", + "summary": "Intermittent marionette [taskcluster:error] exit status 1", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1825562, + "internal_id": 1825562, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 - 1st fxrecord", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1933822, + "internal_id": 1933822, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "WORKSFORME", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | Error: No objdir path", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1966429, + "internal_id": 1959480, + "keywords": "intermittent-failure,regression", + "occurrences": null, + "resolution": "FIXED", + "status": "RESOLVED", + "summary": "Perma esr115 [taskcluster:error] exit status 1 | single tracking bug", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": 1800035, + "id": 1830534, + "internal_id": 1958298, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "DUPLICATE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | Assertion failure", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": 1973629, + "id": 1973539, + "internal_id": 1961280, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "DUPLICATE", + "status": "RESOLVED", + "summary": "Perma [mozilla-beta] mpu [taskcluster:error] exit status 1 | single tracking bug", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1884112, + "internal_id": 1957149, + "keywords": "intermittent-failure,intermittent-testcase", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent Marionette [taskcluster:error] exit status 1 | single tracking bug", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1888377, + "internal_id": 1888377, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent python try [taskcluster:error] exit status 1 | tools/tryselect/test/test_release.py::test_release ERROR", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1905092, + "internal_id": 1905092, + "keywords": "intermittent-failure,regression", + "occurrences": null, + "resolution": "FIXED", + "status": "RESOLVED", + "summary": "Perma [Tier 2] mozlint ERROR at setup of test_lint_codespell_fix[codespell] | [taskcluster:error] exit status 1", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1901903, + "internal_id": 1901903, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent py3 try [taskcluster:error] exit status 1 | in log: ModuleNotFoundError: No module named 'appdirs'", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1881423, + "internal_id": 1881423, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent python try [taskcluster:error] exit status 1 | tools/tryselect/test/test_scriptworker.py::test_release ERROR", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1962983, + "internal_id": 1956575, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 (AssertionError in python/mozperftest/mozperftest/tests/test_shellscript.py)", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1896800, + "internal_id": 1896800, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "FIXED", + "status": "RESOLVED", + "summary": "Intermittent [Sel] error: snap \"firefox\" has \"auto-refresh\" change in progress | [taskcluster:error] exit status 10", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": 1604486, + "id": 1619487, + "internal_id": 1959650, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "DUPLICATE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | pickle.UnpicklingError: pickle data was truncated after mercurial.error.SignalInterrupt", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1952425, + "internal_id": 1952327, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "FIXED", + "status": "RESOLVED", + "summary": "Perma mpu macOS 14.70 [taskcluster:error] exit status 1 | FAILED python/mozperftest/mozperftest/tests/test_shellscript.py::test_shell_script[on_try_setting0]", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1924539, + "internal_id": 1924539, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "WORKSFORME", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | leakcheck large BackstagePass | comm/mail/test/browser/folder-display/browser.ini", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1857381, + "internal_id": 1857381, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent 1st error: a remote error occurred: timed out waiting for CPU and disk to become idle | [taskcluster:error] exit status 1", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1924543, + "internal_id": 1924543, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "WORKSFORME", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | PROCESS-CRASH | application crashed [None] | comm/mail/components/addrbook/test/browser/browser_cardDAV_init.js", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1975782, + "internal_id": 1962065, + "keywords": "intermittent-failure,regression", + "occurrences": null, + "resolution": "FIXED", + "status": "RESOLVED", + "summary": "Perma mpu [taskcluster:error] exit status 1 | FAILED python/mozperftest/mozperftest/tests/test_alert.py::test_alert_exact_command[alert-summary-talos.json-task-info-talos.json-expected_commands1-1] | single tracking bug", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": 1973857, + "id": 1979505, + "internal_id": 1963455, + "keywords": "intermittent-failure,regression", + "occurrences": null, + "resolution": "DUPLICATE", + "status": "RESOLVED", + "summary": "Perma [Tier 2] mozlint [taskcluster:error] exit status 1 | monkeypatch issues with multiprocessing on Windows | signal.CTRL_C_EVENT isn't causing a KeyboardInterrupt on Windows | single tracking bug", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1912980, + "internal_id": 1912980, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent py3 [taskcluster:error] exit status 1 | after tools/tryselect/test/test_release.py::test_release ERROR | ERROR at setup of test_release | FileNotFoundError: [WinError 2] The system cannot find the file specified: 'C:\\\\Users\\\\task_172355900", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1924547, + "internal_id": 1924547, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | after OSError: [WinError 145] The directory is not empty: 'C:\\\\Users\\\\task_172867298287704\\\\.mozbuild\\\\srcdirs\\\\src-47516bc87bc0\\\\_virtualenvs\\\\common'", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1896259, + "internal_id": 1896259, + "keywords": "", + "occurrences": null, + "resolution": "FIXED", + "status": "RESOLVED", + "summary": "Perma fxrecord 1st job [taskcluster:error] exit status 1 | after error: A connection attempt failed because the connected party did not properly respond after a period of time or established connection failed because connected host has failed to respond.", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1924489, + "internal_id": 1924489, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "INCOMPLETE", + "status": "RESOLVED", + "summary": "Intermittent [taskcluster:error] exit status 1 | after FileExistsError: [WinError 183] Cannot create a file when that file already exists: 'C:\\\\Users\\\\task_172670864431210\\\\.mozbuild\\\\srcdirs\\\\src-ca673f75774b\\\\_virtualenvs\\\\common' <- check in log", + "whiteboard": "" + } + ], + "open_recent": [ + { + "crash_signature": "", + "dupe_of": null, + "id": 1891449, + "internal_id": 1891449, + "keywords": "intermittent-failure,intermittent-testcase", + "occurrences": null, + "resolution": "", + "status": "NEW", + "summary": "Intermittent [taskcluster:error] exit status 120", + "whiteboard": "" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1965998, + "internal_id": 1957150, + "keywords": "intermittent-failure", + "occurrences": null, + "resolution": "", + "status": "NEW", + "summary": "Intermittent [taskcluster:error] exit status 1 | single tracking bug", + "whiteboard": "[stockwell unknown]" + }, + { + "crash_signature": "", + "dupe_of": null, + "id": 1973857, + "internal_id": 1961430, + "keywords": "intermittent-failure,regression", + "occurrences": null, + "resolution": "", + "status": "NEW", + "summary": "Perma [tier 2] mozlint [taskcluster:error] exit status 1 | single tracking bug", + "whiteboard": "" + } + ] + }, + "counter": 31054, + "failure_new_in_rev": false, + "line_number": 5127, + "path_end": null, + "search": "[taskcluster:error] exit status 1", + "search_terms": ["[taskcluster:error] exit status 1"] + } +] diff --git a/testing/test/test_failedplatform.py b/testing/test/test_failedplatform.py @@ -149,6 +149,36 @@ def test_get_no_variant_conditions(): == " && !test_variant1 && fission" ) + # Handle composite variants + fp = FailedPlatform( + { + "build_type1": { + "test_variant1": {}, + "socketprocess_networking+!fission": {}, + "no_variant": {}, + } + } + ) + assert ( + fp.get_no_variant_conditions(" && ", "build_type1") + == " && !test_variant1 && !socketprocess_networking && fission" + ) + + # Handle mutually exclusive variants + fp = FailedPlatform( + { + "build_type1": { + "fission": {}, + "socketprocess_networking+!fission": {}, + "no_variant": {}, + } + } + ) + assert ( + fp.get_no_variant_conditions(" && ", "build_type1") + == " && !socketprocess_networking" + ) + def test_get_test_variant_condition(): """Test get_no_variant_conditions""" diff --git a/testing/test/test_skipfails.py b/testing/test/test_skipfails.py @@ -375,6 +375,8 @@ def test_task_to_skip_if(): == "os == 'mac' && os_version == '11.20' && arch == 'aarch64' && debug && swgl" ) + ## The test below is now altered - we WILL use the build_type and test variant + ## regardless of other result permutations (part of deprecating FailedPlatorm) # Do not include build type or test variant if everything failed sf = Skipfails() task_id = "AKYqxtoWStigj_5yHVqAeg" @@ -402,8 +404,11 @@ def test_task_to_skip_if(): skip_if = sf.task_to_skip_if( "test-manifest", task_id, Kind.TOML, "test-path", False ) - assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" + ## The test below is now altered - we WILL use the build_type and test variant + ## regardless of other result permutations (part of deprecating FailedPlatorm) sf = Skipfails() task_id = "QFo2jGFvTKGVcoqHCBpMGw" task_details = { @@ -430,8 +435,14 @@ def test_task_to_skip_if(): skip_if = sf.task_to_skip_if( "test-manifest", task_id, Kind.TOML, "test-path", False ) - assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + assert ( + skip_if + == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && xorigin" + ) + ## The test below is now altered - we WILL NOT negate other variants + ## (part of deprecating FailedPlatorm) # Only the test without variant failed sf = Skipfails() task_id = "Xvdt2gbEQ3iDVAZPddY9PQ" @@ -461,10 +472,11 @@ def test_task_to_skip_if(): skip_if = sf.task_to_skip_if( "test-manifest", task_id, Kind.TOML, "test-path", False ) - assert ( - skip_if - == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && !xorigin" - ) + # assert ( + # skip_if + # == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && !xorigin" + # ) + assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" # Missing platform permutation for the task sf = Skipfails() @@ -522,7 +534,7 @@ def test_task_to_skip_if(): == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt && xorigin" ) - # Full fail with everal tasks + # Full fail with several tasks sf = Skipfails() sf.platform_permutations = { "test-manifest": { @@ -587,6 +599,8 @@ def test_task_to_skip_if(): ) assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" + ## The test below is now altered - we WILL include the build_type + ## regardless of other result permutations (part of deprecating FailedPlatorm) task_id = "ShPeY1F8SY6Gm-1VSjsyUA" task_details = { "expires": "2024-03-19T03:29:11.050Z", @@ -609,7 +623,8 @@ def test_task_to_skip_if(): skip_if = sf.task_to_skip_if( "test-manifest", task_id, Kind.TOML, "test-path", False ) - assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && debug" # Multiple failed tasks allowing for optimized skip if sf = Skipfails() @@ -676,6 +691,8 @@ def test_task_to_skip_if(): ) assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && debug" + ## The test below is now altered - we WILL include the build_type + ## regardless of other result permutations (part of deprecating FailedPlatorm) task_id = "caDMGUmnT7muCqNWj6w3nQ" task_details = { "expires": "2024-03-19T03:29:11.050Z", @@ -698,7 +715,8 @@ def test_task_to_skip_if(): skip_if = sf.task_to_skip_if( "test-manifest", task_id, Kind.TOML, "test-path", False ) - assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + # assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86'" + assert skip_if == "os == 'linux' && os_version == '18.04' && arch == 'x86' && opt" def test_task_to_skip_if_high_freq(): @@ -1586,7 +1604,7 @@ def test_reftest_skip_failure_reorder(capsys): "extra": { "suite": "reftest", "test-setting": { - "build": {"asan": True, "type": "opt"}, + "build": {"type": "opt"}, "platform": { "arch": "64", "os": {"name": "linux", "version": "1804"}, @@ -1630,5 +1648,32 @@ def test_reftest_skip_failure_reorder(capsys): # ) +def test_find_known_intermittent(): + sf = Skipfails() + + repo = "try" + revision = "70613a6f11801e19478238c014b7ca61c28571d0" + task_id = "B64K59oVTlir3gUcdZQiew" + push_id = "1741497" + job_id = "530457469" + sf.push_ids[revision] = push_id # pre-cache push_id + sf.job_ids[f"{push_id}:{task_id}"] = job_id # pre-cache job_id + suggestions_path = DATA_PATH.joinpath(f"suggest-{job_id}.json") + suggestions = sf.read_json(suggestions_path) + sf.suggestions[job_id] = suggestions # pre-cache suggestions + manifest = "dom/midi/tests/mochitest.toml" + filename = "test_midi_device_sysex.html" + skip_if = "os == 'android' && os_version == '14' && arch == 'x86_64' && isolated_process && xorigin" + + (bugid, comment) = sf.find_known_intermittent( + repo, revision, task_id, manifest, filename, skip_if + ) + assert bugid == 1814775 + assert ( + comment + == "Intermittent failure in manifest: \"dom/midi/tests/mochitest.toml\"\n in test: \"[test_midi_device_sysex.html]\"\n added skip-if: \"os == 'android' && os_version == '14' && arch == 'x86_64' && isolated_process && xorigin\"" + ) + + if __name__ == "__main__": main()