failedplatform.py (9359B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 from functools import reduce 6 from typing import Optional 7 8 9 class FailedPlatform: 10 """ 11 Stores all failures on different build types and test variants for a single platform. 12 This allows us to detect when a platform failed on all build types or all test variants to 13 generate a simpler skip-if condition. 14 """ 15 16 def __init__( 17 self, 18 # Keys are build types, values are test variants for this build type 19 # Tests variants can be composite by using the "+" character 20 # eg: a11y_checks+swgl 21 # each build_type[test_variant] has a {'pass': x, 'fail': y} 22 # x and y represent number of times this was run in the last 30 days 23 # See examples in 24 # https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.source.test-info-all/artifacts/public%2Ftest-info-testrun-matrix.json 25 oop_permutations: Optional[ 26 dict[ 27 str, # Build type 28 dict[str, dict[str, int]], # Test Variant # {'pass': x, 'fail': y} 29 ] 30 ], 31 high_freq: bool = False, 32 ) -> None: 33 # Contains all test variants for each build type the task failed on 34 self.failures: dict[str, dict[str, int]] = {} 35 self.oop_permutations = oop_permutations 36 self.high_freq = high_freq 37 38 def get_possible_build_types(self) -> list[str]: 39 return ( 40 list(self.oop_permutations.keys()) 41 if self.oop_permutations is not None 42 else [] 43 ) 44 45 def get_possible_test_variants(self, build_type: str) -> list[str]: 46 permutations = ( 47 self.oop_permutations.get(build_type, {}) 48 if self.oop_permutations is not None 49 else [] 50 ) 51 return [tv for tv in permutations] 52 53 def is_full_fail(self) -> bool: 54 """ 55 Test if failed on every test variant of every build type 56 """ 57 build_types = set(self.failures.keys()) 58 possible_build_types = self.get_possible_build_types() 59 # If we do not have information on possible build types, do not consider it a full fail 60 # This avoids creating a too broad skip-if condition 61 if len(possible_build_types) == 0: 62 return False 63 return all([ 64 bt in build_types and self.is_full_test_variants_fail(bt) 65 for bt in possible_build_types 66 ]) 67 68 def is_full_high_freq_fail(self) -> bool: 69 """ 70 Test if there are at least 7 failures on each build type 71 """ 72 build_types = set(self.failures.keys()) 73 possible_build_types = self.get_possible_build_types() 74 # If we do not have information on possible build types, do not consider it a full fail 75 # This avoids creating a too broad skip-if condition 76 if len(possible_build_types) == 0: 77 return False 78 return all([ 79 bt in build_types and sum(list(self.failures[bt].values())) >= 7 80 for bt in possible_build_types 81 ]) 82 83 def is_full_test_variants_fail(self, build_type: str) -> bool: 84 """ 85 Test if failed on every test variant of given build type 86 """ 87 failed_variants = self.failures.get(build_type, {}).keys() 88 possible_test_variants = self.get_possible_test_variants(build_type) 89 # If we do not have information on possible test variants, do not consider it a full fail 90 # This avoids creating a too broad skip-if condition 91 if len(possible_test_variants) == 0: 92 return False 93 return all([t in failed_variants for t in possible_test_variants]) 94 95 def get_negated_variant(self, test_variant: str): 96 if not test_variant.startswith("!"): 97 return "!" + test_variant 98 return test_variant.replace("!", "", 1) 99 100 def get_no_variant_conditions(self, and_str: str, build_type: str): 101 """ 102 The no_variant test variant does not really exist and is only internal. 103 This function gets all available test variants for the given build type 104 and negates them to create a skip-if that handle tasks without test variants 105 """ 106 variants_to_negate = [] 107 for tv in self.get_possible_test_variants(build_type): 108 if tv != "no_variant": 109 variants_to_negate.extend(tv.split("+")) 110 negated = [] 111 for tv in variants_to_negate: 112 ntv = self.get_negated_variant(tv) 113 if not ntv in variants_to_negate: 114 negated.append(ntv) # ignore mutually exclusive variants 115 return_str = reduce( 116 (lambda rs, npv: rs + and_str + npv), 117 negated, 118 "", 119 ) 120 return return_str 121 122 def get_test_variant_condition( 123 self, and_str: str, build_type: str, test_variant: str 124 ): 125 """ 126 If the given test variant is part of another composite test variant, then add negations matching that composite 127 variant to prevent overlapping in skips. 128 eg: test variant "a11y_checks" is to be added while "a11y_checks+swgl" exists 129 the resulting condition will be "a11y_checks && !swgl" 130 """ 131 all_test_variants_parts = [ 132 tv.split("+") 133 for tv in self.get_possible_test_variants(build_type) 134 if tv not in ["no_variant", test_variant] 135 ] 136 test_variant_parts = test_variant.split("+") 137 # List of composite test variants more specific than the current one 138 matching_variants_parts = [ 139 tv_parts 140 for tv_parts in all_test_variants_parts 141 if all(x in tv_parts for x in test_variant_parts) 142 ] 143 variants_to_negate = [ 144 part 145 for tv_parts in matching_variants_parts 146 for part in tv_parts 147 if part not in test_variant_parts 148 ] 149 150 return_str = reduce((lambda x, y: x + and_str + y), test_variant_parts, "") 151 return_str = reduce( 152 (lambda x, y: x + and_str + self.get_negated_variant(y)), 153 variants_to_negate, 154 return_str, 155 ) 156 return return_str 157 158 def get_full_test_variant_condition( 159 self, and_str: str, build_type: str, test_variant: str 160 ) -> str: 161 if test_variant == "no_variant": 162 return self.get_no_variant_conditions(and_str, build_type) 163 else: 164 return self.get_test_variant_condition(and_str, build_type, test_variant) 165 166 def get_test_variant_string(self, test_variant: str): 167 """ 168 Some test variants strings need to be updated to match what is given in oop_permutations 169 """ 170 if test_variant == "no-fission": 171 return "!fission" 172 if test_variant == "1proc": 173 return "!e10s" 174 return test_variant 175 176 def get_skip_string( 177 self, and_str: str, build_type: str, test_variant: str 178 ) -> Optional[str]: 179 if self.failures.get(build_type) is None: 180 self.failures[build_type] = {test_variant: 1} 181 elif self.failures[build_type].get(test_variant) is None: 182 self.failures[build_type][test_variant] = 1 183 else: 184 self.failures[build_type][test_variant] += 1 185 186 if not self.high_freq: 187 return self._get_skip_string(and_str, build_type, test_variant) 188 return self._get_high_freq_skip_string(and_str, build_type) 189 190 def _get_high_freq_skip_string( 191 self, and_str: str, build_type: str 192 ) -> Optional[str]: 193 return_str: Optional[str] = None 194 195 if self.is_full_high_freq_fail(): 196 return_str = "" 197 else: 198 total_failures = sum(list(self.failures[build_type].values())) 199 most_variant, most_failures = self.get_test_variant_with_most_failures( 200 build_type 201 ) 202 203 if total_failures >= 7: 204 return_str = and_str + build_type 205 if most_failures / total_failures >= 3 / 4: 206 return_str += self.get_full_test_variant_condition( 207 and_str, build_type, most_variant 208 ) 209 elif self.is_full_fail(): 210 return_str = "" 211 212 return return_str 213 214 def get_test_variant_with_most_failures(self, build_type: str) -> tuple[str, int]: 215 most_failures = 0 216 most_variant = "" 217 for variant, failures in self.failures[build_type].items(): 218 if failures > most_failures: 219 most_failures = failures 220 most_variant = variant 221 return most_variant, most_failures 222 223 def _get_skip_string( 224 self, and_str: str, build_type: str, test_variant: str 225 ) -> Optional[str]: 226 return_str = "" 227 # If every test variant of every build type failed, do not add anything 228 if not self.is_full_fail(): 229 return_str += and_str + build_type 230 if not self.is_full_test_variants_fail(build_type): 231 return_str += self.get_full_test_variant_condition( 232 and_str, build_type, test_variant 233 ) 234 235 return return_str