bisection.py (11593B)
1 import math 2 3 import mozinfo 4 5 6 class Bisect: 7 "Class for creating, bisecting and summarizing for --bisect-chunk option." 8 9 def __init__(self, harness): 10 super().__init__() 11 self.summary = [] 12 self.contents = {} 13 self.repeat = 10 14 self.failcount = 0 15 self.max_failures = 3 16 17 def setup(self, tests): 18 """This method is used to initialize various variables that are required 19 for test bisection""" 20 status = 0 21 self.contents.clear() 22 # We need totalTests key in contents for sanity check 23 self.contents["totalTests"] = tests 24 self.contents["tests"] = tests 25 self.contents["loop"] = 0 26 return status 27 28 def reset(self, expectedError, result): 29 """This method is used to initialize self.expectedError and self.result 30 for each loop in runtests.""" 31 self.expectedError = expectedError 32 self.result = result 33 34 def get_tests_for_bisection(self, options, tests): 35 """Make a list of tests for bisection from a given list of tests""" 36 bisectlist = [] 37 for test in tests: 38 bisectlist.append(test) 39 if test.endswith(options.bisectChunk): 40 break 41 42 return bisectlist 43 44 def pre_test(self, options, tests, status): 45 """This method is used to call other methods for setting up variables and 46 getting the list of tests for bisection.""" 47 if options.bisectChunk == "default": 48 return tests 49 # The second condition in 'if' is required to verify that the failing 50 # test is the last one. 51 elif "loop" not in self.contents or not self.contents["tests"][-1].endswith( 52 options.bisectChunk 53 ): 54 tests = self.get_tests_for_bisection(options, tests) 55 status = self.setup(tests) 56 57 return self.next_chunk_binary(options, status) 58 59 def post_test(self, options, expectedError, result): 60 """This method is used to call other methods to summarize results and check whether a 61 sanity check is done or not.""" 62 self.reset(expectedError, result) 63 status = self.summarize_chunk(options) 64 # Check whether sanity check has to be done. Also it is necessary to check whether 65 # options.bisectChunk is present in self.expectedError as we do not want to run 66 # if it is "default". 67 if status == -1 and options.bisectChunk in self.expectedError: 68 # In case we have a debug build, we don't want to run a sanity 69 # check, will take too much time. 70 if mozinfo.info["debug"]: 71 return status 72 73 testBleedThrough = self.contents["testsToRun"][0] 74 tests = self.contents["totalTests"] 75 tests.remove(testBleedThrough) 76 # To make sure that the failing test is dependent on some other 77 # test. 78 if options.bisectChunk in testBleedThrough: 79 return status 80 81 status = self.setup(tests) 82 self.summary.append("Sanity Check:") 83 84 return status 85 86 def next_chunk_reverse(self, options, status): 87 "This method is used to bisect the tests in a reverse search fashion." 88 89 # Base Cases. 90 if self.contents["loop"] <= 1: 91 self.contents["testsToRun"] = self.contents["tests"] 92 if self.contents["loop"] == 1: 93 self.contents["testsToRun"] = [self.contents["tests"][-1]] 94 self.contents["loop"] += 1 95 return self.contents["testsToRun"] 96 97 if "result" in self.contents: 98 if self.contents["result"] == "PASS": 99 chunkSize = self.contents["end"] - self.contents["start"] 100 self.contents["end"] = self.contents["start"] - 1 101 self.contents["start"] = self.contents["end"] - chunkSize 102 103 # self.contents['result'] will be expected error only if it fails. 104 elif self.contents["result"] == "FAIL": 105 self.contents["tests"] = self.contents["testsToRun"] 106 status = 1 # for initializing 107 108 # initialize 109 if status: 110 totalTests = len(self.contents["tests"]) 111 chunkSize = int(math.ceil(totalTests / 10.0)) 112 self.contents["start"] = totalTests - chunkSize - 1 113 self.contents["end"] = totalTests - 2 114 115 start = self.contents["start"] 116 end = self.contents["end"] + 1 117 self.contents["testsToRun"] = self.contents["tests"][start:end] 118 self.contents["testsToRun"].append(self.contents["tests"][-1]) 119 self.contents["loop"] += 1 120 121 return self.contents["testsToRun"] 122 123 def next_chunk_binary(self, options, status): 124 "This method is used to bisect the tests in a binary search fashion." 125 126 # Base cases. 127 if self.contents["loop"] <= 1: 128 self.contents["testsToRun"] = self.contents["tests"] 129 if self.contents["loop"] == 1: 130 self.contents["testsToRun"] = [self.contents["tests"][-1]] 131 self.contents["loop"] += 1 132 return self.contents["testsToRun"] 133 134 # Initialize the contents dict. 135 if status: 136 totalTests = len(self.contents["tests"]) 137 self.contents["start"] = 0 138 self.contents["end"] = totalTests - 2 139 140 # pylint --py3k W1619 141 mid = (self.contents["start"] + self.contents["end"]) / 2 142 if "result" in self.contents: 143 if self.contents["result"] == "PASS": 144 self.contents["end"] = mid 145 146 elif self.contents["result"] == "FAIL": 147 self.contents["start"] = mid + 1 148 149 mid = (self.contents["start"] + self.contents["end"]) / 2 150 start = mid + 1 151 end = self.contents["end"] + 1 152 self.contents["testsToRun"] = self.contents["tests"][start:end] 153 if not self.contents["testsToRun"]: 154 self.contents["testsToRun"].append(self.contents["tests"][mid]) 155 self.contents["testsToRun"].append(self.contents["tests"][-1]) 156 self.contents["loop"] += 1 157 158 return self.contents["testsToRun"] 159 160 def summarize_chunk(self, options): 161 "This method is used summarize the results after the list of tests is run." 162 if options.bisectChunk == "default": 163 # if no expectedError that means all the tests have successfully 164 # passed. 165 if len(self.expectedError) == 0: 166 return -1 167 options.bisectChunk = self.expectedError.keys()[0] 168 self.summary.append("\tFound Error in test: %s" % options.bisectChunk) 169 return 0 170 171 # If options.bisectChunk is not in self.result then we need to move to 172 # the next run. 173 if options.bisectChunk not in self.result: 174 return -1 175 176 self.summary.append("\tPass %d:" % self.contents["loop"]) 177 if len(self.contents["testsToRun"]) > 1: 178 self.summary.append( 179 "\t\t%d test files(start,end,failing). [%s, %s, %s]" 180 % ( 181 len(self.contents["testsToRun"]), 182 self.contents["testsToRun"][0], 183 self.contents["testsToRun"][-2], 184 self.contents["testsToRun"][-1], 185 ) 186 ) 187 else: 188 self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0]) 189 return self.check_for_intermittent(options) 190 191 if self.result[options.bisectChunk] == "PASS": 192 self.summary.append("\t\tno failures found.") 193 if self.contents["loop"] == 1: 194 status = -1 195 else: 196 self.contents["result"] = "PASS" 197 status = 0 198 199 elif self.result[options.bisectChunk] == "FAIL": 200 if "expectedError" not in self.contents: 201 self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1]) 202 self.contents["expectedError"] = self.expectedError[options.bisectChunk] 203 status = 0 204 205 elif ( 206 self.expectedError[options.bisectChunk] 207 == self.contents["expectedError"] 208 ): 209 self.summary.append( 210 "\t\t%s failed with expected error." 211 % self.contents["testsToRun"][-1] 212 ) 213 self.contents["result"] = "FAIL" 214 status = 0 215 216 # This code checks for test-bleedthrough. Should work for any 217 # algorithm. 218 numberOfTests = len(self.contents["testsToRun"]) 219 if numberOfTests < 3: 220 # This means that only 2 tests are run. Since the last test 221 # is the failing test itself therefore the bleedthrough 222 # test is the first test 223 self.summary.append( 224 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the " 225 "root cause for many of the above failures" 226 % self.contents["testsToRun"][0] 227 ) 228 status = -1 229 else: 230 self.summary.append( 231 "\t\t%s failed with different error." 232 % self.contents["testsToRun"][-1] 233 ) 234 status = -1 235 236 return status 237 238 def check_for_intermittent(self, options): 239 "This method is used to check whether a test is an intermittent." 240 if self.result[options.bisectChunk] == "PASS": 241 self.summary.append( 242 "\t\tThe test %s passed." % self.contents["testsToRun"][0] 243 ) 244 if self.repeat > 0: 245 # loop is set to 1 to again run the single test. 246 self.contents["loop"] = 1 247 self.repeat -= 1 248 return 0 249 else: 250 if self.failcount > 0: 251 # -1 is being returned as the test is intermittent, so no need to bisect 252 # further. 253 return -1 254 # If the test does not fail even once, then proceed to next chunk for bisection. 255 # loop is set to 2 to proceed on bisection. 256 self.contents["loop"] = 2 257 return 1 258 elif self.result[options.bisectChunk] == "FAIL": 259 self.summary.append( 260 "\t\tThe test %s failed." % self.contents["testsToRun"][0] 261 ) 262 self.failcount += 1 263 self.contents["loop"] = 1 264 self.repeat -= 1 265 # self.max_failures is the maximum number of times a test is allowed 266 # to fail to be called an intermittent. If a test fails more than 267 # limit set, it is a perma-fail. 268 if self.failcount < self.max_failures: 269 if self.repeat == 0: 270 # -1 is being returned as the test is intermittent, so no need to bisect 271 # further. 272 return -1 273 return 0 274 else: 275 self.summary.append( 276 "TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the " 277 "root cause for many of the above failures" 278 % self.contents["testsToRun"][0] 279 ) 280 return -1 281 282 def print_summary(self): 283 "This method is used to print the recorded summary." 284 print("Bisection summary:") 285 for line in self.summary: 286 print(line)