parse_reftest.py (25235B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import os 6 import os.path 7 import re 8 import sys 9 10 BUILD_TYPES = [ 11 "optimized", 12 "isDebugBuild", 13 "isCoverageBuild", 14 "AddressSanitizer", 15 "ThreadSanitizer", 16 ] 17 EQEQ = "==" 18 FUZZY_IF_REGEX = r"^fuzzy-if\((.*?),(\d+)-(\d+),(\d+)-(\d+)\)$" 19 IMPLICIT = { 20 "fission": True, 21 "is64Bit": True, 22 "useDrawSnapshot": False, 23 "swgl": False, 24 } 25 MARGIN = 0.05 # Increase difference/pixels percentage 26 NOT_EQ = "!=" 27 OSES = ["Android", "cocoaWidget", "appleSilicon", "gtkWidget", "winWidget"] 28 PASS = "PASS" 29 TEST_TYPES = [EQEQ, NOT_EQ] 30 31 32 class ListManifestParser: 33 """ 34 Meta Manifest Parser is the main class for the lmp program. 35 """ 36 37 errfile = sys.stderr 38 outfile = sys.stdout 39 verbose = False 40 41 def __init__( 42 self, implicit_vars=False, verbose=False, error=None, warning=None, info=None 43 ): 44 self.implicit_vars = implicit_vars 45 self.verbose = verbose 46 self._error = error 47 self._warning = warning 48 self._info = info 49 self.parser = None 50 self.fuzzy_if_rx = None 51 52 def error(self, e): 53 if self._error is not None: 54 self._error(e) 55 else: 56 print(f"ERROR: {e}", file=sys.stderr, flush=True) 57 58 def warning(self, e): 59 if self._warning is not None: 60 self._warning(e) 61 else: 62 print(f"WARNING: {e}", file=sys.stderr, flush=True) 63 64 def info(self, e): 65 if self._info is not None: 66 self._info(e) 67 else: 68 print(f"INFO: {e}", file=sys.stderr, flush=True) 69 70 def vinfo(self, e): 71 if self.verbose: 72 self.info(e) 73 74 def should_merge(self, condition, fuzzy_if_condition): 75 """ 76 Return True if existing condition and proposed fuzzy_if 77 differ by one dimension (or less) 78 """ 79 80 c_os = None 81 os = None 82 conditions = condition.split("&&") 83 n = len(conditions) 84 fuzzy_ifs = fuzzy_if_condition.split("&&") 85 m = len(fuzzy_ifs) 86 dimensions = {} 87 delta = 0 # dimensions of difference 88 for i in range(n): 89 if conditions[i].find("||") > 0: 90 disjunctions = conditions[i][1:-1].split("||") 91 if disjunctions[0] in OSES: 92 c_os = disjunctions[0] 93 for j in range(m): 94 if fuzzy_ifs[j] in OSES: 95 os = fuzzy_ifs[j] 96 if c_os != os: 97 return False # do not merge different OSES 98 fuzzy_ifs[j] = "" 99 break 100 conditions[i] = "" 101 elif self.implicit_vars and disjunctions[0] in IMPLICIT: 102 dimensions[disjunctions[0]] = True 103 conditions[i] = "" 104 else: 105 delta += 1 # OTHER adds a dimension 106 elif conditions[i] in OSES: 107 c_os = conditions[i] 108 for j in range(m): 109 if fuzzy_ifs[j] in OSES: 110 os = fuzzy_ifs[j] 111 if c_os != os: 112 return False # do not merge different OSES 113 fuzzy_ifs[j] = "" 114 break # expect only one os variable 115 conditions[i] = "" 116 elif conditions[i] in BUILD_TYPES: 117 for j in range(m): 118 if fuzzy_ifs[j] in BUILD_TYPES: 119 if conditions[i] != fuzzy_ifs[j]: 120 delta += 1 # BUILD_TYPE different 121 fuzzy_ifs[j] = "" 122 break # expect at most one build_type 123 conditions[i] = "" # handles also if BUILD_TYPE is omitted 124 else: 125 negated = False 126 if conditions[i][0] == "!": 127 negated = True 128 cond = conditions[i][1:] 129 else: 130 cond = conditions[i] 131 dimensions[cond] = True 132 if negated: 133 opposite = cond 134 else: 135 opposite = "!" + cond 136 for j in range(m): 137 if conditions[i] == fuzzy_ifs[j]: # same 138 fuzzy_ifs[j] = "" 139 conditions[i] = "" 140 break 141 elif opposite == fuzzy_ifs[j]: # opposite explicit 142 delta += 1 # different 143 fuzzy_ifs[j] = "" 144 conditions[i] = "" 145 break 146 elif fuzzy_ifs[j] == "(" + cond + "||!" + cond + ")": 147 fuzzy_ifs[j] = "" 148 conditions[i] = "" 149 break 150 if ( 151 conditions[i] 152 and self.implicit_vars 153 and not (IMPLICIT[cond] and not negated) 154 and not (not IMPLICIT[cond] and negated) 155 ): # opposite implicit different 156 delta += 1 157 conditions[i] = "" 158 for i in range(n): 159 if conditions[i]: # unhandled 160 delta += 1 # OTHER adds a dimension 161 for j in range(m): 162 if fuzzy_ifs[j]: # unhandled 163 if fuzzy_ifs[j] in OSES: 164 return False # condition doesn't specify OS 165 if fuzzy_ifs[j] in BUILD_TYPES: 166 continue # does not add a dimension b/c condition doesn't specify 167 if fuzzy_ifs[j][0] == "!": 168 cond = fuzzy_ifs[j][1:] 169 else: 170 cond = fuzzy_ifs[j] 171 if not cond in dimensions: 172 delta += 1 # OTHER adds a dimension 173 return delta <= 1 174 175 def merge(self, condition, fuzzy_if_condition): 176 """ 177 A. if 2 of the 5 build-types are present -- eliminate ALL build types 178 (i.e. the condition will apply to all build types) 179 180 B. If both the implicit and explicit (non) default value are present, add 181 an OR like this (swgl || !swgl) -- that way the condition will match 182 any value of swgl. For implicit variables see: 183 https://searchfox.org/mozilla-central/source/layout/tools/reftest/manifest.sys.mjs#30 184 fission: true, 185 is64Bit: true, 186 useDrawSnapshot: false, 187 swgl: false, 188 189 C. for other vars if we have A and !A then remove A from the condition 190 """ 191 192 os = "" 193 build_type = "" 194 conditions = condition.split("&&") 195 n = len(conditions) 196 fuzzy_ifs = fuzzy_if_condition.split("&&") 197 m = len(fuzzy_ifs) 198 conds = {} 199 for i in range(n): 200 if conditions[i].find("||") > 0: 201 disjunctions = conditions[i][1:-1].split("||") 202 cond = disjunctions[0] 203 if cond in OSES: 204 for j in range(m): 205 if fuzzy_ifs[j] in OSES: 206 if fuzzy_ifs[j] not in disjunctions: 207 disjunctions.append(fuzzy_ifs[j]) 208 fuzzy_ifs[j] = "" 209 disjunctions = sorted(disjunctions) 210 os = "(" + "||".join(disjunctions) + ")" 211 conditions[i] = "" 212 elif self.implicit_vars and cond in IMPLICIT: 213 for j in range(m): 214 if not fuzzy_ifs[j]: 215 continue 216 if ( 217 fuzzy_ifs[j] == cond 218 or fuzzy_ifs[j] == "!" + cond 219 or fuzzy_ifs[j] == "(" + cond + "||!" + cond + ")" 220 ): 221 fuzzy_ifs[j] = "" 222 break 223 conds[cond] = conditions[i] 224 conditions[i] = "" 225 elif conditions[i] in OSES: 226 os = conditions[i] 227 conditions[i] = "" 228 for j in range(m): 229 if fuzzy_ifs[j] in OSES: 230 if os < fuzzy_ifs[j]: # add in alpha order 231 os = "(" + os + "||" + fuzzy_ifs[j] + ")" 232 elif os > fuzzy_ifs[j]: 233 os = "(" + fuzzy_ifs[j] + "||" + os + ")" 234 fuzzy_ifs[j] = "" 235 break # expect only one os variable 236 elif conditions[i] in BUILD_TYPES: 237 build_type = conditions[i] 238 for j in range(m): 239 if fuzzy_ifs[j] in BUILD_TYPES: 240 if fuzzy_ifs[j] != build_type: # different 241 build_type = "" 242 fuzzy_ifs[j] = "" 243 conditions[i] = "" 244 break # expect at most one build_type 245 if conditions[i]: # fuzzy_if had build_type _removed_ 246 build_type = "" 247 conditions[i] = "" 248 else: 249 negated = False 250 if conditions[i][0] == "!": 251 negated = True 252 cond = conditions[i][1:] 253 else: 254 cond = conditions[i] 255 if negated: 256 opposite = cond 257 else: 258 opposite = "!" + cond 259 disjunction = "" 260 for j in range(m): 261 if not fuzzy_ifs[j]: 262 continue 263 if conditions[i] == fuzzy_ifs[j]: # same 264 conds[cond] = conditions[i] 265 fuzzy_ifs[j] = "" 266 conditions[i] = "" 267 break 268 if ( 269 self.implicit_vars 270 and cond in IMPLICIT 271 and ( 272 opposite == fuzzy_ifs[j] 273 or fuzzy_ifs[j] == "(" + cond + "||!" + cond + ")" 274 ) 275 ): 276 if negated: 277 disjunction = "(" + opposite + "||" + conditions[i] + ")" 278 else: 279 disjunction = "(" + conditions[i] + "||" + opposite + ")" 280 conds[cond] = disjunction 281 fuzzy_ifs[j] = "" 282 conditions[i] = "" 283 break 284 if opposite == fuzzy_ifs[j]: # remove 285 fuzzy_ifs[j] = "" 286 conditions[i] = "" 287 break 288 if ( 289 self.implicit_vars 290 and conditions[i] 291 and not (IMPLICIT[cond] and not negated) 292 and not (not IMPLICIT[cond] and negated) 293 ): # opposite implicit 294 if negated: 295 disjunction = "(" + opposite + "||" + conditions[i] + ")" 296 else: 297 disjunction = "(" + conditions[i] + "||" + opposite + ")" 298 conds[cond] = disjunction 299 conditions[i] = "" 300 if not self.implicit_vars and conditions[i]: 301 conditions[i] = "" # remove, unspecified in fuzzy_if 302 for i in range(n): 303 if conditions[i]: # unhandled 304 negated = False 305 if conditions[i][0] == "!": 306 negated = True 307 cond = conditions[i][1:] 308 else: 309 cond = conditions[i] 310 if (not (self.implicit_vars and cond in IMPLICIT)) or ( # not implicit 311 (IMPLICIT[cond] and negated) 312 or (not IMPLICIT[cond] and not negated) # or explicit 313 ): 314 conds[cond] = conditions[i] 315 for j in range(m): 316 if fuzzy_ifs[j]: # unhandled 317 if fuzzy_ifs[j] in OSES: 318 os = fuzzy_ifs[j] 319 continue 320 if fuzzy_ifs[j] in BUILD_TYPES and fuzzy_ifs[j] != build_type: 321 build_type = "" 322 continue 323 negated = False 324 if fuzzy_ifs[j][0] == "!": 325 negated = True 326 cond = fuzzy_ifs[j][1:] 327 else: 328 cond = fuzzy_ifs[j] 329 if not (self.implicit_vars and cond in IMPLICIT): # not implicit 330 pass # not present in condition 331 elif (IMPLICIT[cond] and negated) or ( 332 not IMPLICIT[cond] and not negated 333 ): # or opposite of implicit 334 disjunction = "" 335 if negated: 336 opposite = cond 337 else: 338 opposite = "!" + cond 339 if negated: 340 disjunction = "(" + opposite + "||" + fuzzy_ifs[j] + ")" 341 else: 342 disjunction = "(" + fuzzy_ifs[j] + "||" + opposite + ")" 343 conds[cond] = disjunction 344 if os: 345 merged = os 346 else: 347 merged = "" 348 if build_type: 349 if merged: 350 merged += "&&" 351 merged += build_type 352 conds_keys = sorted(list(conds.keys())) 353 for cond in conds_keys: 354 if os != "winWidget" and conds[cond] == "is64Bit": 355 continue # special case: elide is64Bit except on Windows 356 if os != "gtkWidget" and cond == "useDrawSnapshot": 357 continue # special case: elide useDrawSnapshot except on Linux 358 if merged: 359 merged += "&&" 360 merged += conds[cond] 361 return merged 362 363 def get_os_in_condition(self, condition): 364 """Return reftest os variable for condition (or the empty string)""" 365 366 os = "" 367 conditions = condition.split("&&") 368 n = len(conditions) 369 for i in range(n): 370 if conditions[i].find("||") > 0: 371 disjunctions = conditions[i][1:-1].split("||") 372 if disjunctions[0] in OSES: 373 os = disjunctions[0] # returns ONLY first OS if disjunction 374 break 375 if conditions[i] in OSES: 376 os = conditions[i] 377 break 378 return os 379 380 def get_dimensions(self, condition): 381 """Return number of dimensions in condition""" 382 383 dimensions = [] 384 conditions = condition.split("&&") 385 n = len(conditions) 386 for i in range(n): 387 if conditions[i].find("||") > 0: 388 disjunctions = conditions[i][1:-1].split("||") 389 if disjunctions[0] in OSES: 390 if "os" not in dimensions: 391 dimensions.append("os") 392 elif disjunctions[0] not in dimensions: 393 dimensions.append(disjunctions[0]) 394 if conditions[i] in OSES: 395 if "os" not in dimensions: 396 dimensions.append("os") 397 elif conditions[i] in BUILD_TYPES: 398 if "build_type" not in dimensions: 399 dimensions.append("build_type") 400 else: 401 if conditions[i][0] == "!": 402 cond = conditions[i][1:] 403 else: 404 cond = conditions[i] 405 if cond not in dimensions: 406 dimensions.append(cond) 407 if self.implicit_vars: 408 for cond in IMPLICIT: 409 if cond not in dimensions: 410 dimensions.append(cond) 411 return len(dimensions) 412 413 def calc_fuzzy_if( 414 self, modifiers, j, fuzzy_if_condition, d_min, d_max, p_min, p_max 415 ): 416 """ 417 Will analzye modifiers in range(j) and 418 - move non fuzzy-if's to the left 419 - sort fuzzy-ifs by OS and by dimension 420 - merge with an exising fuzzy-if ONLY if differs by one dimension (or less) 421 - else add fuzzy-if in dimension order 422 Returns additional_comment (if added second or subsequent for this OS) 423 """ 424 425 def fuzzy_if_keyfn(fuzzy_if): 426 os = "" 427 dimensions = 0 428 m = self.fuzzy_if_rx.findall(fuzzy_if) 429 if len(m) == 1: # NOT fuzzy-if 430 condition = m[0][0] 431 os = self.get_os_in_condition(condition) 432 dimensions = self.get_dimensions(condition) 433 try: 434 os_i = OSES.index(os) 435 except ValueError: 436 os_i = -1 437 return [os_i, dimensions] 438 439 success = True 440 additional_comment = "" 441 merged = None # index in modifiers of the last merged fuzzy_if 442 os = self.get_os_in_condition(fuzzy_if_condition) 443 fuzzy_if = f"fuzzy-if({fuzzy_if_condition},{d_min}-{d_max},{p_min}-{p_max})" 444 first = j # position of first fuzzy-if 445 if self.fuzzy_if_rx is None: 446 self.fuzzy_if_rx = re.compile(FUZZY_IF_REGEX) 447 i = 0 448 while i < j: 449 m = self.fuzzy_if_rx.findall(modifiers[i]) 450 if len(m) != 1: # NOT fuzzy-if 451 if i > first: # move before fuzzy-if's 452 modifier = modifiers[i] 453 del modifiers[i] 454 modifiers.insert(first, modifier) 455 first += 1 456 else: # fuzzy-if 457 first = min(i, first) 458 condition = m[0][0] 459 dmin = int(m[0][1]) 460 dmax = int(m[0][2]) 461 pmin = int(m[0][3]) 462 pmax = int(m[0][4]) 463 this_os = self.get_os_in_condition(condition) 464 if this_os == os and ( 465 condition == fuzzy_if_condition 466 or self.should_merge(condition, fuzzy_if_condition) 467 ): 468 self.vinfo(f"CONDITION {i:2d} NOW {modifiers[i]}") 469 self.vinfo(f"PROPOSED {fuzzy_if_condition}") 470 fuzzy_if_condition = self.merge(condition, fuzzy_if_condition) 471 d_min = min(dmin, d_min) # dmin, if zero, is kept 472 d_max = max(dmax, d_max) 473 p_min = min(pmin, p_min) # pmin, if zero, is kept 474 p_max = max(pmax, p_max) 475 fuzzy_if = f"fuzzy-if({fuzzy_if_condition},{d_min}-{d_max},{p_min}-{p_max})" 476 if (d_min == 0 and d_max == 0) or (p_min == 0 and p_max == 0): 477 additional_comment = f"fuzzy-if removed as calculated range is {d_min}-{d_max},{p_min}-{p_max}" 478 self.vinfo(f"ABANDONED MERGE {fuzzy_if}") 479 del modifiers[i] 480 i -= 1 481 j -= 1 482 continue 483 if merged is not None: # delete previous 484 self.vinfo(f" Deleting previous: {merged}") 485 del modifiers[merged] 486 i -= 1 487 j -= 1 488 modifiers[i] = fuzzy_if 489 merged = i 490 self.vinfo(f"UPDATED MERGED {fuzzy_if}") 491 i += 1 492 if ( 493 success 494 and merged is None 495 and ((d_min == 0 and d_max == 0) or (p_min == 0 and p_max == 0)) 496 ): 497 if not additional_comment: # this is NOT the result of merging to 0-0 498 self.vinfo(f"ABANDONED ADD {fuzzy_if}") 499 additional_comment = f"fuzzy-if not added as calculated range is {d_min}-{d_max},{p_min}-{p_max}" 500 success = False 501 else: 502 merged = i # avoid adding below 503 if success: 504 if merged is None: 505 self.vinfo(f"UPDATED ADDED {fuzzy_if}") 506 modifiers.insert(j, fuzzy_if) 507 j += 1 508 fuzzy_ifs = modifiers[first:j] 509 if len(fuzzy_ifs) > 0: 510 fuzzy_ifs = sorted(fuzzy_ifs, key=fuzzy_if_keyfn) 511 a = j # first fuzzy_if for os 512 b = j # last fuzzy_if for os 513 for i in range(len(fuzzy_ifs)): 514 modifiers[first + i] = fuzzy_ifs[i] 515 if fuzzy_ifs[i].startswith("fuzzy-if(" + os): 516 if a == j: 517 a = first + i 518 b = first + i 519 if b > a: 520 additional_comment = f"NOTE: more than one fuzzy-if for the OS = {os} ==> may require manual review" 521 return success, additional_comment 522 523 def reftest_add_fuzzy_if( 524 self, 525 manifest_str, 526 filename, 527 fuzzy_if, 528 differences, 529 pixels, 530 lineno, 531 zero, 532 bug_reference, 533 ): 534 """ 535 Edits a reftest manifest string to add disabled condition 536 Returns additional_comment (if any) 537 """ 538 539 result = ("", "") 540 additional_comment = "" 541 words = filename.split() 542 if len(words) < 3: 543 self.error( 544 f"Expected filename in the form '[optional conditions] == url url_ref': {filename}" 545 ) 546 return result 547 test_type = words[-3] 548 url = os.path.basename(words[-2]) 549 url_ref = os.path.basename(words[-1]) 550 lines = manifest_str.splitlines() 551 if lineno == 0 or lineno > len(lines): 552 self.error("cannot determine line to edit in manifest") 553 return result 554 line = lines[lineno - 1] 555 comment = "" 556 comment_start = line.find(" #") # MUST NOT match anchors! 557 if comment_start > 0: 558 comment = line[comment_start + 1 :] 559 line = line[0:comment_start].strip() 560 words = line.split() 561 n = len(words) 562 if n < 3: 563 self.error(f"line {lineno} does not match: {line}") 564 return result 565 if os.path.basename(words[n - 1]) != url_ref: 566 self.error(f"words[n-1] not url_ref: {words[n - 1]} != {url_ref}") 567 return result 568 if os.path.basename(words[n - 2]) != url: 569 self.error(f"words[n-2] not url: {words[n - 2]} != {url}") 570 return result 571 if words[n - 3] != test_type: 572 self.error(f"words[n-3] not '{test_type}': {words[n - 3]}") 573 return result 574 d_min = 0 575 d_max = 0 576 if len(differences) > 0: 577 d_min = min(differences) 578 d_max = max(differences) 579 if d_min == 0 and d_max > 0: # recalc minimum 580 i = 0 581 n = len(differences) 582 while i < n: 583 if differences[i] == 0: 584 del differences[i] 585 n -= 1 586 else: 587 i += 1 588 if n > 0: 589 d_min = min(differences) 590 p_min = 0 591 p_max = 0 592 if len(pixels) > 0: 593 p_min = min(pixels) 594 p_max = max(pixels) 595 if p_min == 0 and p_max > 0: # recalc minimum 596 i = 0 597 n = len(pixels) 598 while i < n: 599 if pixels[i] == 0: 600 del pixels[i] 601 n -= 1 602 else: 603 i += 1 604 if n > 0: 605 p_min = min(pixels) 606 if zero: 607 d_min = 0 608 p_min = 0 609 d_max2 = int((1.0 + MARGIN) * d_max) 610 if d_max2 > d_max: 611 self.info( 612 f"Increased max difference from {d_max} by {int(MARGIN * 100)}% to {d_max2}" 613 ) 614 d_max = d_max2 615 p_max2 = int((1.0 + MARGIN) * p_max) 616 if p_max2 > p_max: 617 self.info( 618 f"Increased differing pixels from {p_max} by {int(MARGIN * 100)}% to {p_max2}" 619 ) 620 p_max = p_max2 621 if comment: 622 bug = bug_reference.split() 623 if comment.find(bug[1]) < 0: # look for bug number only 624 comment += ", " + bug_reference 625 else: 626 comment = "# " + bug_reference 627 j = 0 628 for i in range(n): 629 if words[i].startswith("HTTP") or words[i] == test_type: 630 j = i 631 break 632 success, additional_comment = self.calc_fuzzy_if( 633 words, j, fuzzy_if, d_min, d_max, p_min, p_max 634 ) 635 if success: 636 words.append(comment) 637 lines[lineno - 1] = " ".join(words) 638 manifest_str = "\n".join(lines) 639 if manifest_str[-1] != "\n": 640 manifest_str += "\n" 641 else: 642 manifest_str = "" 643 result = (manifest_str, additional_comment) 644 return result 645 646 647 if __name__ == "__main__": 648 sys.exit(ListManifestParser().run())