test_failedplatform.py (17985B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 from failedplatform import FailedPlatform 6 from mozunit import main 7 8 9 def test_get_possible_build_types(): 10 """Test get_possible_build_types""" 11 12 fp = FailedPlatform({}) 13 assert fp.get_possible_build_types() == [] 14 15 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 16 assert fp.get_possible_build_types() == ["build_type1"] 17 18 fp = FailedPlatform({ 19 "build_type1": {"test_variant1": {}}, 20 "build_type2": {"test_variant1": {}}, 21 }) 22 assert fp.get_possible_build_types() == ["build_type1", "build_type2"] 23 24 25 def test_get_possible_test_variants(): 26 """Test get_possible_test_variants""" 27 28 fp = FailedPlatform({}) 29 assert fp.get_possible_test_variants("") == [] 30 31 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 32 assert fp.get_possible_test_variants("unknown") == [] 33 34 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 35 assert fp.get_possible_test_variants("build_type1") == ["test_variant1"] 36 37 fp = FailedPlatform({ 38 "build_type1": {"test_variant1": {}}, 39 "build_type2": {"test_variant2": {}}, 40 }) 41 assert fp.get_possible_test_variants("build_type2") == ["test_variant2"] 42 43 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) 44 assert fp.get_possible_test_variants("build_type1") == [ 45 "test_variant1", 46 "test_variant2", 47 ] 48 49 50 def test_is_full_test_variants_fail(): 51 """Test is_full_test_variants_fail""" 52 53 fp = FailedPlatform({}) 54 assert not fp.is_full_test_variants_fail("") 55 56 fp = FailedPlatform({"build_type1": {}}) 57 assert not fp.is_full_test_variants_fail("build_type1") 58 59 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 60 assert not fp.is_full_test_variants_fail("build_type1") 61 62 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 63 fp.failures["build_type1"] = {"test_variant2": 1} 64 assert not fp.is_full_test_variants_fail("build_type1") 65 66 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 67 fp.failures["build_type1"] = {"test_variant1": 1} 68 assert fp.is_full_test_variants_fail("build_type1") 69 70 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) 71 fp.failures["build_type1"] = {"test_variant1": 1} 72 assert not fp.is_full_test_variants_fail("build_type1") 73 74 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) 75 fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} 76 assert fp.is_full_test_variants_fail("build_type1") 77 78 79 def test_is_full_fail(): 80 """Test is_full_fail""" 81 82 fp = FailedPlatform({}) 83 assert not fp.is_full_fail() 84 85 fp = FailedPlatform({"build_type1": {}}) 86 assert not fp.is_full_fail() 87 88 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 89 assert not fp.is_full_fail() 90 91 fp = FailedPlatform({"build_type1": {}}) 92 fp.failures["build_type1"] = {} 93 assert not fp.is_full_fail() 94 95 fp = FailedPlatform({"build_type1": {}}) 96 fp.failures["build_type2"] = {} 97 assert not fp.is_full_fail() 98 99 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 100 fp.failures["build_type1"] = {} 101 assert not fp.is_full_fail() 102 103 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 104 fp.failures["build_type1"] = {"test_variant1": 1} 105 assert fp.is_full_fail() 106 107 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) 108 fp.failures["build_type1"] = {"test_variant1": 1} 109 assert not fp.is_full_fail() 110 111 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) 112 fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} 113 assert fp.is_full_fail() 114 115 fp = FailedPlatform({ 116 "build_type1": {"test_variant1": {}, "test_variant2": {}}, 117 "build_type2": {"test_variant1": {}, "test_variant2": {}}, 118 }) 119 fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} 120 assert not fp.is_full_fail() 121 122 fp = FailedPlatform({ 123 "build_type1": {"test_variant1": {}, "test_variant2": {}}, 124 "build_type2": {"test_variant1": {}, "test_variant2": {}}, 125 }) 126 fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} 127 fp.failures["build_type2"] = {"test_variant1": 1, "test_variant2": 1} 128 assert fp.is_full_fail() 129 130 131 def test_get_no_variant_conditions(): 132 """Test get_no_variant_conditions""" 133 134 fp = FailedPlatform({}) 135 assert fp.get_no_variant_conditions(" && ", "build_type1") == "" 136 137 fp = FailedPlatform({"build_type1": {}}) 138 assert fp.get_no_variant_conditions(" && ", "build_type1") == "" 139 140 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 141 assert fp.get_no_variant_conditions(" && ", "build_type1") == " && !test_variant1" 142 143 # Handle already negated test variants 144 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "!fission": {}}}) 145 assert ( 146 fp.get_no_variant_conditions(" && ", "build_type1") 147 == " && !test_variant1 && fission" 148 ) 149 150 # Handle composite variants 151 fp = FailedPlatform({ 152 "build_type1": { 153 "test_variant1": {}, 154 "socketprocess_networking+!fission": {}, 155 "no_variant": {}, 156 } 157 }) 158 assert ( 159 fp.get_no_variant_conditions(" && ", "build_type1") 160 == " && !test_variant1 && !socketprocess_networking && fission" 161 ) 162 163 # Handle mutually exclusive variants 164 fp = FailedPlatform({ 165 "build_type1": { 166 "fission": {}, 167 "socketprocess_networking+!fission": {}, 168 "no_variant": {}, 169 } 170 }) 171 assert ( 172 fp.get_no_variant_conditions(" && ", "build_type1") 173 == " && !socketprocess_networking" 174 ) 175 176 177 def test_get_test_variant_condition(): 178 """Test get_no_variant_conditions""" 179 180 # Simply returns the given variant if no composite variant exists 181 fp = FailedPlatform({}) 182 assert ( 183 fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") 184 == " && test_variant1" 185 ) 186 187 fp = FailedPlatform({"build_type1": {}}) 188 assert ( 189 fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") 190 == " && test_variant1" 191 ) 192 193 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 194 assert ( 195 fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") 196 == " && test_variant1" 197 ) 198 199 # Returns the negation of the composite if any 200 fp = FailedPlatform({ 201 "build_type1": {"test_variant1": {}, "test_variant1+test_variant2": {}} 202 }) 203 assert ( 204 fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") 205 == " && test_variant1 && !test_variant2" 206 ) 207 208 # Ignores composite variants it is not part of 209 fp = FailedPlatform({ 210 "build_type1": {"test_variant1": {}, "test_variant2+test_variant3": {}} 211 }) 212 assert ( 213 fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") 214 == " && test_variant1" 215 ) 216 217 # Convert a composite test variant 218 fp = FailedPlatform({ 219 "build_type1": {"test_variant1": {}, "test_variant1+test_variant2": {}} 220 }) 221 assert ( 222 fp.get_test_variant_condition( 223 " && ", "build_type1", "test_variant1+test_variant2" 224 ) 225 == " && test_variant1 && test_variant2" 226 ) 227 228 # Handles composite variants included in other composites 229 fp = FailedPlatform({ 230 "build_type1": { 231 "test_variant1+test_variant2": {}, 232 "test_variant1+test_variant2+test_variant3": {}, 233 } 234 }) 235 assert ( 236 fp.get_test_variant_condition( 237 " && ", "build_type1", "test_variant1+test_variant2" 238 ) 239 == " && test_variant1 && test_variant2 && !test_variant3" 240 ) 241 242 # Detects if the composite or in scrambled order 243 fp = FailedPlatform({ 244 "build_type1": { 245 "test_variant2+test_variant1": {}, 246 "test_variant1+test_variant3+test_variant2": {}, 247 } 248 }) 249 assert ( 250 fp.get_test_variant_condition( 251 " && ", "build_type1", "test_variant2+test_variant1" 252 ) 253 == " && test_variant2 && test_variant1 && !test_variant3" 254 ) 255 256 # Handles when variant is included in several composites 257 fp = FailedPlatform({ 258 "build_type1": { 259 "test_variant1+test_variant2": {}, 260 "test_variant1+test_variant2+test_variant3": {}, 261 "test_variant1+test_variant4+test_variant2": {}, 262 } 263 }) 264 assert ( 265 fp.get_test_variant_condition( 266 " && ", "build_type1", "test_variant2+test_variant1" 267 ) 268 == " && test_variant2 && test_variant1 && !test_variant3 && !test_variant4" 269 ) 270 271 # Simply returns the current composite variant if it is not fully contained in another 272 fp = FailedPlatform({ 273 "build_type1": { 274 "test_variant1+test_variant2": {}, 275 "test_variant1+test_variant3": {}, 276 } 277 }) 278 assert ( 279 fp.get_test_variant_condition( 280 " && ", "build_type1", "test_variant1+test_variant2" 281 ) 282 == " && test_variant1 && test_variant2" 283 ) 284 285 # Ignore matching composite variants in other build types 286 fp = FailedPlatform({ 287 "build_type1": {"test_variant1": {}}, 288 "build_type2": {"test_variant1+test_variant2": {}}, 289 }) 290 assert ( 291 fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") 292 == " && test_variant1" 293 ) 294 295 296 def test_is_full_high_freq_fail(): 297 """Test is_full_high_freq_fail""" 298 299 # One build type 300 fp = FailedPlatform({"build_type1": {"no_variant": {}}}) 301 assert not fp.is_full_high_freq_fail() 302 303 for i in range(0, 6): 304 fp.get_skip_string(" && ", "build_type1", "no_variant") 305 assert not fp.is_full_high_freq_fail() 306 307 fp.get_skip_string(" && ", "build_type1", "no_variant") 308 assert fp.is_full_high_freq_fail() 309 310 # Several build types 311 fp = FailedPlatform({ 312 "build_type1": {"no_variant": {}}, 313 "build_type2": {"no_variant": {}}, 314 }) 315 assert not fp.is_full_high_freq_fail() 316 317 for i in range(0, 7): 318 fp.get_skip_string(" && ", "build_type1", "no_variant") 319 assert not fp.is_full_high_freq_fail() 320 321 for i in range(0, 6): 322 fp.get_skip_string(" && ", "build_type2", "no_variant") 323 assert not fp.is_full_high_freq_fail() 324 325 fp.get_skip_string(" && ", "build_type2", "no_variant") 326 assert fp.is_full_high_freq_fail() 327 328 # Several build types and test variants 329 fp = FailedPlatform({ 330 "build_type1": {"no_variant": {}, "test_variant1": {}}, 331 "build_type2": {"no_variant": {}, "test_variant1": {}}, 332 }) 333 assert not fp.is_full_high_freq_fail() 334 335 for i in range(0, 7): 336 fp.get_skip_string( 337 " && ", "build_type1", "no_variant" if i < 3 else "test_variant1" 338 ) 339 assert not fp.is_full_high_freq_fail() 340 341 for i in range(0, 6): 342 fp.get_skip_string( 343 " && ", "build_type2", "no_variant" if i < 3 else "test_variant1" 344 ) 345 assert not fp.is_full_high_freq_fail() 346 347 fp.get_skip_string(" && ", "build_type2", "no_variant") 348 assert fp.is_full_high_freq_fail() 349 350 351 def test_get_skip_string(): 352 """Test get_skip_string""" 353 354 # Full fails on single line => nothing returned 355 fp = FailedPlatform({"build_type1": {"no_variant": {}}}) 356 assert fp.get_skip_string(" && ", "build_type1", "no_variant") == "" 357 358 fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) 359 assert fp.get_skip_string(" && ", "build_type1", "test_variant1") == "" 360 361 fp = FailedPlatform({"build_type1": {"test_variant1+test_variant2": {}}}) 362 assert ( 363 fp.get_skip_string(" && ", "build_type1", "test_variant1+test_variant2") == "" 364 ) 365 366 # Fail only on some build types without test variant 367 # => only failed build types returned 368 fp = FailedPlatform({ 369 "build_type1": {"no_variant": {}}, 370 "build_type2": {"no_variant": {}}, 371 }) 372 assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" 373 374 # Fail only on one build type with test variant 375 # => only failed build types returned with negated test variants 376 fp = FailedPlatform({ 377 "build_type1": {"no_variant": {}, "test_variant1": {}}, 378 "build_type2": {"no_variant": {}}, 379 }) 380 assert ( 381 fp.get_skip_string(" && ", "build_type1", "no_variant") 382 == " && build_type1 && !test_variant1" 383 ) 384 385 # Full test variant fail on single line => only build type returned 386 fp = FailedPlatform({ 387 "build_type1": {"test_variant1": {}}, 388 "build_type2": {"no_variant": {}}, 389 }) 390 assert ( 391 fp.get_skip_string(" && ", "build_type1", "test_variant1") == " && build_type1" 392 ) 393 394 fp = FailedPlatform({ 395 "build_type1": {"test_variant1+test_variant2": {}}, 396 "build_type2": {"no_variant": {}}, 397 }) 398 assert ( 399 fp.get_skip_string(" && ", "build_type1", "test_variant1+test_variant2") 400 == " && build_type1" 401 ) 402 403 # Fail only on some test variants => build type and test variant returned 404 fp = FailedPlatform({ 405 "build_type1": {"test_variant1": {}, "test_variant2": {}}, 406 "build_type2": {"no_variant": {}}, 407 }) 408 assert ( 409 fp.get_skip_string(" && ", "build_type1", "test_variant1") 410 == " && build_type1 && test_variant1" 411 ) 412 413 # Full fail on second call 414 fp = FailedPlatform({ 415 "build_type1": {"no_variant": {}}, 416 "build_type2": {"no_variant": {}}, 417 }) 418 assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" 419 assert fp.get_skip_string(" && ", "build_type2", "no_variant") == "" 420 421 # Full fail on second call with test variants 422 fp = FailedPlatform({ 423 "build_type1": {"test_variant1+test_variant2": {}}, 424 "build_type2": {"test_variant1+test_variant2": {}}, 425 }) 426 assert ( 427 fp.get_skip_string(" && ", "build_type1", "test_variant1+test_variant2") 428 == " && build_type1" 429 ) 430 assert ( 431 fp.get_skip_string(" && ", "build_type2", "test_variant1+test_variant2") == "" 432 ) 433 434 fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) 435 assert ( 436 fp.get_skip_string(" && ", "build_type1", "test_variant1") 437 == " && build_type1 && test_variant1" 438 ) 439 assert fp.get_skip_string(" && ", "build_type1", "test_variant2") == "" 440 441 # Fail on variant and no_variant 442 fp = FailedPlatform({ 443 "build_type1": {"test_variant1": {}, "no_variant": {}}, 444 "build_type2": {"no_variant": {}}, 445 }) 446 assert ( 447 fp.get_skip_string(" && ", "build_type1", "test_variant1") 448 == " && build_type1 && test_variant1" 449 ) 450 assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" 451 452 # Complex cases 453 fp = FailedPlatform({ 454 "build_type1": { 455 "test_variant1": {}, 456 "test_variant2": {}, 457 "test_variant1+test_variant2": {}, 458 }, 459 "build_type2": {"no_variant": {}, "test_variant1": {}, "test_variant2": {}}, 460 }) 461 assert ( 462 fp.get_skip_string(" && ", "build_type1", "test_variant1") 463 == " && build_type1 && test_variant1 && !test_variant2" 464 ) 465 assert ( 466 fp.get_skip_string(" && ", "build_type2", "no_variant") 467 == " && build_type2 && !test_variant1 && !test_variant2" 468 ) 469 assert ( 470 fp.get_skip_string(" && ", "build_type1", "test_variant2") 471 == " && build_type1 && test_variant2 && !test_variant1" 472 ) 473 assert ( 474 fp.get_skip_string(" && ", "build_type1", "test_variant1+test_variant2") 475 == " && build_type1" 476 ) 477 assert ( 478 fp.get_skip_string(" && ", "build_type2", "test_variant1") 479 == " && build_type2 && test_variant1" 480 ) 481 assert fp.get_skip_string(" && ", "build_type2", "test_variant2") == "" 482 483 484 def test_get_skip_string_high_freq(): 485 """Test get_skip_string using high freq flag""" 486 487 # Only return skip string if at least 7 failures 488 fp = FailedPlatform( 489 { 490 "build_type1": {}, 491 "build_type2": {}, 492 }, 493 high_freq=True, 494 ) 495 for i in range(0, 6): 496 assert fp.get_skip_string(" && ", "build_type1", "no_variant") is None 497 assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" 498 499 # Skip whole platform if all build types failed 500 fp = FailedPlatform( 501 { 502 "build_type1": {}, 503 "build_type2": {}, 504 }, 505 high_freq=True, 506 ) 507 for i in range(0, 6): 508 assert fp.get_skip_string(" && ", "build_type1", "no_variant") is None 509 assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" 510 for i in range(0, 6): 511 assert fp.get_skip_string(" && ", "build_type2", "no_variant") is None 512 assert fp.get_skip_string(" && ", "build_type2", "no_variant") == "" 513 514 # Skip specific test_variant if it has more than 75% failures 515 fp = FailedPlatform( 516 { 517 "build_type1": {"test_variant1": {}, "test_variant2": {}}, 518 "build_type2": {}, 519 }, 520 high_freq=True, 521 ) 522 for i in range(0, 6): 523 assert fp.get_skip_string(" && ", "build_type1", "test_variant1") is None 524 assert ( 525 fp.get_skip_string(" && ", "build_type1", "test_variant1") 526 == " && build_type1 && test_variant1" 527 ) 528 for i in range(0, 2): 529 assert ( 530 fp.get_skip_string(" && ", "build_type1", "test_variant2") 531 == " && build_type1 && test_variant1" 532 ) 533 assert ( 534 fp.get_skip_string(" && ", "build_type1", "test_variant2") == " && build_type1" 535 ) 536 537 538 if __name__ == "__main__": 539 main()