test_optimize_strategies.py (20717B)
1 # Any copyright is dedicated to the public domain. 2 # http://creativecommons.org/publicdomain/zero/1.0/ 3 4 5 import time 6 from datetime import datetime 7 from time import mktime 8 9 import pytest 10 from mozunit import main 11 from taskgraph.optimize.base import registry 12 from taskgraph.task import Task 13 from taskgraph.util.copy import deepcopy 14 15 from gecko_taskgraph.optimize import project 16 from gecko_taskgraph.optimize.backstop import SkipUnlessBackstop, SkipUnlessPushInterval 17 from gecko_taskgraph.optimize.bugbug import ( 18 FALLBACK, 19 BugBugPushSchedules, 20 DisperseGroups, 21 SkipUnlessDebug, 22 ) 23 from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint 24 from gecko_taskgraph.optimize.strategies import SkipUnlessMissing, SkipUnlessSchedules 25 from gecko_taskgraph.util.backstop import BACKSTOP_PUSH_INTERVAL 26 from gecko_taskgraph.util.bugbug import ( 27 BUGBUG_BASE_URL, 28 BugbugTimeoutException, 29 push_schedules, 30 ) 31 32 33 @pytest.fixture(autouse=True) 34 def clear_push_schedules_memoize(): 35 push_schedules.clear() 36 37 38 @pytest.fixture 39 def params(): 40 return { 41 "branch": "autoland", 42 "head_repository": "https://hg.mozilla.org/integration/autoland", 43 "head_rev": "abcdef", 44 "project": "autoland", 45 "pushlog_id": 1, 46 "pushdate": mktime(datetime.now().timetuple()), 47 } 48 49 50 def generate_tasks(*tasks): 51 for i, task in enumerate(tasks): 52 task.setdefault("label", f"task-{i}-label") 53 task.setdefault("kind", "test") 54 task.setdefault("task", {}) 55 task.setdefault("attributes", {}) 56 task["attributes"].setdefault("e10s", True) 57 58 for attr in ( 59 "optimization", 60 "dependencies", 61 "soft_dependencies", 62 ): 63 task.setdefault(attr, None) 64 65 task["task"].setdefault("label", task["label"]) 66 yield Task.from_json(task) 67 68 69 # task sets 70 71 default_tasks = list( 72 generate_tasks( 73 {"attributes": {"test_manifests": ["foo/test.ini", "bar/test.ini"]}}, 74 {"attributes": {"test_manifests": ["bar/test.ini"], "build_type": "debug"}}, 75 {"attributes": {"build_type": "debug"}}, 76 {"attributes": {"test_manifests": [], "build_type": "opt"}}, 77 {"attributes": {"build_type": "opt"}}, 78 ) 79 ) 80 81 82 disperse_tasks = list( 83 generate_tasks( 84 { 85 "attributes": { 86 "test_manifests": ["foo/test.ini", "bar/test.ini"], 87 "test_platform": "linux/opt", 88 } 89 }, 90 { 91 "attributes": { 92 "test_manifests": ["bar/test.ini"], 93 "test_platform": "linux/opt", 94 } 95 }, 96 { 97 "attributes": { 98 "test_manifests": ["bar/test.ini"], 99 "test_platform": "windows/debug", 100 } 101 }, 102 { 103 "attributes": { 104 "test_manifests": ["bar/test.ini"], 105 "test_platform": "linux/opt", 106 "unittest_variant": "no-fission", 107 } 108 }, 109 { 110 "attributes": { 111 "e10s": False, 112 "test_manifests": ["bar/test.ini"], 113 "test_platform": "linux/opt", 114 } 115 }, 116 ) 117 ) 118 119 120 def idfn(param): 121 if isinstance(param, tuple): 122 try: 123 return param[0].__name__ 124 except AttributeError: 125 return None 126 return None 127 128 129 @pytest.mark.parametrize( 130 "opt,tasks,arg,expected", 131 [ 132 # debug 133 pytest.param( 134 SkipUnlessDebug(), 135 default_tasks, 136 None, 137 ["task-0-label", "task-1-label", "task-2-label"], 138 ), 139 # disperse with no supplied importance 140 pytest.param( 141 DisperseGroups(), 142 disperse_tasks, 143 None, 144 [t.label for t in disperse_tasks], 145 ), 146 # disperse with low importance 147 pytest.param( 148 DisperseGroups(), 149 disperse_tasks, 150 {"bar/test.ini": "low"}, 151 ["task-0-label", "task-2-label"], 152 ), 153 # disperse with medium importance 154 pytest.param( 155 DisperseGroups(), 156 disperse_tasks, 157 {"bar/test.ini": "medium"}, 158 ["task-0-label", "task-1-label", "task-2-label"], 159 ), 160 # disperse with high importance 161 pytest.param( 162 DisperseGroups(), 163 disperse_tasks, 164 {"bar/test.ini": "high"}, 165 ["task-0-label", "task-1-label", "task-2-label", "task-3-label"], 166 ), 167 ], 168 ids=idfn, 169 ) 170 def test_optimization_strategy_remove(params, opt, tasks, arg, expected): 171 labels = [t.label for t in tasks if not opt.should_remove_task(t, params, arg)] 172 assert sorted(labels) == sorted(expected) 173 174 175 @pytest.mark.parametrize( 176 "args,data,expected", 177 [ 178 # empty 179 pytest.param( 180 (0.1,), 181 {}, 182 [], 183 ), 184 # only tasks without test manifests selected 185 pytest.param( 186 (0.1,), 187 {"tasks": {"task-1-label": 0.9, "task-2-label": 0.1, "task-3-label": 0.5}}, 188 ["task-2-label"], 189 ), 190 # tasks which are unknown to bugbug are selected 191 pytest.param( 192 (0.1,), 193 { 194 "tasks": {"task-1-label": 0.9, "task-3-label": 0.5}, 195 "known_tasks": ["task-1-label", "task-3-label", "task-4-label"], 196 }, 197 ["task-2-label"], 198 ), 199 # tasks containing groups selected 200 pytest.param( 201 (0.1,), 202 {"groups": {"foo/test.ini": 0.4}}, 203 ["task-0-label"], 204 ), 205 # tasks matching "tasks" or "groups" selected 206 pytest.param( 207 (0.1,), 208 { 209 "tasks": {"task-2-label": 0.2}, 210 "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, 211 }, 212 ["task-0-label", "task-1-label", "task-2-label"], 213 ), 214 # tasks matching "tasks" or "groups" selected, when they exceed the confidence threshold 215 pytest.param( 216 (0.5,), 217 { 218 "tasks": {"task-2-label": 0.2, "task-4-label": 0.5}, 219 "groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25}, 220 }, 221 ["task-0-label", "task-4-label"], 222 ), 223 # tasks matching "reduced_tasks" are selected, when they exceed the confidence threshold 224 pytest.param( 225 (0.7, True, True), 226 { 227 "tasks": {"task-2-label": 0.7, "task-4-label": 0.7}, 228 "reduced_tasks": {"task-4-label": 0.7}, 229 "groups": {"foo/test.ini": 0.75, "bar/test.ini": 0.25}, 230 }, 231 ["task-4-label"], 232 ), 233 # tasks matching "groups" selected, only on specific platforms. 234 pytest.param( 235 (0.1, False, False, None, 1, True), 236 { 237 "tasks": {"task-2-label": 0.2}, 238 "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, 239 "config_groups": { 240 "foo/test.ini": ["task-1-label", "task-0-label"], 241 "bar/test.ini": ["task-0-label"], 242 }, 243 }, 244 ["task-0-label", "task-2-label"], 245 ), 246 pytest.param( 247 (0.1, False, False, None, 1, True), 248 { 249 "tasks": {"task-2-label": 0.2}, 250 "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, 251 "config_groups": { 252 "foo/test.ini": ["task-1-label", "task-0-label"], 253 "bar/test.ini": ["task-1-label"], 254 }, 255 }, 256 ["task-0-label", "task-1-label", "task-2-label"], 257 ), 258 pytest.param( 259 (0.1, False, False, None, 1, True), 260 { 261 "tasks": {"task-2-label": 0.2}, 262 "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, 263 "config_groups": { 264 "foo/test.ini": ["task-1-label"], 265 "bar/test.ini": ["task-0-label"], 266 }, 267 }, 268 ["task-0-label", "task-2-label"], 269 ), 270 pytest.param( 271 (0.1, False, False, None, 1, True), 272 { 273 "tasks": {"task-2-label": 0.2}, 274 "groups": {"foo/test.ini": 0.25, "bar/test.ini": 0.75}, 275 "config_groups": { 276 "foo/test.ini": ["task-1-label"], 277 "bar/test.ini": ["task-3-label"], 278 }, 279 }, 280 ["task-2-label"], 281 ), 282 ], 283 ids=idfn, 284 ) 285 def test_bugbug_push_schedules(responses, params, args, data, expected): 286 query = "/push/{branch}/{head_rev}/schedules".format(**params) 287 url = BUGBUG_BASE_URL + query 288 289 responses.add( 290 responses.GET, 291 url, 292 json=data, 293 status=200, 294 ) 295 296 opt = BugBugPushSchedules(*args) 297 labels = [ 298 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 299 ] 300 assert sorted(labels) == sorted(expected) 301 302 303 def test_bugbug_multiple_pushes(responses, params): 304 pushes = {str(pid): {"changesets": [f"c{pid}"]} for pid in range(8, 10)} 305 306 responses.add( 307 responses.GET, 308 "https://hg.mozilla.org/integration/autoland/json-pushes/?version=2&startID=8&endID=9", 309 json={"pushes": pushes}, 310 status=200, 311 ) 312 313 responses.add( 314 responses.GET, 315 BUGBUG_BASE_URL + "/push/{}/c9/schedules".format(params["branch"]), 316 json={ 317 "tasks": {"task-2-label": 0.2, "task-4-label": 0.5}, 318 "groups": {"foo/test.ini": 0.2, "bar/test.ini": 0.25}, 319 "config_groups": {"foo/test.ini": ["linux-*"], "bar/test.ini": ["task-*"]}, 320 "known_tasks": ["task-4-label"], 321 }, 322 status=200, 323 ) 324 325 # Tasks with a lower confidence don't override task with a higher one. 326 # Tasks with a higher confidence override tasks with a lower one. 327 # Known tasks are merged. 328 responses.add( 329 responses.GET, 330 BUGBUG_BASE_URL + "/push/{branch}/{head_rev}/schedules".format(**params), 331 json={ 332 "tasks": {"task-2-label": 0.2, "task-4-label": 0.2}, 333 "groups": {"foo/test.ini": 0.65, "bar/test.ini": 0.25}, 334 "config_groups": { 335 "foo/test.ini": ["task-*"], 336 "bar/test.ini": ["windows-*"], 337 }, 338 "known_tasks": ["task-1-label", "task-3-label"], 339 }, 340 status=200, 341 ) 342 343 params["pushlog_id"] = 10 344 345 opt = BugBugPushSchedules(0.3, False, False, False, 2) 346 labels = [ 347 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 348 ] 349 assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"]) 350 351 opt = BugBugPushSchedules(0.3, False, False, False, 2, True) 352 labels = [ 353 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 354 ] 355 assert sorted(labels) == sorted(["task-0-label", "task-2-label", "task-4-label"]) 356 357 opt = BugBugPushSchedules(0.2, False, False, False, 2, True) 358 labels = [ 359 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 360 ] 361 assert sorted(labels) == sorted([ 362 "task-0-label", 363 "task-1-label", 364 "task-2-label", 365 "task-4-label", 366 ]) 367 368 369 def test_bugbug_timeout(monkeypatch, responses, params): 370 query = "/push/{branch}/{head_rev}/schedules".format(**params) 371 url = BUGBUG_BASE_URL + query 372 responses.add( 373 responses.GET, 374 url, 375 json={"ready": False}, 376 status=202, 377 ) 378 379 # Make sure the test runs fast. 380 monkeypatch.setattr(time, "sleep", lambda i: None) 381 382 opt = BugBugPushSchedules(0.5) 383 with pytest.raises(BugbugTimeoutException): 384 opt.should_remove_task(default_tasks[0], params, None) 385 386 387 def test_bugbug_fallback(monkeypatch, responses, params): 388 query = "/push/{branch}/{head_rev}/schedules".format(**params) 389 url = BUGBUG_BASE_URL + query 390 responses.add( 391 responses.GET, 392 url, 393 json={"ready": False}, 394 status=202, 395 ) 396 397 opt = BugBugPushSchedules(0.5, fallback=FALLBACK) 398 399 # Make sure the test runs fast. 400 monkeypatch.setattr(time, "sleep", lambda i: None) 401 402 def fake_should_remove_task(task, params, _): 403 return task.label == default_tasks[0].label 404 405 monkeypatch.setattr( 406 registry[FALLBACK], "should_remove_task", fake_should_remove_task 407 ) 408 409 assert opt.should_remove_task(default_tasks[0], params, None) 410 411 # Make sure we don't hit bugbug more than once. 412 responses.reset() 413 414 assert not opt.should_remove_task(default_tasks[1], params, None) 415 416 417 def test_backstop(params): 418 all_labels = {t.label for t in default_tasks} 419 opt = SkipUnlessBackstop() 420 421 params["backstop"] = False 422 scheduled = { 423 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 424 } 425 assert scheduled == set() 426 427 params["backstop"] = True 428 scheduled = { 429 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 430 } 431 assert scheduled == all_labels 432 433 434 def test_push_interval(params): 435 all_labels = {t.label for t in default_tasks} 436 opt = SkipUnlessPushInterval(10) # every 10th push 437 438 # Only multiples of 10 schedule tasks. 439 params["pushlog_id"] = 9 440 scheduled = { 441 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 442 } 443 assert scheduled == set() 444 445 params["pushlog_id"] = 10 446 scheduled = { 447 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 448 } 449 assert scheduled == all_labels 450 451 452 def test_expanded(params): 453 all_labels = {t.label for t in default_tasks} 454 opt = registry["skip-unless-expanded"] 455 456 params["backstop"] = False 457 params["pushlog_id"] = BACKSTOP_PUSH_INTERVAL / 2 458 scheduled = { 459 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 460 } 461 assert scheduled == all_labels 462 463 params["pushlog_id"] += 1 464 scheduled = { 465 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 466 } 467 assert scheduled == set() 468 469 params["backstop"] = True 470 scheduled = { 471 t.label for t in default_tasks if not opt.should_remove_task(t, params, None) 472 } 473 assert scheduled == all_labels 474 475 476 def test_project_autoland_test(monkeypatch, responses, params): 477 """Tests the behaviour of the `project.autoland["test"]` strategy on 478 various types of pushes. 479 """ 480 # This is meant to test the composition of substrategies, and not the 481 # actual optimization implementations. So mock them out for simplicity. 482 monkeypatch.setattr(SkipUnlessSchedules, "should_remove_task", lambda *args: False) 483 monkeypatch.setattr(DisperseGroups, "should_remove_task", lambda *args: False) 484 485 def fake_bugbug_should_remove_task(self, task, params, importance): 486 if self.num_pushes > 1: 487 return task.label == "task-4-label" 488 return task.label in ("task-2-label", "task-3-label", "task-4-label") 489 490 monkeypatch.setattr( 491 BugBugPushSchedules, "should_remove_task", fake_bugbug_should_remove_task 492 ) 493 494 opt = project.autoland["test"] 495 496 # On backstop pushes, nothing gets optimized. 497 params["backstop"] = True 498 scheduled = { 499 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 500 } 501 assert scheduled == {t.label for t in default_tasks} 502 503 # On expanded pushes, some things are optimized. 504 params["backstop"] = False 505 params["pushlog_id"] = 10 506 scheduled = { 507 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 508 } 509 assert scheduled == {"task-0-label", "task-1-label", "task-2-label", "task-3-label"} 510 511 # On regular pushes, more things are optimized. 512 params["pushlog_id"] = 11 513 scheduled = { 514 t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) 515 } 516 assert scheduled == {"task-0-label", "task-1-label"} 517 518 519 @pytest.mark.parametrize( 520 "pushed_files,to_lint,expected", 521 [ 522 pytest.param( 523 ["a/b/c.txt"], 524 [], 525 True, 526 ), 527 pytest.param( 528 ["python/mozlint/a/support_file.txt", "b/c/d.txt"], 529 ["python/mozlint/a/support_file.txt"], 530 False, 531 ), 532 ], 533 ids=idfn, 534 ) 535 def test_mozlint_should_remove_task( 536 monkeypatch, params, pushed_files, to_lint, expected 537 ): 538 import mozlint.pathutils 539 540 class MockParser: 541 def __call__(self, *args, **kwargs): 542 return [] 543 544 def mock_filterpaths(*args, **kwargs): 545 return to_lint, None 546 547 monkeypatch.setattr(mozlint.pathutils, "filterpaths", mock_filterpaths) 548 549 opt = SkipUnlessMozlint("") 550 monkeypatch.setattr(opt, "mozlint_parser", MockParser()) 551 params["files_changed"] = pushed_files 552 553 result = opt.should_remove_task(default_tasks[0], params, "") 554 assert result == expected 555 556 557 @pytest.mark.parametrize( 558 "pushed_files,linter_config,expected", 559 [ 560 pytest.param( 561 ["a/b/c.txt"], 562 [{"include": ["b/c"]}], 563 True, 564 ), 565 pytest.param( 566 ["a/b/c.txt"], 567 [{"include": ["a/b"], "exclude": ["a/b/c.txt"]}], 568 True, 569 ), 570 pytest.param( 571 ["python/mozlint/a/support_file.txt", "b/c/d.txt"], 572 [{}], 573 False, 574 ), 575 ], 576 ids=idfn, 577 ) 578 def test_mozlint_should_remove_task2( 579 monkeypatch, params, pushed_files, linter_config, expected 580 ): 581 class MockParser: 582 def __call__(self, *args, **kwargs): 583 return linter_config 584 585 opt = SkipUnlessMozlint("") 586 monkeypatch.setattr(opt, "mozlint_parser", MockParser()) 587 params["files_changed"] = pushed_files 588 589 result = opt.should_remove_task(default_tasks[0], params, "") 590 assert result == expected 591 592 593 def test_skip_unless_missing(monkeypatch, responses, params): 594 opt = SkipUnlessMissing() 595 task = deepcopy(default_tasks[0]) 596 task.task["deadline"] = "2024-01-02T00:00:00.000Z" 597 index = "foo.bar.baz" 598 task_id = "abc" 599 root_url = "https://taskcluster.example.com" 600 monkeypatch.delenv("TASKCLUSTER_PROXY_URL", raising=False) 601 monkeypatch.setenv("TASKCLUSTER_ROOT_URL", root_url) 602 603 # Task is missing, don't optimize 604 responses.add( 605 responses.GET, 606 f"{root_url}/api/index/v1/task/{index}", 607 status=404, 608 ) 609 result = opt.should_remove_task(task, params, index) 610 assert result is False 611 612 # Task is found but failed, don't optimize 613 responses.replace( 614 responses.GET, 615 f"{root_url}/api/index/v1/task/{index}", 616 json={"taskId": task_id}, 617 status=200, 618 ) 619 responses.add( 620 responses.GET, 621 f"{root_url}/api/queue/v1/task/{task_id}/status", 622 json={"status": {"state": "failed"}}, 623 status=200, 624 ) 625 result = opt.should_remove_task(task, params, index) 626 assert result is False 627 628 # Task is found and passed but expires before deadline, don't optimize 629 responses.replace( 630 responses.GET, 631 f"{root_url}/api/index/v1/task/{index}", 632 json={"taskId": task_id}, 633 status=200, 634 ) 635 responses.replace( 636 responses.GET, 637 f"{root_url}/api/queue/v1/task/{task_id}/status", 638 json={"status": {"state": "completed", "expires": "2024-01-01T00:00:00.000Z"}}, 639 status=200, 640 ) 641 result = opt.should_remove_task(task, params, index) 642 assert result is False 643 644 # Task is found and passed and expires after deadline, optimize 645 responses.replace( 646 responses.GET, 647 f"{root_url}/api/index/v1/task/{index}", 648 json={"taskId": task_id}, 649 status=200, 650 ) 651 responses.replace( 652 responses.GET, 653 f"{root_url}/api/queue/v1/task/{task_id}/status", 654 json={"status": {"state": "completed", "expires": "2024-01-03T00:00:00.000Z"}}, 655 status=200, 656 ) 657 result = opt.should_remove_task(task, params, index) 658 assert result is True 659 660 # Task has parameterized deadline, does not raise 661 task.task["deadline"] = {"relative-datestamp": "1 day"} 662 responses.replace( 663 responses.GET, 664 f"{root_url}/api/index/v1/task/{index}", 665 json={"taskId": task_id}, 666 status=200, 667 ) 668 responses.replace( 669 responses.GET, 670 f"{root_url}/api/queue/v1/task/{task_id}/status", 671 json={"status": {"state": "completed", "expires": "2024-01-03T00:00:00.000Z"}}, 672 status=200, 673 ) 674 opt.should_remove_task(task, params, index) 675 676 677 if __name__ == "__main__": 678 main()