target_tasks.py (64449B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import itertools 7 import logging 8 import os 9 import re 10 from datetime import datetime, timedelta 11 12 from redo import retry 13 from taskcluster.exceptions import TaskclusterRestFailure 14 from taskgraph import create 15 from taskgraph.target_tasks import filter_for_git_branch, register_target_task 16 from taskgraph.util.attributes import attrmatch 17 from taskgraph.util.parameterization import resolve_timestamps 18 from taskgraph.util.taskcluster import ( 19 find_task_id, 20 get_artifact, 21 get_task_definition, 22 parse_time, 23 ) 24 from taskgraph.util.yaml import load_yaml 25 26 from gecko_taskgraph import GECKO, TEST_CONFIGS 27 from gecko_taskgraph.util.attributes import ( 28 is_try, 29 match_run_on_hg_branches, 30 match_run_on_projects, 31 match_run_on_repo_type, 32 ) 33 from gecko_taskgraph.util.constants import TEST_KINDS 34 from gecko_taskgraph.util.hg import find_hg_revision_push_info, get_hg_commit_message 35 from gecko_taskgraph.util.platforms import platform_family 36 from gecko_taskgraph.util.taskcluster import find_task, insert_index 37 38 logger = logging.getLogger(__name__) 39 40 41 # Some tasks show up in the target task set, but are possibly special cases, 42 # uncommon tasks, or tasks running against limited hardware set that they 43 # should only be selectable with --full. 44 UNCOMMON_TRY_TASK_LABELS = [ 45 # Platforms and/or Build types 46 r"build-.*-gcp", # Bug 1631990 47 r"mingwclang", # Bug 1631990 48 r"valgrind", # Bug 1631990 49 # Android tasks 50 r"android-geckoview-docs", 51 r"android-hw", 52 # Windows tasks 53 r"windows11-64-24h2-hw-ref", 54 r"windows10-aarch64-qr", 55 # Linux tasks 56 r"linux-", # hide all linux32 tasks by default - bug 1599197 57 r"linux1804-32", # hide linux32 tests - bug 1599197 58 # Test tasks 59 r"web-platform-tests.*backlog", # hide wpt jobs that are not implemented yet - bug 1572820 60 r"-ccov", 61 r"-profiling-", # talos/raptor profiling jobs are run too often 62 r"-32-.*-webgpu", # webgpu gets little benefit from these tests. 63 r"-asan-.*-webgpu", 64 r"-tsan-.*-webgpu", 65 # Hide shippable versions of tests we have opt versions of because the non-shippable 66 # versions are faster to run. This is mostly perf tests. 67 r"-shippable(?!.*(awsy|browsertime|marionette-headless|mochitest-devtools-chrome-fis|raptor|talos|web-platform-tests-wdspec-headless|mochitest-plain-headless))", # noqa - too long 68 r"nightly-simulation", 69 # Can't actually run on try 70 r"notarization", 71 ] 72 73 74 def index_exists(index_path, reason=""): 75 print(f"Looking for existing index {index_path} {reason}...") 76 try: 77 task_id = find_task_id(index_path) 78 print(f"Index {index_path} exists: taskId {task_id}") 79 return True 80 except (KeyError, TaskclusterRestFailure) as e: 81 if isinstance(e, TaskclusterRestFailure) and e.status_code != 404: 82 raise 83 print(f"Index {index_path} doesn't exist.") 84 return False 85 86 87 def filter_out_shipping_phase(task, parameters): 88 return ( 89 # nightly still here because of geckodriver 90 not task.attributes.get("nightly") 91 and task.attributes.get("shipping_phase") in (None, "build") 92 ) 93 94 95 def filter_out_devedition(task, parameters): 96 return not task.attributes.get("shipping_product") == "devedition" 97 98 99 def filter_out_cron(task, parameters): 100 """ 101 Filter out tasks that run via cron. 102 """ 103 return not task.attributes.get("cron") 104 105 106 def filter_for_repo_type(task, parameters): 107 """Filter tasks by repository type. 108 109 This filter is temporarily in-place to facilitate the hg.mozilla.org -> 110 Github migration.""" 111 run_on_repo_types = set(task.attributes.get("run_on_repo_type", ["git", "hg"])) 112 return match_run_on_repo_type(parameters["repository_type"], run_on_repo_types) 113 114 115 def filter_for_project(task, parameters): 116 """Filter tasks by project. Optionally enable nightlies.""" 117 run_on_projects = set(task.attributes.get("run_on_projects", [])) 118 return match_run_on_projects(parameters, run_on_projects) 119 120 121 def filter_for_hg_branch(task, parameters): 122 """Filter tasks by hg branch. 123 If `run_on_hg_branch` is not defined, then task runs on all branches""" 124 run_on_hg_branches = set(task.attributes.get("run_on_hg_branches", ["all"])) 125 return match_run_on_hg_branches(parameters["hg_branch"], run_on_hg_branches) 126 127 128 def filter_on_platforms(task, platforms): 129 """Filter tasks on the given platform""" 130 platform = task.attributes.get("build_platform") 131 return platform in platforms 132 133 134 def filter_by_uncommon_try_tasks(task, optional_filters=None): 135 """Filters tasks that should not be commonly run on try. 136 137 Args: 138 task (str): String representing the task name. 139 optional_filters (list, optional): 140 Additional filters to apply to task filtering. 141 142 Returns: 143 (Boolean): True if task does not match any known filters. 144 False otherwise. 145 """ 146 filters = UNCOMMON_TRY_TASK_LABELS 147 if optional_filters: 148 filters = itertools.chain(filters, optional_filters) 149 150 return not any(re.search(pattern, task) for pattern in filters) 151 152 153 def filter_by_regex(task_label, regexes, mode="include"): 154 """Filters tasks according to a list of pre-compiled reguar expressions. 155 156 If mode is "include", a task label must match any regex to pass. 157 If it is "exclude", a task label must _not_ match any regex to pass. 158 """ 159 if not regexes: 160 return True 161 162 assert mode in ["include", "exclude"] 163 164 any_match = any(r.search(task_label) for r in regexes) 165 if any_match: 166 return mode == "include" 167 return mode != "include" 168 169 170 def filter_release_tasks(task, parameters): 171 platform = task.attributes.get("build_platform") 172 if platform in ( 173 "linux64", 174 "linux64-aarch64", 175 "macosx64", 176 "win32", 177 "win64", 178 "win64-aarch64", 179 ): 180 if task.attributes["kind"] == "l10n": 181 # This is on-change l10n 182 return True 183 if ( 184 task.attributes["build_type"] == "opt" 185 and task.attributes.get("unittest_suite") != "talos" 186 and task.attributes.get("unittest_suite") != "raptor" 187 ): 188 return False 189 190 if task.attributes.get("shipping_phase") not in (None, "build"): 191 return False 192 193 """ No debug on release, keep on ESR with 4 week cycles, release 194 will not be too different from central, but ESR will live for a long time. 195 196 From June 2019 -> June 2020, we found 1 unique regression on ESR debug 197 and 5 unique regressions on beta/release. Keeping spidermonkey and linux 198 debug finds all but 1 unique regressions (windows found on try) for beta/release. 199 200 ...but debug-only failures started showing up on ESR (esr-91, esr-102) so 201 desktop debug tests were added back for beta. 202 """ 203 build_type = task.attributes.get("build_type", "") 204 build_platform = task.attributes.get("build_platform", "") 205 test_platform = task.attributes.get("test_platform", "") 206 207 if parameters["release_type"].startswith("esr") or ( 208 parameters["release_type"] == "beta" and "android" not in build_platform 209 ): 210 return True 211 212 # code below here is intended to reduce release debug tasks 213 if task.kind == "hazard" or "toolchain" in build_platform: 214 # keep hazard and toolchain builds around 215 return True 216 217 if build_type == "debug": 218 if "linux" not in build_platform: 219 # filter out windows/mac/android 220 return False 221 if task.kind not in ["spidermonkey"] and "-qr" in test_platform: 222 # filter out linux-qr tests, leave spidermonkey 223 return False 224 if "64" not in build_platform: 225 # filter out linux32 builds 226 return False 227 228 # webrender-android-*-debug doesn't have attributes to find 'debug', using task.label. 229 if task.kind == "webrender" and "debug" in task.label: 230 return False 231 return True 232 233 234 def filter_out_missing_signoffs(task, parameters): 235 for signoff in parameters["required_signoffs"]: 236 if signoff not in parameters["signoff_urls"] and signoff in task.attributes.get( 237 "required_signoffs", [] 238 ): 239 return False 240 return True 241 242 243 def filter_tests_without_manifests(task, parameters): 244 """Remove test tasks that have an empty 'test_manifests' attribute. 245 246 This situation can arise when the test loader (e.g bugbug) decided there 247 weren't any important manifests to run for the given push. We filter tasks 248 out here rather than in the transforms so that the full task graph is still 249 aware that the task exists (which is needed by the backfill action). 250 """ 251 if ( 252 task.kind in TEST_KINDS 253 and "test_manifests" in task.attributes 254 and not task.attributes["test_manifests"] 255 ): 256 return False 257 return True 258 259 260 def standard_filter(task, parameters): 261 return all( 262 filter_func(task, parameters) 263 for filter_func in ( 264 filter_out_cron, 265 filter_for_repo_type, 266 filter_for_project, 267 filter_for_hg_branch, 268 filter_for_git_branch, 269 filter_tests_without_manifests, 270 ) 271 ) 272 273 274 def accept_raptor_android_build(platform): 275 """Helper function for selecting the correct android raptor builds.""" 276 if "android" not in platform: 277 return False 278 if "shippable" not in platform: 279 return False 280 if "p5" in platform and "aarch64" in platform: 281 return False 282 if "p6" in platform and "aarch64" in platform: 283 return True 284 if "s24" in platform and "aarch64" in platform: 285 return True 286 if "a55" in platform and "aarch64" in platform: 287 return True 288 return False 289 290 291 def accept_raptor_desktop_build(platform): 292 """Helper function for selecting correct desktop raptor builds.""" 293 if "android" in platform: 294 return False 295 # ignore all windows 7 perf jobs scheduled automatically 296 if "windows7" in platform or "windows10-32" in platform: 297 return False 298 # Completely ignore all non-shippable platforms 299 if "shippable" in platform: 300 return True 301 return False 302 303 304 def accept_awsy_task(try_name, platform): 305 if accept_raptor_desktop_build(platform): 306 if "windows" in platform and "windows11-64" not in platform: 307 return False 308 if "dmd" in try_name: 309 return False 310 if "awsy-base" in try_name: 311 return True 312 if "awsy-tp6" in try_name: 313 return True 314 return False 315 316 317 def filter_unsupported_artifact_builds(task, parameters): 318 try_config = parameters.get("try_task_config", {}) 319 if not try_config.get("use-artifact-builds", False): 320 return True 321 322 supports_artifact_builds = task.attributes.get("supports-artifact-builds", True) 323 return supports_artifact_builds 324 325 326 def filter_out_shippable(task): 327 return not task.attributes.get("shippable", False) 328 329 330 def _try_task_config(full_task_graph, parameters, graph_config): 331 requested_tasks = parameters["try_task_config"]["tasks"] 332 pattern_tasks = [x for x in requested_tasks if x.endswith("-*")] 333 tasks = list(set(requested_tasks) - set(pattern_tasks)) 334 matched_tasks = [] 335 missing = set() 336 for pattern in pattern_tasks: 337 found = [ 338 t 339 for t in full_task_graph.graph.nodes 340 if t.split(pattern.replace("*", ""))[-1].isnumeric() 341 ] 342 if found: 343 matched_tasks.extend(found) 344 else: 345 missing.add(pattern) 346 347 if "MOZHARNESS_TEST_PATHS" in parameters["try_task_config"].get("env", {}): 348 matched_tasks = [x for x in matched_tasks if x.endswith("-1")] 349 350 if "MOZHARNESS_TEST_TAG" in parameters["try_task_config"].get("env", {}): 351 matched_tasks = [x for x in matched_tasks if x.endswith("-1")] 352 353 selected_tasks = set(tasks) | set(matched_tasks) 354 missing.update(selected_tasks - set(full_task_graph.tasks)) 355 356 if missing: 357 missing_str = "\n ".join(sorted(missing)) 358 logger.warning( 359 f"The following tasks were requested but do not exist in the full task graph and will be skipped:\n {missing_str}" 360 ) 361 return list(selected_tasks - missing) 362 363 364 @register_target_task("try_tasks") 365 def target_tasks_try(full_task_graph, parameters, graph_config): 366 try_mode = parameters["try_mode"] 367 if try_mode == "try_task_config": 368 return _try_task_config(full_task_graph, parameters, graph_config) 369 # With no try mode, we schedule nothing, allowing the user to add tasks 370 # later via treeherder. 371 return [] 372 373 374 @register_target_task("default") 375 def target_tasks_default(full_task_graph, parameters, graph_config): 376 """Target the tasks which have indicated they should be run on this project 377 via the `run_on_projects` attributes.""" 378 return [ 379 l 380 for l, t in full_task_graph.tasks.items() 381 if standard_filter(t, parameters) 382 and filter_out_shipping_phase(t, parameters) 383 and filter_out_devedition(t, parameters) 384 ] 385 386 387 @register_target_task("autoland_tasks") 388 def target_tasks_autoland(full_task_graph, parameters, graph_config): 389 """In addition to doing the filtering by project that the 'default' 390 filter does, also remove any tests running against shippable builds 391 for non-backstop pushes.""" 392 filtered_for_project = target_tasks_default( 393 full_task_graph, parameters, graph_config 394 ) 395 396 def filter(task): 397 if task.kind not in TEST_KINDS: 398 return True 399 400 if parameters["backstop"]: 401 return True 402 403 build_type = task.attributes.get("build_type") 404 405 if not build_type or build_type != "opt" or filter_out_shippable(task): 406 return True 407 408 return False 409 410 return [l for l in filtered_for_project if filter(full_task_graph[l])] 411 412 413 @register_target_task("mozilla_central_tasks") 414 def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): 415 """In addition to doing the filtering by project that the 'default' 416 filter does, also remove any tests running against regular (aka not shippable, 417 asan, etc.) opt builds.""" 418 filtered_for_project = target_tasks_default( 419 full_task_graph, parameters, graph_config 420 ) 421 422 def filter(task): 423 if task.kind not in TEST_KINDS: 424 return True 425 426 build_platform = task.attributes.get("build_platform") 427 build_type = task.attributes.get("build_type") 428 shippable = task.attributes.get("shippable", False) 429 430 if not build_platform or not build_type: 431 return True 432 433 family = platform_family(build_platform) 434 # We need to know whether this test is against a "regular" opt build 435 # (which is to say, not shippable, asan, tsan, or any other opt build 436 # with other properties). There's no positive test for this, so we have to 437 # do it somewhat hackily. Android doesn't have variants other than shippable 438 # so it is pretty straightforward to check for. Other platforms have many 439 # variants, but none of the regular opt builds we're looking for have a "-" 440 # in their platform name, so this works (for now). 441 is_regular_opt = ( 442 family == "android" and not shippable 443 ) or "-" not in build_platform 444 445 if build_type != "opt" or not is_regular_opt: 446 return True 447 448 return False 449 450 return [l for l in filtered_for_project if filter(full_task_graph[l])] 451 452 453 @register_target_task("graphics_tasks") 454 def target_tasks_graphics(full_task_graph, parameters, graph_config): 455 """In addition to doing the filtering by project that the 'default' 456 filter does, also remove artifact builds because we have csets on 457 the graphics branch that aren't on the candidate branches of artifact 458 builds""" 459 filtered_for_project = target_tasks_default( 460 full_task_graph, parameters, graph_config 461 ) 462 463 def filter(task): 464 if task.attributes["kind"] == "artifact-build": 465 return False 466 return True 467 468 return [l for l in filtered_for_project if filter(full_task_graph[l])] 469 470 471 @register_target_task("mozilla_beta_tasks") 472 def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): 473 """Select the set of tasks required for a promotable beta or release build 474 of desktop, plus android CI. The candidates build process involves a pipeline 475 of builds and signing, but does not include beetmover or balrog jobs.""" 476 477 return [ 478 l 479 for l, t in full_task_graph.tasks.items() 480 if filter_release_tasks(t, parameters) and standard_filter(t, parameters) 481 ] 482 483 484 @register_target_task("mozilla_release_tasks") 485 def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): 486 """Select the set of tasks required for a promotable beta or release build 487 of desktop, plus android CI. The candidates build process involves a pipeline 488 of builds and signing, but does not include beetmover or balrog jobs.""" 489 490 return [ 491 l 492 for l, t in full_task_graph.tasks.items() 493 if filter_release_tasks(t, parameters) and standard_filter(t, parameters) 494 ] 495 496 497 @register_target_task("mozilla_esr140_tasks") 498 def target_tasks_mozilla_esr140(full_task_graph, parameters, graph_config): 499 """Select the set of tasks required for a promotable beta or release build 500 of desktop, without android CI. The candidates build process involves a pipeline 501 of builds and signing, but does not include beetmover or balrog jobs.""" 502 503 def filter(task): 504 if not filter_release_tasks(task, parameters): 505 return False 506 507 if not standard_filter(task, parameters): 508 return False 509 510 platform = task.attributes.get("build_platform") 511 512 # Android is not built on esr. 513 if platform and "android" in platform: 514 return False 515 516 return True 517 518 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 519 520 521 @register_target_task("promote_desktop") 522 def target_tasks_promote_desktop(full_task_graph, parameters, graph_config): 523 """Select the superset of tasks required to promote a beta or release build 524 of a desktop product. This should include all non-android 525 mozilla_{beta,release} tasks, plus l10n, beetmover, balrog, etc.""" 526 527 def filter(task): 528 if task.attributes.get("shipping_product") != parameters["release_product"]: 529 return False 530 531 # 'secondary' balrog/update verify/final verify tasks only run for RCs 532 if parameters.get("release_type") != "release-rc": 533 if "secondary" in task.kind: 534 return False 535 536 if not filter_out_missing_signoffs(task, parameters): 537 return False 538 539 if task.attributes.get("shipping_phase") == "promote": 540 return True 541 542 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 543 544 545 @register_target_task("push_desktop") 546 def target_tasks_push_desktop(full_task_graph, parameters, graph_config): 547 """Select the set of tasks required to push a build of desktop to cdns. 548 Previous build deps will be optimized out via action task.""" 549 filtered_for_candidates = target_tasks_promote_desktop( 550 full_task_graph, 551 parameters, 552 graph_config, 553 ) 554 555 def filter(task): 556 if not filter_out_missing_signoffs(task, parameters): 557 return False 558 # Include promotion tasks; these will be optimized out 559 if task.label in filtered_for_candidates: 560 return True 561 562 if ( 563 task.attributes.get("shipping_product") == parameters["release_product"] 564 and task.attributes.get("shipping_phase") == "push" 565 ): 566 return True 567 568 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 569 570 571 @register_target_task("ship_desktop") 572 def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): 573 """Select the set of tasks required to ship desktop. 574 Previous build deps will be optimized out via action task.""" 575 is_rc = parameters.get("release_type") == "release-rc" 576 if is_rc: 577 # ship_firefox_rc runs after `promote` rather than `push`; include 578 # all promote tasks. 579 filtered_for_candidates = target_tasks_promote_desktop( 580 full_task_graph, 581 parameters, 582 graph_config, 583 ) 584 else: 585 # ship_firefox runs after `push`; include all push tasks. 586 filtered_for_candidates = target_tasks_push_desktop( 587 full_task_graph, 588 parameters, 589 graph_config, 590 ) 591 592 def filter(task): 593 if not filter_out_missing_signoffs(task, parameters): 594 return False 595 # Include promotion tasks; these will be optimized out 596 if task.label in filtered_for_candidates: 597 return True 598 599 if ( 600 task.attributes.get("shipping_product") != parameters["release_product"] 601 or task.attributes.get("shipping_phase") != "ship" 602 ): 603 return False 604 605 if "secondary" in task.kind: 606 return is_rc 607 return not is_rc 608 609 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 610 611 612 @register_target_task("cypress_tasks") 613 def target_tasks_cypress(full_task_graph, parameters, graph_config): 614 filtered_for_project = target_tasks_default( 615 full_task_graph, parameters, graph_config 616 ) 617 618 def filter(task): 619 # bug 1899403: no need for android tasks 620 if "android" in task.attributes.get("build_platform", ""): 621 return False 622 return True 623 624 return [l for l in filtered_for_project if filter(full_task_graph[l])] 625 626 627 @register_target_task("pine_tasks") 628 def target_tasks_pine(full_task_graph, parameters, graph_config): 629 """Bug 1879960 - no reftests or wpt needed""" 630 filtered_for_project = target_tasks_default( 631 full_task_graph, parameters, graph_config 632 ) 633 634 def filter(task): 635 if "android" in task.attributes.get("build_platform", ""): 636 return False 637 suite = task.attributes.get("unittest_suite", "") 638 if "reftest" in suite or "web-platform" in suite: 639 return False 640 return True 641 642 return [l for l in filtered_for_project if filter(full_task_graph[l])] 643 644 645 @register_target_task("larch_tasks") 646 def target_tasks_larch(full_task_graph, parameters, graph_config): 647 """Bug 1879213 - only run necessary tasks on larch""" 648 filtered_for_project = target_tasks_default( 649 full_task_graph, parameters, graph_config 650 ) 651 652 def filter(task): 653 # no localized builds, no android 654 if ( 655 "l10n" in task.kind 656 or "msix" in task.kind 657 or "android" in task.attributes.get("build_platform", "") 658 or (task.kind == "test" and "msix" in task.label) 659 ): 660 return False 661 # otherwise reduce tests only 662 if task.kind not in TEST_KINDS: 663 return True 664 return "browser-chrome" in task.label or "xpcshell" in task.label 665 666 return [l for l in filtered_for_project if filter(full_task_graph[l])] 667 668 669 @register_target_task("kaios_tasks") 670 def target_tasks_kaios(full_task_graph, parameters, graph_config): 671 """The set of tasks to run for kaios integration""" 672 673 def filter(task): 674 # We disable everything in central, and adjust downstream. 675 return False 676 677 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 678 679 680 @register_target_task("custom-car_perf_testing") 681 def target_tasks_custom_car_perf_testing(full_task_graph, parameters, graph_config): 682 """Select tasks required for running daily performance tests for custom chromium-as-release.""" 683 684 def filter(task): 685 platform = task.attributes.get("test_platform") 686 attributes = task.attributes 687 if attributes.get("unittest_suite") != "raptor": 688 return False 689 690 try_name = attributes.get("raptor_try_name") 691 if "network-bench" in try_name and "linux" not in platform: 692 return False 693 694 # Desktop and Android selection for CaR 695 if accept_raptor_desktop_build(platform): 696 if "browsertime" in try_name and "custom-car" in try_name: 697 # Bug 1898514: avoid tp6m or non-essential tp6 jobs in cron 698 if "tp6" in try_name and "essential" not in try_name: 699 return False 700 # Bug 1928416 701 # For ARM coverage, this will only run on M2 machines at the moment. 702 if "jetstream2" in try_name: 703 # Bug 1963732 - Disable js2 on 1500 mac for custom-car due to near perma 704 if "m-car" in try_name and "1500" in platform: 705 return False 706 return True 707 return True 708 elif accept_raptor_android_build(platform): 709 if "browsertime" in try_name and "cstm-car-m" in try_name: 710 if "-nofis" not in try_name: 711 return False 712 if "hw-s24" in platform and "speedometer3" not in try_name: 713 return False 714 if "jetstream2" in try_name: 715 return True 716 if "jetstream3" in try_name: 717 return True 718 # Bug 1898514 - Avoid tp6m or non-essential tp6 jobs in cron on non-a55 platform 719 # Bug 1961831 - Disable pageload tests temporarily during provider switch 720 if "tp6m" in try_name: 721 return False 722 # Bug 1945165 - Disable ebay-kleinanzeigen on cstm-car-m because of permafail 723 if ( 724 "ebay-kleinanzeigen" in try_name 725 and "ebay-kleinanzeigen-search" not in try_name 726 ): 727 return False 728 # Bug 1960921 Disable ebay-kleinanzeigen-search-nofis on custom-car 729 if "ebay-kleinanzeigen-search-nofis" in try_name: 730 return False 731 return True 732 return False 733 734 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 735 736 737 @register_target_task("general_perf_testing") 738 def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config): 739 """ 740 Select tasks required for running performance tests 3 times a week. 741 """ 742 743 def filter(task): 744 platform = task.attributes.get("test_platform") 745 attributes = task.attributes 746 if attributes.get("unittest_suite") != "raptor": 747 return False 748 749 try_name = attributes.get("raptor_try_name") 750 751 if "tp6-bench" in try_name: 752 return False 753 754 if "tp7" in try_name: 755 return False 756 757 # Bug 1867669 - Temporarily disable all live site tests 758 if "live" in try_name and "sheriffed" not in try_name: 759 return False 760 761 if "network-bench" in try_name: 762 return False 763 764 # Desktop selection 765 if accept_raptor_desktop_build(platform): 766 # Select some browsertime tasks as desktop smoke-tests 767 if "responsiveness" in try_name and "chrome" in try_name: 768 # Disabled chrome responsiveness tests temporarily in bug 1898351 769 # due to frequent failures 770 return False 771 # Bug 1961141 - Disable unity webgl for chrome windows 772 if "chrome-unity-webgl" in try_name and "windows11" in platform: 773 return False 774 # Bug 1961145 - Disable bing-search for test-windows11-64-24h2-shippable 775 if "windows11" in platform and "bing-search" in try_name: 776 return False 777 if "browsertime" in try_name: 778 if "chrome" in try_name: 779 if "tp6" in try_name and "essential" not in try_name: 780 return False 781 return True 782 # chromium-as-release has its own cron 783 if "custom-car" in try_name: 784 return False 785 if "-live" in try_name: 786 return True 787 if "-fis" in try_name: 788 return False 789 if "linux" in platform: 790 if "speedometer" in try_name: 791 return True 792 if "safari" and "benchmark" in try_name: 793 if "jetstream2" in try_name and "safari" in try_name: 794 return False 795 # JetStream 3 fails with Safari 18.3 but not Safari-TP. 796 # See bug 1996277. 797 if ( 798 "safari-jetstream3" in try_name 799 and "macosx1500-aarch64" in platform 800 ): 801 return False 802 return True 803 # Android selection 804 elif accept_raptor_android_build(platform): 805 if "hw-s24" in platform and "speedometer3" not in try_name: 806 return False 807 if "chrome-m" in try_name and "-nofis" not in try_name: 808 return False 809 # Bug 1929960 - Enable all chrome-m tp6m tests on a55 only 810 if "chrome-m" in try_name and "tp6m" in try_name and "hw-a55" in platform: 811 # Bug 1954923 - Disable ebay-kleinanzeigen 812 if ( 813 "ebay-kleinanzeigen" in try_name 814 and "search" not in try_name 815 and "nofis" in try_name 816 ): 817 return False 818 return True 819 if "chrome-m" in try_name and ( 820 ("ebay" in try_name and "live" not in try_name) 821 or ( 822 "live" in try_name 823 and ("facebook" in try_name or "dailymail" in try_name) 824 ) 825 ): 826 return False 827 # Ignore all fennec tests here, we run those weekly 828 if "fennec" in try_name: 829 return False 830 # Select live site tests 831 if "-live" in try_name: 832 return True 833 # Select fenix resource usage tests 834 if "fenix" in try_name: 835 if "-power" in try_name: 836 return True 837 838 if "geckoview" in try_name: 839 return False 840 # Select browsertime-specific tests 841 if "browsertime" in try_name: 842 # Don't run android CaR sp tests as we already have a cron for this. 843 if "m-car" in try_name: 844 return False 845 if "jetstream2" in try_name: 846 return True 847 if "jetstream3" in try_name: 848 return True 849 if "fenix" in try_name: 850 return False 851 if "speedometer" in try_name: 852 return True 853 if "motionmark" in try_name and "1-3" in try_name: 854 if "chrome-m" in try_name: 855 return True 856 return False 857 858 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 859 860 861 @register_target_task("geckoview-perftest") 862 def target_tasks_geckoview_perftest(full_task_graph, parameters, graph_config): 863 """ 864 Select tasks required for running geckoview tests 2 times a week. 865 """ 866 867 def filter(task): 868 platform = task.attributes.get("test_platform") 869 attributes = task.attributes 870 if attributes.get("unittest_suite") != "raptor": 871 return False 872 873 if accept_raptor_android_build(platform): 874 try_name = attributes.get("raptor_try_name") 875 if "geckoview" in try_name and "browsertime" in try_name: 876 if "hw-s24" in platform and "speedometer" not in try_name: 877 return False 878 if "live" in try_name and "cnn-amp" not in try_name: 879 return False 880 return True 881 882 return False 883 884 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 885 886 887 def make_desktop_nightly_filter(platforms): 888 """Returns a filter that gets all nightly tasks on the given platform.""" 889 890 def filter(task, parameters): 891 return all([ 892 filter_on_platforms(task, platforms), 893 filter_for_project(task, parameters), 894 task.attributes.get("shippable", False), 895 # Tests and nightly only builds don't have `shipping_product` set 896 task.attributes.get("shipping_product") in {None, "firefox", "thunderbird"}, 897 task.kind not in {"l10n"}, # no on-change l10n 898 ]) 899 900 return filter 901 902 903 @register_target_task("sp-perftests") 904 def target_tasks_speedometer_tests(full_task_graph, parameters, graph_config): 905 def filter(task): 906 platform = task.attributes.get("test_platform") 907 attributes = task.attributes 908 if attributes.get("unittest_suite") != "raptor": 909 return False 910 911 try_name = attributes.get("raptor_try_name") 912 if accept_raptor_desktop_build(platform): 913 if ( 914 "browsertime" in try_name 915 and "speedometer" in try_name 916 and "chrome" in try_name 917 ): 918 return True 919 if accept_raptor_android_build(platform): 920 if "hw-s24" in platform and "speedometer3" not in try_name: 921 return False 922 if ( 923 "browsertime" in try_name 924 and "speedometer" in try_name 925 and "chrome-m" in try_name 926 ): 927 if "-nofis" not in try_name: 928 return False 929 return True 930 931 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 932 933 934 @register_target_task("nightly_linux") 935 def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): 936 """Select the set of tasks required for a nightly build of linux. The 937 nightly build process involves a pipeline of builds, signing, 938 and, eventually, uploading the tasks to balrog.""" 939 filter = make_desktop_nightly_filter({ 940 "linux64-shippable", 941 "linux64-aarch64-shippable", 942 }) 943 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 944 945 946 @register_target_task("nightly_macosx") 947 def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): 948 """Select the set of tasks required for a nightly build of macosx. The 949 nightly build process involves a pipeline of builds, signing, 950 and, eventually, uploading the tasks to balrog.""" 951 filter = make_desktop_nightly_filter({"macosx64-shippable"}) 952 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 953 954 955 @register_target_task("nightly_win32") 956 def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): 957 """Select the set of tasks required for a nightly build of win32 and win64. 958 The nightly build process involves a pipeline of builds, signing, 959 and, eventually, uploading the tasks to balrog.""" 960 filter = make_desktop_nightly_filter({"win32-shippable"}) 961 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 962 963 964 @register_target_task("nightly_win64") 965 def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): 966 """Select the set of tasks required for a nightly build of win32 and win64. 967 The nightly build process involves a pipeline of builds, signing, 968 and, eventually, uploading the tasks to balrog.""" 969 filter = make_desktop_nightly_filter({"win64-shippable"}) 970 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 971 972 973 @register_target_task("nightly_win64_aarch64") 974 def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config): 975 """Select the set of tasks required for a nightly build of win32 and win64. 976 The nightly build process involves a pipeline of builds, signing, 977 and, eventually, uploading the tasks to balrog.""" 978 filter = make_desktop_nightly_filter({"win64-aarch64-shippable"}) 979 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 980 981 982 @register_target_task("nightly_asan") 983 def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): 984 """Select the set of tasks required for a nightly build of asan. The 985 nightly build process involves a pipeline of builds, signing, 986 and, eventually, uploading the tasks to balrog.""" 987 filter = make_desktop_nightly_filter({ 988 "linux64-asan-reporter-shippable", 989 "win64-asan-reporter-shippable", 990 }) 991 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 992 993 994 @register_target_task("daily_releases") 995 def target_tasks_daily_releases(full_task_graph, parameters, graph_config): 996 """Select the set of tasks required to identify if we should release. 997 If we determine that we should the task will communicate to ship-it to 998 schedule the release itself.""" 999 1000 def filter(task): 1001 return task.kind in ["maybe-release"] 1002 1003 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1004 1005 1006 @register_target_task("nightly_desktop") 1007 def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): 1008 """Select the set of tasks required for a nightly build of linux, mac, 1009 windows.""" 1010 for platform in ("desktop", "all"): 1011 index_path = ( 1012 f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." 1013 f"{parameters['head_rev']}.taskgraph.decision-nightly-{platform}" 1014 ) 1015 if os.environ.get("MOZ_AUTOMATION") and retry( 1016 index_exists, 1017 args=(index_path,), 1018 kwargs={ 1019 "reason": "to avoid triggering multiple nightlies off the same revision", 1020 }, 1021 ): 1022 return [] 1023 1024 # Tasks that aren't platform specific 1025 release_filter = make_desktop_nightly_filter({None}) 1026 release_tasks = [ 1027 l for l, t in full_task_graph.tasks.items() if release_filter(t, parameters) 1028 ] 1029 # Avoid duplicate tasks. 1030 return list( 1031 set(target_tasks_nightly_win32(full_task_graph, parameters, graph_config)) 1032 | set(target_tasks_nightly_win64(full_task_graph, parameters, graph_config)) 1033 | set( 1034 target_tasks_nightly_win64_aarch64( 1035 full_task_graph, parameters, graph_config 1036 ) 1037 ) 1038 | set(target_tasks_nightly_macosx(full_task_graph, parameters, graph_config)) 1039 | set(target_tasks_nightly_linux(full_task_graph, parameters, graph_config)) 1040 | set(target_tasks_nightly_asan(full_task_graph, parameters, graph_config)) 1041 | set(release_tasks) 1042 ) 1043 1044 1045 @register_target_task("nightly_all") 1046 def target_tasks_nightly_all(full_task_graph, parameters, graph_config): 1047 """Select the set of tasks required for a nightly build of firefox desktop and android""" 1048 index_path = ( 1049 f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." 1050 f"{parameters['head_rev']}.taskgraph.decision-nightly-all" 1051 ) 1052 if os.environ.get("MOZ_AUTOMATION") and retry( 1053 index_exists, 1054 args=(index_path,), 1055 kwargs={ 1056 "reason": "to avoid triggering multiple nightlies off the same revision", 1057 }, 1058 ): 1059 return [] 1060 1061 return list( 1062 set(target_tasks_nightly_desktop(full_task_graph, parameters, graph_config)) 1063 | set(target_tasks_nightly_android(full_task_graph, parameters, graph_config)) 1064 ) 1065 1066 1067 # Run Searchfox analysis once daily. 1068 @register_target_task("searchfox_index") 1069 def target_tasks_searchfox(full_task_graph, parameters, graph_config): 1070 """Select tasks required for indexing Firefox for Searchfox web site each day""" 1071 index_path = ( 1072 f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." 1073 f"{parameters['head_rev']}.searchfox-index" 1074 ) 1075 if os.environ.get("MOZ_AUTOMATION"): 1076 print( 1077 f"Looking for existing index {index_path} to avoid triggering redundant indexing off the same revision..." 1078 ) 1079 try: 1080 task = find_task(index_path) 1081 print(f"Index {index_path} exists: taskId {task['taskId']}") 1082 except TaskclusterRestFailure as e: 1083 if e.status_code != 404: 1084 raise 1085 print(f"Index {index_path} doesn't exist.") 1086 else: 1087 # Find the earlier expiration time of existing tasks 1088 taskdef = get_task_definition(task["taskId"]) 1089 try: 1090 task_graph = get_artifact(task["taskId"], "public/task-graph.json") 1091 except TaskclusterRestFailure as e: 1092 if e.status_code != 404: 1093 raise 1094 task_graph = None 1095 if task_graph: 1096 base_time = parse_time(taskdef["created"]) 1097 first_expiry = min( 1098 resolve_timestamps(base_time, t["task"]["expires"]) 1099 for t in task_graph.values() 1100 ) 1101 expiry = parse_time(first_expiry) 1102 if expiry > datetime.utcnow() + timedelta(days=7): 1103 print("Skipping index tasks") 1104 return [] 1105 if not create.testing: 1106 insert_index(index_path, os.environ["TASK_ID"]) 1107 1108 return [ 1109 "searchfox-linux64-searchfox/opt", 1110 "searchfox-linux64-searchfox/debug", 1111 "searchfox-macosx64-searchfox/debug", 1112 "searchfox-macosx64-aarch64-searchfox/opt", 1113 "searchfox-macosx64-aarch64-searchfox/debug", 1114 "searchfox-win64-searchfox/opt", 1115 "searchfox-win64-searchfox/debug", 1116 "searchfox-android-aarch64-searchfox/debug", 1117 "searchfox-ios-searchfox/debug", 1118 "source-test-file-metadata-bugzilla-components", 1119 "source-test-file-metadata-test-info-all", 1120 "source-test-wpt-metadata-summary", 1121 ] 1122 1123 1124 # Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests 1125 @register_target_task("linux64_clang_trunk_perf") 1126 def target_tasks_build_linux64_clang_trunk_perf( 1127 full_task_graph, parameters, graph_config 1128 ): 1129 """Select tasks required to run perf test on linux64 build with clang trunk""" 1130 1131 # Only keep tasks generated from platform `linux1804-64-clang-trunk-qr/opt` 1132 def filter(task_label): 1133 # Bug 1961141 - Disable unity webgl for linux1804-64-clang-trunk-qr 1134 if "linux1804-64-clang-trunk-qr/opt" in task_label and "unity" in task_label: 1135 return False 1136 if "linux1804-64-clang-trunk-qr/opt" in task_label and "live" not in task_label: 1137 return True 1138 return False 1139 1140 return [l for l, t in full_task_graph.tasks.items() if filter(t.label)] 1141 1142 1143 # Run Updatebot's cron job 4 times daily. 1144 @register_target_task("updatebot_cron") 1145 def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config): 1146 """Select tasks required to run Updatebot's cron job""" 1147 return ["updatebot-cron"] 1148 1149 1150 @register_target_task("customv8_update") 1151 def target_tasks_customv8_update(full_task_graph, parameters, graph_config): 1152 """Select tasks required for building latest d8/v8 version.""" 1153 return ["toolchain-linux64-custom-v8"] 1154 1155 1156 @register_target_task("file_update") 1157 def target_tasks_file_update(full_task_graph, parameters, graph_config): 1158 """Select the set of tasks required to perform nightly in-tree file updates""" 1159 1160 def filter(task): 1161 # For now any task in the repo-update kind is ok 1162 return task.kind in ["repo-update"] 1163 1164 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1165 1166 1167 @register_target_task("l10n_bump") 1168 def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): 1169 """Select the set of tasks required to perform l10n bumping.""" 1170 1171 def filter(task): 1172 # For now any task in the repo-update kind is ok 1173 return task.kind in ["l10n-bump"] 1174 1175 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1176 1177 1178 @register_target_task("merge_automation") 1179 def target_tasks_merge_automation(full_task_graph, parameters, graph_config): 1180 """Select the set of tasks required to perform repository merges.""" 1181 1182 def filter(task): 1183 # For now any task in the repo-update kind is ok 1184 return task.kind in ["merge-automation"] 1185 1186 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1187 1188 1189 @register_target_task("scriptworker_canary") 1190 def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): 1191 """Select the set of tasks required to run scriptworker canaries.""" 1192 1193 def filter(task): 1194 # For now any task in the repo-update kind is ok 1195 return task.kind in ["scriptworker-canary"] 1196 1197 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1198 1199 1200 @register_target_task("cron_bouncer_check") 1201 def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): 1202 """Select the set of tasks required to perform bouncer version verification.""" 1203 1204 def filter(task): 1205 if not filter_for_project(task, parameters): 1206 return False 1207 # For now any task in the repo-update kind is ok 1208 return task.kind in ["cron-bouncer-check"] 1209 1210 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1211 1212 1213 def _filter_by_release_project(parameters): 1214 project_by_release = { 1215 "nightly": "mozilla-central", 1216 "beta": "mozilla-beta", 1217 "release": "mozilla-release", 1218 "esr140": "mozilla-esr140", 1219 } 1220 target_project = project_by_release.get(parameters["release_type"]) 1221 if target_project is None: 1222 raise Exception("Unknown or unspecified release type in simulation run.") 1223 1224 params = parameters.copy() 1225 params["project"] = target_project 1226 return lambda task: filter_for_project(task, params) 1227 1228 1229 def filter_out_android_on_esr(parameters, task): 1230 return not parameters["release_type"].startswith( 1231 "esr" 1232 ) or "android" not in task.attributes.get("build_platform", "") 1233 1234 1235 @register_target_task("staging_release_builds") 1236 def target_tasks_staging_release(full_task_graph, parameters, graph_config): 1237 """ 1238 Select all builds that are part of releases. 1239 """ 1240 filter_for_target_project = _filter_by_release_project(parameters) 1241 1242 return [ 1243 l 1244 for l, t in full_task_graph.tasks.items() 1245 if t.attributes.get("shipping_product") 1246 and filter_out_android_on_esr(parameters, t) 1247 and filter_for_target_project(t) 1248 and t.attributes.get("shipping_phase") == "build" 1249 ] 1250 1251 1252 @register_target_task("release_simulation") 1253 def target_tasks_release_simulation(full_task_graph, parameters, graph_config): 1254 """ 1255 Select tasks that would run on push on a release branch. 1256 """ 1257 filter_for_target_project = _filter_by_release_project(parameters) 1258 1259 return [ 1260 l 1261 for l, t in full_task_graph.tasks.items() 1262 if filter_release_tasks(t, parameters) 1263 and filter_out_cron(t, parameters) 1264 and filter_for_target_project(t) 1265 and filter_out_android_on_esr(parameters, t) 1266 ] 1267 1268 1269 @register_target_task("codereview") 1270 def target_tasks_codereview(full_task_graph, parameters, graph_config): 1271 """Select all code review tasks needed to produce a report""" 1272 1273 def filter(task): 1274 # Ending tasks 1275 if task.kind in ["code-review"]: 1276 return True 1277 1278 # Analyzer tasks 1279 if task.attributes.get("code-review") is True: 1280 return True 1281 1282 return False 1283 1284 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1285 1286 1287 @register_target_task("nothing") 1288 def target_tasks_nothing(full_task_graph, parameters, graph_config): 1289 """Select nothing, for DONTBUILD pushes""" 1290 return [] 1291 1292 1293 @register_target_task("daily_beta_perf") 1294 def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): 1295 """ 1296 Select performance tests on the beta branch to be run daily 1297 """ 1298 index_path = ( 1299 f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." 1300 f"{parameters['head_rev']}.taskgraph.decision-daily-beta-perf" 1301 ) 1302 if os.environ.get("MOZ_AUTOMATION") and retry( 1303 index_exists, 1304 args=(index_path,), 1305 kwargs={ 1306 "reason": "to avoid triggering multiple daily beta perftests off of the same revision", 1307 }, 1308 ): 1309 return [] 1310 1311 def filter(task): 1312 platform = task.attributes.get("test_platform") 1313 attributes = task.attributes 1314 try_name = attributes.get("raptor_try_name") or task.label 1315 1316 unittest_suite = attributes.get("unittest_suite") 1317 if unittest_suite not in ("raptor", "awsy", "talos"): 1318 return False 1319 if not platform: 1320 return False 1321 1322 # Select beta tasks for awsy 1323 if "awsy" in try_name: 1324 if accept_awsy_task(try_name, platform): 1325 return True 1326 return False 1327 1328 # Select beta tasks for talos 1329 if "talos" == unittest_suite: 1330 if accept_raptor_desktop_build(platform): 1331 if "windows11-64" in platform: 1332 if "xperf" in try_name: 1333 return True 1334 return False 1335 if ("mac" in platform or "windows" in platform) and "g3" in try_name: 1336 return False 1337 if "-swr" in try_name: 1338 if "dromaeo" in try_name: 1339 return False 1340 if "perf-reftest-singletons" in try_name: 1341 return False 1342 if "realworldweb" in try_name: 1343 return False 1344 if any( 1345 x in try_name 1346 for x in ("prof", "ipc", "gli", "sessionrestore", "tabswitch") 1347 ): 1348 return False 1349 return True 1350 return False 1351 1352 if accept_raptor_desktop_build(platform): 1353 if "browsertime" and "firefox" in try_name: 1354 if "profiling" in try_name: 1355 return False 1356 if "bytecode" in try_name: 1357 return False 1358 if "live" in try_name: 1359 return False 1360 if "webext" in try_name: 1361 return False 1362 if "unity" in try_name: 1363 return False 1364 if "wasm" in try_name: 1365 return False 1366 if "tp6-bench" in try_name: 1367 return False 1368 if "tp6" in try_name: 1369 return True 1370 if "benchmark" in try_name: 1371 return True 1372 elif accept_raptor_android_build(platform): 1373 if "browsertime" and "geckoview" in try_name: 1374 return False 1375 1376 return False 1377 1378 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1379 1380 1381 @register_target_task("weekly_release_perf") 1382 def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): 1383 """ 1384 Select performance tests on the release branch to be run weekly 1385 """ 1386 1387 def filter(task): 1388 platform = task.attributes.get("test_platform") 1389 attributes = task.attributes 1390 try_name = attributes.get("raptor_try_name") or task.label 1391 1392 if attributes.get("unittest_suite") not in ("raptor", "awsy"): 1393 return False 1394 if not platform: 1395 return False 1396 1397 # Select release tasks for awsy 1398 if "awsy" in try_name: 1399 if accept_awsy_task(try_name, platform): 1400 return True 1401 return False 1402 1403 # Select browsertime tests 1404 if accept_raptor_desktop_build(platform): 1405 if "browsertime" and "firefox" in try_name: 1406 if "power" in try_name: 1407 return False 1408 if "profiling" in try_name: 1409 return False 1410 if "bytecode" in try_name: 1411 return False 1412 if "live" in try_name: 1413 return False 1414 if "webext" in try_name: 1415 return False 1416 if "tp6-bench" in try_name: 1417 return False 1418 if "tp6" in try_name: 1419 return True 1420 if "benchmark" in try_name: 1421 return True 1422 if "youtube-playback" in try_name: 1423 return True 1424 elif accept_raptor_android_build(platform): 1425 if "browsertime" and "geckoview" in try_name: 1426 return False 1427 1428 return False 1429 1430 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1431 1432 1433 @register_target_task("raptor_tp6m") 1434 def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): 1435 """ 1436 Select tasks required for running raptor cold page-load tests on fenix and refbrow 1437 """ 1438 1439 def filter(task): 1440 platform = task.attributes.get("build_platform") 1441 attributes = task.attributes 1442 1443 if platform and "android" not in platform: 1444 return False 1445 if attributes.get("unittest_suite") != "raptor": 1446 return False 1447 try_name = attributes.get("raptor_try_name") 1448 if "-cold" in try_name and "shippable" in platform: 1449 # Get browsertime amazon smoke tests 1450 if ( 1451 "browsertime" in try_name 1452 and "amazon" in try_name 1453 and "search" not in try_name 1454 ): 1455 return True 1456 1457 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1458 1459 1460 @register_target_task("backfill_all_browsertime") 1461 def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config): 1462 """ 1463 Search for revisions that contains patches that were reviewed by perftest reviewers 1464 and landed the day before the cron is running. Trigger backfill-all-browsertime action 1465 task on each of them. 1466 """ 1467 from gecko_taskgraph.actions.util import get_decision_task_id, get_pushes 1468 1469 def date_is_yesterday(date): 1470 yesterday = datetime.today() - timedelta(days=1) 1471 date = datetime.fromtimestamp(date) 1472 return date.date() == yesterday.date() 1473 1474 def reviewed_by_perftest(push): 1475 try: 1476 commit_message = get_hg_commit_message( 1477 os.path.join(GECKO, graph_config["product-dir"]), rev=push 1478 ) 1479 except Exception as e: 1480 print(e) 1481 return False 1482 1483 for line in commit_message.split("\n\n"): 1484 if line.lower().startswith("bug ") and "r=" in line: 1485 if "perftest-reviewers" in line.split("r=")[-1]: 1486 print(line) 1487 return True 1488 return False 1489 1490 pushes = get_pushes( 1491 project=parameters["head_repository"], 1492 end_id=int(parameters["pushlog_id"]), 1493 depth=200, 1494 full_response=True, 1495 ) 1496 for push_id in sorted([int(p) for p in pushes.keys()], reverse=True): 1497 push_rev = pushes[str(push_id)]["changesets"][-1] 1498 push_info = find_hg_revision_push_info( 1499 "https://hg.mozilla.org/integration/" + parameters["project"], push_rev 1500 ) 1501 pushdate = int(push_info["pushdate"]) 1502 if date_is_yesterday(pushdate) and reviewed_by_perftest(push_rev): 1503 from gecko_taskgraph.actions.util import trigger_action 1504 1505 print( 1506 f"Revision {push_rev} was created yesterday and was reviewed by " 1507 f"#perftest-reviewers." 1508 ) 1509 try: 1510 push_decision_task_id = get_decision_task_id( 1511 parameters["project"], push_id 1512 ) 1513 except Exception: 1514 print(f"Could not find decision task for push {push_id}") 1515 continue 1516 try: 1517 trigger_action( 1518 action_name="backfill-all-browsertime", 1519 # This lets the action know on which push we want to add a new task 1520 decision_task_id=push_decision_task_id, 1521 ) 1522 except Exception as e: 1523 print(f"Failed to trigger action for {push_rev}: {e}") 1524 1525 return [] 1526 1527 1528 @register_target_task("condprof") 1529 def target_tasks_condprof(full_task_graph, parameters, graph_config): 1530 """ 1531 Select tasks required for building conditioned profiles. 1532 """ 1533 for name, task in full_task_graph.tasks.items(): 1534 if task.kind == "condprof": 1535 if "a51" not in name: # bug 1765348 1536 yield name 1537 1538 1539 @register_target_task("system_symbols") 1540 def target_tasks_system_symbols(full_task_graph, parameters, graph_config): 1541 """ 1542 Select tasks for scraping and uploading system symbols. 1543 """ 1544 for name, task in full_task_graph.tasks.items(): 1545 if task.kind in [ 1546 "system-symbols", 1547 "system-symbols-upload", 1548 "system-symbols-reprocess", 1549 ]: 1550 yield name 1551 1552 1553 @register_target_task("perftest") 1554 def target_tasks_perftest(full_task_graph, parameters, graph_config): 1555 """ 1556 Select perftest tasks we want to run daily 1557 """ 1558 for name, task in full_task_graph.tasks.items(): 1559 if task.kind != "perftest": 1560 continue 1561 if task.attributes.get("cron", False): 1562 yield name 1563 1564 1565 @register_target_task("perftest-fenix-startup") 1566 def target_tasks_perftest_fenix_startup(full_task_graph, parameters, graph_config): 1567 """ 1568 Select perftest tasks we want to run daily for fenix startup 1569 """ 1570 for name, task in full_task_graph.tasks.items(): 1571 if task.kind != "perftest": 1572 continue 1573 if "fenix" in name and "startup" in name and "profiling" not in name: 1574 yield name 1575 1576 1577 @register_target_task("perftest-on-autoland") 1578 def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): 1579 """ 1580 Select perftest tasks we want to run daily 1581 """ 1582 for name, task in full_task_graph.tasks.items(): 1583 if task.kind != "perftest": 1584 continue 1585 if task.attributes.get("cron", False) and any( 1586 test_name in name for test_name in ["view"] 1587 ): 1588 yield name 1589 1590 1591 @register_target_task("retrigger-perftests-autoland") 1592 def retrigger_perftests_autoland_commits(full_task_graph, parameters, graph_config): 1593 """ 1594 In this transform we are trying to do retrigger the following tasks 4 times every 20 commits in autoland: 1595 - "perftest-android-hw-a55-aarch64-shippable-startup-fenix-cold-main-first-frame", 1596 - "perftest-android-hw-a55-aarch64-shippable-startup-fenix-cold-view-nav-start", 1597 - "perftest-android-hw-a55-aarch64-shippable-startup-fenix-homeview-startup", 1598 - "perftest-android-hw-a55-aarch64-shippable-startup-fenix-newssite-applink-startup", 1599 - "perftest-android-hw-a55-aarch64-shippable-startup-fenix-shopify-applink-startup", 1600 - "perftest-android-hw-a55-aarch64-shippable-startup-fenix-tab-restore-shopify" 1601 - "test-windows11-64-24h2-shippable/opt-browsertime-benchmark-firefox-speedometer3", 1602 """ 1603 retrigger_count = 4 1604 for name, task in full_task_graph.tasks.items(): 1605 if ( 1606 ( 1607 "perftest-android-hw-a55-aarch64-shippable-startup-fenix" in task.label 1608 and "simple" not in task.label 1609 ) 1610 or "test-windows11-64-24h2-shippable/opt-browsertime-benchmark-firefox-speedometer3" 1611 in task.label 1612 ): 1613 task.attributes["task_duplicates"] = retrigger_count 1614 yield name 1615 1616 1617 @register_target_task("eslint-build") 1618 def target_tasks_eslint_build(full_task_graph, parameters, graph_config): 1619 """Select the task to run additional ESLint rules which require a build.""" 1620 1621 for name, task in full_task_graph.tasks.items(): 1622 if task.kind != "source-test": 1623 continue 1624 if "eslint-build" in name: 1625 yield name 1626 1627 1628 @register_target_task("holly_tasks") 1629 def target_tasks_holly(full_task_graph, parameters, graph_config): 1630 """Bug 1814661: only run updatebot tasks on holly""" 1631 1632 def filter(task): 1633 return task.kind == "updatebot" 1634 1635 return [l for l, t in full_task_graph.tasks.items() if filter(t)] 1636 1637 1638 @register_target_task("snap_upstream_tasks") 1639 def target_tasks_snap_upstream_tasks(full_task_graph, parameters, graph_config): 1640 """ 1641 Select tasks for building/testing Snap package built as upstream. Omit -try 1642 because it does not really make sense on a m-c cron 1643 1644 Use test tasks for linux64 builds and only builds for arm* until there is 1645 support for running tests (bug 1855463) 1646 """ 1647 for name, task in full_task_graph.tasks.items(): 1648 if "snap-upstream" in name and not "-local" in name: 1649 yield name 1650 1651 1652 @register_target_task("nightly-android") 1653 def target_tasks_nightly_android(full_task_graph, parameters, graph_config): 1654 def filter(task, parameters): 1655 # geckoview 1656 if task.attributes.get("shipping_product") == "fennec" and task.kind in ( 1657 "beetmover-geckoview", 1658 "upload-symbols", 1659 ): 1660 return True 1661 1662 # fenix/focus/a-c 1663 build_type = task.attributes.get("build-type", "") 1664 return build_type in ( 1665 "nightly", 1666 "focus-nightly", 1667 "fenix-nightly", 1668 "fenix-nightly-firebase", 1669 "focus-nightly-firebase", 1670 ) 1671 1672 for platform in ("android", "all"): 1673 index_path = ( 1674 f"{graph_config['trust-domain']}.v2.{parameters['project']}.revision." 1675 f"{parameters['head_rev']}.taskgraph.decision-nightly-{platform}" 1676 ) 1677 if os.environ.get("MOZ_AUTOMATION") and retry( 1678 index_exists, 1679 args=(index_path,), 1680 kwargs={ 1681 "reason": "to avoid triggering multiple nightlies off the same revision", 1682 }, 1683 ): 1684 return [] 1685 1686 return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] 1687 1688 1689 @register_target_task("android-l10n-import") 1690 def target_tasks_android_l10n_import(full_task_graph, parameters, graph_config): 1691 return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-import"] 1692 1693 1694 @register_target_task("android-l10n-sync") 1695 def target_tasks_android_l10n_sync(full_task_graph, parameters, graph_config): 1696 return [l for l, t in full_task_graph.tasks.items() if l == "android-l10n-sync"] 1697 1698 1699 @register_target_task("os-integration") 1700 def target_tasks_os_integration(full_task_graph, parameters, graph_config): 1701 candidate_attrs = load_yaml(os.path.join(TEST_CONFIGS, "os-integration.yml")) 1702 1703 labels = [] 1704 for label, task in full_task_graph.tasks.items(): 1705 if task.kind not in TEST_KINDS + ("source-test", "perftest", "startup-test"): 1706 continue 1707 1708 # Match tasks against attribute sets defined in os-integration.yml. 1709 if not any(attrmatch(task.attributes, **c) for c in candidate_attrs): 1710 continue 1711 1712 if not is_try(parameters): 1713 # Only run hardware tasks if scheduled from try. We do this because 1714 # the `cron` task is designed to provide a base for testing worker 1715 # images, which isn't something that impacts our hardware pools. 1716 if ( 1717 task.attributes.get("build_platform") == "macosx64" 1718 or "android-hw" in label 1719 ): 1720 continue 1721 1722 # Perform additional filtering for non-try repos. We don't want to 1723 # limit what can be scheduled on try as os-integration tests are still 1724 # useful for manual verification of things. 1725 if not ( 1726 filter_for_project(task, parameters) 1727 and filter_for_hg_branch(task, parameters) 1728 and filter_tests_without_manifests(task, parameters) 1729 ): 1730 continue 1731 1732 labels.append(label) 1733 return labels 1734 1735 1736 @register_target_task("weekly-test-info") 1737 def target_tasks_weekly_test_info(full_task_graph, parameters, graph_config): 1738 return ["source-test-file-metadata-test-info-all"] 1739 1740 1741 @register_target_task("test-info-xpcshell-timings-daily") 1742 def target_tasks_test_info_xpcshell_timings_daily( 1743 full_task_graph, parameters, graph_config 1744 ): 1745 return ["source-test-file-metadata-test-info-xpcshell-timings-daily"]