decision.py (18749B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import logging 6 import os 7 import shutil 8 import sys 9 import time 10 from collections import defaultdict 11 from pathlib import Path 12 13 import yaml 14 from redo import retry 15 from taskgraph import create 16 from taskgraph.create import create_tasks 17 from taskgraph.generator import TaskGraphGenerator 18 from taskgraph.main import format_kind_graph_mermaid 19 from taskgraph.parameters import Parameters 20 from taskgraph.taskgraph import TaskGraph 21 from taskgraph.util import json 22 from taskgraph.util.python_path import find_object 23 from taskgraph.util.taskcluster import get_artifact 24 from taskgraph.util.vcs import get_repository 25 from taskgraph.util.yaml import load_yaml 26 27 from . import GECKO 28 from .actions import render_actions_json 29 from .files_changed import get_changed_files 30 from .parameters import get_app_version, get_version 31 from .util.backstop import ANDROID_PERFTEST_BACKSTOP_INDEX, BACKSTOP_INDEX, is_backstop 32 from .util.bugbug import push_schedules 33 from .util.hg import get_hg_revision_branch, get_hg_revision_info 34 from .util.partials import populate_release_history 35 from .util.taskcluster import insert_index 36 from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds 37 38 logger = logging.getLogger(__name__) 39 40 ARTIFACTS_DIR = os.environ.get("MOZ_UPLOAD_DIR", "artifacts") 41 42 # For each project, this gives a set of parameters specific to the project. 43 # See `taskcluster/docs/parameters.rst` for information on parameters. 44 PER_PROJECT_PARAMETERS = { 45 "try": { 46 "enable_always_target": True, 47 "target_tasks_method": "try_tasks", 48 "release_type": "nightly", 49 }, 50 "kaios-try": { 51 "target_tasks_method": "try_tasks", 52 }, 53 "ash": { 54 "target_tasks_method": "default", 55 }, 56 "cedar": { 57 "target_tasks_method": "default", 58 }, 59 "holly": { 60 "enable_always_target": True, 61 "target_tasks_method": "holly_tasks", 62 }, 63 "oak": { 64 "target_tasks_method": "default", 65 "release_type": "nightly-oak", 66 }, 67 "graphics": { 68 "target_tasks_method": "graphics_tasks", 69 }, 70 "autoland": { 71 "optimize_strategies": "gecko_taskgraph.optimize:project.autoland", 72 "target_tasks_method": "autoland_tasks", 73 "test_manifest_loader": "bugbug", # Remove this line to disable "manifest scheduling". 74 }, 75 "mozilla-central": { 76 "target_tasks_method": "mozilla_central_tasks", 77 "release_type": "nightly", 78 }, 79 "mozilla-beta": { 80 "target_tasks_method": "mozilla_beta_tasks", 81 "release_type": "beta", 82 }, 83 "mozilla-release": { 84 "target_tasks_method": "mozilla_release_tasks", 85 "release_type": "release", 86 }, 87 "mozilla-esr140": { 88 "target_tasks_method": "mozilla_esr140_tasks", 89 "release_type": "esr140", 90 }, 91 "pine": { 92 "target_tasks_method": "pine_tasks", 93 "release_type": "nightly-pine", 94 }, 95 "maple": { 96 # Prevent it from running everything - For now just use "Add New Jobs" 97 "target_tasks_method": "nothing", 98 "release_type": "release", 99 }, 100 "cypress": { 101 "target_tasks_method": "cypress_tasks", 102 "release_type": "nightly-cypress", 103 }, 104 "larch": { 105 "target_tasks_method": "larch_tasks", 106 "release_type": "nightly-larch", 107 }, 108 "kaios": { 109 "target_tasks_method": "kaios_tasks", 110 }, 111 "toolchains": { 112 "target_tasks_method": "mozilla_central_tasks", 113 }, 114 # git projects 115 "staging-firefox": { 116 "target_tasks_method": "default", 117 }, 118 # the default parameters are used for projects that do not match above. 119 "default": { 120 "target_tasks_method": "default", 121 }, 122 } 123 124 125 def full_task_graph_to_runnable_jobs(full_task_json): 126 runnable_jobs = {} 127 for label, node in full_task_json.items(): 128 if not ("extra" in node["task"] and "treeherder" in node["task"]["extra"]): 129 continue 130 131 th = node["task"]["extra"]["treeherder"] 132 runnable_jobs[label] = {"symbol": th["symbol"]} 133 134 for i in ("groupName", "groupSymbol", "collection"): 135 if i in th: 136 runnable_jobs[label][i] = th[i] 137 if th.get("machine", {}).get("platform"): 138 runnable_jobs[label]["platform"] = th["machine"]["platform"] 139 return runnable_jobs 140 141 142 def full_task_graph_to_manifests_by_task(full_task_json): 143 manifests_by_task = defaultdict(list) 144 for label, node in full_task_json.items(): 145 manifests = node["attributes"].get("test_manifests") 146 if not manifests: 147 continue 148 149 manifests_by_task[label].extend(manifests) 150 return manifests_by_task 151 152 153 def try_syntax_from_message(message): 154 """ 155 Parse the try syntax out of a commit message, returning '' if none is 156 found. 157 """ 158 try_idx = message.find("try:") 159 if try_idx == -1: 160 return "" 161 return message[try_idx:].split("\n", 1)[0] 162 163 164 def taskgraph_decision(options, parameters=None): 165 """ 166 Run the decision task. This function implements `mach taskgraph decision`, 167 and is responsible for 168 169 * processing decision task command-line options into parameters 170 * running task-graph generation exactly the same way the other `mach 171 taskgraph` commands do 172 * generating a set of artifacts to memorialize the graph 173 * calling TaskCluster APIs to create the graph 174 """ 175 176 parameters = parameters or ( 177 lambda graph_config: get_decision_parameters(graph_config, options) 178 ) 179 180 decision_task_id = os.environ["TASK_ID"] 181 182 # create a TaskGraphGenerator instance 183 tgg = TaskGraphGenerator( 184 root_dir=options.get("root"), 185 parameters=parameters, 186 decision_task_id=decision_task_id, 187 write_artifacts=True, 188 enable_verifications=options.get("verify", True), 189 ) 190 191 if not create.testing: 192 # set additional index paths for the decision task 193 set_decision_indexes(decision_task_id, tgg.parameters, tgg.graph_config) 194 195 # write out the parameters used to generate this graph 196 write_artifact("parameters.yml", dict(**tgg.parameters)) 197 198 # write out the public/actions.json file 199 write_artifact( 200 "actions.json", 201 render_actions_json(tgg.parameters, tgg.graph_config, decision_task_id), 202 ) 203 204 # write out the full graph for reference 205 full_task_json = tgg.full_task_graph.to_json() 206 write_artifact("full-task-graph.json", full_task_json) 207 208 # write out kind graph 209 write_artifact("kind-graph.mm", format_kind_graph_mermaid(tgg.kind_graph)) 210 211 # write out the public/runnable-jobs.json file 212 write_artifact( 213 "runnable-jobs.json", full_task_graph_to_runnable_jobs(full_task_json) 214 ) 215 216 # write out the public/manifests-by-task.json file 217 write_artifact( 218 "manifests-by-task.json.gz", 219 full_task_graph_to_manifests_by_task(full_task_json), 220 ) 221 222 # `tests-by-manifest.json.gz` was previously written out here 223 # it was moved to `loader/test.py` because its contents now depend on 224 # data generated in a subprocess which we do not have access to here 225 # see https://bugzilla.mozilla.org/show_bug.cgi?id=1989038 for additional 226 # details 227 228 # this is just a test to check whether the from_json() function is working 229 _, _ = TaskGraph.from_json(full_task_json) 230 231 # write out the target task set to allow reproducing this as input 232 write_artifact("target-tasks.json", list(tgg.target_task_set.tasks.keys())) 233 234 # write out the optimized task graph to describe what will actually happen, 235 # and the map of labels to taskids 236 write_artifact("task-graph.json", tgg.morphed_task_graph.to_json()) 237 write_artifact("label-to-taskid.json", tgg.label_to_taskid) 238 239 # write bugbug scheduling information if it was invoked 240 if len(push_schedules) > 0: 241 write_artifact("bugbug-push-schedules.json", push_schedules.popitem()[1]) 242 243 # upload run-task, fetch-content, robustcheckout.py and more as artifacts 244 mozharness_dir = Path(GECKO, "testing", "mozharness") 245 scripts_dir = Path(GECKO, "taskcluster", "scripts") 246 taskgraph_dir = Path( 247 GECKO, "third_party", "python", "taskcluster_taskgraph", "taskgraph" 248 ) 249 to_copy = { 250 scripts_dir / "run-task": f"{ARTIFACTS_DIR}/run-task-hg", 251 scripts_dir / "tester" / "test-linux.sh": ARTIFACTS_DIR, 252 taskgraph_dir / "run-task" / "fetch-content": ARTIFACTS_DIR, 253 taskgraph_dir / "run-task" / "run-task": f"{ARTIFACTS_DIR}/run-task-git", 254 mozharness_dir / "external_tools" / "robustcheckout.py": ARTIFACTS_DIR, 255 } 256 for target, dest in to_copy.items(): 257 shutil.copy2(target, dest) 258 259 # actually create the graph 260 create_tasks( 261 tgg.graph_config, 262 tgg.morphed_task_graph, 263 tgg.label_to_taskid, 264 tgg.parameters, 265 decision_task_id=decision_task_id, 266 ) 267 268 269 def get_decision_parameters(graph_config, options): 270 """ 271 Load parameters from the command-line options for 'taskgraph decision'. 272 This also applies per-project parameters, based on the given project. 273 274 """ 275 product_dir = graph_config["product-dir"] 276 277 parameters = { 278 n: options[n] 279 for n in [ 280 "base_repository", 281 "base_ref", 282 "base_rev", 283 "head_repository", 284 "head_rev", 285 "head_ref", 286 "head_tag", 287 "project", 288 "pushlog_id", 289 "pushdate", 290 "owner", 291 "level", 292 "repository_type", 293 "target_tasks_method", 294 "tasks_for", 295 ] 296 if n in options 297 } 298 299 repo_path = os.getcwd() 300 repo = get_repository(repo_path) 301 302 try: 303 commit_message = repo.get_commit_message() 304 except UnicodeDecodeError: 305 commit_message = "" 306 307 # Set some vcs specific parameters 308 if parameters["repository_type"] == "hg": 309 if head_git_rev := get_hg_revision_info( 310 GECKO, revision=parameters["head_rev"], info="extras.git_commit" 311 ): 312 parameters["head_git_rev"] = head_git_rev 313 314 parameters["hg_branch"] = get_hg_revision_branch( 315 GECKO, revision=parameters["head_rev"] 316 ) 317 318 parameters["files_changed"] = sorted( 319 get_changed_files(parameters["head_repository"], parameters["head_rev"]) 320 ) 321 322 elif parameters["repository_type"] == "git": 323 parameters["hg_branch"] = None 324 parameters["files_changed"] = repo.get_changed_files( 325 rev=parameters["head_rev"], base=parameters["base_rev"] 326 ) 327 328 # Define default filter list, as most configurations shouldn't need 329 # custom filters. 330 parameters["filters"] = [ 331 "target_tasks_method", 332 ] 333 parameters["enable_always_target"] = ["docker-image"] 334 parameters["existing_tasks"] = {} 335 parameters["do_not_optimize"] = [] 336 parameters["build_number"] = 1 337 parameters["version"] = get_version(product_dir) 338 parameters["app_version"] = get_app_version(product_dir) 339 parameters["message"] = try_syntax_from_message(commit_message) 340 parameters["next_version"] = None 341 parameters["optimize_strategies"] = None 342 parameters["optimize_target_tasks"] = True 343 parameters["phabricator_diff"] = None 344 parameters["release_type"] = "" 345 parameters["release_eta"] = "" 346 parameters["release_enable_partner_repack"] = False 347 parameters["release_enable_partner_attribution"] = False 348 parameters["release_partners"] = [] 349 parameters["release_partner_config"] = {} 350 parameters["release_partner_build_number"] = 1 351 parameters["release_enable_emefree"] = False 352 parameters["release_product"] = None 353 parameters["required_signoffs"] = [] 354 parameters["signoff_urls"] = {} 355 parameters["test_manifest_loader"] = "default" 356 parameters["try_mode"] = None 357 parameters["try_task_config"] = {} 358 359 # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which 360 # case, fake it 361 if "@" not in parameters["owner"]: 362 parameters["owner"] += "@noreply.mozilla.org" 363 364 # use the pushdate as build_date if given, else use current time 365 parameters["build_date"] = parameters["pushdate"] or int(time.time()) 366 # moz_build_date is the build identifier based on build_date 367 parameters["moz_build_date"] = time.strftime( 368 "%Y%m%d%H%M%S", time.gmtime(parameters["build_date"]) 369 ) 370 371 project = parameters["project"] 372 try: 373 parameters.update(PER_PROJECT_PARAMETERS[project]) 374 except KeyError: 375 logger.warning( 376 f"using default project parameters; add {project} to " 377 f"PER_PROJECT_PARAMETERS in {__file__} to customize behavior " 378 "for this project" 379 ) 380 parameters.update(PER_PROJECT_PARAMETERS["default"]) 381 382 # `target_tasks_method` has higher precedence than `project` parameters 383 if options.get("target_tasks_method"): 384 parameters["target_tasks_method"] = options["target_tasks_method"] 385 386 # ..but can be overridden by the commit message: if it contains the special 387 # string "DONTBUILD" and this is an on-push decision task, then use the 388 # special 'nothing' target task method. 389 if "DONTBUILD" in commit_message and options["tasks_for"] == "hg-push": 390 parameters["target_tasks_method"] = "nothing" 391 392 if options.get("include_push_tasks"): 393 get_existing_tasks(options.get("rebuild_kinds", []), parameters, graph_config) 394 395 # If the target method is nightly, we should build partials. This means 396 # knowing what has been released previously. 397 # An empty release_history is fine, it just means no partials will be built 398 parameters.setdefault("release_history", dict()) 399 if "nightly" in parameters.get("target_tasks_method", ""): 400 parameters["release_history"] = populate_release_history("Firefox", project) 401 402 if options.get("try_task_config_file"): 403 task_config_file = os.path.abspath(options.get("try_task_config_file")) 404 else: 405 # if try_task_config.json is present, load it 406 task_config_file = os.path.join(os.getcwd(), "try_task_config.json") 407 408 # load try settings 409 if "try" in project and options["tasks_for"] == "hg-push": 410 set_try_config(parameters, task_config_file) 411 412 if options.get("optimize_target_tasks") is not None: 413 parameters["optimize_target_tasks"] = options["optimize_target_tasks"] 414 415 # Determine if this should be a backstop push. 416 parameters["backstop"] = is_backstop(parameters) 417 418 # For the android perf tasks, run them 50% less often 419 parameters["android_perftest_backstop"] = is_backstop( 420 parameters, 421 push_interval=30, 422 time_interval=60 * 6, 423 backstop_strategy="android_perftest_backstop", 424 ) 425 426 if "decision-parameters" in graph_config["taskgraph"]: 427 find_object(graph_config["taskgraph"]["decision-parameters"])( 428 graph_config, parameters 429 ) 430 431 result = Parameters(**parameters) 432 result.check() 433 return result 434 435 436 def get_existing_tasks(rebuild_kinds, parameters, graph_config): 437 """ 438 Find the decision task corresponding to the on-push graph, and return 439 a mapping of labels to task-ids from it. This will skip the kinds specificed 440 by `rebuild_kinds`. 441 """ 442 try: 443 decision_task = retry( 444 find_decision_task, 445 args=(parameters, graph_config), 446 attempts=4, 447 sleeptime=5 * 60, 448 ) 449 except Exception: 450 logger.exception("Didn't find existing push task.") 451 sys.exit(1) 452 _, task_graph = TaskGraph.from_json( 453 get_artifact(decision_task, "public/full-task-graph.json") 454 ) 455 parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( 456 task_graph, [decision_task], rebuild_kinds 457 ) 458 459 460 def set_try_config(parameters, task_config_file): 461 if os.path.isfile(task_config_file): 462 logger.info(f"using try tasks from {task_config_file}") 463 with open(task_config_file) as fh: 464 task_config = json.load(fh) 465 task_config_version = task_config.pop("version", 1) 466 if task_config_version == 1: 467 parameters["try_mode"] = "try_task_config" 468 parameters["try_task_config"] = task_config 469 elif task_config_version == 2: 470 parameters.update(task_config["parameters"]) 471 parameters["try_mode"] = "try_task_config" 472 else: 473 raise Exception( 474 f"Unknown `try_task_config.json` version: {task_config_version}" 475 ) 476 477 478 def set_decision_indexes(decision_task_id, params, graph_config): 479 index_paths = [] 480 if params["android_perftest_backstop"]: 481 index_paths.insert(0, ANDROID_PERFTEST_BACKSTOP_INDEX) 482 if params["backstop"]: 483 # When two Decision tasks run at nearly the same time, it's possible 484 # they both end up being backstops if the second checks the backstop 485 # index before the first inserts it. Insert this index first to reduce 486 # the chances of that happening. 487 index_paths.insert(0, BACKSTOP_INDEX) 488 489 subs = params.copy() 490 subs["trust-domain"] = graph_config["trust-domain"] 491 492 for index_path in index_paths: 493 insert_index(index_path.format(**subs), decision_task_id) 494 495 496 def write_artifact(filename, data): 497 logger.info(f"writing artifact file `{filename}`") 498 if not os.path.isdir(ARTIFACTS_DIR): 499 os.mkdir(ARTIFACTS_DIR) 500 path = os.path.join(ARTIFACTS_DIR, filename) 501 if filename.endswith(".yml"): 502 with open(path, "w") as f: 503 yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False) 504 elif filename.endswith(".json"): 505 with open(path, "w") as f: 506 json.dump(data, f) 507 elif filename.endswith(".json.gz"): 508 import gzip 509 510 with gzip.open(path, "wb") as f: 511 f.write(json.dumps(data).encode("utf-8")) 512 else: 513 with open(path, "w") as f: 514 f.write(data) 515 516 517 def read_artifact(filename): 518 path = os.path.join(ARTIFACTS_DIR, filename) 519 if filename.endswith(".yml"): 520 return load_yaml(path, filename) 521 if filename.endswith(".json"): 522 with open(path) as f: 523 return json.load(f) 524 if filename.endswith(".json.gz"): 525 import gzip 526 527 with gzip.open(path, "rb") as f: 528 return json.load(f) 529 else: 530 raise TypeError(f"Don't know how to read {filename}") 531 532 533 def rename_artifact(src, dest): 534 os.rename(os.path.join(ARTIFACTS_DIR, src), os.path.join(ARTIFACTS_DIR, dest))