release_promotion.py (16599B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import json 7 import os 8 9 from taskcluster.exceptions import TaskclusterRestFailure 10 from taskgraph.parameters import Parameters 11 from taskgraph.taskgraph import TaskGraph 12 from taskgraph.util.taskcluster import get_artifact, list_task_group_incomplete_tasks 13 14 from gecko_taskgraph.actions.registry import register_callback_action 15 from gecko_taskgraph.decision import taskgraph_decision 16 from gecko_taskgraph.util.attributes import RELEASE_PROMOTION_PROJECTS, release_level 17 from gecko_taskgraph.util.partials import populate_release_history 18 from gecko_taskgraph.util.partners import ( 19 fix_partner_config, 20 get_partner_config_by_url, 21 get_partner_url_config, 22 get_token, 23 ) 24 from gecko_taskgraph.util.taskgraph import ( 25 find_decision_task, 26 find_existing_tasks_from_previous_kinds, 27 ) 28 29 RELEASE_PROMOTION_SIGNOFFS = ("mar-signing",) 30 31 32 def is_release_promotion_available(parameters): 33 return parameters["project"] in RELEASE_PROMOTION_PROJECTS 34 35 36 def get_partner_config(partner_url_config, github_token): 37 partner_config = {} 38 for kind, url in partner_url_config.items(): 39 if url: 40 partner_config[kind] = get_partner_config_by_url(url, kind, github_token) 41 return partner_config 42 43 44 def get_signoff_properties(): 45 props = {} 46 for signoff in RELEASE_PROMOTION_SIGNOFFS: 47 props[signoff] = { 48 "type": "string", 49 } 50 return props 51 52 53 def get_required_signoffs(input, parameters): 54 input_signoffs = set(input.get("required_signoffs", [])) 55 params_signoffs = set(parameters["required_signoffs"] or []) 56 return sorted(list(input_signoffs | params_signoffs)) 57 58 59 def get_signoff_urls(input, parameters): 60 signoff_urls = parameters["signoff_urls"] 61 signoff_urls.update(input.get("signoff_urls", {})) 62 return signoff_urls 63 64 65 def get_flavors(graph_config, param): 66 """ 67 Get all flavors with the given parameter enabled. 68 """ 69 promotion_flavors = graph_config["release-promotion"]["flavors"] 70 return sorted( 71 flavor 72 for (flavor, config) in promotion_flavors.items() 73 if config.get(param, False) 74 ) 75 76 77 @register_callback_action( 78 name="release-promotion", 79 title="Release Promotion", 80 symbol="${input.release_promotion_flavor}", 81 description="Promote a release.", 82 permission="release-promotion", 83 order=500, 84 context=[], 85 available=is_release_promotion_available, 86 schema=lambda graph_config: { 87 "type": "object", 88 "properties": { 89 "build_number": { 90 "type": "integer", 91 "default": 1, 92 "minimum": 1, 93 "title": "The release build number", 94 "description": ( 95 "The release build number. Starts at 1 per " 96 "release version, and increments on rebuild." 97 ), 98 }, 99 "do_not_optimize": { 100 "type": "array", 101 "description": ( 102 "Optional: a list of labels to avoid optimizing out " 103 "of the graph (to force a rerun of, say, " 104 "funsize docker-image tasks)." 105 ), 106 "items": { 107 "type": "string", 108 }, 109 }, 110 "revision": { 111 "type": "string", 112 "title": "Optional: revision to promote", 113 "description": ( 114 "Optional: the revision to promote. If specified, " 115 "and `previous_graph_kinds is not specified, find the " 116 "push graph to promote based on the revision." 117 ), 118 }, 119 "release_promotion_flavor": { 120 "type": "string", 121 "description": "The flavor of release promotion to perform.", 122 "default": "FILL ME OUT", 123 "enum": sorted(graph_config["release-promotion"]["flavors"].keys()), 124 }, 125 "rebuild_kinds": { 126 "type": "array", 127 "description": ( 128 "Optional: an array of kinds to ignore from the previous graph(s)." 129 ), 130 "default": graph_config["release-promotion"].get("rebuild-kinds", []), 131 "items": { 132 "type": "string", 133 }, 134 }, 135 "previous_graph_ids": { 136 "type": "array", 137 "description": ( 138 "Optional: an array of taskIds of decision or action " 139 "tasks from the previous graph(s) to use to populate " 140 "our `previous_graph_kinds`." 141 ), 142 "items": { 143 "type": "string", 144 }, 145 }, 146 "version": { 147 "type": "string", 148 "description": ( 149 "Optional: override the version for release promotion. " 150 "Occasionally we'll land a taskgraph fix in a later " 151 "commit, but want to act on a build from a previous " 152 "commit. If a version bump has landed in the meantime, " 153 "relying on the in-tree version will break things." 154 ), 155 "default": "", 156 }, 157 "next_version": { 158 "type": "string", 159 "description": ( 160 "Next version. Required in the following flavors: {}".format( 161 get_flavors(graph_config, "version-bump") 162 ) 163 ), 164 "default": "", 165 }, 166 # Example: 167 # 'partial_updates': { 168 # '38.0': { 169 # 'buildNumber': 1, 170 # 'locales': ['de', 'en-GB', 'ru', 'uk', 'zh-TW'] 171 # }, 172 # '37.0': { 173 # 'buildNumber': 2, 174 # 'locales': ['de', 'en-GB', 'ru', 'uk'] 175 # } 176 # } 177 "partial_updates": { 178 "type": "object", 179 "description": ( 180 "Partial updates. Required in the following flavors: {}".format( 181 get_flavors(graph_config, "partial-updates") 182 ) 183 ), 184 "default": {}, 185 "additionalProperties": { 186 "type": "object", 187 "properties": { 188 "buildNumber": { 189 "type": "number", 190 }, 191 "locales": { 192 "type": "array", 193 "items": { 194 "type": "string", 195 }, 196 }, 197 }, 198 "required": [ 199 "buildNumber", 200 "locales", 201 ], 202 "additionalProperties": False, 203 }, 204 }, 205 "release_eta": { 206 "type": "string", 207 "default": "", 208 }, 209 "release_enable_partner_repack": { 210 "type": "boolean", 211 "default": False, 212 "description": "Toggle for creating partner repacks", 213 }, 214 "release_enable_partner_attribution": { 215 "type": "boolean", 216 "default": False, 217 "description": "Toggle for creating partner attribution", 218 }, 219 "release_partner_build_number": { 220 "type": "integer", 221 "default": 1, 222 "minimum": 1, 223 "description": ( 224 "The partner build number. This translates to, e.g. " 225 "`v1` in the path. We generally only have to " 226 "bump this on off-cycle partner rebuilds." 227 ), 228 }, 229 "release_partners": { 230 "type": "array", 231 "description": ( 232 "A list of partners to repack, or if null or empty then use " 233 "the current full set" 234 ), 235 "items": { 236 "type": "string", 237 }, 238 }, 239 "release_partner_config": { 240 "type": "object", 241 "description": "Partner configuration to use for partner repacks.", 242 "properties": {}, 243 "additionalProperties": True, 244 }, 245 "release_enable_emefree": { 246 "type": "boolean", 247 "default": False, 248 "description": "Toggle for creating EME-free repacks", 249 }, 250 "required_signoffs": { 251 "type": "array", 252 "description": ("The flavor of release promotion to perform."), 253 "items": { 254 "enum": RELEASE_PROMOTION_SIGNOFFS, 255 }, 256 }, 257 "signoff_urls": { 258 "type": "object", 259 "default": {}, 260 "additionalProperties": False, 261 "properties": get_signoff_properties(), 262 }, 263 }, 264 "required": ["release_promotion_flavor", "build_number"], 265 }, 266 ) 267 def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): 268 release_promotion_flavor = input["release_promotion_flavor"] 269 promotion_config = graph_config["release-promotion"]["flavors"][ 270 release_promotion_flavor 271 ] 272 release_history = {} 273 product = promotion_config["product"] 274 275 next_version = str(input.get("next_version") or "") 276 if promotion_config.get("version-bump", False): 277 # We force str() the input, hence the 'None' 278 if next_version in ["", "None"]: 279 raise Exception( 280 f"`next_version` property needs to be provided for `{release_promotion_flavor}` " 281 "target." 282 ) 283 284 if promotion_config.get("partial-updates", False): 285 partial_updates = input.get("partial_updates", {}) 286 if not partial_updates and release_level(parameters) == "production": 287 raise Exception( 288 f"`partial_updates` property needs to be provided for `{release_promotion_flavor}`" 289 "target." 290 ) 291 balrog_prefix = product.title() 292 os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True) 293 release_history = populate_release_history( 294 balrog_prefix, parameters["project"], partial_updates=partial_updates 295 ) 296 297 target_tasks_method = promotion_config["target-tasks-method"].format( 298 project=parameters["project"] 299 ) 300 rebuild_kinds = input.get( 301 "rebuild_kinds", promotion_config.get("rebuild-kinds", []) 302 ) 303 do_not_optimize = input.get( 304 "do_not_optimize", promotion_config.get("do-not-optimize", []) 305 ) 306 307 # Make sure no pending tasks remain from a previous run 308 own_task_id = os.environ.get("TASK_ID", "") 309 try: 310 for t in list_task_group_incomplete_tasks(own_task_id): 311 if t == own_task_id: 312 continue 313 raise Exception( 314 f"task group has unexpected pre-existing incomplete tasks (e.g. {t})" 315 ) 316 except TaskclusterRestFailure as e: 317 # 404 means the task group doesn't exist yet, and we're fine 318 if e.status_code != 404: 319 raise 320 321 # Build previous_graph_ids from ``previous_graph_ids``, ``revision``, 322 # or the action parameters. 323 previous_graph_ids = input.get("previous_graph_ids") 324 if not previous_graph_ids: 325 revision = input.get("revision") 326 if revision: 327 head_rev_param = "{}head_rev".format( 328 graph_config["project-repo-param-prefix"] 329 ) 330 push_parameters = { 331 head_rev_param: revision, 332 "project": parameters["project"], 333 } 334 else: 335 push_parameters = parameters 336 previous_graph_ids = [find_decision_task(push_parameters, graph_config)] 337 338 # Download parameters from the first decision task 339 parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") 340 # Download and combine full task graphs from each of the previous_graph_ids. 341 # Sometimes previous relpro action tasks will add tasks, like partials, 342 # that didn't exist in the first full_task_graph, so combining them is 343 # important. The rightmost graph should take precedence in the case of 344 # conflicts. 345 combined_full_task_graph = {} 346 for graph_id in previous_graph_ids: 347 full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") 348 combined_full_task_graph.update(full_task_graph) 349 _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) 350 parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( 351 combined_full_task_graph, previous_graph_ids, rebuild_kinds 352 ) 353 parameters["do_not_optimize"] = do_not_optimize 354 parameters["target_tasks_method"] = target_tasks_method 355 parameters["build_number"] = int(input["build_number"]) 356 parameters["next_version"] = next_version 357 parameters["release_history"] = release_history 358 if promotion_config.get("is-rc"): 359 parameters["release_type"] += "-rc" 360 parameters["release_eta"] = input.get("release_eta", "") 361 parameters["release_product"] = product 362 # When doing staging releases on try, we still want to re-use tasks from 363 # previous graphs. 364 parameters["optimize_target_tasks"] = True 365 366 if release_promotion_flavor == "promote_firefox_partner_repack": 367 release_enable_partner_repack = True 368 release_enable_partner_attribution = False 369 release_enable_emefree = False 370 elif release_promotion_flavor == "promote_firefox_partner_attribution": 371 release_enable_partner_repack = False 372 release_enable_partner_attribution = True 373 release_enable_emefree = False 374 else: 375 # for promotion or ship phases, we use the action input to turn the repacks/attribution off 376 release_enable_partner_repack = input["release_enable_partner_repack"] 377 release_enable_partner_attribution = input["release_enable_partner_attribution"] 378 release_enable_emefree = input["release_enable_emefree"] 379 380 partner_url_config = get_partner_url_config(parameters, graph_config) 381 if ( 382 release_enable_partner_repack 383 and not partner_url_config["release-partner-repack"] 384 ): 385 raise Exception("Can't enable partner repacks when no config url found") 386 if ( 387 release_enable_partner_attribution 388 and not partner_url_config["release-partner-attribution"] 389 ): 390 raise Exception("Can't enable partner attribution when no config url found") 391 if release_enable_emefree and not partner_url_config["release-eme-free-repack"]: 392 raise Exception("Can't enable EMEfree repacks when no config url found") 393 parameters["release_enable_partner_repack"] = release_enable_partner_repack 394 parameters["release_enable_partner_attribution"] = ( 395 release_enable_partner_attribution 396 ) 397 parameters["release_enable_emefree"] = release_enable_emefree 398 399 partner_config = input.get("release_partner_config") 400 if not partner_config and any([ 401 release_enable_partner_repack, 402 release_enable_partner_attribution, 403 release_enable_emefree, 404 ]): 405 github_token = get_token(parameters) 406 partner_config = get_partner_config(partner_url_config, github_token) 407 if partner_config: 408 parameters["release_partner_config"] = fix_partner_config(partner_config) 409 parameters["release_partners"] = input.get("release_partners") 410 if input.get("release_partner_build_number"): 411 parameters["release_partner_build_number"] = input[ 412 "release_partner_build_number" 413 ] 414 415 if input["version"]: 416 parameters["version"] = input["version"] 417 418 parameters["required_signoffs"] = get_required_signoffs(input, parameters) 419 parameters["signoff_urls"] = get_signoff_urls(input, parameters) 420 421 # make parameters read-only 422 parameters = Parameters(**parameters) 423 424 taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)