scriptworker.py (33170B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """Make scriptworker.cot.verify more user friendly by making scopes dynamic. 5 6 Scriptworker uses certain scopes to determine which sets of credentials to use. 7 Certain scopes are restricted by branch in chain of trust verification, and are 8 checked again at the script level. This file provides functions to adjust 9 these scopes automatically by project; this makes pushing to try, forking a 10 project branch, and merge day uplifts more user friendly. 11 12 In the future, we may adjust scopes by other settings as well, e.g. different 13 scopes for `push-to-candidates` rather than `push-to-releases`, even if both 14 happen on mozilla-beta and mozilla-release. 15 16 Additional configuration is found in the :ref:`graph config <taskgraph-graph-config>`. 17 """ 18 19 import functools 20 import itertools 21 import json 22 import os 23 from datetime import datetime 24 25 import jsone 26 from mozbuild.util import memoize 27 from taskgraph.util.copy import deepcopy 28 from taskgraph.util.schema import resolve_keyed_by 29 from taskgraph.util.taskcluster import get_artifact_prefix 30 from taskgraph.util.yaml import load_yaml 31 32 # constants {{{1 33 """Map signing scope aliases to sets of projects. 34 35 Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r 36 use release signing. These data structures aren't set-up to handle different 37 scopes on the same repo, so we use a different set of them for DevEdition, and 38 callers are responsible for using the correct one (by calling the appropriate 39 helper below). More context on this in https://bugzilla.mozilla.org/show_bug.cgi?id=1358601. 40 41 We will need to add esr support at some point. Eventually we want to add 42 nuance so certain m-b and m-r tasks use dep or nightly signing, and we only 43 release sign when we have a signed-off set of candidate builds. This current 44 approach works for now, though. 45 46 This is a list of list-pairs, for ordering. 47 """ 48 SIGNING_SCOPE_ALIAS_TO_PROJECT = [ 49 [ 50 "all-nightly-branches", 51 { 52 "mozilla-central", 53 "comm-central", 54 # bug 1845368: pine is a permanent project branch used for testing 55 # nightly updates 56 "pine", 57 # bug 1877483: larch has similar needs for nightlies 58 "larch", 59 # maple is also an L3 branch: https://phabricator.services.mozilla.com/D184833 60 "maple", 61 # bug 1988213: cypress project branch 62 "cypress", 63 }, 64 ], 65 [ 66 "all-release-branches", 67 { 68 "mozilla-beta", 69 "mozilla-release", 70 "mozilla-esr115", 71 "mozilla-esr128", 72 "mozilla-esr140", 73 "comm-beta", 74 "comm-release", 75 "comm-esr115", 76 "comm-esr128", 77 "comm-esr140", 78 }, 79 ], 80 ] 81 82 """Map the signing scope aliases to the actual scopes. 83 """ 84 SIGNING_TYPES = { 85 "all-release-branches": "release-signing", 86 "all-nightly-branches": "nightly-signing", 87 "default": "dep-signing", 88 } 89 90 DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [ 91 [ 92 "beta", 93 { 94 "mozilla-beta", 95 }, 96 ] 97 ] 98 99 DEVEDITION_SIGNING_TYPES = { 100 "beta": "nightly-signing", 101 "default": "dep-signing", 102 } 103 104 """Map beetmover scope aliases to sets of projects. 105 """ 106 BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [ 107 [ 108 "all-nightly-branches", 109 { 110 "mozilla-central", 111 "comm-central", 112 # bug 1845368: pine is a permanent project branch used for testing 113 # nightly updates 114 "pine", 115 # bug 1877483: larch has similar needs for nightlies 116 "larch", 117 # bug 1988213: cypress project branch 118 "cypress", 119 }, 120 ], 121 [ 122 "all-release-branches", 123 { 124 "mozilla-beta", 125 "mozilla-release", 126 "mozilla-esr115", 127 "mozilla-esr128", 128 "mozilla-esr140", 129 "comm-beta", 130 "comm-release", 131 "comm-esr115", 132 "comm-esr128", 133 "comm-esr140", 134 }, 135 ], 136 ] 137 138 """Map the beetmover scope aliases to the actual scopes. 139 """ 140 BEETMOVER_BUCKET_SCOPES = { 141 "all-release-branches": "beetmover:bucket:release", 142 "all-nightly-branches": "beetmover:bucket:nightly", 143 "default": "beetmover:bucket:dep", 144 } 145 146 """Map the beetmover scope aliases to the actual scopes. 147 These are the scopes needed to import artifacts into the product delivery APT repos. 148 """ 149 BEETMOVER_APT_REPO_SCOPES = { 150 "all-release-branches": "beetmover:apt-repo:release", 151 "all-nightly-branches": "beetmover:apt-repo:nightly", 152 "default": "beetmover:apt-repo:dep", 153 } 154 155 """Map the beetmover scope aliases to the actual scopes. 156 These are the scopes needed to import artifacts into the product delivery YUM repos. 157 """ 158 BEETMOVER_YUM_REPO_SCOPES = { 159 "all-release-branches": "beetmover:yum-repo:release", 160 "all-nightly-branches": "beetmover:yum-repo:nightly", 161 "default": "beetmover:yum-repo:dep", 162 } 163 164 """Map the beetmover tasks aliases to the actual action scopes. 165 """ 166 BEETMOVER_ACTION_SCOPES = { 167 "nightly": "beetmover:action:push-to-nightly", 168 # bug 1845368: pine is a permanent project branch used for testing 169 # nightly updates 170 "nightly-pine": "beetmover:action:push-to-nightly", 171 # bug 1877483: larch has similar needs for nightlies 172 "nightly-larch": "beetmover:action:push-to-nightly", 173 # bug 1988213: cypress project branch 174 "nightly-cypress": "beetmover:action:push-to-nightly", 175 "default": "beetmover:action:push-to-candidates", 176 } 177 178 """Map the beetmover tasks aliases to the actual action scopes. 179 The action scopes are generic across different repo types. 180 """ 181 BEETMOVER_REPO_ACTION_SCOPES = { 182 "default": "beetmover:action:import-from-gcs-to-artifact-registry", 183 } 184 185 """Known balrog actions.""" 186 BALROG_ACTIONS = ( 187 "submit-locale", 188 "submit-toplevel", 189 "schedule", 190 "v2-submit-locale", 191 "v2-submit-toplevel", 192 ) 193 194 """Map balrog scope aliases to sets of projects. 195 196 This is a list of list-pairs, for ordering. 197 """ 198 BALROG_SCOPE_ALIAS_TO_PROJECT = [ 199 [ 200 "nightly", 201 { 202 "mozilla-central", 203 "comm-central", 204 # bug 1845368: pine is a permanent project branch used for testing 205 # nightly updates 206 "pine", 207 # bug 1877483: larch has similar needs for nightlies 208 "larch", 209 # bug 1988213: cypress project branch 210 "cypress", 211 }, 212 ], 213 [ 214 "beta", 215 { 216 "mozilla-beta", 217 "comm-beta", 218 }, 219 ], 220 [ 221 "release", 222 { 223 "mozilla-release", 224 "comm-release", 225 }, 226 ], 227 [ 228 "esr115", 229 { 230 "mozilla-esr115", 231 "comm-esr115", 232 }, 233 ], 234 [ 235 "esr128", 236 { 237 "mozilla-esr128", 238 "comm-esr128", 239 }, 240 ], 241 [ 242 "esr140", 243 { 244 "mozilla-esr140", 245 "comm-esr140", 246 }, 247 ], 248 ] 249 250 """Map the balrog scope aliases to the actual scopes. 251 """ 252 BALROG_SERVER_SCOPES = { 253 "nightly": "balrog:server:nightly", 254 "aurora": "balrog:server:aurora", 255 "beta": "balrog:server:beta", 256 "release": "balrog:server:release", 257 "esr115": "balrog:server:esr", 258 "esr128": "balrog:server:esr", 259 "esr140": "balrog:server:esr", 260 "default": "balrog:server:dep", 261 } 262 263 264 """ The list of the release promotion phases which we send notifications for 265 """ 266 RELEASE_NOTIFICATION_PHASES = ("promote", "push", "ship") 267 268 269 def add_scope_prefix(config, scope): 270 """ 271 Prepends the scriptworker scope prefix from the :ref:`graph config 272 <taskgraph-graph-config>`. 273 274 Args: 275 config (TransformConfig): The configuration for the kind being transformed. 276 scope (string): The suffix of the scope 277 278 Returns: 279 string: the scope to use. 280 """ 281 return "{prefix}:{scope}".format( 282 prefix=config.graph_config["scriptworker"]["scope-prefix"], 283 scope=scope, 284 ) 285 286 287 def with_scope_prefix(f): 288 """ 289 Wraps a function, calling :py:func:`add_scope_prefix` on the result of 290 calling the wrapped function. 291 292 Args: 293 f (callable): A function that takes a ``config`` and some keyword 294 arguments, and returns a scope suffix. 295 296 Returns: 297 callable: the wrapped function 298 """ 299 300 @functools.wraps(f) 301 def wrapper(config, **kwargs): 302 scope_or_scopes = f(config, **kwargs) 303 if isinstance(scope_or_scopes, list): 304 return map(functools.partial(add_scope_prefix, config), scope_or_scopes) 305 return add_scope_prefix(config, scope_or_scopes) 306 307 return wrapper 308 309 310 def get_signing_type_from_project( 311 config, alias_to_project_map, alias_to_signing_type_map 312 ): 313 """Determine the restricted scope from `config.params['project']`. 314 315 Args: 316 config (TransformConfig): The configuration for the kind being transformed. 317 alias_to_project_map (list of lists): each list pair contains the 318 alias and the set of projects that match. This is ordered. 319 alias_to_signing_type_map (dict): the alias to signing type 320 321 Returns: 322 string: the scope to use. 323 """ 324 for alias, projects in alias_to_project_map: 325 if config.params["project"] in projects and alias in alias_to_signing_type_map: 326 return alias_to_signing_type_map[alias] 327 return alias_to_signing_type_map["default"] 328 329 330 # scope functions {{{1 331 @with_scope_prefix 332 def get_scope_from_project(config, alias_to_project_map, alias_to_scope_map): 333 """Determine the restricted scope from `config.params['project']`. 334 335 Args: 336 config (TransformConfig): The configuration for the kind being transformed. 337 alias_to_project_map (list of lists): each list pair contains the 338 alias and the set of projects that match. This is ordered. 339 alias_to_scope_map (dict): the alias alias to scope 340 341 Returns: 342 string: the scope to use. 343 """ 344 for alias, projects in alias_to_project_map: 345 if config.params["project"] in projects and alias in alias_to_scope_map: 346 return alias_to_scope_map[alias] 347 return alias_to_scope_map["default"] 348 349 350 @with_scope_prefix 351 def get_scope_from_release_type(config, release_type_to_scope_map): 352 """Determine the restricted scope from `config.params['target_tasks_method']`. 353 354 Args: 355 config (TransformConfig): The configuration for the kind being transformed. 356 release_type_to_scope_map (dict): the maps release types to scopes 357 358 Returns: 359 string: the scope to use. 360 """ 361 return release_type_to_scope_map.get( 362 config.params["release_type"], release_type_to_scope_map["default"] 363 ) 364 365 366 def get_phase_from_target_method(config, alias_to_tasks_map, alias_to_phase_map): 367 """Determine the phase from `config.params['target_tasks_method']`. 368 369 Args: 370 config (TransformConfig): The configuration for the kind being transformed. 371 alias_to_tasks_map (list of lists): each list pair contains the 372 alias and the set of target methods that match. This is ordered. 373 alias_to_phase_map (dict): the alias to phase map 374 375 Returns: 376 string: the phase to use. 377 """ 378 for alias, tasks in alias_to_tasks_map: 379 if ( 380 config.params["target_tasks_method"] in tasks 381 and alias in alias_to_phase_map 382 ): 383 return alias_to_phase_map[alias] 384 return alias_to_phase_map["default"] 385 386 387 get_signing_type = functools.partial( 388 get_signing_type_from_project, 389 alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT, 390 alias_to_signing_type_map=SIGNING_TYPES, 391 ) 392 393 get_devedition_signing_type = functools.partial( 394 get_signing_type_from_project, 395 alias_to_project_map=DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT, 396 alias_to_signing_type_map=DEVEDITION_SIGNING_TYPES, 397 ) 398 399 get_beetmover_bucket_scope = functools.partial( 400 get_scope_from_project, 401 alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, 402 alias_to_scope_map=BEETMOVER_BUCKET_SCOPES, 403 ) 404 405 get_beetmover_apt_repo_scope = functools.partial( 406 get_scope_from_project, 407 alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, 408 alias_to_scope_map=BEETMOVER_APT_REPO_SCOPES, 409 ) 410 411 get_beetmover_yum_repo_scope = functools.partial( 412 get_scope_from_project, 413 alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT, 414 alias_to_scope_map=BEETMOVER_YUM_REPO_SCOPES, 415 ) 416 417 get_beetmover_repo_action_scope = functools.partial( 418 get_scope_from_release_type, 419 release_type_to_scope_map=BEETMOVER_REPO_ACTION_SCOPES, 420 ) 421 422 get_beetmover_action_scope = functools.partial( 423 get_scope_from_release_type, 424 release_type_to_scope_map=BEETMOVER_ACTION_SCOPES, 425 ) 426 427 get_balrog_server_scope = functools.partial( 428 get_scope_from_project, 429 alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT, 430 alias_to_scope_map=BALROG_SERVER_SCOPES, 431 ) 432 433 cached_load_yaml = memoize(load_yaml) 434 435 436 # release_config {{{1 437 def get_release_config(config): 438 """Get the build number and version for a release task. 439 440 Currently only applies to beetmover tasks. 441 442 Args: 443 config (TransformConfig): The configuration for the kind being transformed. 444 445 Returns: 446 dict: containing both `build_number` and `version`. This can be used to 447 update `task.payload`. 448 """ 449 release_config = {} 450 451 partial_updates = os.environ.get("PARTIAL_UPDATES", "") 452 if partial_updates != "" and config.kind in ( 453 "release-bouncer-sub", 454 "release-bouncer-check", 455 "release-update-verify-config", 456 "release-secondary-update-verify-config", 457 "release-balrog-submit-toplevel", 458 "release-secondary-balrog-submit-toplevel", 459 ): 460 partial_updates = json.loads(partial_updates) 461 release_config["partial_versions"] = ", ".join([ 462 "{}build{}".format(v, info["buildNumber"]) 463 for v, info in partial_updates.items() 464 ]) 465 if release_config["partial_versions"] == "{}": 466 del release_config["partial_versions"] 467 468 release_config["version"] = config.params["version"] 469 release_config["appVersion"] = config.params["app_version"] 470 471 release_config["next_version"] = config.params["next_version"] 472 release_config["build_number"] = config.params["build_number"] 473 return release_config 474 475 476 def get_signing_type_per_platform(build_platform, is_shippable, config): 477 if "devedition" in build_platform: 478 return get_devedition_signing_type(config) 479 if is_shippable: 480 return get_signing_type(config) 481 return "dep-signing" 482 483 484 # generate_beetmover_upstream_artifacts {{{1 485 def generate_beetmover_upstream_artifacts( 486 config, job, platform, locale=None, dependencies=None, **kwargs 487 ): 488 """Generate the upstream artifacts for beetmover, using the artifact map. 489 490 Currently only applies to beetmover tasks. 491 492 Args: 493 job (dict): The current job being generated 494 dependencies (list): A list of the job's dependency labels. 495 platform (str): The current build platform 496 locale (str): The current locale being beetmoved. 497 498 Returns: 499 list: A list of dictionaries conforming to the upstream_artifacts spec. 500 """ 501 base_artifact_prefix = get_artifact_prefix(job) 502 resolve_keyed_by( 503 job, 504 "attributes.artifact_map", 505 "artifact map", 506 **{ 507 "release-type": config.params["release_type"], 508 "platform": platform, 509 }, 510 ) 511 map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) 512 upstream_artifacts = list() 513 514 if not locale: 515 locales = map_config["default_locales"] 516 elif isinstance(locale, list): 517 locales = locale 518 else: 519 locales = [locale] 520 521 if not dependencies: 522 if job.get("dependencies"): 523 dependencies = job["dependencies"].keys() 524 else: 525 raise Exception(f"Unsupported type of dependency. Got job: {job}") 526 527 for current_locale, dep in itertools.product(locales, dependencies): 528 paths = list() 529 530 for filename in map_config["mapping"]: 531 resolve_keyed_by( 532 map_config["mapping"][filename], 533 "from", 534 f"beetmover filename {filename}", 535 platform=platform, 536 ) 537 if dep not in map_config["mapping"][filename]["from"]: 538 continue 539 if ( 540 current_locale != "en-US" 541 and not map_config["mapping"][filename]["all_locales"] 542 ): 543 continue 544 if ( 545 "only_for_platforms" in map_config["mapping"][filename] 546 and platform 547 not in map_config["mapping"][filename]["only_for_platforms"] 548 ): 549 continue 550 if ( 551 "not_for_platforms" in map_config["mapping"][filename] 552 and platform in map_config["mapping"][filename]["not_for_platforms"] 553 ): 554 continue 555 if ( 556 "not_for_locales" in map_config["mapping"][filename] 557 and current_locale in map_config["mapping"][filename]["not_for_locales"] 558 ): 559 continue 560 if "partials_only" in map_config["mapping"][filename]: 561 continue 562 # The next time we look at this file it might be a different locale. 563 file_config = deepcopy(map_config["mapping"][filename]) 564 resolve_keyed_by( 565 file_config, 566 "source_path_modifier", 567 "source path modifier", 568 locale=current_locale, 569 ) 570 571 kwargs["locale"] = current_locale 572 573 paths.append( 574 os.path.join( 575 base_artifact_prefix, 576 jsone.render(file_config["source_path_modifier"], kwargs), 577 jsone.render(filename, kwargs), 578 ) 579 ) 580 581 if ( 582 job.get("dependencies") 583 and getattr(job["dependencies"][dep], "attributes", None) 584 and job["dependencies"][dep].attributes.get("release_artifacts") 585 ): 586 paths = [ 587 path 588 for path in paths 589 if path in job["dependencies"][dep].attributes["release_artifacts"] 590 ] 591 592 if not paths: 593 continue 594 595 upstream_artifacts.append({ 596 "taskId": {"task-reference": f"<{dep}>"}, 597 "taskType": map_config["tasktype_map"].get(dep), 598 "paths": sorted(paths), 599 "locale": current_locale, 600 }) 601 602 upstream_artifacts.sort(key=lambda u: u["paths"]) 603 return upstream_artifacts 604 605 606 def generate_artifact_registry_gcs_sources(dep): 607 gcs_sources = [] 608 locale = dep.attributes.get("locale") 609 if not locale: 610 repackage_deb_reference = "<repackage-deb>" 611 repackage_deb_artifact = "public/build/target.deb" 612 else: 613 repackage_deb_reference = "<repackage-deb-l10n>" 614 repackage_deb_artifact = f"public/build/{locale}/target.langpack.deb" 615 for config in dep.task["payload"]["artifactMap"]: 616 if ( 617 config["taskId"]["task-reference"] == repackage_deb_reference 618 and repackage_deb_artifact in config["paths"] 619 ): 620 gcs_sources.append( 621 config["paths"][repackage_deb_artifact]["destinations"][0] 622 ) 623 return gcs_sources 624 625 626 def generate_artifact_registry_gcs_sources_rpm(dep): 627 """Generate GCS sources for RPM packages from beetmover-repackage-rpm task. 628 629 The beetmover-repackage-rpm task contains all RPM packages (firefox + langpacks) 630 for a given platform in its artifactMap. This function extracts all destinations 631 from that artifactMap to upload to the YUM repository. 632 """ 633 gcs_sources = [] 634 for config in dep.task["payload"]["artifactMap"]: 635 if config["taskId"]["task-reference"] == "<repackage-rpm>": 636 for path_info in config["paths"].values(): 637 if "destinations" in path_info and path_info["destinations"]: 638 gcs_sources.append(path_info["destinations"][0]) 639 return gcs_sources 640 641 642 # generate_beetmover_artifact_map {{{1 643 def generate_beetmover_artifact_map(config, job, **kwargs): 644 """Generate the beetmover artifact map. 645 646 Currently only applies to beetmover tasks. 647 648 Args: 649 config (): Current taskgraph configuration. 650 job (dict): The current job being generated 651 Common kwargs: 652 platform (str): The current build platform 653 locale (str): The current locale being beetmoved. 654 655 Returns: 656 list: A list of dictionaries containing source->destination 657 maps for beetmover. 658 """ 659 platform = kwargs.get("platform", "") 660 resolve_keyed_by( 661 job, 662 "attributes.artifact_map", 663 job["label"], 664 **{ 665 "release-type": config.params["release_type"], 666 "platform": platform, 667 }, 668 ) 669 map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) 670 base_artifact_prefix = map_config.get( 671 "base_artifact_prefix", get_artifact_prefix(job) 672 ) 673 674 artifacts = list() 675 676 dependencies = job["dependencies"].keys() 677 678 if kwargs.get("locale"): 679 if isinstance(kwargs["locale"], list): 680 locales = kwargs["locale"] 681 else: 682 locales = [kwargs["locale"]] 683 else: 684 locales = map_config["default_locales"] 685 686 resolve_keyed_by(map_config, "s3_bucket_paths", job["label"], platform=platform) 687 688 for locale, dep in sorted(itertools.product(locales, dependencies)): 689 paths = dict() 690 for filename in map_config["mapping"]: 691 # Relevancy checks 692 resolve_keyed_by( 693 map_config["mapping"][filename], "from", "blah", platform=platform 694 ) 695 if dep not in map_config["mapping"][filename]["from"]: 696 # We don't get this file from this dependency. 697 continue 698 if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: 699 # This locale either doesn't produce or shouldn't upload this file. 700 continue 701 if ( 702 "only_for_platforms" in map_config["mapping"][filename] 703 and platform 704 not in map_config["mapping"][filename]["only_for_platforms"] 705 ): 706 # This platform either doesn't produce or shouldn't upload this file. 707 continue 708 if ( 709 "not_for_platforms" in map_config["mapping"][filename] 710 and platform in map_config["mapping"][filename]["not_for_platforms"] 711 ): 712 # This platform either doesn't produce or shouldn't upload this file. 713 continue 714 if ( 715 "not_for_locales" in map_config["mapping"][filename] 716 and locale in map_config["mapping"][filename]["not_for_locales"] 717 ): 718 # This locale either doesn't produce or shouldn't upload this file 719 continue 720 if "partials_only" in map_config["mapping"][filename]: 721 continue 722 723 # deepcopy because the next time we look at this file the locale will differ. 724 file_config = deepcopy(map_config["mapping"][filename]) 725 726 for field in [ 727 "destinations", 728 "locale_prefix", 729 "source_path_modifier", 730 "update_balrog_manifest", 731 "pretty_name", 732 "checksums_path", 733 ]: 734 resolve_keyed_by( 735 file_config, field, job["label"], locale=locale, platform=platform 736 ) 737 738 # This format string should ideally be in the configuration file, 739 # but this would mean keeping variable names in sync between code + config. 740 destinations = [ 741 "{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format( 742 s3_bucket_path=bucket_path, 743 dest_path=dest_path, 744 locale_prefix=file_config["locale_prefix"], 745 filename=file_config.get("pretty_name", filename), 746 ) 747 for dest_path, bucket_path in itertools.product( 748 file_config["destinations"], map_config["s3_bucket_paths"] 749 ) 750 ] 751 # Creating map entries 752 # Key must be artifact path, to avoid trampling duplicates, such 753 # as public/build/target.apk and public/build/en-US/target.apk 754 key = os.path.join( 755 base_artifact_prefix, 756 file_config["source_path_modifier"], 757 filename, 758 ) 759 760 paths[key] = { 761 "destinations": destinations, 762 } 763 if file_config.get("checksums_path"): 764 paths[key]["checksums_path"] = file_config["checksums_path"] 765 766 # optional flag: balrog manifest 767 if file_config.get("update_balrog_manifest"): 768 paths[key]["update_balrog_manifest"] = True 769 if file_config.get("balrog_format"): 770 paths[key]["balrog_format"] = file_config["balrog_format"] 771 772 # optional flag: expiry 773 if file_config.get("expiry"): 774 paths[key]["expiry"] = {"relative-datestamp": file_config["expiry"]} 775 776 if not paths: 777 # No files for this dependency/locale combination. 778 continue 779 780 # Render all variables for the artifact map 781 platforms = deepcopy(map_config.get("platform_names", {})) 782 if platform: 783 for key in platforms.keys(): 784 resolve_keyed_by(platforms, key, job["label"], platform=platform) 785 786 upload_date = datetime.fromtimestamp(config.params["build_date"]) 787 788 kwargs.update({ 789 "locale": locale, 790 "version": config.params["version"], 791 "branch": config.params["project"], 792 "build_number": config.params["build_number"], 793 "year": upload_date.year, 794 "month": upload_date.strftime("%m"), # zero-pad the month 795 "day": upload_date.strftime("%d"), 796 "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), 797 "head_rev": config.params["head_rev"], 798 }) 799 kwargs.update(**platforms) 800 paths = jsone.render(paths, kwargs) 801 artifacts.append({ 802 "taskId": {"task-reference": f"<{dep}>"}, 803 "locale": locale, 804 "paths": paths, 805 }) 806 807 return artifacts 808 809 810 # generate_beetmover_partials_artifact_map {{{1 811 def generate_beetmover_partials_artifact_map(config, job, partials_info, **kwargs): 812 """Generate the beetmover partials artifact map. 813 814 Currently only applies to beetmover tasks. 815 816 Args: 817 config (): Current taskgraph configuration. 818 job (dict): The current job being generated 819 partials_info (dict): Current partials and information about them in a dict 820 Common kwargs: 821 platform (str): The current build platform 822 locale (str): The current locale being beetmoved. 823 824 Returns: 825 list: A list of dictionaries containing source->destination 826 maps for beetmover. 827 """ 828 platform = kwargs.get("platform", "") 829 resolve_keyed_by( 830 job, 831 "attributes.artifact_map", 832 "artifact map", 833 **{ 834 "release-type": config.params["release_type"], 835 "platform": platform, 836 }, 837 ) 838 map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) 839 base_artifact_prefix = map_config.get( 840 "base_artifact_prefix", get_artifact_prefix(job) 841 ) 842 843 artifacts = list() 844 dependencies = job["dependencies"].keys() 845 846 if kwargs.get("locale"): 847 locales = [kwargs["locale"]] 848 else: 849 locales = map_config["default_locales"] 850 851 resolve_keyed_by( 852 map_config, "s3_bucket_paths", "s3_bucket_paths", platform=platform 853 ) 854 855 platforms = deepcopy(map_config.get("platform_names", {})) 856 if platform: 857 for key in platforms.keys(): 858 resolve_keyed_by(platforms, key, key, platform=platform) 859 upload_date = datetime.fromtimestamp(config.params["build_date"]) 860 861 for locale, dep in itertools.product(locales, dependencies): 862 paths = dict() 863 for filename in map_config["mapping"]: 864 # Relevancy checks 865 if dep not in map_config["mapping"][filename]["from"]: 866 # We don't get this file from this dependency. 867 continue 868 if locale != "en-US" and not map_config["mapping"][filename]["all_locales"]: 869 # This locale either doesn't produce or shouldn't upload this file. 870 continue 871 if "partials_only" not in map_config["mapping"][filename]: 872 continue 873 # deepcopy because the next time we look at this file the locale will differ. 874 file_config = deepcopy(map_config["mapping"][filename]) 875 876 for field in [ 877 "destinations", 878 "locale_prefix", 879 "source_path_modifier", 880 "update_balrog_manifest", 881 "from_buildid", 882 "pretty_name", 883 "checksums_path", 884 ]: 885 resolve_keyed_by( 886 file_config, field, field, locale=locale, platform=platform 887 ) 888 889 # This format string should ideally be in the configuration file, 890 # but this would mean keeping variable names in sync between code + config. 891 destinations = [ 892 "{s3_bucket_path}/{dest_path}/{locale_prefix}{filename}".format( 893 s3_bucket_path=bucket_path, 894 dest_path=dest_path, 895 locale_prefix=file_config["locale_prefix"], 896 filename=file_config.get("pretty_name", filename), 897 ) 898 for dest_path, bucket_path in itertools.product( 899 file_config["destinations"], map_config["s3_bucket_paths"] 900 ) 901 ] 902 # Creating map entries 903 # Key must be artifact path, to avoid trampling duplicates, such 904 # as public/build/target.apk and public/build/en-US/target.apk 905 key = os.path.join( 906 base_artifact_prefix, 907 file_config["source_path_modifier"], 908 filename, 909 ) 910 partials_paths = {} 911 for pname, info in partials_info.items(): 912 partials_paths[key] = { 913 "destinations": destinations, 914 } 915 if file_config.get("checksums_path"): 916 partials_paths[key]["checksums_path"] = file_config[ 917 "checksums_path" 918 ] 919 920 # optional flag: balrog manifest 921 if file_config.get("update_balrog_manifest"): 922 partials_paths[key]["update_balrog_manifest"] = True 923 if file_config.get("balrog_format"): 924 partials_paths[key]["balrog_format"] = file_config[ 925 "balrog_format" 926 ] 927 # optional flag: from_buildid 928 if file_config.get("from_buildid"): 929 partials_paths[key]["from_buildid"] = file_config["from_buildid"] 930 931 # optional flag: expiry 932 if file_config.get("expiry"): 933 partials_paths[key]["expiry"] = { 934 "relative-datestamp": file_config["expiry"] 935 } 936 937 # render buildid 938 kwargs.update({ 939 "partial": pname, 940 "from_buildid": info["buildid"], 941 "previous_version": info.get("previousVersion"), 942 "buildid": str(config.params["moz_build_date"]), 943 "locale": locale, 944 "version": config.params["version"], 945 "branch": config.params["project"], 946 "build_number": config.params["build_number"], 947 "year": upload_date.year, 948 "month": upload_date.strftime("%m"), # zero-pad the month 949 "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), 950 }) 951 kwargs.update(**platforms) 952 paths.update(jsone.render(partials_paths, kwargs)) 953 954 if not paths: 955 continue 956 957 artifacts.append({ 958 "taskId": {"task-reference": f"<{dep}>"}, 959 "locale": locale, 960 "paths": paths, 961 }) 962 963 artifacts.sort(key=lambda a: sorted(a["paths"].items())) 964 return artifacts