partner_repack.py (6143B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the partner repack task into an actual task description. 6 """ 7 8 from taskgraph.transforms.base import TransformSequence 9 from taskgraph.util.dependencies import get_dependencies 10 from taskgraph.util.schema import resolve_keyed_by 11 12 from gecko_taskgraph.util.attributes import release_level 13 from gecko_taskgraph.util.partners import ( 14 apply_partner_priority, 15 check_if_partners_enabled, 16 get_partner_config_by_kind, 17 get_partner_url_config, 18 get_repack_ids_by_platform, 19 ) 20 from gecko_taskgraph.util.scriptworker import get_release_config 21 22 transforms = TransformSequence() 23 transforms.add(check_if_partners_enabled) 24 transforms.add(apply_partner_priority) 25 26 27 @transforms.add 28 def skip_unnecessary_platforms(config, tasks): 29 for task in tasks: 30 if config.kind == "release-partner-repack": 31 platform = task["attributes"]["build_platform"] 32 repack_ids = get_repack_ids_by_platform(config, platform) 33 if not repack_ids: 34 continue 35 yield task 36 37 38 @transforms.add 39 def remove_mac_dependency(config, tasks): 40 """Remove mac dependency depending on current level 41 to accomodate for mac notarization not running on level 1 42 """ 43 level = int(config.params.get("level", 0)) 44 for task in tasks: 45 if "macosx" not in task["attributes"]["build_platform"]: 46 yield task 47 continue 48 skipped_kind = "mac-signing" if level == 3 else "mac-notarization" 49 for dep_label in list(task["dependencies"].keys()): 50 if skipped_kind in dep_label: 51 del task["dependencies"][dep_label] 52 yield task 53 54 55 @transforms.add 56 def populate_repack_manifests_url(config, tasks): 57 for task in tasks: 58 partner_url_config = get_partner_url_config(config.params, config.graph_config) 59 60 for k in partner_url_config: 61 if config.kind.startswith(k): 62 task["worker"].setdefault("env", {})["REPACK_MANIFESTS_URL"] = ( 63 partner_url_config[k] 64 ) 65 break 66 else: 67 raise Exception("Can't find partner REPACK_MANIFESTS_URL") 68 69 for property in ("limit-locales",): 70 property = f"extra.{property}" 71 resolve_keyed_by( 72 task, 73 property, 74 property, 75 **{"release-level": release_level(config.params)}, 76 ) 77 78 if task["worker"]["env"]["REPACK_MANIFESTS_URL"].startswith("git@"): 79 task.setdefault("scopes", []).append( 80 "secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh".format( 81 **config.params 82 ) 83 ) 84 85 yield task 86 87 88 @transforms.add 89 def make_label(config, tasks): 90 for task in tasks: 91 task["label"] = "{}-{}".format(config.kind, task["name"]) 92 yield task 93 94 95 @transforms.add 96 def add_command_arguments(config, tasks): 97 release_config = get_release_config(config) 98 99 # staging releases - pass reduced set of locales to the repacking script 100 all_locales = set() 101 partner_config = get_partner_config_by_kind(config, config.kind) 102 for partner in partner_config.values(): 103 for sub_partner in partner.values(): 104 all_locales.update(sub_partner.get("locales", [])) 105 106 for task in tasks: 107 # add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64 108 if not task["attributes"]["build_platform"].endswith("-shippable"): 109 raise Exception( 110 "Unexpected partner repack platform: {}".format( 111 task["attributes"]["build_platform"], 112 ), 113 ) 114 platform = task["attributes"]["build_platform"].partition("-shippable")[0] 115 task["run"]["options"] = [ 116 "version={}".format(release_config["version"]), 117 "build-number={}".format(release_config["build_number"]), 118 f"platform={platform}", 119 ] 120 if task["extra"]["limit-locales"]: 121 for locale in all_locales: 122 task["run"]["options"].append(f"limit-locale={locale}") 123 if "partner" in config.kind and config.params["release_partners"]: 124 for partner in config.params["release_partners"]: 125 task["run"]["options"].append(f"partner={partner}") 126 127 # The upstream taskIds are stored a special environment variable, because we want to use 128 # task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS 129 # blocks that. It's space-separated string of ids in the end. 130 task["worker"]["env"]["UPSTREAM_TASKIDS"] = { 131 # We only want signing related tasks here, not build (used by mac builds for signing artifact resolution) 132 "task-reference": " ".join([ 133 f"<{dep}>" 134 for dep in task["dependencies"] 135 if ("signing" in dep or "notarization" in dep) 136 ]) 137 } 138 139 # Forward the release type for bouncer product construction 140 task["worker"]["env"]["RELEASE_TYPE"] = config.params["release_type"] 141 142 yield task 143 144 145 @transforms.add 146 def add_macos_signing_artifacts(config, tasks): 147 for task in tasks: 148 if "macosx" not in task["name"]: 149 yield task 150 continue 151 build_dep = None 152 for dep_task in get_dependencies(config, task): 153 if dep_task.kind == "build": 154 build_dep = dep_task 155 break 156 assert build_dep, f"repackage job {task['name']} has no build dependency" 157 for path, artifact in build_dep.task["payload"]["artifacts"].items(): 158 if path.startswith("public/build/security/"): 159 task["worker"].setdefault("artifacts", []).append({ 160 "name": path, 161 "path": artifact["path"], 162 "type": "file", 163 }) 164 yield task