release_beetmover_signed_addons.py (8684B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the beetmover task into an actual task description. 6 """ 7 8 import copy 9 import logging 10 11 from taskgraph.transforms.base import TransformSequence 12 from taskgraph.util.dependencies import get_primary_dependency 13 from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by 14 from taskgraph.util.treeherder import inherit_treeherder_from_dep 15 from voluptuous import Optional, Required 16 17 from gecko_taskgraph.transforms.beetmover import craft_release_properties 18 from gecko_taskgraph.transforms.task import task_description_schema 19 from gecko_taskgraph.util.attributes import ( 20 copy_attributes_from_dependent_job, 21 release_level, 22 ) 23 from gecko_taskgraph.util.scriptworker import ( 24 generate_beetmover_artifact_map, 25 generate_beetmover_upstream_artifacts, 26 get_beetmover_action_scope, 27 get_beetmover_bucket_scope, 28 ) 29 30 logger = logging.getLogger(__name__) 31 32 33 transforms = TransformSequence() 34 35 36 beetmover_description_schema = Schema({ 37 # attributes is used for enabling artifact-map by declarative artifacts 38 Required("attributes"): {str: object}, 39 # unique label to describe this beetmover task, defaults to {dep.label}-beetmover 40 Optional("label"): str, 41 # treeherder is allowed here to override any defaults we use for beetmover. See 42 # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the 43 # below transforms for defaults of various values. 44 Optional("treeherder"): task_description_schema["treeherder"], 45 Required("description"): str, 46 Required("worker-type"): optionally_keyed_by("release-level", str), 47 Required("run-on-projects"): [], 48 # locale is passed only for l10n beetmoving 49 Optional("locale"): str, 50 Optional("shipping-phase"): task_description_schema["shipping-phase"], 51 Optional("task-from"): task_description_schema["task-from"], 52 Optional("dependencies"): task_description_schema["dependencies"], 53 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 54 }) 55 56 57 @transforms.add 58 def remove_name(config, jobs): 59 for job in jobs: 60 if "name" in job: 61 del job["name"] 62 yield job 63 64 65 transforms.add_validate(beetmover_description_schema) 66 67 68 @transforms.add 69 def resolve_keys(config, jobs): 70 for job in jobs: 71 for field in ("worker-type", "attributes.artifact_map"): 72 resolve_keyed_by( 73 job, 74 field, 75 item_name=job["label"], 76 **{ 77 "release-level": release_level(config.params), 78 "release-type": config.params["release_type"], 79 "project": config.params["project"], 80 }, 81 ) 82 yield job 83 84 85 @transforms.add 86 def make_task_description(config, jobs): 87 for job in jobs: 88 dep_job = get_primary_dependency(config, job) 89 assert dep_job 90 91 attributes = dep_job.attributes 92 93 treeherder = inherit_treeherder_from_dep(job, dep_job) 94 treeherder.setdefault( 95 "symbol", "langpack(BM{})".format(attributes.get("l10n_chunk", "")) 96 ) 97 98 job["attributes"].update(copy_attributes_from_dependent_job(dep_job)) 99 job["attributes"]["chunk_locales"] = dep_job.attributes.get( 100 "chunk_locales", ["en-US"] 101 ) 102 103 job["description"] = job["description"].format( 104 locales="/".join(job["attributes"]["chunk_locales"]), 105 platform=job["attributes"]["build_platform"], 106 ) 107 108 job["scopes"] = [ 109 get_beetmover_bucket_scope(config), 110 get_beetmover_action_scope(config), 111 ] 112 113 job["dependencies"] = {"langpack-copy": dep_job.label} 114 115 job["run-on-projects"] = job.get( 116 "run_on_projects", dep_job.attributes["run_on_projects"] 117 ) 118 job["treeherder"] = treeherder 119 job["shipping-phase"] = job.get( 120 "shipping-phase", dep_job.attributes["shipping_phase"] 121 ) 122 job["shipping-product"] = dep_job.attributes["shipping_product"] 123 124 yield job 125 126 127 @transforms.add 128 def make_task_worker(config, jobs): 129 for job in jobs: 130 platform = job["attributes"]["build_platform"] 131 locale = job["attributes"]["chunk_locales"] 132 133 job["worker"] = { 134 "implementation": "beetmover", 135 "release-properties": craft_release_properties(config, job), 136 "upstream-artifacts": generate_beetmover_upstream_artifacts( 137 config, 138 job, 139 platform, 140 locale, 141 ), 142 "artifact-map": generate_beetmover_artifact_map( 143 config, job, platform=platform, locale=locale 144 ), 145 } 146 147 yield job 148 149 150 @transforms.add 151 def yield_all_platform_jobs(config, jobs): 152 # Even though langpacks are now platform independent, we keep beetmoving them at old 153 # platform-specific locations. That's why this transform exist 154 # The linux64 and mac specific ja-JP-mac are beetmoved along with the signing beetmover 155 # So while the dependent jobs are linux here, we only yield jobs for other platforms 156 for job in jobs: 157 platforms = ("linux64-aarch64", "macosx64", "win32", "win64") 158 if "devedition" in job["attributes"]["build_platform"]: 159 platforms = (f"{plat}-devedition" for plat in platforms) 160 for platform in platforms: 161 platform_job = copy.deepcopy(job) 162 if "ja" in platform_job["attributes"]["chunk_locales"] and platform in ( 163 "macosx64", 164 "macosx64-devedition", 165 ): 166 platform_job = _strip_ja_data_from_linux_job(platform_job) 167 168 platform_job = _change_platform_data(config, platform_job, platform) 169 170 yield platform_job 171 172 173 def _strip_ja_data_from_linux_job(platform_job): 174 # Let's take "ja" out the description. This locale is in a substring like "aa/bb/cc/dd", where 175 # "ja" could be any of "aa", "bb", "cc", "dd" 176 platform_job["description"] = platform_job["description"].replace("ja/", "") 177 platform_job["description"] = platform_job["description"].replace("/ja", "") 178 179 platform_job["worker"]["upstream-artifacts"] = [ 180 artifact 181 for artifact in platform_job["worker"]["upstream-artifacts"] 182 if artifact["locale"] != "ja" 183 ] 184 185 return platform_job 186 187 188 def _change_platform_in_artifact_map_paths(paths, orig_platform, new_platform): 189 amended_paths = {} 190 for artifact, artifact_info in paths.items(): 191 amended_artifact_info = { 192 "checksums_path": artifact_info["checksums_path"].replace( 193 orig_platform, new_platform 194 ), 195 "destinations": [ 196 d.replace(orig_platform, new_platform) 197 for d in artifact_info["destinations"] 198 ], 199 } 200 amended_paths[artifact] = amended_artifact_info 201 202 return amended_paths 203 204 205 def _change_platform_data(config, platform_job, platform): 206 orig_platform = "linux64" 207 if "devedition" in platform: 208 orig_platform = "linux64-devedition" 209 platform_job["attributes"]["build_platform"] = platform 210 platform_job["label"] = platform_job["label"].replace(orig_platform, platform) 211 platform_job["description"] = platform_job["description"].replace( 212 orig_platform, platform 213 ) 214 platform_job["treeherder"]["platform"] = platform_job["treeherder"][ 215 "platform" 216 ].replace(orig_platform, platform) 217 platform_job["worker"]["release-properties"]["platform"] = platform 218 219 # amend artifactMap entries as well 220 platform_mapping = { 221 "linux64": "linux-x86_64", 222 "linux64-aarch64": "linux-aarch64", 223 "macosx64": "mac", 224 "win32": "win32", 225 "win64": "win64", 226 "linux64-devedition": "linux-x86_64", 227 "linux64-aarch64-devedition": "linux-aarch64", 228 "macosx64-devedition": "mac", 229 "win32-devedition": "win32", 230 "win64-devedition": "win64", 231 } 232 orig_platform = platform_mapping.get(orig_platform, orig_platform) 233 platform = platform_mapping.get(platform, platform) 234 platform_job["worker"]["artifact-map"] = [ 235 { 236 "locale": entry["locale"], 237 "taskId": entry["taskId"], 238 "paths": _change_platform_in_artifact_map_paths( 239 entry["paths"], orig_platform, platform 240 ), 241 } 242 for entry in platform_job["worker"]["artifact-map"] 243 ] 244 245 return platform_job