beetmover_repackage.py (13477B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the beetmover task into an actual task description. 6 """ 7 8 import logging 9 10 from taskgraph.transforms.base import TransformSequence 11 from taskgraph.util.dependencies import get_dependencies, get_primary_dependency 12 from taskgraph.util.schema import Schema 13 from taskgraph.util.taskcluster import get_artifact_prefix 14 from taskgraph.util.treeherder import inherit_treeherder_from_dep, replace_group 15 from voluptuous import Optional, Required 16 17 from gecko_taskgraph.transforms.beetmover import craft_release_properties 18 from gecko_taskgraph.transforms.task import task_description_schema 19 from gecko_taskgraph.util.attributes import ( 20 copy_attributes_from_dependent_job, 21 sorted_unique_list, 22 ) 23 from gecko_taskgraph.util.partials import ( 24 get_balrog_platform_name, 25 get_partials_artifacts_from_params, 26 get_partials_info_from_params, 27 ) 28 from gecko_taskgraph.util.scriptworker import ( 29 generate_beetmover_artifact_map, 30 generate_beetmover_partials_artifact_map, 31 generate_beetmover_upstream_artifacts, 32 get_beetmover_action_scope, 33 get_beetmover_bucket_scope, 34 ) 35 36 logger = logging.getLogger(__name__) 37 38 39 beetmover_description_schema = Schema({ 40 # unique label to describe this beetmover task, defaults to {dep.label}-beetmover 41 Required("label"): str, 42 Required("dependencies"): task_description_schema["dependencies"], 43 # treeherder is allowed here to override any defaults we use for beetmover. See 44 # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the 45 # below transforms for defaults of various values. 46 Optional("treeherder"): task_description_schema["treeherder"], 47 Optional("attributes"): task_description_schema["attributes"], 48 # locale is passed only for l10n beetmoving 49 Optional("locale"): str, 50 Required("shipping-phase"): task_description_schema["shipping-phase"], 51 Optional("task-from"): task_description_schema["task-from"], 52 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 53 }) 54 55 transforms = TransformSequence() 56 57 58 @transforms.add 59 def remove_name(config, jobs): 60 for job in jobs: 61 if "name" in job: 62 del job["name"] 63 yield job 64 65 66 transforms.add_validate(beetmover_description_schema) 67 68 69 def get_label_by_suffix(labels: list, suffix: str): 70 """ 71 Given list of labels, returns the label with provided suffix 72 Raises exception if more than one label is found. 73 74 Args: 75 labels (list): List of labels 76 suffix (str): Suffix for the desired label 77 78 Returns 79 str: The desired label 80 """ 81 labels = [l for l in labels if l.endswith(suffix)] 82 if len(labels) > 1: 83 raise Exception( 84 f"There should only be a single label with suffix: {suffix} - found {len(labels)}" 85 ) 86 return labels[0] 87 88 89 @transforms.add 90 def gather_required_signoffs(config, jobs): 91 for job in jobs: 92 job.setdefault("attributes", {})["required_signoffs"] = sorted_unique_list( 93 *( 94 dep.attributes.get("required_signoffs", []) 95 for dep in get_dependencies(config, job) 96 ) 97 ) 98 yield job 99 100 101 @transforms.add 102 def make_task_description(config, jobs): 103 for job in jobs: 104 dep_job = get_primary_dependency(config, job) 105 assert dep_job 106 107 attributes = dep_job.attributes 108 109 treeherder = inherit_treeherder_from_dep(job, dep_job) 110 upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"] 111 if "build" in job["dependencies"]: 112 build_label = job["dependencies"]["build"] 113 build_dep = config.kind_dependencies_tasks[build_label] 114 upstream_symbol = build_dep.task["extra"]["treeherder"]["symbol"] 115 treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR")) 116 label = job["label"] 117 description = ( 118 "Beetmover submission for locale '{locale}' for build '" 119 "{build_platform}/{build_type}'".format( 120 locale=attributes.get("locale", "en-US"), 121 build_platform=attributes.get("build_platform"), 122 build_type=attributes.get("build_type"), 123 ) 124 ) 125 126 upstream_deps = { 127 k: config.kind_dependencies_tasks[v] for k, v in job["dependencies"].items() 128 } 129 130 signing_name = "build-signing" 131 build_name = "build" 132 repackage_name = "repackage" 133 repackage_signing_name = "repackage-signing" 134 msi_signing_name = "repackage-signing-msi" 135 msix_signing_name = "repackage-signing-shippable-l10n-msix" 136 mar_signing_name = "mar-signing" 137 attribution_name = "attribution" 138 repackage_deb_name = "repackage-deb" 139 if job.get("locale"): 140 signing_name = "shippable-l10n-signing" 141 build_name = "shippable-l10n" 142 repackage_name = "repackage-l10n" 143 repackage_signing_name = "repackage-signing-l10n" 144 mar_signing_name = "mar-signing-l10n" 145 attribution_name = "attribution-l10n" 146 repackage_deb_name = "repackage-deb-l10n" 147 148 # The upstream "signing" task for macosx is either *-mac-signing or *-mac-notarization 149 if attributes.get("build_platform", "").startswith("macosx"): 150 signing_name = None 151 # We use the signing task on level 1 and notarization on level 3 152 if int(config.params.get("level", 0)) < 3: 153 signing_name = get_label_by_suffix(job["dependencies"], "-mac-signing") 154 else: 155 signing_name = get_label_by_suffix( 156 job["dependencies"], "-mac-notarization" 157 ) 158 if not signing_name: 159 raise Exception("Could not find upstream kind for mac signing.") 160 161 dependencies = { 162 "build": upstream_deps[build_name], 163 "signing": upstream_deps[signing_name], 164 } 165 if repackage_name in upstream_deps: 166 dependencies["repackage"] = upstream_deps[repackage_name] 167 if mar_signing_name in upstream_deps: 168 dependencies["mar-signing"] = upstream_deps[mar_signing_name] 169 if "partials-signing" in upstream_deps: 170 dependencies["partials-signing"] = upstream_deps["partials-signing"] 171 if msi_signing_name in upstream_deps: 172 dependencies[msi_signing_name] = upstream_deps[msi_signing_name] 173 if msix_signing_name in upstream_deps: 174 dependencies[msix_signing_name] = upstream_deps[msix_signing_name] 175 if repackage_signing_name in upstream_deps: 176 dependencies["repackage-signing"] = upstream_deps[repackage_signing_name] 177 if attribution_name in upstream_deps: 178 dependencies[attribution_name] = upstream_deps[attribution_name] 179 if repackage_deb_name in upstream_deps: 180 dependencies[repackage_deb_name] = upstream_deps[repackage_deb_name] 181 for kind in ("upload-symbols", "upload-symbols-dummy"): 182 if kind in upstream_deps: 183 dependencies[kind] = upstream_deps[kind] 184 185 attributes = copy_attributes_from_dependent_job(dep_job) 186 attributes.update(job.get("attributes", {})) 187 if job.get("locale"): 188 attributes["locale"] = job["locale"] 189 190 bucket_scope = get_beetmover_bucket_scope(config) 191 action_scope = get_beetmover_action_scope(config) 192 193 task = { 194 "label": label, 195 "description": description, 196 "worker-type": "beetmover", 197 "scopes": [bucket_scope, action_scope], 198 "dependencies": dependencies, 199 "attributes": attributes, 200 "run-on-projects": dep_job.attributes.get("run_on_projects"), 201 "run-on-repo-type": job.get("run-on-repo-type", ["git", "hg"]), 202 "treeherder": treeherder, 203 "shipping-phase": job["shipping-phase"], 204 "shipping-product": job.get("shipping-product"), 205 } 206 207 yield task 208 209 210 def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None): 211 artifact_prefix = get_artifact_prefix(job) 212 if locale and locale != "en-US": 213 artifact_prefix = f"{artifact_prefix}/{locale}" 214 215 upstream_artifacts = [ 216 { 217 "taskId": {"task-reference": "<partials-signing>"}, 218 "taskType": "signing", 219 "paths": [f"{artifact_prefix}/{path}" for path, _ in artifacts], 220 "locale": locale or "en-US", 221 } 222 ] 223 224 return upstream_artifacts 225 226 227 @transforms.add 228 def make_task_worker(config, jobs): 229 for job in jobs: 230 locale = job["attributes"].get("locale") 231 platform = job["attributes"]["build_platform"] 232 233 worker = { 234 "implementation": "beetmover", 235 "release-properties": craft_release_properties(config, job), 236 "upstream-artifacts": generate_beetmover_upstream_artifacts( 237 config, job, platform, locale 238 ), 239 "artifact-map": generate_beetmover_artifact_map( 240 config, job, platform=platform, locale=locale 241 ), 242 } 243 244 if locale: 245 worker["locale"] = locale 246 job["worker"] = worker 247 248 yield job 249 250 251 @transforms.add 252 def strip_unwanted_langpacks_from_worker(config, jobs): 253 """Strips out langpacks where we didn't sign them. 254 255 This explicitly deletes langpacks from upstream artifacts and from artifact-maps. 256 Due to limitations in declarative artifacts, doing this was our easiest way right now. 257 """ 258 ALWAYS_OK_PLATFORMS = {"linux64-shippable", "linux64-devedition"} 259 OSX_OK_PLATFORMS = {"macosx64-shippable", "macosx64-devedition"} 260 for job in jobs: 261 platform = job["attributes"].get("build_platform") 262 if platform in ALWAYS_OK_PLATFORMS: 263 # No need to strip anything 264 yield job 265 continue 266 267 for map in job["worker"].get("artifact-map", [])[:]: 268 if not any([path.endswith("target.langpack.xpi") for path in map["paths"]]): 269 continue 270 if map["locale"] == "ja-JP-mac": 271 # This locale should only exist on mac 272 assert platform in OSX_OK_PLATFORMS 273 continue 274 # map[paths] is being modified while iterating, so we need to resolve the 275 # ".keys()" iterator up front by throwing it into a list. 276 for path in list(map["paths"].keys()): 277 if path.endswith("target.langpack.xpi"): 278 del map["paths"][path] 279 if map["paths"] == {}: 280 job["worker"]["artifact-map"].remove(map) 281 282 for artifact in job["worker"].get("upstream-artifacts", []): 283 if not any([ 284 path.endswith("target.langpack.xpi") for path in artifact["paths"] 285 ]): 286 continue 287 if artifact["locale"] == "ja-JP-mac": 288 # This locale should only exist on mac 289 assert platform in OSX_OK_PLATFORMS 290 continue 291 artifact["paths"] = [ 292 path 293 for path in artifact["paths"] 294 if not path.endswith("target.langpack.xpi") 295 ] 296 if artifact["paths"] == []: 297 job["worker"]["upstream-artifacts"].remove(artifact) 298 299 yield job 300 301 302 @transforms.add 303 def make_partials_artifacts(config, jobs): 304 for job in jobs: 305 locale = job["attributes"].get("locale") 306 if not locale: 307 locale = "en-US" 308 309 platform = job["attributes"]["build_platform"] 310 311 if "partials-signing" not in job["dependencies"]: 312 yield job 313 continue 314 315 balrog_platform = get_balrog_platform_name(platform) 316 artifacts = get_partials_artifacts_from_params( 317 config.params.get("release_history"), balrog_platform, locale 318 ) 319 320 upstream_artifacts = generate_partials_upstream_artifacts( 321 job, artifacts, balrog_platform, locale 322 ) 323 324 job["worker"]["upstream-artifacts"].extend(upstream_artifacts) 325 326 extra = list() 327 328 partials_info = get_partials_info_from_params( 329 config.params.get("release_history"), balrog_platform, locale 330 ) 331 332 job["worker"]["artifact-map"].extend( 333 generate_beetmover_partials_artifact_map( 334 config, job, partials_info, platform=platform, locale=locale 335 ) 336 ) 337 338 for artifact in partials_info: 339 artifact_extra = { 340 "locale": locale, 341 "artifact_name": artifact, 342 "buildid": partials_info[artifact]["buildid"], 343 "platform": balrog_platform, 344 } 345 for rel_attr in ("previousBuildNumber", "previousVersion"): 346 if partials_info[artifact].get(rel_attr): 347 artifact_extra[rel_attr] = partials_info[artifact][rel_attr] 348 extra.append(artifact_extra) 349 350 job.setdefault("extra", {}) 351 job["extra"]["partials"] = extra 352 353 yield job 354 355 356 @transforms.add 357 def convert_deps(config, jobs): 358 for job in jobs: 359 job["dependencies"] = { 360 name: dep_job.label for name, dep_job in job["dependencies"].items() 361 } 362 yield job