beetmover_repackage_partner.py (9679B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the beetmover task into an actual task description. 6 """ 7 8 import logging 9 10 from taskgraph.transforms.base import TransformSequence 11 from taskgraph.util.dependencies import get_primary_dependency 12 from taskgraph.util.schema import Schema 13 from taskgraph.util.taskcluster import get_artifact_prefix 14 from voluptuous import Optional, Required 15 16 from gecko_taskgraph.transforms.beetmover import craft_release_properties 17 from gecko_taskgraph.transforms.task import task_description_schema 18 from gecko_taskgraph.util.attributes import ( 19 copy_attributes_from_dependent_job, 20 ) 21 from gecko_taskgraph.util.partners import get_ftp_platform, get_partner_config_by_kind 22 from gecko_taskgraph.util.scriptworker import ( 23 add_scope_prefix, 24 get_beetmover_bucket_scope, 25 ) 26 27 logger = logging.getLogger(__name__) 28 29 30 beetmover_description_schema = Schema({ 31 # unique label to describe this beetmover task, defaults to {dep.label}-beetmover 32 Optional("label"): str, 33 Required("partner-path"): str, 34 Optional("extra"): object, 35 Optional("attributes"): task_description_schema["attributes"], 36 Optional("dependencies"): task_description_schema["dependencies"], 37 Required("shipping-phase"): task_description_schema["shipping-phase"], 38 Optional("shipping-product"): task_description_schema["shipping-product"], 39 Optional("priority"): task_description_schema["priority"], 40 Optional("task-from"): task_description_schema["task-from"], 41 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 42 }) 43 44 transforms = TransformSequence() 45 46 47 @transforms.add 48 def remove_name(config, jobs): 49 for job in jobs: 50 if "name" in job: 51 del job["name"] 52 yield job 53 54 55 transforms.add_validate(beetmover_description_schema) 56 57 58 @transforms.add 59 def make_task_description(config, jobs): 60 for job in jobs: 61 dep_job = get_primary_dependency(config, job) 62 assert dep_job 63 64 repack_id = dep_job.task.get("extra", {}).get("repack_id") 65 if not repack_id: 66 raise Exception("Cannot find repack id!") 67 68 attributes = dep_job.attributes 69 build_platform = attributes.get("build_platform") 70 if not build_platform: 71 raise Exception("Cannot find build platform!") 72 73 label = dep_job.label.replace("repackage-signing-l10n", "beetmover-") 74 label = dep_job.label.replace("repackage-signing-", "beetmover-") 75 label = label.replace("repackage-", "beetmover-") 76 label = label.replace("chunking-dummy-", "beetmover-") 77 description = ( 78 "Beetmover submission for repack_id '{repack_id}' for build '" 79 "{build_platform}/{build_type}'".format( 80 repack_id=repack_id, 81 build_platform=build_platform, 82 build_type=attributes.get("build_type"), 83 ) 84 ) 85 86 dependencies = {} 87 88 base_label = "release-partner-repack" 89 if "eme" in config.kind: 90 base_label = "release-eme-free-repack" 91 dependencies["build"] = f"{base_label}-{build_platform}" 92 if "macosx" in build_platform or "win" in build_platform: 93 dependencies["repackage"] = "{}-repackage-{}-{}".format( 94 base_label, build_platform, repack_id.replace("/", "-") 95 ) 96 dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format( 97 base_label, build_platform, repack_id.replace("/", "-") 98 ) 99 100 attributes = copy_attributes_from_dependent_job(dep_job) 101 102 task = { 103 "label": label, 104 "description": description, 105 "dependencies": dependencies, 106 "attributes": attributes, 107 "run-on-projects": dep_job.attributes.get("run_on_projects"), 108 "shipping-phase": job["shipping-phase"], 109 "shipping-product": job.get("shipping-product"), 110 "partner-path": job["partner-path"], 111 "extra": { 112 "repack_id": repack_id, 113 }, 114 } 115 # we may have reduced the priority for partner jobs, otherwise task.py will set it 116 if job.get("priority"): 117 task["priority"] = job["priority"] 118 119 yield task 120 121 122 @transforms.add 123 def populate_scopes_and_worker_type(config, jobs): 124 bucket_scope = get_beetmover_bucket_scope(config) 125 action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") 126 127 for job in jobs: 128 job["scopes"] = [bucket_scope, action_scope] 129 job["worker-type"] = "beetmover" 130 yield job 131 132 133 def generate_upstream_artifacts( 134 job, 135 build_task_ref, 136 repackage_task_ref, 137 repackage_signing_task_ref, 138 platform, 139 repack_id, 140 partner_path, 141 repack_stub_installer=False, 142 ): 143 upstream_artifacts = [] 144 artifact_prefix = get_artifact_prefix(job) 145 146 if "linux" in platform: 147 upstream_artifacts.append({ 148 "taskId": {"task-reference": build_task_ref}, 149 "taskType": "build", 150 "paths": [f"{artifact_prefix}/{repack_id}/target.tar.xz"], 151 "locale": partner_path, 152 }) 153 upstream_artifacts.append({ 154 "taskId": {"task-reference": repackage_signing_task_ref}, 155 "taskType": "repackage", 156 "paths": [f"{artifact_prefix}/{repack_id}/target.tar.xz.asc"], 157 "locale": partner_path, 158 }) 159 elif "macosx" in platform: 160 upstream_artifacts.append({ 161 "taskId": {"task-reference": repackage_task_ref}, 162 "taskType": "repackage", 163 "paths": [f"{artifact_prefix}/{repack_id}/target.dmg"], 164 "locale": partner_path, 165 }) 166 upstream_artifacts.append({ 167 "taskId": {"task-reference": repackage_signing_task_ref}, 168 "taskType": "repackage", 169 "paths": [f"{artifact_prefix}/{repack_id}/target.dmg.asc"], 170 "locale": partner_path, 171 }) 172 elif "win" in platform: 173 upstream_artifacts.append({ 174 "taskId": {"task-reference": repackage_signing_task_ref}, 175 "taskType": "repackage", 176 "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe"], 177 "locale": partner_path, 178 }) 179 upstream_artifacts.append({ 180 "taskId": {"task-reference": repackage_signing_task_ref}, 181 "taskType": "repackage", 182 "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe.asc"], 183 "locale": partner_path, 184 }) 185 if platform.startswith("win32") and repack_stub_installer: 186 upstream_artifacts.append({ 187 "taskId": {"task-reference": repackage_signing_task_ref}, 188 "taskType": "repackage", 189 "paths": [f"{artifact_prefix}/{repack_id}/target.stub-installer.exe"], 190 "locale": partner_path, 191 }) 192 upstream_artifacts.append({ 193 "taskId": {"task-reference": repackage_signing_task_ref}, 194 "taskType": "repackage", 195 "paths": [ 196 f"{artifact_prefix}/{repack_id}/target.stub-installer.exe.asc" 197 ], 198 "locale": partner_path, 199 }) 200 201 if not upstream_artifacts: 202 raise Exception("Couldn't find any upstream artifacts.") 203 204 return upstream_artifacts 205 206 207 @transforms.add 208 def make_task_worker(config, jobs): 209 for job in jobs: 210 platform = job["attributes"]["build_platform"] 211 repack_id = job["extra"]["repack_id"] 212 partner, subpartner, locale = job["extra"]["repack_id"].split("/") 213 partner_config = get_partner_config_by_kind(config, config.kind) 214 repack_stub_installer = partner_config[partner][subpartner].get( 215 "repack_stub_installer" 216 ) 217 build_task = None 218 repackage_task = None 219 repackage_signing_task = None 220 221 for dependency in job["dependencies"].keys(): 222 if "repackage-signing" in dependency: 223 repackage_signing_task = dependency 224 elif "repackage" in dependency: 225 repackage_task = dependency 226 else: 227 build_task = "build" 228 229 build_task_ref = "<" + str(build_task) + ">" 230 repackage_task_ref = "<" + str(repackage_task) + ">" 231 repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">" 232 233 # generate the partner path; we'll send this to beetmover as the "locale" 234 ftp_platform = get_ftp_platform(platform) 235 repl_dict = { 236 "build_number": config.params["build_number"], 237 "locale": locale, 238 "partner": partner, 239 "platform": ftp_platform, 240 "release_partner_build_number": config.params[ 241 "release_partner_build_number" 242 ], 243 "subpartner": subpartner, 244 "version": config.params["version"], 245 } 246 partner_path = job["partner-path"].format(**repl_dict) 247 del job["partner-path"] 248 249 worker = { 250 "implementation": "beetmover", 251 "release-properties": craft_release_properties(config, job), 252 "upstream-artifacts": generate_upstream_artifacts( 253 job, 254 build_task_ref, 255 repackage_task_ref, 256 repackage_signing_task_ref, 257 platform, 258 repack_id, 259 partner_path, 260 repack_stub_installer, 261 ), 262 } 263 job["worker"] = worker 264 265 yield job