bouncer_submission.py (10758B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Add from parameters.yml into bouncer submission tasks. 6 """ 7 8 import copy 9 import logging 10 11 import attr 12 from taskgraph.transforms.base import TransformSequence 13 from taskgraph.util.schema import resolve_keyed_by 14 15 from gecko_taskgraph.transforms.l10n import parse_locales_file 16 from gecko_taskgraph.util.attributes import release_level 17 from gecko_taskgraph.util.scriptworker import get_release_config 18 19 logger = logging.getLogger(__name__) 20 21 22 FTP_PLATFORMS_PER_BOUNCER_PLATFORM = { 23 "linux64": "linux-x86_64", 24 "linux64-aarch64": "linux-aarch64", 25 "osx": "mac", 26 "win": "win32", 27 "win64": "win64", 28 "win64-aarch64": "win64-aarch64", 29 } 30 31 # :lang is interpolated by bouncer at runtime 32 CANDIDATES_PATH_TEMPLATE = ( 33 "/{ftp_product}/candidates/{version}-candidates/build{build_number}/\ 34 {update_folder}{ftp_platform}/:lang/{file}" 35 ) 36 RELEASES_PATH_TEMPLATE = "/{ftp_product}/releases/{version}/\ 37 {update_folder}{ftp_platform}/:lang/{file}" 38 39 40 CONFIG_PER_BOUNCER_PRODUCT = { 41 "complete-mar": { 42 "name_postfix": "-Complete", 43 "path_template": RELEASES_PATH_TEMPLATE, 44 "file_names": { 45 "default": "{product}-{version}.complete.mar", 46 }, 47 }, 48 "complete-mar-candidates": { 49 "name_postfix": "build{build_number}-Complete", 50 "path_template": CANDIDATES_PATH_TEMPLATE, 51 "file_names": { 52 "default": "{product}-{version}.complete.mar", 53 }, 54 }, 55 "installer": { 56 "path_template": RELEASES_PATH_TEMPLATE, 57 "file_names": { 58 "linux64": "{product}-{version}.tar.xz", 59 "linux64-aarch64": "{product}-{version}.tar.xz", 60 "osx": "{pretty_product}%20{version}.dmg", 61 "win": "{pretty_product}%20Setup%20{version}.exe", 62 "win64": "{pretty_product}%20Setup%20{version}.exe", 63 "win64-aarch64": "{pretty_product}%20Setup%20{version}.exe", 64 }, 65 }, 66 "partial-mar": { 67 "name_postfix": "-Partial-{previous_version}", 68 "path_template": RELEASES_PATH_TEMPLATE, 69 "file_names": { 70 "default": "{product}-{previous_version}-{version}.partial.mar", 71 }, 72 }, 73 "partial-mar-candidates": { 74 "name_postfix": "build{build_number}-Partial-{previous_version}build{previous_build}", 75 "path_template": CANDIDATES_PATH_TEMPLATE, 76 "file_names": { 77 "default": "{product}-{previous_version}-{version}.partial.mar", 78 }, 79 }, 80 "stub-installer": { 81 "name_postfix": "-stub", 82 # We currently have a sole win32 stub installer that is to be used 83 # in all windows platforms to toggle between full installers 84 "path_template": RELEASES_PATH_TEMPLATE.replace("{ftp_platform}", "win32"), 85 "file_names": { 86 "win": "{pretty_product}%20Installer.exe", 87 "win64": "{pretty_product}%20Installer.exe", 88 "win64-aarch64": "{pretty_product}%20Installer.exe", 89 }, 90 }, 91 "msi": { 92 "name_postfix": "-msi-SSL", 93 "path_template": RELEASES_PATH_TEMPLATE, 94 "file_names": { 95 "win": "{pretty_product}%20Setup%20{version}.msi", 96 "win64": "{pretty_product}%20Setup%20{version}.msi", 97 }, 98 }, 99 "msix": { 100 "name_postfix": "-msix-SSL", 101 "path_template": RELEASES_PATH_TEMPLATE.replace(":lang", "multi"), 102 "file_names": { 103 "win": "{pretty_product}%20Setup%20{version}.msix", 104 "win64": "{pretty_product}%20Setup%20{version}.msix", 105 "win64-aarch64": "{pretty_product}%20Setup%20{version}.msix", 106 }, 107 }, 108 "pkg": { 109 "name_postfix": "-pkg-SSL", 110 "path_template": RELEASES_PATH_TEMPLATE, 111 "file_names": { 112 "osx": "{pretty_product}%20{version}.pkg", 113 }, 114 }, 115 "langpack": { 116 "name_postfix": "-langpack-SSL", 117 "path_template": RELEASES_PATH_TEMPLATE.replace(":lang", "xpi"), 118 "file_names": {"default": ":lang.xpi"}, 119 }, 120 } 121 CONFIG_PER_BOUNCER_PRODUCT["installer-ssl"] = copy.deepcopy( 122 CONFIG_PER_BOUNCER_PRODUCT["installer"] 123 ) 124 CONFIG_PER_BOUNCER_PRODUCT["installer-ssl"]["name_postfix"] = "-SSL" 125 126 transforms = TransformSequence() 127 128 129 @transforms.add 130 def make_task_worker(config, jobs): 131 for job in jobs: 132 resolve_keyed_by( 133 job, 134 "worker-type", 135 item_name=job["name"], 136 **{"release-level": release_level(config.params)}, 137 ) 138 resolve_keyed_by( 139 job, 140 "scopes", 141 item_name=job["name"], 142 **{"release-level": release_level(config.params)}, 143 ) 144 resolve_keyed_by( 145 job, 146 "bouncer-products", 147 item_name=job["name"], 148 **{"release-type": config.params["release_type"]}, 149 ) 150 151 # No need to filter out ja-JP-mac, we need to upload both; but we do 152 # need to filter out the platforms they come with 153 all_locales = sorted( 154 locale 155 for locale in parse_locales_file(job["locales-file"]).keys() 156 if locale not in ("linux", "win32", "osx") 157 ) 158 159 job["worker"]["locales"] = all_locales 160 job["worker"]["entries"] = craft_bouncer_entries(config, job) 161 162 del job["locales-file"] 163 del job["bouncer-platforms"] 164 del job["bouncer-products"] 165 166 if job["worker"]["entries"]: 167 yield job 168 else: 169 logger.warning( 170 'No bouncer entries defined in bouncer submission task for "{}". \ 171 Job deleted.'.format(job["name"]) 172 ) 173 174 175 def craft_bouncer_entries(config, job): 176 release_config = get_release_config(config) 177 178 product = job["shipping-product"] 179 bouncer_platforms = job["bouncer-platforms"] 180 181 current_version = release_config["version"] 182 current_build_number = release_config["build_number"] 183 184 bouncer_products = job["bouncer-products"] 185 previous_versions_string = release_config.get("partial_versions", None) 186 if previous_versions_string: 187 previous_versions = previous_versions_string.split(", ") 188 else: 189 logger.warning( 190 'No partials defined! Bouncer submission task won\'t send any \ 191 partial-related entry for "{}"'.format(job["name"]) 192 ) 193 bouncer_products = [ 194 bouncer_product 195 for bouncer_product in bouncer_products 196 if "partial" not in bouncer_product 197 ] 198 previous_versions = [None] 199 200 project = config.params["project"] 201 202 return { 203 craft_bouncer_product_name( 204 product, 205 bouncer_product, 206 current_version, 207 current_build_number, 208 previous_version, 209 ): { 210 "options": { 211 "add_locales": False if "msix" in bouncer_product else True, 212 "ssl_only": craft_ssl_only(bouncer_product, project), 213 }, 214 "paths_per_bouncer_platform": craft_paths_per_bouncer_platform( 215 product, 216 bouncer_product, 217 bouncer_platforms, 218 current_version, 219 current_build_number, 220 previous_version, 221 ), 222 } 223 for bouncer_product in bouncer_products 224 for previous_version in previous_versions 225 } 226 227 228 def craft_paths_per_bouncer_platform( 229 product, 230 bouncer_product, 231 bouncer_platforms, 232 current_version, 233 current_build_number, 234 previous_version=None, 235 ): 236 paths_per_bouncer_platform = {} 237 for bouncer_platform in bouncer_platforms: 238 file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product][ 239 "file_names" 240 ] 241 file_name_template = file_names_per_platform.get( 242 bouncer_platform, file_names_per_platform.get("default", None) 243 ) 244 if not file_name_template: 245 # Some bouncer product like stub-installer are only meant to be on Windows. 246 # Thus no default value is defined there 247 continue 248 249 file_name_product = _craft_filename_product(product) 250 file_name = file_name_template.format( 251 product=file_name_product, 252 pretty_product=file_name_product.capitalize(), 253 version=current_version, 254 previous_version=split_build_data(previous_version)[0], 255 ) 256 257 path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]["path_template"] 258 file_relative_location = path_template.format( 259 ftp_product=_craft_ftp_product(product), 260 version=current_version, 261 build_number=current_build_number, 262 update_folder="update/" if "-mar" in bouncer_product else "", 263 ftp_platform=FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform], 264 file=file_name, 265 ) 266 267 paths_per_bouncer_platform[bouncer_platform] = file_relative_location 268 269 return paths_per_bouncer_platform 270 271 272 def _craft_ftp_product(product): 273 return product.lower() 274 275 276 def _craft_filename_product(product): 277 return "firefox" if product == "devedition" else product 278 279 280 @attr.s 281 class InvalidSubstitution: 282 error = attr.ib(type=str) 283 284 def __str__(self): 285 raise Exception("Partial is being processed, but no previous version defined.") 286 287 288 def craft_bouncer_product_name( 289 product, 290 bouncer_product, 291 current_version, 292 current_build_number=None, 293 previous_version=None, 294 ): 295 if previous_version is None: 296 previous_version = previous_build = InvalidSubstitution( 297 "Partial is being processed, but no previous version defined." 298 ) 299 else: 300 previous_version, previous_build = split_build_data(previous_version) 301 postfix = ( 302 CONFIG_PER_BOUNCER_PRODUCT[bouncer_product] 303 .get("name_postfix", "") 304 .format( 305 build_number=current_build_number, 306 previous_version=previous_version, 307 previous_build=previous_build, 308 ) 309 ) 310 311 return f"{product.capitalize()}-{current_version}{postfix}" 312 313 314 def craft_ssl_only(bouncer_product, project): 315 # XXX ESR is the only channel where we force serve the installer over SSL 316 if "-esr" in project and bouncer_product == "installer": 317 return True 318 319 return bouncer_product not in ( 320 "complete-mar", 321 "complete-mar-candidates", 322 "installer", 323 "partial-mar", 324 "partial-mar-candidates", 325 ) 326 327 328 def split_build_data(version): 329 if version and "build" in version: 330 return version.split("build") 331 return version, InvalidSubstitution("k")