update_test.py (7904B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the update-test suite to parametrize by locale, source version, machine 6 """ 7 8 from enum import Enum 9 10 from taskgraph.transforms.base import TransformSequence 11 from taskgraph.util.copy import deepcopy 12 from taskgraph.util.schema import resolve_keyed_by 13 from typing_extensions import final 14 15 from gecko_taskgraph.util.attributes import is_try, task_name 16 17 18 @final 19 class ReleaseType(Enum): 20 """Release type""" 21 22 release = 0 23 beta = 1 24 esr = 2 25 other = 3 26 27 28 transforms = TransformSequence() 29 30 APPLICATIONS = ["fx"] 31 32 PLATFORM_TO_DOCKER = { 33 "linux2404-64-shippable": "ubuntu2404-test", 34 } 35 36 TOP_LOCALES = [ 37 "en-US", 38 "zh-CN", 39 "de", 40 "fr", 41 "it", 42 "es-ES", 43 "pt-BR", 44 "ru", 45 "pl", 46 "en-GB", 47 ] 48 49 BASE_TYPE_COMMAND = "./mach update-test" 50 51 UPDATE_ARTIFACT_NAME = "public/update-test" 52 53 DEFAULT_VERSIONS_BACK = 3 54 55 ESR_SUPPORT_CUTOFF = 140 56 57 58 def infix_treeherder_symbol(symbol, infix): 59 head, tail = symbol.split("(", 1) 60 if infix.startswith("-"): 61 infix = infix[1:] 62 return f"{head}({tail[:-1]}-{infix})" 63 64 65 @transforms.add 66 def set_task_configuration(config, tasks): 67 release_type = ReleaseType.release 68 if config.params["release_type"] == "beta": 69 release_type = ReleaseType.beta 70 elif config.params["release_type"].startswith("esr"): 71 esr_version = int(config.params["release_type"].split("esr")[1]) 72 release_type = ReleaseType.esr 73 if esr_version < ESR_SUPPORT_CUTOFF: 74 yield None 75 76 config_tasks = {} 77 for dep in config.kind_dependencies_tasks.values(): 78 if "update-verify-config" in dep.kind: 79 config_tasks[task_name(dep)] = dep 80 81 for task in tasks: 82 for platform in task["test-platforms"]: 83 this_task = deepcopy(task) 84 if config_tasks: 85 if "linux" in platform: 86 config_task = config_tasks["firefox-linux64"] 87 elif "win" in platform: 88 config_task = config_tasks["firefox-win64"] 89 elif "osx" in platform: 90 config_task = config_tasks["firefox-macosx64"] 91 this_task.setdefault("fetches", {})[config_task.label] = [ 92 "update-verify.cfg", 93 ] 94 if "linux" in platform: 95 this_task["worker"]["docker-image"] = {} 96 this_task["worker"]["docker-image"]["in-tree"] = PLATFORM_TO_DOCKER[ 97 platform 98 ] 99 100 this_task.setdefault("attributes", {}) 101 this_task["attributes"]["build_platform"] = get_build_platform(platform) 102 name_segments = this_task["name"].split("-") 103 this_task["name"] = "-".join([platform, *name_segments[2:]]) 104 this_task["description"] = f"Test updates on {platform}" 105 this_task["treeherder"]["platform"] = f"{platform}/opt" 106 this_task["run"]["cwd"] = "{checkout}" 107 del this_task["test-platforms"] 108 109 if this_task["shipping-product"] == "firefox": 110 product_channel = release_type.name.lower() 111 if this_task["shipping-phase"] == "promote": 112 update_channel = "-localtest" 113 elif this_task["shipping-phase"] == "push": 114 update_channel = "-cdntest" 115 else: 116 raise OSError("Expected promote or push shipping-phase.") 117 118 if this_task["shipping-phase"] != "promote": 119 this_task["treeherder"]["symbol"] = infix_treeherder_symbol( 120 this_task["treeherder"]["symbol"], 121 this_task["shipping-phase"][:6], 122 ) 123 if release_type != ReleaseType.release: 124 this_task["name"] = this_task["name"] + f"-{product_channel}" 125 this_task["run"]["command"] = ( 126 this_task["run"]["command"] 127 + f" --channel {product_channel}{update_channel}" 128 ) 129 130 if release_type == ReleaseType.esr: 131 this_task["run"]["command"] = ( 132 this_task["run"]["command"] + f" --esr-version {esr_version}" 133 ) 134 135 if is_try(config.params): 136 this_task["run"]["command"] = ( 137 this_task["run"]["command"] + " --use-balrog-staging" 138 ) 139 this_task["worker"]["env"]["BALROG_STAGING"] = "1" 140 this_task["name"] = this_task["name"].replace("linux-docker-", "") 141 this_task["index"]["job-name"] = "update-test-" + this_task["name"] 142 143 resolve_keyed_by( 144 item=this_task, 145 field="worker-type", 146 item_name=this_task["name"], 147 **{"test-platform": platform}, 148 ) 149 150 for app in APPLICATIONS: 151 if f"-{app}" in this_task["name"]: 152 break 153 154 if f"{app}-" in this_task["name"]: 155 (_, infix) = this_task["name"].split(app) 156 this_task["treeherder"]["symbol"] = infix_treeherder_symbol( 157 this_task["treeherder"]["symbol"], infix 158 ) 159 yield this_task 160 161 162 @transforms.add 163 def parametrize_by_locale(config, tasks): 164 for task in tasks: 165 if "locale" not in task.get("name"): 166 yield task 167 continue 168 for locale in TOP_LOCALES: 169 this_task = deepcopy(task) 170 this_task["run"]["command"] = ( 171 this_task["run"]["command"] + f" --source-locale {locale}" 172 ) 173 this_task["description"] = ( 174 f"{this_task['description']}, locale coverage: {locale}" 175 ) 176 this_task["name"] = this_task["name"].replace("locale", locale) 177 this_task["index"]["job-name"] = ( 178 f"{this_task['index']['job-name']}-{locale}" 179 ) 180 this_task["treeherder"]["symbol"] = infix_treeherder_symbol( 181 this_task["treeherder"]["symbol"], locale.replace("-", "") 182 ) 183 yield this_task 184 185 186 @transforms.add 187 def parametrize_by_source_version(config, tasks): 188 for task in tasks: 189 if "source-version" not in task.get("name"): 190 yield task 191 continue 192 if "-esr" in task.get("name"): 193 yield task 194 continue 195 # NB: We actually want source_versions_back = 0, because it gives us oldest usable ver 196 for v in range(5): 197 # avoid tasks with different names, same defs 198 if v == DEFAULT_VERSIONS_BACK: 199 continue 200 this_task = deepcopy(task) 201 this_task["run"]["command"] = ( 202 this_task["run"]["command"] + f" --source-versions-back {v}" 203 ) 204 description_tag = ( 205 " from 3 major versions back" if v == 0 else f" from {v} releases back" 206 ) 207 this_task["description"] = this_task["description"] + description_tag 208 ago_tag = "-from-oldest" if v == 0 else f"-from-{v}-ago" 209 this_task["name"] = this_task["name"].replace("-source-version", ago_tag) 210 this_task["index"]["job-name"] = this_task["index"]["job-name"] + ago_tag 211 this_task["treeherder"]["symbol"] = infix_treeherder_symbol( 212 this_task["treeherder"]["symbol"], "oldst" if v == 0 else f"bk{v}" 213 ) 214 yield this_task 215 216 217 def get_build_platform(platform): 218 build_platforms = { 219 "win": "win64-shippable", 220 "mac": "macosx64-shippable", 221 "lin": "linux64-shippable", 222 } 223 return build_platforms[platform[:3]]