partials.py (11091B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import logging 7 8 import redo 9 import requests 10 11 from gecko_taskgraph.util.scriptworker import ( 12 BALROG_SCOPE_ALIAS_TO_PROJECT, 13 BALROG_SERVER_SCOPES, 14 ) 15 16 logger = logging.getLogger(__name__) 17 18 PLATFORM_RENAMES = { 19 "windows2012-32": "win32", 20 "windows2012-64": "win64", 21 "windows2012-aarch64": "win64-aarch64", 22 "osx-cross": "macosx64", 23 "osx": "macosx64", 24 } 25 26 BALROG_PLATFORM_MAP = { 27 "linux64": ["Linux_x86_64-gcc3"], 28 "linux64-aarch64": ["Linux_aarch64-gcc3"], 29 "linux64-asan-reporter": ["Linux_x86_64-gcc3-asan"], 30 "macosx64": [ 31 "Darwin_x86_64-gcc3-u-i386-x86_64", 32 "Darwin_x86-gcc3-u-i386-x86_64", 33 "Darwin_aarch64-gcc3", 34 "Darwin_x86-gcc3", 35 "Darwin_x86_64-gcc3", 36 ], 37 "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"], 38 "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"], 39 "win64-asan-reporter": ["WINNT_x86_64-msvc-x64-asan"], 40 "win64-aarch64": [ 41 "WINNT_aarch64-msvc-aarch64", 42 ], 43 } 44 45 FTP_PLATFORM_MAP = { 46 "Darwin_x86-gcc3": "mac", 47 "Darwin_x86-gcc3-u-i386-x86_64": "mac", 48 "Darwin_x86_64-gcc3": "mac", 49 "Darwin_x86_64-gcc3-u-i386-x86_64": "mac", 50 "Darwin_aarch64-gcc3": "mac", 51 "Linux_x86_64-gcc3": "linux-x86_64", 52 "Linux_aarch64-gcc3": "linux-aarch64", 53 "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter", 54 "WINNT_x86_64-msvc-x64-asan": "win64-asan-reporter", 55 "WINNT_x86-msvc": "win32", 56 "WINNT_x86-msvc-x64": "win32", 57 "WINNT_x86-msvc-x86": "win32", 58 "WINNT_x86_64-msvc": "win64", 59 "WINNT_x86_64-msvc-x64": "win64", 60 "WINNT_aarch64-msvc-aarch64": "win64-aarch64", 61 } 62 63 64 def get_balrog_platform_name(platform): 65 """Convert build platform names into balrog platform names. 66 67 Remove known values instead to catch aarch64 and other platforms 68 that may be added. 69 """ 70 removals = ["-devedition", "-shippable"] 71 for remove in removals: 72 platform = platform.replace(remove, "") 73 return PLATFORM_RENAMES.get(platform, platform) 74 75 76 def _sanitize_platform(platform): 77 platform = get_balrog_platform_name(platform) 78 return BALROG_PLATFORM_MAP[platform][0] 79 80 81 def get_builds(release_history, platform, locale): 82 """Examine cached balrog release history and return the list of 83 builds we need to generate diffs from""" 84 platform = _sanitize_platform(platform) 85 return release_history.get(platform, {}).get(locale, {}) 86 87 88 def get_partials_artifacts_from_params(release_history, platform, locale): 89 platform = _sanitize_platform(platform) 90 return [ 91 (artifact, details.get("previousVersion", None)) 92 for artifact, details in release_history.get(platform, {}) 93 .get(locale, {}) 94 .items() 95 ] 96 97 98 def get_partials_info_from_params(release_history, platform, locale): 99 platform = _sanitize_platform(platform) 100 101 artifact_map = {} 102 for k in release_history.get(platform, {}).get(locale, {}): 103 details = release_history[platform][locale][k] 104 attributes = ("buildid", "previousBuildNumber", "previousVersion") 105 artifact_map[k] = { 106 attr: details[attr] for attr in attributes if attr in details 107 } 108 return artifact_map 109 110 111 def _retry_on_http_errors(url, verify, params, errors): 112 if params: 113 params_str = "&".join("=".join([k, str(v)]) for k, v in params.items()) 114 else: 115 params_str = "" 116 logger.info("Connecting to %s?%s", url, params_str) 117 for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10): 118 try: 119 req = requests.get(url, verify=verify, params=params, timeout=10) 120 req.raise_for_status() 121 return req 122 except requests.HTTPError as e: 123 if e.response.status_code in errors: 124 logger.exception( 125 "Got HTTP %s trying to reach %s", e.response.status_code, url 126 ) 127 else: 128 raise 129 raise Exception(f"Cannot connect to {url}!") 130 131 132 def get_sorted_releases(product, branch): 133 """Returns a list of release names from Balrog. 134 :param product: product name, AKA appName 135 :param branch: branch name, e.g. mozilla-central 136 :return: a sorted list of release names, most recent first. 137 """ 138 url = f"{_get_balrog_api_root(branch)}/releases" 139 params = { 140 "product": product, 141 # Adding -nightly-2 (2 stands for the beginning of build ID 142 # based on date) should filter out release and latest blobs. 143 # This should be changed to -nightly-3 in 3000 ;) 144 "name_prefix": f"{product}-{branch}-nightly-2", 145 "names_only": True, 146 } 147 req = _retry_on_http_errors(url=url, verify=True, params=params, errors=[500, 502]) 148 releases = req.json()["names"] 149 releases = sorted(releases, reverse=True) 150 return releases 151 152 153 def get_release_builds(release, branch): 154 url = f"{_get_balrog_api_root(branch)}/releases/{release}" 155 req = _retry_on_http_errors(url=url, verify=True, params=None, errors=[500, 502]) 156 return req.json() 157 158 159 def _get_balrog_api_root(branch): 160 # Query into the scopes scriptworker uses to make sure we check against the same balrog server 161 # That our jobs would use. 162 scope = None 163 for alias, projects in BALROG_SCOPE_ALIAS_TO_PROJECT: 164 if branch in projects and alias in BALROG_SERVER_SCOPES: 165 scope = BALROG_SERVER_SCOPES[alias] 166 break 167 else: 168 scope = BALROG_SERVER_SCOPES["default"] 169 170 if scope == "balrog:server:dep": 171 return "https://stage.balrog.nonprod.cloudops.mozgcp.net/api/v1" 172 return "https://aus5.mozilla.org/api/v1" 173 174 175 def find_localtest(fileUrls): 176 for channel in fileUrls: 177 if "-localtest" in channel: 178 return channel 179 180 181 def populate_release_history( 182 product, branch, maxbuilds=4, maxsearch=10, partial_updates=None 183 ): 184 # Assuming we are using release branches when we know the list of previous 185 # releases in advance 186 if partial_updates is not None: 187 return _populate_release_history( 188 product, branch, partial_updates=partial_updates 189 ) 190 return _populate_nightly_history( 191 product, branch, maxbuilds=maxbuilds, maxsearch=maxsearch 192 ) 193 194 195 def _populate_nightly_history(product, branch, maxbuilds=4, maxsearch=10): 196 """Find relevant releases in Balrog 197 Not all releases have all platforms and locales, due 198 to Taskcluster migration. 199 200 Args: 201 product (str): capitalized product name, AKA appName, e.g. Firefox 202 branch (str): branch name (mozilla-central) 203 maxbuilds (int): Maximum number of historical releases to populate 204 maxsearch(int): Traverse at most this many releases, to avoid 205 working through the entire history. 206 Returns: 207 json object based on data from balrog api 208 209 results = { 210 'platform1': { 211 'locale1': { 212 'buildid1': mar_url, 213 'buildid2': mar_url, 214 'buildid3': mar_url, 215 }, 216 'locale2': { 217 'target.partial-1.mar': {'buildid1': 'mar_url'}, 218 } 219 }, 220 'platform2': { 221 } 222 } 223 """ 224 last_releases = get_sorted_releases(product, branch) 225 226 partial_mar_tmpl = "target.partial-{}.mar" 227 228 builds = dict() 229 for release in last_releases[:maxsearch]: 230 # maxbuilds in all categories, don't make any more queries 231 full = len(builds) > 0 and all( 232 len(builds[platform][locale]) >= maxbuilds 233 for platform in builds 234 for locale in builds[platform] 235 ) 236 if full: 237 break 238 history = get_release_builds(release, branch) 239 240 for platform in history["platforms"]: 241 if "alias" in history["platforms"][platform]: 242 continue 243 if platform not in builds: 244 builds[platform] = dict() 245 for locale in history["platforms"][platform]["locales"]: 246 if locale not in builds[platform]: 247 builds[platform][locale] = dict() 248 if len(builds[platform][locale]) >= maxbuilds: 249 continue 250 if "buildID" not in history["platforms"][platform]["locales"][locale]: 251 continue 252 buildid = history["platforms"][platform]["locales"][locale]["buildID"] 253 if ( 254 "completes" not in history["platforms"][platform]["locales"][locale] 255 or len( 256 history["platforms"][platform]["locales"][locale]["completes"] 257 ) 258 == 0 259 ): 260 continue 261 url = history["platforms"][platform]["locales"][locale]["completes"][0][ 262 "fileUrl" 263 ] 264 nextkey = len(builds[platform][locale]) + 1 265 builds[platform][locale][partial_mar_tmpl.format(nextkey)] = { 266 "buildid": buildid, 267 "mar_url": url, 268 } 269 return builds 270 271 272 def _populate_release_history(product, branch, partial_updates): 273 builds = dict() 274 for version, release in partial_updates.items(): 275 prev_release_blob = "{product}-{version}-build{build_number}".format( 276 product=product, version=version, build_number=release["buildNumber"] 277 ) 278 partial_mar_key = f"target-{version}.partial.mar" 279 history = get_release_builds(prev_release_blob, branch) 280 # use one of the localtest channels to avoid relying on bouncer 281 localtest = find_localtest(history["fileUrls"]) 282 url_pattern = history["fileUrls"][localtest]["completes"]["*"] 283 284 for platform in history["platforms"]: 285 if platform not in FTP_PLATFORM_MAP: 286 # skip EOL platforms 287 continue 288 if "alias" in history["platforms"][platform]: 289 continue 290 if platform not in builds: 291 builds[platform] = dict() 292 for locale in history["platforms"][platform]["locales"]: 293 if locale not in builds[platform]: 294 builds[platform][locale] = dict() 295 buildid = history["platforms"][platform]["locales"][locale]["buildID"] 296 url = url_pattern.replace( 297 "%OS_FTP%", FTP_PLATFORM_MAP[platform] 298 ).replace("%LOCALE%", locale) 299 builds[platform][locale][partial_mar_key] = { 300 "buildid": buildid, 301 "mar_url": url, 302 "previousVersion": version, 303 "previousBuildNumber": release["buildNumber"], 304 "product": product, 305 } 306 return builds