scriptworker.py (10157B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import itertools 6 import os 7 from datetime import datetime 8 from functools import lru_cache 9 10 import jsone 11 from taskgraph.util.copy import deepcopy 12 from taskgraph.util.schema import resolve_keyed_by 13 from taskgraph.util.taskcluster import get_artifact_prefix 14 from taskgraph.util.yaml import load_yaml 15 16 cached_load_yaml = lru_cache(maxsize=None)(load_yaml) 17 18 19 def generate_beetmover_upstream_artifacts( 20 config, job, platform, locale=None, dependencies=None, **kwargs 21 ): 22 """Generate the upstream artifacts for beetmover, using the artifact map. 23 24 Currently only applies to beetmover tasks. 25 26 Args: 27 job (dict): The current job being generated 28 dependencies (list): A list of the job's dependency labels. 29 platform (str): The current build platform 30 locale (str): The current locale being beetmoved. 31 32 Returns: 33 list: A list of dictionaries conforming to the upstream_artifacts spec. 34 """ 35 base_artifact_prefix = get_artifact_prefix(job) 36 resolve_keyed_by( 37 job, 38 "attributes.artifact_map", 39 "artifact map", 40 **{ 41 "release-type": config.params["release_type"], 42 "platform": platform, 43 }, 44 ) 45 map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) 46 upstream_artifacts = list() 47 48 if not locale: 49 locales = map_config["default_locales"] 50 elif isinstance(locale, list): 51 locales = locale 52 else: 53 locales = [locale] 54 55 if not dependencies: 56 if job.get("dependencies"): 57 dependencies = job["dependencies"].keys() 58 elif job.get("primary-dependency"): 59 dependencies = [job["primary-dependency"].kind] 60 else: 61 raise Exception(f"Unsupported type of dependency. Got job: {job}") 62 63 for current_locale, dep in itertools.product(locales, dependencies): 64 paths = list() 65 66 for filename in map_config["mapping"]: 67 if dep not in map_config["mapping"][filename]["from"]: 68 continue 69 if ( 70 current_locale != "multi" 71 and not map_config["mapping"][filename]["all_locales"] 72 ): 73 continue 74 if ( 75 "only_for_platforms" in map_config["mapping"][filename] 76 and platform 77 not in map_config["mapping"][filename]["only_for_platforms"] 78 ): 79 continue 80 if ( 81 "not_for_platforms" in map_config["mapping"][filename] 82 and platform in map_config["mapping"][filename]["not_for_platforms"] 83 ): 84 continue 85 if "partials_only" in map_config["mapping"][filename]: 86 continue 87 # The next time we look at this file it might be a different locale. 88 file_config = deepcopy(map_config["mapping"][filename]) 89 resolve_keyed_by( 90 file_config, 91 "source_path_modifier", 92 "source path modifier", 93 locale=current_locale, 94 ) 95 96 kwargs["locale"] = current_locale 97 98 paths.append( 99 os.path.join( 100 base_artifact_prefix, 101 jsone.render(file_config["source_path_modifier"], kwargs), 102 jsone.render(filename, kwargs), 103 ) 104 ) 105 106 if job.get("dependencies") and getattr( 107 job["dependencies"][dep], "release_artifacts", None 108 ): 109 paths = [ 110 path 111 for path in paths 112 if path in job["dependencies"][dep].release_artifacts 113 ] 114 115 if not paths: 116 continue 117 118 upstream_artifacts.append({ 119 "taskId": {"task-reference": f"<{dep}>"}, 120 "taskType": map_config["tasktype_map"].get(dep), 121 "paths": sorted(paths), 122 "locale": current_locale, 123 }) 124 125 upstream_artifacts.sort(key=lambda u: u["paths"]) 126 return upstream_artifacts 127 128 129 def generate_beetmover_artifact_map(config, job, **kwargs): 130 """Generate the beetmover artifact map. 131 132 Currently only applies to beetmover tasks. 133 134 Args: 135 config (): Current taskgraph configuration. 136 job (dict): The current job being generated 137 Common kwargs: 138 platform (str): The current build platform 139 locale (str): The current locale being beetmoved. 140 141 Returns: 142 list: A list of dictionaries containing source->destination 143 maps for beetmover. 144 """ 145 platform = kwargs.get("platform", "") 146 resolve_keyed_by( 147 job, 148 "attributes.artifact_map", 149 job["label"], 150 **{ 151 "release-type": config.params["release_type"], 152 "platform": platform, 153 }, 154 ) 155 map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) 156 base_artifact_prefix = map_config.get( 157 "base_artifact_prefix", get_artifact_prefix(job) 158 ) 159 160 artifacts = list() 161 162 dependencies = job["dependencies"].keys() 163 164 if kwargs.get("locale"): 165 if isinstance(kwargs["locale"], list): 166 locales = kwargs["locale"] 167 else: 168 locales = [kwargs["locale"]] 169 else: 170 locales = map_config["default_locales"] 171 172 resolve_keyed_by( 173 map_config, 174 "s3_bucket_paths", 175 job["label"], 176 **{"build-type": job["attributes"]["build-type"]}, 177 ) 178 179 for locale, dep in sorted(itertools.product(locales, dependencies)): 180 paths = dict() 181 for filename in map_config["mapping"]: 182 # Relevancy checks 183 if dep not in map_config["mapping"][filename]["from"]: 184 # We don't get this file from this dependency. 185 continue 186 if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: 187 # This locale either doesn't produce or shouldn't upload this file. 188 continue 189 if ( 190 "only_for_platforms" in map_config["mapping"][filename] 191 and platform 192 not in map_config["mapping"][filename]["only_for_platforms"] 193 ): 194 # This platform either doesn't produce or shouldn't upload this file. 195 continue 196 if ( 197 "not_for_platforms" in map_config["mapping"][filename] 198 and platform in map_config["mapping"][filename]["not_for_platforms"] 199 ): 200 # This platform either doesn't produce or shouldn't upload this file. 201 continue 202 if "partials_only" in map_config["mapping"][filename]: 203 continue 204 205 # deepcopy because the next time we look at this file the locale will differ. 206 file_config = deepcopy(map_config["mapping"][filename]) 207 208 for field in [ 209 "destinations", 210 "locale_prefix", 211 "source_path_modifier", 212 "update_balrog_manifest", 213 "pretty_name", 214 "checksums_path", 215 ]: 216 resolve_keyed_by(file_config, field, job["label"], locale=locale) 217 218 # This format string should ideally be in the configuration file, 219 # but this would mean keeping variable names in sync between code + config. 220 destinations = [ 221 "{s3_bucket_path}/{dest_path}/{filename}".format( 222 s3_bucket_path=bucket_path, 223 dest_path=dest_path, 224 filename=file_config.get("pretty_name", filename), 225 ) 226 for dest_path, bucket_path in itertools.product( 227 file_config["destinations"], map_config["s3_bucket_paths"] 228 ) 229 ] 230 # Creating map entries 231 # Key must be artifact path, to avoid trampling duplicates, such 232 # as public/build/target.apk and public/build/multi/target.apk 233 key = os.path.join( 234 base_artifact_prefix, 235 file_config["source_path_modifier"], 236 filename, 237 ) 238 239 paths[key] = { 240 "destinations": destinations, 241 } 242 if file_config.get("checksums_path"): 243 paths[key]["checksums_path"] = file_config["checksums_path"] 244 245 # optional flag: balrog manifest 246 if file_config.get("update_balrog_manifest"): 247 paths[key]["update_balrog_manifest"] = True 248 if file_config.get("balrog_format"): 249 paths[key]["balrog_format"] = file_config["balrog_format"] 250 251 if not paths: 252 # No files for this dependency/locale combination. 253 continue 254 255 # Render all variables for the artifact map 256 platforms = deepcopy(map_config.get("platform_names", {})) 257 if platform: 258 for key in platforms.keys(): 259 resolve_keyed_by(platforms, key, job["label"], platform=platform) 260 261 version = config.params["version"] 262 build_number = config.params["build_number"] 263 upload_date = datetime.fromtimestamp(config.params["build_date"]) 264 265 if "nightly" in job["attributes"].get("build-type", ""): 266 folder_prefix = upload_date.strftime("%Y/%m/%Y-%m-%d-%H-%M-%S-") 267 # TODO: Remove this when version.txt has versioning fixed 268 version = version.split("-")[0] 269 else: 270 folder_prefix = f"{version}-candidates/build{build_number}/android/" 271 272 kwargs.update({ 273 "locale": locale, 274 "version": version, 275 "folder_prefix": folder_prefix, 276 }) 277 kwargs.update(**platforms) 278 paths = jsone.render(paths, kwargs) 279 artifacts.append({ 280 "taskId": {"task-reference": f"<{dep}>"}, 281 "locale": locale, 282 "paths": paths, 283 }) 284 285 return artifacts