artifact.py (4782B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml) 6 """ 7 8 import logging 9 import os 10 import sys 11 12 import yaml 13 from taskgraph.transforms.base import TransformSequence 14 from yaml import YAMLError 15 16 from gecko_taskgraph.transforms.job.common import get_expiration 17 from gecko_taskgraph.util.workertypes import worker_type_implementation 18 19 logger = logging.getLogger(__name__) 20 21 transforms = TransformSequence() 22 23 24 def read_artifact_manifest(manifest_path): 25 """Read the artifacts.yml manifest file and return it.""" 26 # logger.info(f"The current directory is {os.getcwd()}") 27 try: 28 with open(manifest_path) as ymlf: 29 yml = yaml.safe_load(ymlf.read()) 30 return yml 31 except YAMLError as ye: 32 err = 'Failed to parse manifest "{manifest_path}". Invalid Yaml:' 33 err += ye 34 raise SystemExit(err) 35 except FileNotFoundError: 36 err = f'Failed to load manifest "{manifest_path}". File not found' 37 raise SystemExit(err) 38 except PermissionError: 39 err = f'Failed to load manifest "{manifest_path}". Permission Error' 40 raise SystemExit(err) 41 42 43 @transforms.add 44 def set_artifact_expiration(config, jobs): 45 """Set the expiration for certain artifacts based on a manifest file.""" 46 """--- 47 win: 48 - build_resources.json: short 49 50 linux: 51 - target.crashreporter-symbols-full.tar.zst: medium 52 """ 53 transform_dir = os.path.dirname(__file__) 54 manifest = read_artifact_manifest(os.path.join(transform_dir, "artifacts.yml")) 55 56 for job in jobs: 57 try: 58 platform = job["attributes"]["build_platform"] 59 except KeyError: 60 err = "Tried to get build_platfrom for job, but it does not exist. Exiting." 61 raise SystemExit(err) 62 if "worker" in job: 63 if "env" in job["worker"]: 64 if isinstance(job["worker"]["env"], dict): 65 job["worker"]["env"]["MOZ_ARTIFACT_PLATFORM"] = platform 66 else: 67 raise SystemExit( 68 f"Expected env to be a dict, but it was {type(job['worker']['env'])}" 69 ) 70 if "artifacts" in job["worker"]: 71 plat = platform.lower() 72 if "plain" in plat or "ccov" in plat or "rusttest" in plat: 73 art_dict = None 74 elif plat in { 75 "toolchain-wasm32-wasi-compiler-rt-trunk", 76 "toolchain-linux64-x64-compiler-rt-trunk", 77 "toolchain-linux64-x86-compiler-rt-trunk", 78 "android-geckoview-docs", 79 }: 80 art_dict = None 81 elif plat.startswith("win"): 82 art_dict = manifest["win"] 83 elif plat.startswith("linux"): 84 art_dict = manifest["linux"] 85 elif plat.startswith("mac"): 86 art_dict = manifest["macos"] 87 elif plat.startswith("android"): 88 art_dict = manifest["android"] 89 elif plat.startswith("ios"): 90 art_dict = manifest["ios"] 91 else: 92 print( 93 f'The platform name "{plat}" didn\'t start with', 94 '"win", "mac", "android", or "linux".', 95 file=sys.stderr, 96 ) 97 art_dict = None 98 worker_implementation, _ = worker_type_implementation( 99 config.graph_config, config.params, job["worker-type"] 100 ) 101 if worker_implementation == "docker-worker": 102 artifact_dest = "/builds/worker/cidata/{}" 103 else: 104 artifact_dest = "cidata/{}" 105 106 if art_dict is not None: 107 for art_name in art_dict.keys(): 108 # The 'artifacts' key of a job is a list at this stage. 109 # So, must append a new dict to the list 110 expiry_policy = art_dict[art_name] 111 expires = get_expiration(config, policy=expiry_policy) 112 new_art = { 113 "name": f"public/cidata/{art_name}", 114 "path": artifact_dest.format(art_name), 115 "type": "file", 116 "expires-after": expires, 117 } 118 job["worker"]["artifacts"].append(new_art) 119 yield job