update_product_bundles.py (7740B)
1 #!/usr/bin/env vpython3 2 # Copyright 2022 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 """Updates the Fuchsia product bundles to the given revision. Should be used 6 in a 'hooks_os' entry so that it only runs when .gclient's target_os includes 7 'fuchsia'.""" 8 9 import argparse 10 import json 11 import logging 12 import os 13 import sys 14 15 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 16 'test'))) 17 18 import common 19 import update_sdk 20 from compatible_utils import running_unattended 21 22 23 # TODO(crbug.com/40863468): Remove when the old scripts have been deprecated. 24 _IMAGE_TO_PRODUCT_BUNDLE = { 25 'qemu.arm64': 'terminal.qemu-arm64', 26 'qemu.x64': 'terminal.x64', 27 } 28 29 30 # TODO(crbug.com/40863468): Remove when the old scripts have been deprecated. 31 def convert_to_products(images_list): 32 """Convert image names in the SDK to product bundle names.""" 33 34 product_bundle_list = [] 35 for image in images_list: 36 if image in _IMAGE_TO_PRODUCT_BUNDLE: 37 logging.warning(f'Image name {image} has been deprecated. Use ' 38 f'{_IMAGE_TO_PRODUCT_BUNDLE.get(image)} instead.') 39 product_bundle_list.append(_IMAGE_TO_PRODUCT_BUNDLE[image]) 40 else: 41 if image.endswith('-release'): 42 image = image[:-len('-release')] 43 logging.warning(f'Image name {image}-release has been deprecated. Use ' 44 f'{image} instead.') 45 product_bundle_list.append(image) 46 return product_bundle_list 47 48 49 def remove_repositories(repo_names_to_remove): 50 """Removes given repos from repo list. 51 Repo MUST be present in list to succeed. 52 53 Args: 54 repo_names_to_remove: List of repo names (as strings) to remove. 55 """ 56 for repo_name in repo_names_to_remove: 57 common.run_ffx_command(cmd=('repository', 'remove', repo_name)) 58 59 60 def get_repositories(): 61 """Lists repositories that are available on disk. 62 63 Also prunes repositories that are listed, but do not have an actual packages 64 directory. 65 66 Returns: 67 List of dictionaries containing info about the repositories. They have the 68 following structure: 69 { 70 'name': <repo name>, 71 'spec': { 72 'type': <type, usually pm>, 73 'path': <path to packages directory> 74 }, 75 } 76 """ 77 78 repos = json.loads( 79 common.run_ffx_command(cmd=('--machine', 'json', 'repository', 'list'), 80 capture_output=True).stdout.strip()) 81 to_prune = set() 82 sdk_root_abspath = os.path.abspath(os.path.dirname(common.SDK_ROOT)) 83 for repo in repos: 84 # Confirm the path actually exists. If not, prune list. 85 # Also assert the product-bundle repository is for the current repo 86 # (IE within the same directory). 87 if not os.path.exists(repo['spec']['path']): 88 to_prune.add(repo['name']) 89 90 if not repo['spec']['path'].startswith(sdk_root_abspath): 91 to_prune.add(repo['name']) 92 93 repos = [repo for repo in repos if repo['name'] not in to_prune] 94 95 remove_repositories(to_prune) 96 return repos 97 98 99 def get_current_signature(image_dir): 100 """Determines the current version of the image, if it exists. 101 102 Returns: 103 The current version, or None if the image is non-existent. 104 """ 105 106 version_file = os.path.join(image_dir, 'product_bundle.json') 107 if os.path.exists(version_file): 108 with open(version_file) as f: 109 try: 110 data = json.load(f) 111 except json.decoder.JSONDecodeError: 112 logging.warning('product_bundle.json is not at the JSON format and may be empty.') 113 return None 114 if 'product_version' not in data: 115 logging.warning('The key "product_version" does not exist in product_bundle.json') 116 return None 117 return data['product_version'] 118 return None 119 120 121 # VisibleForTesting 122 def internal_hash(): 123 hash_filename = os.path.join(os.path.dirname(__file__), 124 'linux_internal.sdk.sha1') 125 return (open(hash_filename, 'r').read().strip() 126 if os.path.exists(hash_filename) else '') 127 128 129 def main(): 130 parser = argparse.ArgumentParser() 131 parser.add_argument('--verbose', 132 '-v', 133 action='store_true', 134 help='Enable debug-level logging.') 135 parser.add_argument( 136 'products', 137 type=str, 138 help='List of product bundles to download, represented as a comma ' 139 'separated list.') 140 parser.add_argument( 141 '--internal', 142 action='store_true', 143 help='Whether the images are coming from internal, it impacts version ' 144 'file, bucket and download location.') 145 args = parser.parse_args() 146 147 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) 148 149 # Check whether there's Fuchsia support for this platform. 150 common.get_host_os() 151 152 new_products = convert_to_products(args.products.split(',')) 153 logging.debug('Searching for the following products: %s', str(new_products)) 154 155 logging.debug('Getting new SDK hash') 156 if args.internal: 157 new_hash = internal_hash() 158 else: 159 new_hash = common.get_hash_from_sdk() 160 161 auth_args = [ 162 '--auth', 163 os.path.join(os.path.dirname(__file__), 'get_auth_token.py') 164 ] if running_unattended() else [] 165 for product in new_products: 166 prod, board = product.split('.', 1) 167 if prod.startswith('smart_display_') and board in [ 168 'astro', 'sherlock', 'nelson' 169 ]: 170 # This is a hacky way of keeping the files into the folders matching 171 # the original image name, since the definition is unfortunately in 172 # src-internal. Likely we can download two copies for a smooth 173 # transition, but it would be easier to keep it as-is during the ffx 174 # product v2 migration. 175 # TODO(crbug.com/40938340): Migrate the image download folder away from 176 # the following hack. 177 prod, board = board + '-release', prod 178 if args.internal: 179 # sdk_override.txt does not work for internal images. 180 override_url = None 181 image_dir = os.path.join(common.INTERNAL_IMAGES_ROOT, prod, board) 182 else: 183 override_url = update_sdk.GetSDKOverrideGCSPath() 184 if override_url: 185 # TODO(zijiehe): Convert to removesuffix once python 3.9 is supported. 186 if override_url.endswith('/sdk'): 187 override_url = override_url[:-len('/sdk')] 188 logging.debug(f'Using {override_url} from override file.') 189 image_dir = os.path.join(common.IMAGES_ROOT, prod, board) 190 curr_signature = get_current_signature(image_dir) 191 192 if not override_url and curr_signature == new_hash: 193 continue 194 195 common.make_clean_directory(image_dir) 196 base_url = override_url or 'gs://{bucket}/development/{new_hash}'.format( 197 bucket='fuchsia-sdk' if args.internal else 'fuchsia', new_hash=new_hash) 198 effective_auth_args = auth_args if base_url.startswith( 199 'gs://fuchsia-artifacts-internal/') or base_url.startswith( 200 'gs://fuchsia-sdk/') else [] 201 lookup_output = common.run_ffx_command(cmd=[ 202 '--machine', 'json', 'product', 'lookup', product, new_hash, 203 '--base-url', base_url 204 ] + effective_auth_args, 205 capture_output=True).stdout.strip() 206 download_url = json.loads(lookup_output)['transfer_manifest_url'] 207 # The download_url is purely a timestamp based gs location and is fairly 208 # meaningless, so we log the base_url instead which contains the sdk version 209 # if it's not coming from the sdk_override.txt file. 210 logging.info(f'Downloading {product} from {base_url} and {download_url}.') 211 common.run_ffx_command( 212 cmd=['product', 'download', download_url, image_dir] + 213 effective_auth_args) 214 215 return 0 216 217 218 if __name__ == '__main__': 219 sys.exit(main())