common.py (7615B)
1 # Copyright 2024 The Chromium Authors 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import argparse 6 import logging 7 import os 8 import pathlib 9 import shlex 10 import shutil 11 import subprocess 12 import sys 13 import tarfile 14 import tempfile 15 import time 16 import urllib.request 17 18 import scripthash 19 20 _THIS_DIR = pathlib.Path(__file__).resolve().parent 21 _SRC_ROOT = _THIS_DIR.parents[1] 22 _CHECKOUT_SRC_ROOT_SUBDIR = '.3pp/chromium' 23 24 25 def parse_args(): 26 parser = argparse.ArgumentParser() 27 # TODO(agrieve): Add required=True once 3pp builds with > python3.6. 28 subparsers = parser.add_subparsers() 29 30 subparser = subparsers.add_parser( 31 'latest', help='Prints the version as $LATEST.$RUNTIME_DEPS_HASH') 32 subparser.set_defaults(action='latest') 33 34 subparser = subparsers.add_parser( 35 'checkout', help='Copies files into the workdir used by docker') 36 subparser.add_argument('checkout_dir') 37 subparser.add_argument('--version', help='Output from "latest"') 38 subparser.set_defaults(action='checkout') 39 40 subparser = subparsers.add_parser( 41 'install', 42 help=('Run from workdir, inside docker container. ' 43 'Builds & copies outputs into |output_prefix| directory')) 44 subparser.add_argument('output_prefix', 45 help='The path to install the compiled package to.') 46 subparser.add_argument('deps_prefix', 47 help='The path to a directory containing all deps.') 48 subparser.add_argument('--version', help='Output from "latest"') 49 subparser.add_argument('--checkout-dir', help='Directory to use as CWD') 50 subparser.set_defaults(action='install') 51 52 subparser = subparsers.add_parser( 53 'local-test', help='Run latest / checkout / install locally') 54 subparser.add_argument('--checkout-dir', 55 default='3pp_workdir', 56 help='Workdir to use') 57 subparser.add_argument('--output-prefix', 58 default='3pp_out', 59 help='Directory for final artifacts') 60 subparser.set_defaults(action='local-test') 61 62 args = parser.parse_args() 63 if not hasattr(args, 'action'): 64 parser.print_help() 65 sys.exit(1) 66 67 if hasattr(args, 'version'): 68 args.version = args.version or os.environ.get('_3PP_VERSION') 69 if not args.version: 70 parser.error('Must set --version or _3PP_VERSION') 71 if hasattr(args, 'output_prefix') and args.output_prefix: 72 args.output_prefix = os.path.abspath(args.output_prefix) 73 if hasattr(args, 'checkout_dir') and args.checkout_dir: 74 args.checkout_dir = os.path.abspath(args.checkout_dir) 75 76 if args.action == 'checkout': 77 # 3pp bot recipe does this, so needed only when running locally. 78 os.makedirs(args.checkout_dir, exist_ok=True) 79 80 if args.action == 'install': 81 if args.checkout_dir: 82 logging.info('Setting CWD=%s', args.checkout_dir) 83 os.chdir(args.checkout_dir) 84 85 if not os.path.exists(_CHECKOUT_SRC_ROOT_SUBDIR): 86 parser.error(f'Does not exist: {_CHECKOUT_SRC_ROOT_SUBDIR}.' 87 f' Use --checkout-dir?') 88 89 # 3pp bot recipe does this, so needed only when running locally. 90 os.makedirs(args.output_prefix, exist_ok=True) 91 92 return args 93 94 95 def path_within_checkout(subpath): 96 return os.path.abspath(os.path.join(_CHECKOUT_SRC_ROOT_SUBDIR, subpath)) 97 98 99 def _all_files(path): 100 if os.path.isfile(path): 101 return [path] 102 assert os.path.isdir(path), 'Not a file or dir: ' + path 103 all_paths = pathlib.Path(path).glob('**/*') 104 return [str(f) for f in all_paths if f.is_file()] 105 106 107 def _resolve_runtime_deps(runtime_deps): 108 ret = [] 109 for p in runtime_deps: 110 if p.startswith('//'): 111 ret.append(os.path.relpath(str(_SRC_ROOT / p[2:]))) 112 elif os.path.isabs(p): 113 ret.append(os.path.relpath(p)) 114 else: 115 ret.append(p) 116 return ret 117 118 119 def copy_runtime_deps(checkout_dir, runtime_deps): 120 # Make 3pp_common scripts available in the docker container install.py 121 # will run in. 122 dest_dir = os.path.join(checkout_dir, _CHECKOUT_SRC_ROOT_SUBDIR) 123 124 for src_path in _resolve_runtime_deps(runtime_deps): 125 relpath = os.path.relpath(src_path, _SRC_ROOT) 126 dest_path = os.path.join(dest_dir, relpath) 127 os.makedirs(os.path.dirname(dest_path), exist_ok=True) 128 if os.path.isfile(src_path): 129 shutil.copy(src_path, dest_path) 130 else: 131 shutil.copytree(src_path, 132 dest_path, 133 ignore=shutil.ignore_patterns('.*', '__pycache__')) 134 logging.info('Runtime deps:') 135 sys.stderr.write('\n'.join(_all_files(checkout_dir)) + '\n') 136 137 138 def download_file(url, dest): 139 logging.info('Downloading %s', url) 140 with urllib.request.urlopen(url) as r: 141 with open(dest, 'wb') as f: 142 shutil.copyfileobj(r, f) 143 144 145 def extract_tar(path, dest): 146 logging.info('Extracting %s to %s', path, dest) 147 with tarfile.open(path) as f: 148 f.extractall(dest) 149 150 151 def run_cmd(cmd, check=True, *args, **kwargs): 152 logging.info('Running: %s', shlex.join(cmd)) 153 return subprocess.run(cmd, check=check, *args, **kwargs) 154 155 156 def apply_patches(patches_dir, checkout_dir): 157 for path in sorted(pathlib.Path(patches_dir).glob('*.patch')): 158 cmd = ['git', 'apply', '-v', str(path)] 159 run_cmd(cmd, cwd=checkout_dir) 160 161 162 def main(*, do_latest, do_install, runtime_deps): 163 logging.basicConfig( 164 level=logging.DEBUG, 165 format='%(levelname).1s %(relativeCreated)6d %(message)s') 166 args = parse_args() 167 runtime_deps = [str(_THIS_DIR)] + runtime_deps 168 169 if args.action == 'local-test': 170 logging.warning('Will use work dir: %s', args.checkout_dir) 171 logging.warning('Will use output dir: %s', args.output_prefix) 172 if os.path.exists(args.checkout_dir) and os.listdir(args.checkout_dir): 173 logging.warning( 174 '*** Work dir not empty. This often causes failures. ***') 175 time.sleep(4) 176 # Approximates what 3pp recipe does for minimal configs. 177 # https://source.chromium.org/search?q=symbol:Chromium3ppApi.execute&ss=chromium 178 prog = os.path.abspath(sys.argv[0]) 179 cmd = [prog, 'latest'] 180 version = run_cmd(cmd, stdout=subprocess.PIPE, text=True).stdout 181 os.environ['_3PP_VERSION'] = version 182 checkout_dir = args.checkout_dir 183 run_cmd([prog, 'checkout', checkout_dir]) 184 run_cmd([prog, 'install', args.output_prefix, 'UNUSED-DEPS-DIR'], 185 cwd=checkout_dir) 186 logging.warning('Local test complete.') 187 return 188 189 if args.action == 'latest': 190 version = do_latest() 191 assert version, 'do_latest() returned ' + repr(version) 192 extra_paths = [] 193 for p in _resolve_runtime_deps(runtime_deps): 194 extra_paths += _all_files(p) 195 deps_hash = scripthash.compute(extra_paths=extra_paths) 196 print(f'{version}.{deps_hash}') 197 return 198 199 # Remove the hash at the end: 30.4.0-alpha05.HASH => 30.4.0-alpha05 200 args.version = args.version.rsplit('.', 1)[0] 201 if args.action == 'checkout': 202 copy_runtime_deps(args.checkout_dir, runtime_deps) 203 return 204 205 assert args.action == 'install' 206 do_install(args) 207 prefix_len = len(args.output_prefix) + 1 208 logging.info( 209 'Contents of %s: \n%s\n', args.output_prefix, 210 '\n'.join(p[prefix_len:] for p in _all_files(args.output_prefix)))