manifestupdate.py (8956B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import configparser 7 import errno 8 import hashlib 9 import os 10 import sys 11 12 import manifestdownload 13 from mach.util import get_state_dir 14 from mozfile import load_source 15 from mozlog.structured import commandline 16 from wptrunner import wptcommandline 17 18 manifest = None 19 20 21 def do_delayed_imports(wpt_dir): 22 global manifest 23 load_source("localpaths", os.path.join(wpt_dir, "tests", "tools", "localpaths.py")) 24 sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest")) 25 import manifest 26 27 28 def create_parser(): 29 p = argparse.ArgumentParser() 30 p.add_argument( 31 "--rebuild", action="store_true", help="Rebuild manifest from scratch" 32 ) 33 download_group = p.add_mutually_exclusive_group() 34 download_group.add_argument( 35 "--download", 36 dest="download", 37 action="store_true", 38 default=None, 39 help="Always download even if the local manifest is recent", 40 ) 41 download_group.add_argument( 42 "--no-download", 43 dest="download", 44 action="store_false", 45 help="Don't try to download the manifest", 46 ) 47 p.add_argument( 48 "--no-update", 49 action="store_false", 50 dest="update", 51 default=True, 52 help="Just download the manifest, don't update", 53 ) 54 p.add_argument( 55 "--config", 56 action="store", 57 dest="config_path", 58 default=None, 59 help="Path to wptrunner config file", 60 ) 61 p.add_argument( 62 "--rewrite-config", 63 action="store_true", 64 default=False, 65 help="Force the local configuration to be regenerated", 66 ) 67 p.add_argument( 68 "--cache-root", 69 action="store", 70 default=os.path.join(get_state_dir(), "cache", "wpt"), 71 help="Path to use for the metadata cache", 72 ) 73 commandline.add_logging_group(p) 74 75 return p 76 77 78 def ensure_kwargs(kwargs): 79 _kwargs = vars(create_parser().parse_args([])) 80 _kwargs.update(kwargs) 81 return _kwargs 82 83 84 def run(src_root, obj_root, logger=None, **kwargs): 85 kwargs = ensure_kwargs(kwargs) 86 87 if logger is None: 88 from wptrunner import wptlogging 89 90 logger = wptlogging.setup(kwargs, {"mach": sys.stdout}) 91 92 src_wpt_dir = os.path.join(src_root, "testing", "web-platform") 93 94 do_delayed_imports(src_wpt_dir) 95 96 if not kwargs["config_path"]: 97 config_path = generate_config( 98 logger, 99 src_root, 100 src_wpt_dir, 101 os.path.join(obj_root, "_tests", "web-platform"), 102 kwargs["rewrite_config"], 103 ) 104 else: 105 config_path = kwargs["config_path"] 106 107 if not os.path.exists(config_path): 108 logger.critical("Config file %s does not exist" % config_path) 109 return None 110 111 logger.debug("Using config path %s" % config_path) 112 113 test_paths = wptcommandline.get_test_paths(wptcommandline.config.read(config_path)) 114 115 for paths in test_paths.values(): 116 if isinstance(paths, dict) and "manifest_path" not in paths: 117 paths["manifest_path"] = os.path.join( 118 paths["metadata_path"], "MANIFEST.json" 119 ) 120 121 ensure_manifest_directories(logger, test_paths) 122 123 local_config = read_local_config(os.path.join(src_wpt_dir, "wptrunner.ini")) 124 for section in ["manifest:upstream", "manifest:mozilla"]: 125 url_base = local_config.get(section, "url_base") 126 manifest_rel_path = os.path.join( 127 local_config.get(section, "metadata"), "MANIFEST.json" 128 ) 129 if isinstance(test_paths[url_base], dict): 130 test_paths[url_base]["manifest_rel_path"] = manifest_rel_path 131 else: 132 test_paths[url_base].manifest_rel_path = manifest_rel_path 133 134 if not kwargs["rebuild"] and kwargs["download"] is not False: 135 force_download = False if kwargs["download"] is None else True 136 manifestdownload.download_from_taskcluster( 137 logger, src_root, test_paths, force=force_download 138 ) 139 else: 140 logger.debug("Skipping manifest download") 141 142 update = kwargs["update"] or kwargs["rebuild"] 143 manifests = load_and_update( 144 logger, 145 src_wpt_dir, 146 test_paths, 147 update=update, 148 rebuild=kwargs["rebuild"], 149 cache_root=kwargs["cache_root"], 150 ) 151 152 return manifests 153 154 155 def ensure_manifest_directories(logger, test_paths): 156 for paths in test_paths.values(): 157 manifest_path = ( 158 paths["manifest_path"] if isinstance(paths, dict) else paths.manifest_path 159 ) 160 manifest_dir = os.path.dirname(manifest_path) 161 if not os.path.exists(manifest_dir): 162 logger.info("Creating directory %s" % manifest_dir) 163 # Even though we just checked the path doesn't exist, there's a chance 164 # of race condition with another process or thread having created it in 165 # between. This happens during tests. 166 try: 167 os.makedirs(manifest_dir) 168 except OSError as e: 169 if e.errno != errno.EEXIST: 170 raise 171 elif not os.path.isdir(manifest_dir): 172 raise OSError("Manifest directory is a file") 173 174 175 def read_local_config(src_config_path): 176 parser = configparser.ConfigParser() 177 success = parser.read(src_config_path) 178 assert src_config_path in success 179 return parser 180 181 182 def generate_config(logger, repo_root, wpt_dir, dest_path, force_rewrite=False): 183 """Generate the local wptrunner.ini file to use locally""" 184 if not os.path.exists(dest_path): 185 # Even though we just checked the path doesn't exist, there's a chance 186 # of race condition with another process or thread having created it in 187 # between. This happens during tests. 188 try: 189 os.makedirs(dest_path) 190 except OSError as e: 191 if e.errno != errno.EEXIST: 192 raise 193 194 src_config_path = os.path.join(wpt_dir, "wptrunner.ini") 195 dest_config_path = os.path.join(dest_path, "wptrunner.local.ini") 196 197 if ( 198 not force_rewrite 199 and os.path.exists(dest_config_path) 200 and os.stat(dest_config_path).st_mtime >= os.stat(src_config_path).st_mtime 201 ): 202 logger.debug("Config is up to date, not regenerating") 203 return dest_config_path 204 205 logger.info(f"Creating config file {dest_config_path}") 206 207 parser = read_local_config(src_config_path) 208 209 for section in ["manifest:upstream", "manifest:mozilla"]: 210 meta_rel_path = parser.get(section, "metadata") 211 tests_rel_path = parser.get(section, "tests") 212 213 parser.set( 214 section, "manifest", os.path.join(dest_path, meta_rel_path, "MANIFEST.json") 215 ) 216 parser.set(section, "metadata", os.path.join(wpt_dir, meta_rel_path)) 217 parser.set(section, "tests", os.path.join(wpt_dir, tests_rel_path)) 218 219 parser.set( 220 "paths", 221 "prefs", 222 os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))), 223 ) 224 ws_extra_paths = ";".join( 225 os.path.abspath(os.path.join(wpt_dir, path)) 226 for path in parser.get("paths", "ws_extra").split(";") 227 ) 228 parser.set("paths", "ws_extra", ws_extra_paths) 229 230 with open(dest_config_path, "w") as config_file: 231 parser.write(config_file) 232 233 return dest_config_path 234 235 236 def load_and_update( 237 logger, 238 wpt_dir, 239 test_paths, 240 rebuild=False, 241 config_dir=None, 242 cache_root=None, 243 update=True, 244 ): 245 rv = {} 246 wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir).encode()).hexdigest() 247 for url_base, paths in test_paths.items(): 248 manifest_path = ( 249 paths["manifest_path"] if isinstance(paths, dict) else paths.manifest_path 250 ) 251 manifest_rel_path = ( 252 paths["manifest_rel_path"] 253 if isinstance(paths, dict) 254 else paths.manifest_rel_path 255 ) 256 tests_path = ( 257 paths["tests_path"] if isinstance(paths, dict) else paths.tests_path 258 ) 259 this_cache_root = os.path.join( 260 cache_root, wptdir_hash, os.path.dirname(manifest_rel_path) 261 ) 262 m = manifest.manifest.load_and_update( 263 tests_path, 264 manifest_path, 265 url_base, 266 update=update, 267 rebuild=rebuild, 268 working_copy=True, 269 cache_root=this_cache_root, 270 ) 271 path_data = {"url_base": url_base} 272 if isinstance(paths, dict): 273 path_data.update(paths) 274 else: 275 for key, value in paths.__dict__.items(): 276 path_data[key] = value 277 278 rv[m] = path_data 279 280 return rv 281 282 283 def log_error(logger, manifest_path, msg): 284 logger.lint_error( 285 path=manifest_path, message=msg, lineno=0, source="", linter="wpt-manifest" 286 )