metasummary.py (15517B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import json 7 import logging 8 import os 9 import re 10 from collections import defaultdict 11 from urllib import parse as urlparse 12 13 import manifestupdate 14 from wptrunner import expected 15 from wptrunner.wptmanifest.backends import base 16 from wptrunner.wptmanifest.serializer import serialize 17 18 here = os.path.dirname(__file__) 19 logger = logging.getLogger(__name__) 20 yaml = None 21 22 23 class Compiler(base.Compiler): 24 def visit_KeyValueNode(self, node): 25 key_name = node.data 26 values = [] 27 for child in node.children: 28 values.append(self.visit(child)) 29 30 self.output_node.set(key_name, values) 31 32 def visit_ConditionalNode(self, node): 33 assert len(node.children) == 2 34 # For conditional nodes, just return the subtree 35 return node.children[0], self.visit(node.children[1]) 36 37 def visit_UnaryExpressionNode(self, node): 38 raise NotImplementedError 39 40 def visit_BinaryExpressionNode(self, node): 41 raise NotImplementedError 42 43 def visit_UnaryOperatorNode(self, node): 44 raise NotImplementedError 45 46 def visit_BinaryOperatorNode(self, node): 47 raise NotImplementedError 48 49 50 class ExpectedManifest(base.ManifestItem): 51 def __init__(self, node, test_path, url_base): 52 """Object representing all the tests in a particular manifest 53 54 :param name: Name of the AST Node associated with this object. 55 Should always be None since this should always be associated with 56 the root node of the AST. 57 :param test_path: Path of the test file associated with this manifest. 58 :param url_base: Base url for serving the tests in this manifest 59 """ 60 if test_path is None: 61 raise ValueError("ExpectedManifest requires a test path") 62 if url_base is None: 63 raise ValueError("ExpectedManifest requires a base url") 64 base.ManifestItem.__init__(self, node) 65 self.child_map = {} 66 self.test_path = test_path 67 self.url_base = url_base 68 69 def append(self, child): 70 """Add a test to the manifest""" 71 base.ManifestItem.append(self, child) 72 self.child_map[child.id] = child 73 74 @property 75 def url(self): 76 return urlparse.urljoin( 77 self.url_base, "/".join(self.test_path.split(os.path.sep)) 78 ) 79 80 81 class DirectoryManifest(base.ManifestItem): 82 pass 83 84 85 class TestManifestItem(base.ManifestItem): 86 def __init__(self, node, **kwargs): 87 """Tree node associated with a particular test in a manifest 88 89 :param name: name of the test""" 90 base.ManifestItem.__init__(self, node) 91 self.subtests = {} 92 93 @property 94 def id(self): 95 return urlparse.urljoin(self.parent.url, self.name) 96 97 def append(self, node): 98 """Add a subtest to the current test 99 100 :param node: AST Node associated with the subtest""" 101 child = base.ManifestItem.append(self, node) 102 self.subtests[child.name] = child 103 104 def get_subtest(self, name): 105 """Get the SubtestNode corresponding to a particular subtest, by name 106 107 :param name: Name of the node to return""" 108 if name in self.subtests: 109 return self.subtests[name] 110 return None 111 112 113 class SubtestManifestItem(TestManifestItem): 114 pass 115 116 117 def data_cls_getter(output_node, visited_node): 118 # visited_node is intentionally unused 119 if output_node is None: 120 return ExpectedManifest 121 if isinstance(output_node, ExpectedManifest): 122 return TestManifestItem 123 if isinstance(output_node, TestManifestItem): 124 return SubtestManifestItem 125 raise ValueError 126 127 128 def get_manifest(metadata_root, test_path, url_base): 129 """Get the ExpectedManifest for a particular test path, or None if there is no 130 metadata stored for that test path. 131 132 :param metadata_root: Absolute path to the root of the metadata directory 133 :param test_path: Path to the test(s) relative to the test root 134 :param url_base: Base url for serving the tests in this manifest 135 :param run_info: Dictionary of properties of the test run for which the expectation 136 values should be computed. 137 """ 138 manifest_path = expected.expected_path(metadata_root, test_path) 139 try: 140 with open(manifest_path, "rb") as f: 141 return compile( 142 f, 143 data_cls_getter=data_cls_getter, 144 test_path=test_path, 145 url_base=url_base, 146 ) 147 except OSError: 148 return None 149 150 151 def get_dir_manifest(path): 152 """Get the ExpectedManifest for a particular test path, or None if there is no 153 metadata stored for that test path. 154 155 :param path: Full path to the ini file 156 :param run_info: Dictionary of properties of the test run for which the expectation 157 values should be computed. 158 """ 159 try: 160 with open(path, "rb") as f: 161 return compile(f, data_cls_getter=lambda x, y: DirectoryManifest) 162 except OSError: 163 return None 164 165 166 def compile(stream, data_cls_getter=None, **kwargs): 167 return base.compile(Compiler, stream, data_cls_getter=data_cls_getter, **kwargs) 168 169 170 def create_parser(): 171 parser = argparse.ArgumentParser() 172 parser.add_argument("--out-dir", help="Directory to store output files") 173 parser.add_argument( 174 "--meta-dir", help="Directory containing wpt-metadata checkout to update." 175 ) 176 return parser 177 178 179 def run(src_root, obj_root, logger_=None, **kwargs): 180 logger_obj = logger_ if logger_ is not None else logger 181 182 manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs) 183 184 rv = {} 185 dirs_seen = set() 186 187 for meta_root, test_path, test_metadata in iter_tests(manifests): 188 for dir_path in get_dir_paths(meta_root, test_path): 189 if dir_path not in dirs_seen: 190 dirs_seen.add(dir_path) 191 dir_manifest = get_dir_manifest(dir_path) 192 rel_path = os.path.relpath(dir_path, meta_root) 193 if dir_manifest: 194 add_manifest(rv, rel_path, dir_manifest) 195 else: 196 break 197 add_manifest(rv, test_path, test_metadata) 198 199 if kwargs["out_dir"]: 200 if not os.path.exists(kwargs["out_dir"]): 201 os.makedirs(kwargs["out_dir"]) 202 out_path = os.path.join(kwargs["out_dir"], "summary.json") 203 with open(out_path, "w") as f: 204 json.dump(rv, f) 205 else: 206 print(json.dumps(rv, indent=2)) 207 208 if kwargs["meta_dir"]: 209 update_wpt_meta(logger_obj, kwargs["meta_dir"], rv) 210 211 212 def get_dir_paths(test_root, test_path): 213 if not os.path.isabs(test_path): 214 test_path = os.path.join(test_root, test_path) 215 dir_path = os.path.dirname(test_path) 216 while dir_path != test_root: 217 yield os.path.join(dir_path, "__dir__.ini") 218 dir_path = os.path.dirname(dir_path) 219 assert len(dir_path) >= len(test_root) 220 221 222 def iter_tests(manifests): 223 for manifest in manifests.keys(): 224 for test_type, test_path, tests in manifest: 225 url_base = manifests[manifest]["url_base"] 226 metadata_base = manifests[manifest]["metadata_path"] 227 expected_manifest = get_manifest(metadata_base, test_path, url_base) 228 if expected_manifest: 229 yield metadata_base, test_path, expected_manifest 230 231 232 def add_manifest(target, path, metadata): 233 dir_name, file_name = path.rsplit(os.sep, 1) 234 key = [dir_name] 235 236 add_metadata(target, key, metadata) 237 238 key.append("_tests") 239 240 for test_metadata in metadata.children: 241 key.append(test_metadata.name) 242 add_metadata(target, key, test_metadata) 243 add_filename(target, key, file_name) 244 key.append("_subtests") 245 for subtest_metadata in test_metadata.children: 246 key.append(subtest_metadata.name) 247 add_metadata(target, key, subtest_metadata) 248 key.pop() 249 key.pop() 250 key.pop() 251 252 253 simple_props = [ 254 "disabled", 255 "min-asserts", 256 "max-asserts", 257 "lsan-allowed", 258 "leak-allowed", 259 "bug", 260 ] 261 statuses = set(["CRASH"]) 262 263 264 def add_filename(target, key, filename): 265 for part in key: 266 if part not in target: 267 target[part] = {} 268 target = target[part] 269 270 target["_filename"] = filename 271 272 273 def add_metadata(target, key, metadata): 274 if not is_interesting(metadata): 275 return 276 277 for part in key: 278 if part not in target: 279 target[part] = {} 280 target = target[part] 281 282 for prop in simple_props: 283 if metadata.has_key(prop): # noqa W601 284 target[prop] = get_condition_value_list(metadata, prop) 285 286 if metadata.has_key("expected"): # noqa W601 287 intermittent = [] 288 values = metadata.get("expected") 289 by_status = defaultdict(list) 290 for item in values: 291 if isinstance(item, tuple): 292 condition, status = item 293 else: 294 condition = None 295 status = item 296 if isinstance(status, list): 297 intermittent.append((condition, status)) 298 expected_status = status[0] 299 else: 300 expected_status = status 301 by_status[expected_status].append(condition) 302 for status in statuses: 303 if status in by_status: 304 target["expected_%s" % status] = [ 305 serialize(item) if item else None for item in by_status[status] 306 ] 307 if intermittent: 308 target["intermittent"] = [ 309 [serialize(cond) if cond else None, intermittent_statuses] 310 for cond, intermittent_statuses in intermittent 311 ] 312 313 314 def get_condition_value_list(metadata, key): 315 conditions = [] 316 for item in metadata.get(key): 317 if isinstance(item, tuple): 318 assert len(item) == 2 319 conditions.append((serialize(item[0]), item[1])) 320 else: 321 conditions.append((None, item)) 322 return conditions 323 324 325 def is_interesting(metadata): 326 if any(metadata.has_key(prop) for prop in simple_props): # noqa W601 327 return True 328 329 if metadata.has_key("expected"): # noqa W601 330 for expected_value in metadata.get("expected"): 331 # Include both expected and known intermittent values 332 if isinstance(expected_value, tuple): 333 expected_value = expected_value[1] 334 if isinstance(expected_value, list): 335 return True 336 if expected_value in statuses: 337 return True 338 return True 339 return False 340 341 342 def update_wpt_meta(logger, meta_root, data): 343 global yaml 344 import yaml 345 346 if not os.path.exists(meta_root) or not os.path.isdir(meta_root): 347 raise ValueError("%s is not a directory" % (meta_root,)) 348 349 with WptMetaCollection(meta_root) as wpt_meta: 350 for dir_path, dir_data in sorted(data.items()): 351 for test, test_data in dir_data.get("_tests", {}).items(): 352 add_test_data(logger, wpt_meta, dir_path, test, None, test_data) 353 for subtest, subtest_data in test_data.get("_subtests", {}).items(): 354 add_test_data( 355 logger, wpt_meta, dir_path, test, subtest, subtest_data 356 ) 357 358 359 def add_test_data(logger, wpt_meta, dir_path, test, subtest, test_data): 360 triage_keys = ["bug"] 361 362 for key in triage_keys: 363 if key in test_data: 364 value = test_data[key] 365 for cond_value in value: 366 if cond_value[0] is not None: 367 logger.info("Skipping conditional metadata") 368 continue 369 cond_value = cond_value[1] 370 if not isinstance(cond_value, list): 371 cond_value = [cond_value] 372 for bug_value in cond_value: 373 bug_link = get_bug_link(bug_value) 374 if bug_link is None: 375 logger.info("Could not extract bug: %s" % value) 376 continue 377 meta = wpt_meta.get(dir_path) 378 meta.set(test, subtest, product="firefox", bug_url=bug_link) 379 380 381 bugzilla_re = re.compile(r"https://bugzilla\.mozilla\.org/show_bug\.cgi\?id=\d+") 382 bug_re = re.compile(r"(?:[Bb][Uu][Gg])?\s*(\d+)") 383 384 385 def get_bug_link(value): 386 value = value.strip() 387 m = bugzilla_re.match(value) 388 if m: 389 return m.group(0) 390 m = bug_re.match(value) 391 if m: 392 return "https://bugzilla.mozilla.org/show_bug.cgi?id=%s" % m.group(1) 393 394 395 class WptMetaCollection: 396 def __init__(self, root): 397 self.root = root 398 self.loaded = {} 399 400 def __enter__(self): 401 return self 402 403 def __exit__(self, *args, **kwargs): 404 for item in self.loaded.itervalues(): 405 item.write(self.root) 406 self.loaded = {} 407 408 def get(self, dir_path): 409 if dir_path not in self.loaded: 410 meta = WptMeta.get_or_create(self.root, dir_path) 411 self.loaded[dir_path] = meta 412 return self.loaded[dir_path] 413 414 415 class WptMeta: 416 def __init__(self, dir_path, data): 417 assert "links" in data and isinstance(data["links"], list) 418 self.dir_path = dir_path 419 self.data = data 420 421 @staticmethod 422 def meta_path(meta_root, dir_path): 423 return os.path.join(meta_root, dir_path, "META.yml") 424 425 def path(self, meta_root): 426 return self.meta_path(meta_root, self.dir_path) 427 428 @classmethod 429 def get_or_create(cls, meta_root, dir_path): 430 if os.path.exists(cls.meta_path(meta_root, dir_path)): 431 return cls.load(meta_root, dir_path) 432 return cls(dir_path, {"links": []}) 433 434 @classmethod 435 def load(cls, meta_root, dir_path): 436 with open(cls.meta_path(meta_root, dir_path)) as f: 437 data = yaml.safe_load(f) 438 return cls(dir_path, data) 439 440 def set(self, test, subtest, product, bug_url): 441 target_link = None 442 for link in self.data["links"]: 443 link_product = link.get("product") 444 if link_product: 445 link_product = link_product.split("-", 1)[0] 446 if link_product is None or link_product == product: 447 if link["url"] == bug_url: 448 target_link = link 449 break 450 451 if target_link is None: 452 target_link = { 453 "product": product.encode("utf8"), 454 "url": bug_url.encode("utf8"), 455 "results": [], 456 } 457 self.data["links"].append(target_link) 458 459 if "results" not in target_link: 460 target_link["results"] = [] 461 462 has_result = any( 463 (result["test"] == test and result.get("subtest") == subtest) 464 for result in target_link["results"] 465 ) 466 if not has_result: 467 data = {"test": test.encode("utf8")} 468 if subtest: 469 data["subtest"] = subtest.encode("utf8") 470 target_link["results"].append(data) 471 472 def write(self, meta_root): 473 path = self.path(meta_root) 474 dirname = os.path.dirname(path) 475 if not os.path.exists(dirname): 476 os.makedirs(dirname) 477 with open(path, "wb") as f: 478 yaml.safe_dump(self.data, f, default_flow_style=False, allow_unicode=True)