categorize_results.py (3940B)
1 import os 2 import sys 3 import json 4 import fnmatch 5 6 TEST_DIR = "/webvtt/" 7 CATEGORIES_FILE = "../categories.json" 8 9 class Test: 10 def __init__(self, file, name, status, message): 11 self.file = file 12 self.name = name 13 self.status = status 14 self.message = message 15 self.passed = status == 'PASS' 16 self.categories = [] 17 18 @classmethod 19 def from_json(cls, json): 20 file = json["test"] 21 if not file.startswith(TEST_DIR): 22 return [] 23 file = file[len(TEST_DIR):] 24 25 status = json["status"] 26 message = json["message"] 27 28 tests = [] 29 30 for test in json["subtests"]: 31 name = test["name"] 32 if status == 'OK': 33 test_status = test["status"] 34 test_message = test["message"] 35 else: 36 test_status, test_message = status, message 37 38 tests.append(Test(file, name, test_status, test_message)) 39 40 return tests 41 42 class Category: 43 def __init__(self, names): 44 self.names = set(names) 45 self.tests = {} 46 47 @classmethod 48 def from_json(cls, json): 49 return Category(json) 50 51 def add_test(self, name, test): 52 self.tests[test] = name 53 54 def __contains__(self, name): 55 return name in self.names 56 57 def parse_results(file): 58 data = json.load(file) 59 60 results = data["results"] 61 tests = [] 62 for result in results: 63 tests += Test.from_json(result) 64 65 return tests 66 67 def parse_categories(file, tests, categories = None, categories_map = None): 68 data = json.load(file) 69 basepath = os.path.dirname(file.name) 70 71 categories = categories or [] 72 73 if categories_map: 74 categories_map = dict(categories_map) 75 else: 76 categories_map = {} 77 78 if ":categories" in data: 79 for cat_data in data[":categories"]: 80 category = Category.from_json(cat_data) 81 82 categories.append(category) 83 for name in category.names: 84 categories_map[name] = category 85 86 for pattern, category_name in data.items(): 87 if pattern.startswith(":"): 88 continue 89 category = categories_map[category_name] 90 91 file_pattern = os.path.normpath(os.path.join(basepath, pattern)) 92 for test in tests: 93 if fnmatch.fnmatch(test.name, file_pattern) or fnmatch.fnmatch(test.file, file_pattern): 94 category.add_test(category_name, test) 95 test.categories.append(category) 96 97 if ":subcategories" in data: 98 for subcat_name in data[":subcategories"]: 99 path = os.path.join(basepath, subcat_name) 100 file = open(path, "r") 101 parse_categories(file, tests, categories, categories_map) 102 103 return categories 104 105 def main(argv): 106 if len(argv) == 1: 107 if argv[0] == '-': 108 results_file = sys.stdin 109 else: 110 results_file = open(argv[0], "r") 111 else: 112 print("USAGE: python3 categorize_results.py <file>") 113 print("<file>\tA file containing wpt results. Or `-` for reading results from stdin.") 114 return 115 116 filepath = os.path.dirname(__file__) 117 categories_path = os.path.join(filepath, CATEGORIES_FILE) 118 categories_file = open(categories_path, "r") 119 120 tests = parse_results(results_file) 121 categories = parse_categories(categories_file, tests) 122 123 for category in categories: 124 tests_by_name = { name: [] for name in category.names } 125 for test, name in category.tests.items(): 126 tests_by_name[name].append(test) 127 128 for name in category.names: 129 test_group = tests_by_name[name] 130 amount = len(test_group) 131 if amount == 0: 132 continue 133 passed = sum(1 for test in test_group if test.passed) 134 print("{}:\t{}/{} - {}%".format(name, passed, amount, round(passed / amount * 100, 2))) 135 136 if __name__ == "__main__": 137 main(sys.argv[1:])