report.py (9526B)
1 # flake8: noqa 2 # mypy: ignore-errors 3 4 import argparse 5 import json 6 import sys 7 import types 8 9 from cgi import escape 10 from collections import defaultdict 11 12 13 def html_escape(item, escape_quote=False): 14 if isinstance(item, types.StringTypes): 15 rv = escape(item) 16 if escape_quote: 17 rv = rv.replace('"', """) 18 return rv 19 else: 20 return item 21 22 23 class Raw: 24 """Simple wrapper around a string to stop it being escaped by html_escape""" 25 def __init__(self, value): 26 self.value = value 27 28 def __unicode__(self): 29 return unicode(self.value) 30 31 32 class Node: 33 """Node structure used when building HTML""" 34 def __init__(self, name, attrs, children): 35 #Need list of void elements 36 self.name = name 37 self.attrs = attrs 38 self.children = children 39 40 def __unicode__(self): 41 if self.attrs: 42 #Need to escape 43 attrs_unicode = " " + " ".join("%s=\"%s\"" % (html_escape(key), 44 html_escape(value, 45 escape_quote=True)) 46 for key, value in self.attrs.items()) 47 else: 48 attrs_unicode = "" 49 return "<%s%s>%s</%s>\n" % (self.name, 50 attrs_unicode, 51 "".join(unicode(html_escape(item)) 52 for item in self.children), 53 self.name) 54 55 def __str__(self): 56 return unicode(self).encode("utf8") 57 58 59 class RootNode: 60 """Special Node representing the document root""" 61 def __init__(self, *children): 62 self.children = ["<!DOCTYPE html>"] + list(children) 63 64 def __unicode__(self): 65 return "".join(unicode(item) for item in self.children) 66 67 def __str__(self): 68 return unicode(self).encode("utf8") 69 70 71 def flatten(iterable): 72 """Flatten a list of lists by one level so that 73 [1,["abc"], "def",[2, [3]]] 74 becomes 75 [1, "abc", "def", 2, [3]]""" 76 rv = [] 77 for item in iterable: 78 if hasattr(item, "__iter__") and not isinstance(item, types.StringTypes): 79 rv.extend(item) 80 else: 81 rv.append(item) 82 return rv 83 84 85 class HTML: 86 """Simple HTML templating system. An instance of this class can create 87 element nodes by calling methods with the same name as the element, 88 passing in children as positional arguments or as a list, and attributes 89 as keyword arguments, with _ replacing - and trailing _ for python keywords 90 91 e.g. 92 93 h = HTML() 94 print(h.html( 95 html.head(), 96 html.body([html.h1("Hello World!")], class_="body-class") 97 )) 98 Would give 99 <!DOCTYPE html><html><head></head><body class="body-class"><h1>Hello World!</h1></body></html>""" 100 def __getattr__(self, name): 101 def make_html(self, *content, **attrs): 102 for attr_name in attrs.keys(): 103 if "_" in attr_name: 104 new_name = attr_name.replace("_", "-") 105 if new_name.endswith("-"): 106 new_name = new_name[:-1] 107 attrs[new_name] = attrs.pop(attr_name) 108 return Node(name, attrs, flatten(content)) 109 110 method = types.MethodType(make_html, self, HTML) 111 setattr(self, name, method) 112 return method 113 114 def __call__(self, *children): 115 return RootNode(*flatten(children)) 116 117 118 h = HTML() 119 120 121 class TestResult: 122 """Simple holder for the results of a single test in a single UA""" 123 def __init__(self, test): 124 self.test = test 125 self.results = {} 126 127 def __cmp__(self, other): 128 return self.test == other.test 129 130 def __hash__(self): 131 return hash(self.test) 132 133 134 def load_data(args): 135 """Load data treating args as a list of UA name, filename pairs""" 136 pairs = [] 137 for i in xrange(0, len(args), 2): 138 pairs.append(args[i:i+2]) 139 140 rv = {} 141 for UA, filename in pairs: 142 with open(filename) as f: 143 rv[UA] = json.load(f) 144 145 return rv 146 147 148 def test_id(id): 149 """Convert a test id in JSON into an immutable object that 150 can be used as a dictionary key""" 151 if isinstance(id, list): 152 return tuple(id) 153 else: 154 return id 155 156 157 def all_tests(data): 158 tests = defaultdict(set) 159 for UA, results in iteritems(data): 160 for result in results["results"]: 161 id = test_id(result["test"]) 162 tests[id] |= {subtest["name"] for subtest in result["subtests"]} 163 return tests 164 165 166 def group_results(data): 167 """Produce a list of UAs and a dictionary mapping specific tests to their 168 status in all UAs e.g. 169 ["UA1", "UA2"], {"test_id":{"harness":{"UA1": (status1, message1), 170 "UA2": (status2, message2)}, 171 "subtests":{"subtest1": "UA1": (status1-1, message1-1), 172 "UA2": (status2-1, message2-1)}}} 173 Status and message are None if the test didn't run in a particular UA. 174 Message is None if the test didn't produce a message""" 175 tests = all_tests(data) 176 177 UAs = data.keys() 178 179 def result(): 180 return { 181 "harness": {UA: (None, None) for UA in UAs}, 182 "subtests": None # init this later 183 } 184 185 results_by_test = defaultdict(result) 186 187 for UA, results in iteritems(data): 188 for test_data in results["results"]: 189 id = test_id(test_data["test"]) 190 result = results_by_test[id] 191 192 if result["subtests"] is None: 193 result["subtests"] = { 194 name: {UA: (None, None) for UA in UAs} for name in tests[id] 195 } 196 197 result["harness"][UA] = (test_data["status"], test_data["message"]) 198 for subtest in test_data["subtests"]: 199 result["subtests"][subtest["name"]][UA] = (subtest["status"], 200 subtest["message"]) 201 202 return UAs, results_by_test 203 204 205 def status_cell(status, message=None): 206 """Produce a table cell showing the status of a test""" 207 status = status if status is not None else "NONE" 208 kwargs = {} 209 if message: 210 kwargs["title"] = message 211 status_text = status.title() 212 return h.td(status_text, class_="status " + status, 213 **kwargs) 214 215 216 def test_link(test_id, subtest=None): 217 """Produce an <a> element linking to a test""" 218 if isinstance(test_id, types.StringTypes): 219 rv = [h.a(test_id, href=test_id)] 220 else: 221 rv = [h.a(test_id[0], href=test_id[0]), 222 " %s " % test_id[1], 223 h.a(test_id[2], href=test_id[2])] 224 if subtest is not None: 225 rv.append(" [%s]" % subtest) 226 return rv 227 228 229 def summary(UAs, results_by_test): 230 """Render the implementation report summary""" 231 not_passing = [] 232 for test, results in iteritems(results_by_test): 233 if not any(item[0] in ("PASS", "OK") for item in results["harness"].values()): 234 not_passing.append((test, None)) 235 for subtest_name, subtest_results in iteritems(results["subtests"]): 236 if not any(item[0] == "PASS" for item in subtest_results.values()): 237 not_passing.append((test, subtest_name)) 238 if not_passing: 239 rv = [ 240 h.p("The following tests failed to pass in all UAs:"), 241 h.ul([h.li(test_link(test, subtest)) 242 for test, subtest in not_passing]) 243 ] 244 else: 245 rv = "All tests passed in at least one UA" 246 return rv 247 248 249 def result_rows(UAs, test, result): 250 """Render the results for each test run""" 251 yield h.tr( 252 h.td( 253 test_link(test), 254 rowspan=(1 + len(result["subtests"])) 255 ), 256 h.td(), 257 [status_cell(status, message) 258 for UA, (status, message) in sorted(result["harness"].items())], 259 class_="test" 260 ) 261 262 for name, subtest_result in sorted(iteritems(result["subtests"])): 263 yield h.tr( 264 h.td(name), 265 [status_cell(status, message) 266 for UA, (status, message) in sorted(subtest_result.items())], 267 class_="subtest" 268 ) 269 270 271 def result_bodies(UAs, results_by_test): 272 return [h.tbody(result_rows(UAs, test, result)) 273 for test, result in sorted(iteritems(results_by_test))] 274 275 276 def generate_html(UAs, results_by_test): 277 """Generate all the HTML output""" 278 return h(h.html( 279 h.head( 280 h.meta(charset="utf8"), 281 h.title("Implementation Report"), 282 h.link(href="report.css", rel="stylesheet")), 283 h.body( 284 h.h1("Implementation Report"), 285 h.h2("Summary"), 286 summary(UAs, results_by_test), 287 h.h2("Full Results"), 288 h.table( 289 h.thead( 290 h.tr( 291 h.th("Test"), 292 h.th("Subtest"), 293 [h.th(UA) for UA in sorted(UAs)])), 294 result_bodies(UAs, results_by_test))))) 295 296 297 def main(filenames): 298 data = load_data(filenames) 299 UAs, results_by_test = group_results(data) 300 return generate_html(UAs, results_by_test) 301 302 303 if __name__ == "__main__": 304 if not sys.argv[1:]: 305 print("""Please supply a list of UA name, filename pairs e.g. 306 307 python report.py Firefox firefox.json Chrome chrome.json IE internet_explorer.json""") 308 print(main(sys.argv[1:]))