test_results_presentation.py (18479B)
1 #!/usr/bin/env python3 2 # 3 # Copyright 2017 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 9 import argparse 10 import collections 11 import contextlib 12 import json 13 import logging 14 import tempfile 15 import os 16 import sys 17 try: 18 from urllib.parse import urlencode 19 from urllib.request import urlopen 20 except ImportError: 21 from urllib import urlencode 22 from urllib2 import urlopen 23 24 25 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) 26 BASE_DIR = os.path.abspath(os.path.join( 27 CURRENT_DIR, '..', '..', '..', '..', '..')) 28 29 sys.path.append(os.path.join(BASE_DIR, 'build', 'android')) 30 from pylib.results.presentation import standard_gtest_merge 31 from pylib.utils import google_storage_helper # pylint: disable=import-error 32 33 sys.path.append(os.path.join(BASE_DIR, 'third_party')) 34 import jinja2 # pylint: disable=import-error 35 JINJA_ENVIRONMENT = jinja2.Environment( 36 loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), 37 autoescape=True) 38 39 40 def cell(data, html_class='center'): 41 """Formats table cell data for processing in jinja template.""" 42 return { 43 'data': data, 44 'class': html_class, 45 } 46 47 48 def pre_cell(data, html_class='center'): 49 """Formats table <pre> cell data for processing in jinja template.""" 50 return { 51 'cell_type': 'pre', 52 'data': data, 53 'class': html_class, 54 } 55 56 57 class LinkTarget: 58 # Opens the linked document in a new window or tab. 59 NEW_TAB = '_blank' 60 # Opens the linked document in the same frame as it was clicked. 61 CURRENT_TAB = '_self' 62 63 64 def link(data, href, target=LinkTarget.CURRENT_TAB): 65 """Formats <a> tag data for processing in jinja template. 66 67 Args: 68 data: String link appears as on HTML page. 69 href: URL where link goes. 70 target: Where link should be opened (e.g. current tab or new tab). 71 """ 72 return { 73 'data': data, 74 'href': href, 75 'target': target, 76 } 77 78 79 def links_cell(links, html_class='center', rowspan=None): 80 """Formats table cell with links for processing in jinja template. 81 82 Args: 83 links: List of link dictionaries. Use |link| function to generate them. 84 html_class: Class for table cell. 85 rowspan: Rowspan HTML attribute. 86 """ 87 return { 88 'cell_type': 'links', 89 'class': html_class, 90 'links': links, 91 'rowspan': rowspan, 92 } 93 94 95 def action_cell(action, data, html_class): 96 """Formats table cell with javascript actions. 97 98 Args: 99 action: Javscript action. 100 data: Data in cell. 101 class: Class for table cell. 102 """ 103 return { 104 'cell_type': 'action', 105 'action': action, 106 'data': data, 107 'class': html_class, 108 } 109 110 111 def flakiness_dashbord_link(test_name, suite_name, bucket): 112 # Assume the bucket will be like "foo-bar-baz", we will take "foo" 113 # as the test_project. 114 # Fallback to "chromium" if bucket is not passed, e.g. local_output=True 115 test_project = bucket.split('-')[0] if bucket else 'chromium' 116 query = '%s/%s' % (suite_name, test_name) 117 url_args = urlencode([('t', 'TESTS'), ('q', query), ('tp', test_project)]) 118 return 'https://ci.chromium.org/ui/search?%s' % url_args 119 120 121 def logs_cell(result, test_name, suite_name, bucket): 122 """Formats result logs data for processing in jinja template.""" 123 link_list = [] 124 result_link_dict = result.get('links', {}) 125 result_link_dict['flakiness'] = flakiness_dashbord_link( 126 test_name, suite_name, bucket) 127 for name, href in sorted(result_link_dict.items()): 128 link_list.append(link( 129 data=name, 130 href=href, 131 target=LinkTarget.NEW_TAB)) 132 if link_list: 133 return links_cell(link_list) 134 return cell('(no logs)') 135 136 137 def code_search(test, cs_base_url): 138 """Returns URL for test on codesearch.""" 139 search = test.replace('#', '.') 140 return '%s/search/?q=%s&type=cs' % (cs_base_url, search) 141 142 143 def status_class(status): 144 """Returns HTML class for test status.""" 145 if not status: 146 return 'failure unknown' 147 status = status.lower() 148 if status not in ('success', 'skipped'): 149 return 'failure %s' % status 150 return status 151 152 153 def create_test_table(results_dict, cs_base_url, suite_name, bucket): 154 """Format test data for injecting into HTML table.""" 155 156 header_row = [ 157 cell(data='test_name', html_class='text'), 158 cell(data='status', html_class='flaky'), 159 cell(data='elapsed_time_ms', html_class='number'), 160 cell(data='logs', html_class='text'), 161 cell(data='output_snippet', html_class='text'), 162 ] 163 164 test_row_blocks = [] 165 for test_name, test_results in results_dict.items(): 166 test_runs = [] 167 for index, result in enumerate(test_results): 168 if index == 0: 169 test_run = [links_cell( 170 links=[ 171 link(href=code_search(test_name, cs_base_url), 172 target=LinkTarget.NEW_TAB, 173 data=test_name)], 174 rowspan=len(test_results), 175 html_class='left %s' % test_name 176 )] # test_name 177 else: 178 test_run = [] 179 180 test_run.extend([ 181 cell(data=result['status'] or 'UNKNOWN', 182 # status 183 html_class=('center %s' % 184 status_class(result['status']))), 185 cell(data=result['elapsed_time_ms']), # elapsed_time_ms 186 logs_cell(result, test_name, suite_name, bucket), 187 # logs 188 pre_cell(data=result['output_snippet'], # output_snippet 189 html_class='left'), 190 ]) 191 test_runs.append(test_run) 192 test_row_blocks.append(test_runs) 193 return header_row, test_row_blocks 194 195 196 def create_suite_table(results_dict): 197 """Format test suite data for injecting into HTML table.""" 198 199 SUCCESS_COUNT_INDEX = 1 200 FAIL_COUNT_INDEX = 2 201 ALL_COUNT_INDEX = 3 202 TIME_INDEX = 4 203 204 header_row = [ 205 cell(data='suite_name', html_class='text'), 206 cell(data='number_success_tests', html_class='number'), 207 cell(data='number_fail_tests', html_class='number'), 208 cell(data='all_tests', html_class='number'), 209 cell(data='elapsed_time_ms', html_class='number'), 210 ] 211 212 footer_row = [ 213 action_cell( 214 'showTestsOfOneSuiteOnlyWithNewState("TOTAL")', 215 'TOTAL', 216 'center' 217 ), # TOTAL 218 cell(data=0), # number_success_tests 219 cell(data=0), # number_fail_tests 220 cell(data=0), # all_tests 221 cell(data=0), # elapsed_time_ms 222 ] 223 224 suite_row_dict = collections.defaultdict(lambda: [ 225 # Note: |suite_name| will be given in the following for loop. 226 # It is not assigned yet here. 227 action_cell('showTestsOfOneSuiteOnlyWithNewState("%s")' % suite_name, 228 suite_name, 'left'), # suite_name 229 cell(data=0), # number_success_tests 230 cell(data=0), # number_fail_tests 231 cell(data=0), # all_tests 232 cell(data=0), # elapsed_time_ms 233 ]) 234 for test_name, test_results in results_dict.items(): 235 # TODO(mikecase): This logic doesn't work if there are multiple test runs. 236 # That is, if 'per_iteration_data' has multiple entries. 237 # Since we only care about the result of the last test run. 238 result = test_results[-1] 239 240 suite_name = (test_name.split('#')[0] 241 if '#' in test_name else test_name.split('.')[0]) 242 suite_row = suite_row_dict[suite_name] 243 244 suite_row[ALL_COUNT_INDEX]['data'] += 1 245 footer_row[ALL_COUNT_INDEX]['data'] += 1 246 247 if result['status'] == 'SUCCESS': 248 suite_row[SUCCESS_COUNT_INDEX]['data'] += 1 249 footer_row[SUCCESS_COUNT_INDEX]['data'] += 1 250 elif result['status'] != 'SKIPPED': 251 suite_row[FAIL_COUNT_INDEX]['data'] += 1 252 footer_row[FAIL_COUNT_INDEX]['data'] += 1 253 254 # Some types of crashes can have 'null' values for elapsed_time_ms. 255 if result['elapsed_time_ms'] is not None: 256 suite_row[TIME_INDEX]['data'] += result['elapsed_time_ms'] 257 footer_row[TIME_INDEX]['data'] += result['elapsed_time_ms'] 258 259 for suite in list(suite_row_dict.values()): 260 if suite[FAIL_COUNT_INDEX]['data'] > 0: 261 suite[FAIL_COUNT_INDEX]['class'] += ' failure' 262 else: 263 suite[FAIL_COUNT_INDEX]['class'] += ' success' 264 265 if footer_row[FAIL_COUNT_INDEX]['data'] > 0: 266 footer_row[FAIL_COUNT_INDEX]['class'] += ' failure' 267 else: 268 footer_row[FAIL_COUNT_INDEX]['class'] += ' success' 269 270 return (header_row, [[suite_row] 271 for suite_row in list(suite_row_dict.values())], 272 footer_row) 273 274 275 def feedback_url(result_details_link): 276 url_args = [ 277 ('labels', 'Pri-2,Type-Bug,Restrict-View-Google'), 278 ('summary', 'Result Details Feedback:'), 279 ('components', 'Test>Android'), 280 ] 281 if result_details_link: 282 url_args.append(('comment', 'Please check out: %s' % result_details_link)) 283 url_args = urlencode(url_args) 284 return 'https://bugs.chromium.org/p/chromium/issues/entry?%s' % url_args 285 286 287 def results_to_html(results_dict, cs_base_url, bucket, test_name, 288 builder_name, build_number, local_output): 289 """Convert list of test results into html format. 290 291 Args: 292 local_output: Whether this results file is uploaded to Google Storage or 293 just a local file. 294 """ 295 test_rows_header, test_rows = create_test_table( 296 results_dict, cs_base_url, test_name, bucket) 297 suite_rows_header, suite_rows, suite_row_footer = create_suite_table( 298 results_dict) 299 300 suite_table_values = { 301 'table_id': 'suite-table', 302 'table_headers': suite_rows_header, 303 'table_row_blocks': suite_rows, 304 'table_footer': suite_row_footer, 305 } 306 307 test_table_values = { 308 'table_id': 'test-table', 309 'table_headers': test_rows_header, 310 'table_row_blocks': test_rows, 311 } 312 313 main_template = JINJA_ENVIRONMENT.get_template( 314 os.path.join('template', 'main.html')) 315 316 if local_output: 317 html_render = main_template.render( # pylint: disable=no-member 318 { 319 'tb_values': [suite_table_values, test_table_values], 320 'feedback_url': feedback_url(None), 321 }) 322 return (html_render, None, None) 323 dest = google_storage_helper.unique_name( 324 '%s_%s_%s' % (test_name, builder_name, build_number)) 325 result_details_link = google_storage_helper.get_url_link( 326 dest, '%s/html' % bucket) 327 html_render = main_template.render( # pylint: disable=no-member 328 { 329 'tb_values': [suite_table_values, test_table_values], 330 'feedback_url': feedback_url(result_details_link), 331 }) 332 return (html_render, dest, result_details_link) 333 334 335 def result_details(json_path, test_name, cs_base_url, bucket=None, 336 builder_name=None, build_number=None, local_output=False): 337 """Get result details from json path and then convert results to html. 338 339 Args: 340 local_output: Whether this results file is uploaded to Google Storage or 341 just a local file. 342 """ 343 344 with open(json_path) as json_file: 345 json_object = json.loads(json_file.read()) 346 347 if not 'per_iteration_data' in json_object: 348 return 'Error: json file missing per_iteration_data.' 349 350 results_dict = collections.defaultdict(list) 351 for testsuite_run in json_object['per_iteration_data']: 352 for test, test_runs in testsuite_run.items(): 353 results_dict[test].extend(test_runs) 354 return results_to_html(results_dict, cs_base_url, bucket, test_name, 355 builder_name, build_number, local_output) 356 357 358 def upload_to_google_bucket(html, bucket, dest): 359 with tempfile.NamedTemporaryFile(suffix='.html') as temp_file: 360 temp_file.write(html) 361 temp_file.flush() 362 return google_storage_helper.upload( 363 name=dest, 364 filepath=temp_file.name, 365 bucket='%s/html' % bucket, 366 content_type='text/html', 367 authenticated_link=True) 368 369 370 def ui_screenshot_set(json_path): 371 with open(json_path) as json_file: 372 json_object = json.loads(json_file.read()) 373 if not 'per_iteration_data' in json_object: 374 # This will be reported as an error by result_details, no need to duplicate. 375 return None 376 ui_screenshots = [] 377 # pylint: disable=too-many-nested-blocks 378 for testsuite_run in json_object['per_iteration_data']: 379 for _, test_runs in testsuite_run.items(): 380 for test_run in test_runs: 381 if 'ui screenshot' in test_run['links']: 382 screenshot_link = test_run['links']['ui screenshot'] 383 if screenshot_link.startswith('file:'): 384 with contextlib.closing(urlopen(screenshot_link)) as f: 385 test_screenshots = json.load(f) 386 else: 387 # Assume anything that isn't a file link is a google storage link 388 screenshot_string = google_storage_helper.read_from_link( 389 screenshot_link) 390 if not screenshot_string: 391 logging.error('Bad screenshot link %s', screenshot_link) 392 continue 393 test_screenshots = json.loads( 394 screenshot_string) 395 ui_screenshots.extend(test_screenshots) 396 # pylint: enable=too-many-nested-blocks 397 398 if ui_screenshots: 399 return json.dumps(ui_screenshots) 400 return None 401 402 403 def upload_screenshot_set(json_path, test_name, bucket, builder_name, 404 build_number): 405 screenshot_set = ui_screenshot_set(json_path) 406 if not screenshot_set: 407 return None 408 dest = google_storage_helper.unique_name( 409 'screenshots_%s_%s_%s' % (test_name, builder_name, build_number), 410 suffix='.json') 411 with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as temp_file: 412 temp_file.write(screenshot_set) 413 temp_file.flush() 414 return google_storage_helper.upload( 415 name=dest, 416 filepath=temp_file.name, 417 bucket='%s/json' % bucket, 418 content_type='application/json', 419 authenticated_link=True) 420 421 422 def main(): 423 parser = argparse.ArgumentParser() 424 parser.add_argument('--json-file', help='Path of json file.') 425 parser.add_argument('--cs-base-url', help='Base url for code search.', 426 default='http://cs.chromium.org') 427 parser.add_argument('--bucket', help='Google storage bucket.', required=True) 428 parser.add_argument('--builder-name', help='Builder name.') 429 parser.add_argument('--build-number', help='Build number.') 430 parser.add_argument('--test-name', help='The name of the test.', 431 required=True) 432 parser.add_argument( 433 '-o', '--output-json', 434 help='(Swarming Merge Script API) ' 435 'Output JSON file to create.') 436 parser.add_argument( 437 '--build-properties', 438 help='(Swarming Merge Script API) ' 439 'Build property JSON file provided by recipes.') 440 parser.add_argument( 441 '--summary-json', 442 help='(Swarming Merge Script API) ' 443 'Summary of shard state running on swarming. ' 444 '(Output of the swarming.py collect ' 445 '--task-summary-json=XXX command.)') 446 parser.add_argument( 447 '--task-output-dir', 448 help='(Swarming Merge Script API) ' 449 'Directory containing all swarming task results.') 450 parser.add_argument( 451 'positional', nargs='*', 452 help='output.json from shards.') 453 454 args = parser.parse_args() 455 456 if ((args.build_properties is None) == 457 (args.build_number is None or args.builder_name is None)): 458 raise parser.error('Exactly one of build_perperties or ' 459 '(build_number or builder_name) should be given.') 460 461 if (args.build_number is None) != (args.builder_name is None): 462 raise parser.error('args.build_number and args.builder_name ' 463 'has to be be given together' 464 'or not given at all.') 465 466 if len(args.positional) == 0 and args.json_file is None: 467 if args.output_json: 468 with open(args.output_json, 'w') as f: 469 json.dump({}, f) 470 return 471 if len(args.positional) != 0 and args.json_file: 472 raise parser.error('Exactly one of args.positional and ' 473 'args.json_file should be given.') 474 475 if args.build_properties: 476 build_properties = json.loads(args.build_properties) 477 if ((not 'buildnumber' in build_properties) or 478 (not 'buildername' in build_properties)): 479 raise parser.error('Build number/builder name not specified.') 480 build_number = build_properties['buildnumber'] 481 builder_name = build_properties['buildername'] 482 elif args.build_number and args.builder_name: 483 build_number = args.build_number 484 builder_name = args.builder_name 485 486 if args.positional: 487 if len(args.positional) == 1: 488 json_file = args.positional[0] 489 else: 490 if args.output_json and args.summary_json: 491 standard_gtest_merge.standard_gtest_merge( 492 args.output_json, args.summary_json, args.positional) 493 json_file = args.output_json 494 elif not args.output_json: 495 raise Exception('output_json required by merge API is missing.') 496 else: 497 raise Exception('summary_json required by merge API is missing.') 498 elif args.json_file: 499 json_file = args.json_file 500 501 if not os.path.exists(json_file): 502 raise IOError('--json-file %s not found.' % json_file) 503 504 # Link to result details presentation page is a part of the page. 505 result_html_string, dest, result_details_link = result_details( 506 json_file, args.test_name, args.cs_base_url, args.bucket, 507 builder_name, build_number) 508 509 result_details_link_2 = upload_to_google_bucket( 510 result_html_string.encode('UTF-8'), 511 args.bucket, dest) 512 assert result_details_link == result_details_link_2, ( 513 'Result details link do not match. The link returned by get_url_link' 514 ' should be the same as that returned by upload.') 515 516 ui_screenshot_set_link = upload_screenshot_set(json_file, args.test_name, 517 args.bucket, builder_name, build_number) 518 519 if ui_screenshot_set_link: 520 ui_catalog_url = 'https://chrome-ui-catalog.appspot.com/' 521 ui_catalog_query = urlencode({'screenshot_source': ui_screenshot_set_link}) 522 ui_screenshot_link = '%s?%s' % (ui_catalog_url, ui_catalog_query) 523 524 if args.output_json: 525 with open(json_file) as original_json_file: 526 json_object = json.load(original_json_file) 527 json_object['links'] = { 528 'result_details (logcats, flakiness links)': result_details_link 529 } 530 531 if ui_screenshot_set_link: 532 json_object['links']['ui screenshots'] = ui_screenshot_link 533 534 with open(args.output_json, 'w') as f: 535 json.dump(json_object, f) 536 else: 537 print('Result Details: %s' % result_details_link) 538 539 if ui_screenshot_set_link: 540 print('UI Screenshots %s' % ui_screenshot_link) 541 542 543 if __name__ == '__main__': 544 sys.exit(main())