parse-junit-results.py (6050B)
1 #!/usr/bin/python3 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 """ 8 Script to parse and print UI test JUnit results from a FullJUnitReport.xml file. 9 10 This script processes JUnit test results generated by Flank and Firebase Test Lab. 11 It reads test results from a specified directory containing a full JUnit report, 12 parses the results to identify test failures and flaky tests, and prints a formatted 13 table summarizing these results to the console. 14 15 - Parses JUnit XML test result files, including custom 'flaky' attributes. 16 - Identifies and displays unique test failures and flaky tests. 17 - Prints the results in a readable table format using BeautifulTable. 18 - Provides detailed failure messages and test case information. 19 - Designed for use in Taskcluster following a Firebase Test Lab test execution. 20 21 Flank: https://flank.github.io/flank/ 22 23 Usage: 24 python3 parse-junit-results.py --results <path_to_results_directory> 25 """ 26 27 import argparse 28 import sys 29 import xml 30 from pathlib import Path 31 32 from beautifultable import BeautifulTable 33 from junitparser import Attr, Failure, JUnitXml, TestCase, TestSuite 34 35 36 def parse_args(cmdln_args): 37 parser = argparse.ArgumentParser( 38 description="Parse and print UI test JUnit results" 39 ) 40 parser.add_argument( 41 "--results", 42 type=Path, 43 help="Directory containing task artifact results", 44 required=True, 45 ) 46 return parser.parse_args(args=cmdln_args) 47 48 49 class test_suite(TestSuite): 50 flakes = Attr() 51 52 53 class test_case(TestCase): 54 flaky = Attr() 55 56 57 def parse_print_failure_results(results): 58 """ 59 Parses the given JUnit test results and prints a formatted table of failures and flaky tests. 60 61 Args: 62 results (JUnitXml): Parsed JUnit XML results. 63 64 Returns: 65 int: The number of test failures. 66 67 The function processes each test suite and each test case within the suite. 68 If a test case has a result that is an instance of Failure, it is added to the table. 69 The test case is marked as 'Flaky' if the flaky attribute is set to "true", otherwise it is marked as 'Failure'. 70 71 Example of possible JUnit XML (FullJUnitReport.xml): 72 <testsuites> 73 <testsuite name="ExampleSuite" tests="2" failures="1" flakes="1" time="0.003"> 74 <testcase classname="example.TestClass" name="testSuccess" flaky="true" time="0.001"> 75 <failure message="Assertion failed">Expected true but was false</failure> 76 </testcase> 77 <testcase classname="example.TestClass" name="testFailure" time="0.002"> 78 <failure message="Assertion failed">Expected true but was false</failure> 79 <failure message="Assertion failed">Expected true but was false</failure> 80 </testcase> 81 </testsuite> 82 </testsuites> 83 """ 84 85 table = BeautifulTable(maxwidth=256) 86 table.columns.header = ["UI Test", "Outcome", "Details"] 87 table.columns.alignment = BeautifulTable.ALIGN_LEFT 88 table.set_style(BeautifulTable.STYLE_GRID) 89 90 failure_count = 0 91 92 # Dictionary to store the last seen failure details for each test case 93 last_seen_failures = {} 94 95 for suite in results: 96 cur_suite = test_suite.fromelem(suite) 97 for case in cur_suite: 98 cur_case = test_case.fromelem(case) 99 if cur_case.result: 100 for entry in case.result: 101 if isinstance(entry, Failure): 102 flaky_status = getattr(cur_case, "flaky", "false") == "true" 103 if flaky_status: 104 test_id = "%s#%s" % (case.classname, case.name) 105 details = ( 106 entry.text.replace("\t", " ") if entry.text else "" 107 ) 108 # Check if the current failure details are different from the last seen ones 109 if details != last_seen_failures.get(test_id, ""): 110 table.rows.append([ 111 test_id, 112 "Flaky", 113 details, 114 ]) 115 last_seen_failures[test_id] = details 116 else: 117 test_id = "%s#%s" % (case.classname, case.name) 118 details = ( 119 entry.text.replace("\t", " ") if entry.text else "" 120 ) 121 # Check if the current failure details are different from the last seen ones 122 if details != last_seen_failures.get(test_id, ""): 123 table.rows.append([ 124 test_id, 125 "Failure", 126 details, 127 ]) 128 print(f"TEST-UNEXPECTED-FAIL | {test_id} | {details}") 129 failure_count += 1 130 # Update the last seen failure details for this test case 131 last_seen_failures[test_id] = details 132 133 print(table) 134 return failure_count 135 136 137 def load_results_file(filename): 138 ret = None 139 try: 140 f = open(filename) 141 try: 142 ret = JUnitXml.fromfile(f) 143 except xml.etree.ElementTree.ParseError as e: 144 print(f"Error parsing {filename} file: {e}") 145 finally: 146 f.close() 147 except OSError as e: 148 print(e) 149 150 return ret 151 152 153 def main(): 154 args = parse_args(sys.argv[1:]) 155 156 failure_count = 0 157 junitxml = load_results_file(args.results.joinpath("FullJUnitReport.xml")) 158 if junitxml: 159 failure_count = parse_print_failure_results(junitxml) 160 return failure_count 161 162 163 if __name__ == "__main__": 164 sys.exit(main())