generate-flaky-report-from-ftl.py (10145B)
1 #!/usr/bin/env python3 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 """ 8 Script to generate an HTML report for flaky tests detected in JUnit XML results. 9 10 This script processes JUnit test results generated by Flank and Firebase Test Lab, 11 specifically looking for tests marked with the flaky="true" attribute. It creates 12 a styled HTML report with links to Firebase Test Lab for detailed test analysis. 13 14 - Searches for the FullJUnitReport.xml file in the results directory. 15 - Identifies test cases with flaky="true" attribute. 16 - Extracts failure information and Firebase Test Lab web links. 17 - Generates a formatted HTML report with clickable links to test executions. 18 - Only creates the report if flaky tests are found. 19 - Designed for use in Taskcluster following a Firebase Test Lab test execution. 20 21 Flank: https://flank.github.io/flank/ 22 23 Usage: 24 python3 generate-flaky-report-from-ftl.py --results <path_to_results_directory> 25 python3 generate-flaky-report-from-ftl.py --results <path_to_results_directory> 26 --output <path_to_output_html> 27 """ 28 29 import argparse 30 import logging 31 import sys 32 from pathlib import Path 33 from typing import Any 34 35 from junitparser import Attr, JUnitXml, TestCase 36 37 38 class test_case(TestCase): 39 flaky = Attr() 40 41 42 def setup_logging(): 43 """Configure logging for the script.""" 44 log_format = "%(levelname)s: %(message)s" 45 logging.basicConfig(level=logging.INFO, format=log_format) 46 47 48 def find_junit_xml_files(results_dir: Path) -> list[Path]: 49 """Find the FullJUnitReport.xml file in the results directory. 50 51 Args: 52 results_dir: Path to the results directory 53 54 Returns: 55 List containing path to FullJUnitReport.xml if it exists 56 """ 57 full_report = results_dir / "FullJUnitReport.xml" 58 if full_report.exists(): 59 logging.info(f"Found FullJUnitReport.xml in {results_dir}") 60 return [full_report] 61 else: 62 logging.info(f"FullJUnitReport.xml not found in {results_dir}") 63 return [] 64 65 66 def extract_flaky_tests(xml_files: list[Path]) -> list[dict[str, Any]]: 67 """Parse JUnit XML files and extract tests marked as flaky. 68 69 Args: 70 xml_files: List of paths to JUnit XML files 71 72 Returns: 73 List of dictionaries containing flaky test information 74 """ 75 flaky_tests = [] 76 77 for xml_file in xml_files: 78 try: 79 xml = JUnitXml.fromfile(str(xml_file)) 80 for suite in xml: 81 for case in suite: 82 # Use custom test_case class to access flaky attribute 83 cur_case = test_case.fromelem(case) 84 if isinstance(cur_case, TestCase) and cur_case.flaky: 85 flaky_info = { 86 "name": cur_case.name, 87 "classname": cur_case.classname, 88 "time": cur_case.time, 89 "file": xml_file.name, 90 } 91 92 # Extract webLink if present 93 web_link = cur_case._elem.find("webLink") 94 if web_link is not None and web_link.text: 95 flaky_info["weblink"] = web_link.text 96 97 # Capture failure/error information if present 98 if cur_case.result: 99 result_info = [] 100 for result in cur_case.result: 101 result_info.append({ 102 "type": ( 103 result.type 104 if result.type 105 else "Failure Stack Trace" 106 ), 107 "message": ( 108 result.message 109 if result.message 110 else result.text 111 ), 112 }) 113 flaky_info["results"] = result_info 114 115 flaky_tests.append(flaky_info) 116 logging.info( 117 f"Found flaky test: {cur_case.classname}.{cur_case.name}" 118 ) 119 120 except Exception as e: 121 logging.error(f"Failed to parse {xml_file}: {e}") 122 continue 123 124 return flaky_tests 125 126 127 def generate_html_report(flaky_tests: list[dict[str, Any]], output_path: Path) -> None: 128 """Generate an HTML report for flaky tests. 129 130 Args: 131 flaky_tests: List of flaky test information 132 output_path: Path where the HTML report should be written 133 """ 134 html_content = f"""<!DOCTYPE html> 135 <html lang="en"> 136 <head> 137 <meta charset="UTF-8"> 138 <meta name="viewport" content="width=device-width, initial-scale=1.0"> 139 <title>Flaky Test Report</title> 140 <style> 141 body {{ 142 font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; 143 line-height: 1.6; 144 color: #333; 145 max-width: 1200px; 146 margin: 0 auto; 147 padding: 20px; 148 background-color: #f5f5f5; 149 }} 150 h1 {{ 151 color: #e65100; 152 border-bottom: 3px solid #e65100; 153 padding-bottom: 10px; 154 }} 155 .summary {{ 156 background-color: #fff3e0; 157 border-left: 4px solid #ff9800; 158 padding: 15px; 159 margin: 20px 0; 160 border-radius: 4px; 161 }} 162 .test-case {{ 163 background-color: white; 164 border: 1px solid #ddd; 165 border-radius: 4px; 166 padding: 15px; 167 margin: 15px 0; 168 box-shadow: 0 2px 4px rgba(0,0,0,0.1); 169 }} 170 .test-name {{ 171 font-size: 1.1em; 172 font-weight: bold; 173 color: #d84315; 174 margin-bottom: 8px; 175 }} 176 .test-class {{ 177 color: #666; 178 font-size: 0.95em; 179 margin-bottom: 8px; 180 }} 181 .test-meta {{ 182 color: #888; 183 font-size: 0.9em; 184 margin-bottom: 10px; 185 }} 186 .firebase-link {{ 187 display: inline-block; 188 background-color: #1976d2; 189 color: white; 190 padding: 8px 16px; 191 text-decoration: none; 192 border-radius: 4px; 193 margin-top: 10px; 194 font-size: 0.9em; 195 }} 196 .firebase-link:hover {{ 197 background-color: #1565c0; 198 }} 199 .result-info {{ 200 background-color: #ffebee; 201 border-left: 3px solid #f44336; 202 padding: 10px; 203 margin-top: 10px; 204 font-family: monospace; 205 font-size: 0.9em; 206 white-space: pre-wrap; 207 word-wrap: break-word; 208 }} 209 .result-type {{ 210 font-weight: bold; 211 color: #c62828; 212 }} 213 .footer {{ 214 margin-top: 30px; 215 padding-top: 20px; 216 border-top: 1px solid #ddd; 217 text-align: center; 218 color: #666; 219 font-size: 0.9em; 220 }} 221 </style> 222 </head> 223 <body> 224 <h1>⚠️ Flaky Test Report</h1> 225 226 <div class="summary"> 227 <strong>Total Flaky Tests Found:</strong> {len(flaky_tests)} 228 </div> 229 """ 230 231 for test in flaky_tests: 232 html_content += f""" 233 <div class="test-case"> 234 <div class="test-name">{test["name"]}</div> 235 <div class="test-class">Class: {test["classname"]}</div> 236 <div class="test-meta"> 237 Execution Time: {test.get("time", "N/A")}s | Source: {test["file"]} 238 </div> 239 """ 240 241 if "weblink" in test: 242 html_content += f""" 243 <a href="{test["weblink"]}" class="firebase-link" target="_blank">View in Firebase Test Lab</a> 244 """ 245 246 if "results" in test: 247 for result in test["results"]: 248 html_content += f""" 249 <div class="result-info"> 250 <div class="result-type">{result.get("type", "Unknown")}</div> 251 <div>{result.get("message", "No message available")}</div> 252 </div> 253 """ 254 255 html_content += """ 256 </div> 257 """ 258 259 html_content += """ 260 <div class="footer"> 261 Generated by Mozilla Mobile Test Engineering 262 </div> 263 </body> 264 </html> 265 """ 266 267 output_path.write_text(html_content, encoding="utf-8") 268 logging.info(f"HTML flaky report written to {output_path}") 269 270 271 def main(): 272 """Parse arguments and generate flaky test report.""" 273 parser = argparse.ArgumentParser( 274 description="Generate an HTML report for flaky tests from JUnit XML results" 275 ) 276 parser.add_argument( 277 "--results", 278 required=True, 279 help="Path to the results directory containing JUnit XML files", 280 ) 281 parser.add_argument( 282 "--output", 283 default="/builds/worker/artifacts/HtmlFlakyReport.html", 284 help="Path where the HTML report should be written", 285 ) 286 args = parser.parse_args() 287 288 results_dir = Path(args.results) 289 output_path = Path(args.output) 290 291 if not results_dir.exists(): 292 logging.warning(f"Results directory does not exist: {results_dir}") 293 sys.exit(0) 294 295 # Find all JUnit XML files 296 xml_files = find_junit_xml_files(results_dir) 297 298 if not xml_files: 299 logging.info("No JUnit XML files found, skipping flaky report generation") 300 sys.exit(0) 301 302 # Extract flaky tests 303 flaky_tests = extract_flaky_tests(xml_files) 304 305 if not flaky_tests: 306 logging.info("No flaky tests detected, skipping report generation") 307 sys.exit(0) 308 309 # Generate HTML report 310 output_path.parent.mkdir(parents=True, exist_ok=True) 311 generate_html_report(flaky_tests, output_path) 312 313 logging.info( 314 f"Flaky test report generation complete: {len(flaky_tests)} flaky tests found" 315 ) 316 sys.exit(0) 317 318 319 if __name__ == "__main__": 320 setup_logging() 321 main()