commit 2c1f3bfa2add7630fbbfbd145534f95cea1645a8
parent 237a6bb3f6de5cb41e7b12f85e1e5108b522337b
Author: Aaron Train <aaron.train@gmail.com>
Date: Wed, 3 Dec 2025 19:27:49 +0000
Bug 2003037 - Generate a Flaky HTML Report r=isabel_rios
Differential Revision: https://phabricator.services.mozilla.com/D274473
Diffstat:
2 files changed, 333 insertions(+), 0 deletions(-)
diff --git a/taskcluster/scripts/tests/generate-flaky-report-from-ftl.py b/taskcluster/scripts/tests/generate-flaky-report-from-ftl.py
@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Script to generate an HTML report for flaky tests detected in JUnit XML results.
+
+This script processes JUnit test results generated by Flank and Firebase Test Lab,
+specifically looking for tests marked with the flaky="true" attribute. It creates
+a styled HTML report with links to Firebase Test Lab for detailed test analysis.
+
+- Searches for the FullJUnitReport.xml file in the results directory.
+- Identifies test cases with flaky="true" attribute.
+- Extracts failure information and Firebase Test Lab web links.
+- Generates a formatted HTML report with clickable links to test executions.
+- Only creates the report if flaky tests are found.
+- Designed for use in Taskcluster following a Firebase Test Lab test execution.
+
+Flank: https://flank.github.io/flank/
+
+Usage:
+ python3 generate-flaky-report-from-ftl.py --results <path_to_results_directory>
+ python3 generate-flaky-report-from-ftl.py --results <path_to_results_directory>
+ --output <path_to_output_html>
+"""
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+from junitparser import Attr, JUnitXml, TestCase
+
+
+class test_case(TestCase):
+ flaky = Attr()
+
+
+def setup_logging():
+ """Configure logging for the script."""
+ log_format = "%(levelname)s: %(message)s"
+ logging.basicConfig(level=logging.INFO, format=log_format)
+
+
+def find_junit_xml_files(results_dir: Path) -> list[Path]:
+ """Find the FullJUnitReport.xml file in the results directory.
+
+ Args:
+ results_dir: Path to the results directory
+
+ Returns:
+ List containing path to FullJUnitReport.xml if it exists
+ """
+ full_report = results_dir / "FullJUnitReport.xml"
+ if full_report.exists():
+ logging.info(f"Found FullJUnitReport.xml in {results_dir}")
+ return [full_report]
+ else:
+ logging.info(f"FullJUnitReport.xml not found in {results_dir}")
+ return []
+
+
+def extract_flaky_tests(xml_files: list[Path]) -> list[dict[str, Any]]:
+ """Parse JUnit XML files and extract tests marked as flaky.
+
+ Args:
+ xml_files: List of paths to JUnit XML files
+
+ Returns:
+ List of dictionaries containing flaky test information
+ """
+ flaky_tests = []
+
+ for xml_file in xml_files:
+ try:
+ xml = JUnitXml.fromfile(str(xml_file))
+ for suite in xml:
+ for case in suite:
+ # Use custom test_case class to access flaky attribute
+ cur_case = test_case.fromelem(case)
+ if isinstance(cur_case, TestCase) and cur_case.flaky:
+ flaky_info = {
+ "name": cur_case.name,
+ "classname": cur_case.classname,
+ "time": cur_case.time,
+ "file": xml_file.name,
+ }
+
+ # Extract webLink if present
+ web_link = cur_case._elem.find("webLink")
+ if web_link is not None and web_link.text:
+ flaky_info["weblink"] = web_link.text
+
+ # Capture failure/error information if present
+ if cur_case.result:
+ result_info = []
+ for result in cur_case.result:
+ result_info.append(
+ {
+ "type": (
+ result.type
+ if result.type
+ else "Failure Stack Trace"
+ ),
+ "message": (
+ result.message
+ if result.message
+ else result.text
+ ),
+ }
+ )
+ flaky_info["results"] = result_info
+
+ flaky_tests.append(flaky_info)
+ logging.info(
+ f"Found flaky test: {cur_case.classname}.{cur_case.name}"
+ )
+
+ except Exception as e:
+ logging.error(f"Failed to parse {xml_file}: {e}")
+ continue
+
+ return flaky_tests
+
+
+def generate_html_report(flaky_tests: list[dict[str, Any]], output_path: Path) -> None:
+ """Generate an HTML report for flaky tests.
+
+ Args:
+ flaky_tests: List of flaky test information
+ output_path: Path where the HTML report should be written
+ """
+ html_content = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Flaky Test Report</title>
+ <style>
+ body {{
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+ line-height: 1.6;
+ color: #333;
+ max-width: 1200px;
+ margin: 0 auto;
+ padding: 20px;
+ background-color: #f5f5f5;
+ }}
+ h1 {{
+ color: #e65100;
+ border-bottom: 3px solid #e65100;
+ padding-bottom: 10px;
+ }}
+ .summary {{
+ background-color: #fff3e0;
+ border-left: 4px solid #ff9800;
+ padding: 15px;
+ margin: 20px 0;
+ border-radius: 4px;
+ }}
+ .test-case {{
+ background-color: white;
+ border: 1px solid #ddd;
+ border-radius: 4px;
+ padding: 15px;
+ margin: 15px 0;
+ box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+ }}
+ .test-name {{
+ font-size: 1.1em;
+ font-weight: bold;
+ color: #d84315;
+ margin-bottom: 8px;
+ }}
+ .test-class {{
+ color: #666;
+ font-size: 0.95em;
+ margin-bottom: 8px;
+ }}
+ .test-meta {{
+ color: #888;
+ font-size: 0.9em;
+ margin-bottom: 10px;
+ }}
+ .firebase-link {{
+ display: inline-block;
+ background-color: #1976d2;
+ color: white;
+ padding: 8px 16px;
+ text-decoration: none;
+ border-radius: 4px;
+ margin-top: 10px;
+ font-size: 0.9em;
+ }}
+ .firebase-link:hover {{
+ background-color: #1565c0;
+ }}
+ .result-info {{
+ background-color: #ffebee;
+ border-left: 3px solid #f44336;
+ padding: 10px;
+ margin-top: 10px;
+ font-family: monospace;
+ font-size: 0.9em;
+ white-space: pre-wrap;
+ word-wrap: break-word;
+ }}
+ .result-type {{
+ font-weight: bold;
+ color: #c62828;
+ }}
+ .footer {{
+ margin-top: 30px;
+ padding-top: 20px;
+ border-top: 1px solid #ddd;
+ text-align: center;
+ color: #666;
+ font-size: 0.9em;
+ }}
+ </style>
+</head>
+<body>
+ <h1>⚠️ Flaky Test Report</h1>
+
+ <div class="summary">
+ <strong>Total Flaky Tests Found:</strong> {len(flaky_tests)}
+ </div>
+"""
+
+ for test in flaky_tests:
+ html_content += f"""
+ <div class="test-case">
+ <div class="test-name">{test['name']}</div>
+ <div class="test-class">Class: {test['classname']}</div>
+ <div class="test-meta">
+ Execution Time: {test.get('time', 'N/A')}s | Source: {test['file']}
+ </div>
+"""
+
+ if "weblink" in test:
+ html_content += f"""
+ <a href="{test['weblink']}" class="firebase-link" target="_blank">View in Firebase Test Lab</a>
+"""
+
+ if "results" in test:
+ for result in test["results"]:
+ html_content += f"""
+ <div class="result-info">
+ <div class="result-type">{result.get('type', 'Unknown')}</div>
+ <div>{result.get('message', 'No message available')}</div>
+ </div>
+"""
+
+ html_content += """
+ </div>
+"""
+
+ html_content += """
+ <div class="footer">
+ Generated by Mozilla Mobile Test Engineering
+ </div>
+</body>
+</html>
+"""
+
+ output_path.write_text(html_content, encoding="utf-8")
+ logging.info(f"HTML flaky report written to {output_path}")
+
+
+def main():
+ """Parse arguments and generate flaky test report."""
+ parser = argparse.ArgumentParser(
+ description="Generate an HTML report for flaky tests from JUnit XML results"
+ )
+ parser.add_argument(
+ "--results",
+ required=True,
+ help="Path to the results directory containing JUnit XML files",
+ )
+ parser.add_argument(
+ "--output",
+ default="/builds/worker/artifacts/HtmlFlakyReport.html",
+ help="Path where the HTML report should be written",
+ )
+ args = parser.parse_args()
+
+ results_dir = Path(args.results)
+ output_path = Path(args.output)
+
+ if not results_dir.exists():
+ logging.warning(f"Results directory does not exist: {results_dir}")
+ sys.exit(0)
+
+ # Find all JUnit XML files
+ xml_files = find_junit_xml_files(results_dir)
+
+ if not xml_files:
+ logging.info("No JUnit XML files found, skipping flaky report generation")
+ sys.exit(0)
+
+ # Extract flaky tests
+ flaky_tests = extract_flaky_tests(xml_files)
+
+ if not flaky_tests:
+ logging.info("No flaky tests detected, skipping report generation")
+ sys.exit(0)
+
+ # Generate HTML report
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ generate_html_report(flaky_tests, output_path)
+
+ logging.info(
+ f"Flaky test report generation complete: {len(flaky_tests)} flaky tests found"
+ )
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ setup_logging()
+ main()
diff --git a/taskcluster/scripts/tests/test-lab.py b/taskcluster/scripts/tests/test-lab.py
@@ -153,15 +153,20 @@ def process_results(flank_config: str, test_type: str = "instrumentation") -> No
Args:
flank_config: The YML configuration for Flank to use e.g, automation/taskcluster/androidTest/flank-<config>.yml
+ test_type: The type of test executed: 'instrumentation' or 'robo'
"""
parse_junit_results_artifact = os.path.join(SCRIPT_DIR, "parse-junit-results.py")
copy_robo_crash_artifacts_script = os.path.join(
SCRIPT_DIR, "copy-artifacts-from-ftl.py"
)
+ generate_flaky_report_script = os.path.join(
+ SCRIPT_DIR, "generate-flaky-report-from-ftl.py"
+ )
os.chmod(parse_junit_results_artifact, 0o755)
os.chmod(copy_robo_crash_artifacts_script, 0o755)
+ os.chmod(generate_flaky_report_script, 0o755)
# Process the results differently based on the test type: instrumentation or robo
#
@@ -172,6 +177,11 @@ def process_results(flank_config: str, test_type: str = "instrumentation") -> No
[parse_junit_results_artifact, "--results", Worker.RESULTS_DIR.value],
"flank.log",
)
+ # Generate flaky test report if flaky tests exist
+ run_command(
+ [generate_flaky_report_script, "--results", Worker.RESULTS_DIR.value],
+ "flank.log",
+ )
if test_type == "robo":
run_command([copy_robo_crash_artifacts_script, "crash_log"])
