test-lab.py (8329B)
1 #!/usr/bin/env python3 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 # Firebase Test Lab (Flank) test runner script for Taskcluster 8 # This script is used to run UI tests on Firebase Test Lab using Flank 9 # It requires a service account key file to authenticate with Firebase Test Lab 10 # It also requires the `gcloud` command line tool to be installed and configured 11 # Lastly it requires the `flank.jar` file to be present in the `test-tools` directory set up in the task definition 12 # The service account key file is stored in the `secrets` section of the task definition 13 14 # Flank: https://flank.github.io/flank/ 15 16 import argparse 17 import logging 18 import os 19 import subprocess 20 import sys 21 from enum import Enum 22 from pathlib import Path 23 from typing import Optional, Union 24 from urllib.parse import urlparse 25 26 27 # Worker paths and binaries 28 class Worker(Enum): 29 JAVA_BIN = "/usr/bin/java" 30 FLANK_BIN = "/builds/worker/test-tools/flank.jar" 31 RESULTS_DIR = "/builds/worker/artifacts/results" 32 33 34 # Locate other scripts and configs relative to this script. The actual 35 # invocation of Flank will be relative to ANDROID_TEST path below. 36 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) 37 TOPSRCDIR = os.path.join(SCRIPT_DIR, "../../..") 38 ANDROID_TEST = os.path.join(TOPSRCDIR, "mobile/android/test_infra") 39 40 41 def setup_logging(): 42 """Configure logging for the script.""" 43 log_format = "%(message)s" 44 logging.basicConfig(level=logging.INFO, format=log_format) 45 46 47 def run_command( 48 command: list[Union[str, bytes]], log_path: Optional[str] = None 49 ) -> int: 50 """Execute a command, log its output, and check for errors. 51 52 Args: 53 command: The command to execute 54 log_path: The path to a log file to write the command output to 55 Returns: 56 int: The exit code of the command 57 """ 58 59 with subprocess.Popen( 60 command, 61 stdout=subprocess.PIPE, 62 stderr=subprocess.STDOUT, 63 text=True, 64 cwd=ANDROID_TEST, 65 ) as process: 66 if log_path: 67 with open(log_path, "a") as log_file: 68 for line in process.stdout: 69 sys.stdout.write(line) 70 log_file.write(line) 71 else: 72 for line in process.stdout: 73 sys.stdout.write(line) 74 process.wait() 75 sys.stdout.flush() 76 if process.returncode != 0: 77 error_message = f"Command {' '.join(command)} failed with exit code {process.returncode}" 78 logging.error(error_message) 79 return process.returncode 80 81 82 def setup_environment(): 83 """Configure Google Cloud project and authenticate with the service account.""" 84 project_id = os.getenv("GOOGLE_PROJECT") 85 credentials_file = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") 86 if not project_id or not credentials_file: 87 logging.error( 88 "Error: GOOGLE_PROJECT and GOOGLE_APPLICATION_CREDENTIALS environment variables must be set." 89 ) 90 sys.exit(1) 91 92 run_command(["gcloud", "config", "set", "project", project_id]) 93 run_command([ 94 "gcloud", 95 "auth", 96 "activate-service-account", 97 "--key-file", 98 credentials_file, 99 ]) 100 101 102 def execute_tests( 103 flank_config: str, apk_app: Path, apk_test: Optional[Path] = None 104 ) -> int: 105 """Run UI tests on Firebase Test Lab using Flank. 106 107 Args: 108 flank_config: The YML configuration for Flank to use e.g, automation/taskcluster/androidTest/flank-<config>.yml 109 apk_app: Absolute path to a Android APK application package (optional) for robo test or instrumentation test 110 apk_test: Absolute path to a Android APK androidTest package 111 Returns: 112 int: The exit code of the command 113 """ 114 115 run_command([Worker.JAVA_BIN.value, "-jar", Worker.FLANK_BIN.value, "--version"]) 116 117 flank_command = [ 118 Worker.JAVA_BIN.value, 119 "-jar", 120 Worker.FLANK_BIN.value, 121 "android", 122 "run", 123 "--config", 124 f"{ANDROID_TEST}/flank-configs/{flank_config}", 125 "--app", 126 str(apk_app), 127 "--local-result-dir", 128 Worker.RESULTS_DIR.value, 129 "--project", 130 os.environ.get("GOOGLE_PROJECT"), 131 ] 132 133 # Add a client details parameter using the repository name 134 matrixLabel = os.environ.get("GECKO_HEAD_REPOSITORY") 135 geckoRev = os.environ.get("GECKO_HEAD_REV") 136 137 if matrixLabel is not None and geckoRev is not None: 138 flank_command.extend([ 139 "--client-details", 140 f"matrixLabel={urlparse(matrixLabel).path.rpartition('/')[-1]},geckoRev={geckoRev}", 141 ]) 142 143 # Add androidTest APK if provided (optional) as robo test or instrumentation test 144 if apk_test: 145 flank_command.extend(["--test", str(apk_test)]) 146 147 exit_code = run_command(flank_command, "flank.log") 148 if exit_code == 0: 149 logging.info("All UI test(s) have passed!") 150 return exit_code 151 152 153 def process_results( 154 flank_config: str, test_type: str = "instrumentation", artifact_type: str = None 155 ) -> None: 156 """Process and parse test results. 157 158 Args: 159 flank_config: The YML configuration for Flank to use e.g, automation/taskcluster/androidTest/flank-<config>.yml 160 test_type: The type of test executed: 'instrumentation' or 'robo' 161 artifact_type: The type of the artifacts to copy after the test run 162 """ 163 164 parse_junit_results_artifact = os.path.join(SCRIPT_DIR, "parse-junit-results.py") 165 copy_artifacts_script = os.path.join(SCRIPT_DIR, "copy-artifacts-from-ftl.py") 166 generate_flaky_report_script = os.path.join( 167 SCRIPT_DIR, "generate-flaky-report-from-ftl.py" 168 ) 169 170 os.chmod(parse_junit_results_artifact, 0o755) 171 os.chmod(copy_artifacts_script, 0o755) 172 os.chmod(generate_flaky_report_script, 0o755) 173 174 # Process the results differently based on the test type: instrumentation or robo 175 # 176 # Instrumentation (i.e, Android UI Tests): parse the JUnit results for CI logging 177 # Robo Test (i.e, self-crawling): copy crash artifacts from Google Cloud Storage over 178 if test_type == "instrumentation": 179 run_command( 180 [parse_junit_results_artifact, "--results", Worker.RESULTS_DIR.value], 181 "flank.log", 182 ) 183 # Generate flaky test report if flaky tests exist 184 run_command( 185 [generate_flaky_report_script, "--results", Worker.RESULTS_DIR.value], 186 "flank.log", 187 ) 188 189 # Copy artifacts if specified 190 if artifact_type: 191 run_command([copy_artifacts_script, artifact_type]) 192 193 if test_type == "robo": 194 run_command([copy_artifacts_script, "crash_log"]) 195 196 197 def main(): 198 """Parse command line arguments and execute the test runner.""" 199 parser = argparse.ArgumentParser( 200 description="Run UI tests on Firebase Test Lab using Flank as a test runner" 201 ) 202 parser.add_argument( 203 "flank_config", 204 help="The YML configuration for Flank to use e.g, 'fenix/flank-arm-debug.yml'." 205 + " This is relative to 'mobile/android/test_infra/flank-configs'.", 206 ) 207 parser.add_argument( 208 "apk_app", help="Absolute path to a Android APK application package" 209 ) 210 parser.add_argument( 211 "--apk_test", 212 help="Absolute path to a Android APK androidTest package", 213 default=None, 214 ) 215 parser.add_argument( 216 "--artifact_type", 217 help="Type of artifact to copy after running the tests", 218 default=None, 219 ) 220 args = parser.parse_args() 221 222 setup_environment() 223 224 # Only resolve apk_test if it is provided 225 apk_test_path = Path(args.apk_test).resolve() if args.apk_test else None 226 exit_code = execute_tests( 227 flank_config=args.flank_config, 228 apk_app=Path(args.apk_app).resolve(), 229 apk_test=apk_test_path, 230 ) 231 232 # Determine the instrumentation type to process the results differently 233 instrumentation_type = "instrumentation" if args.apk_test else "robo" 234 process_results( 235 flank_config=args.flank_config, 236 test_type=instrumentation_type, 237 artifact_type=args.artifact_type, 238 ) 239 240 sys.exit(exit_code) 241 242 243 if __name__ == "__main__": 244 setup_logging() 245 main()