runcppunittests.py (13385B)
1 #!/usr/bin/env python 2 # 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 import os 8 import sys 9 from optparse import OptionParser 10 from os import environ as env 11 12 import manifestparser 13 import mozcrash 14 import mozfile 15 import mozinfo 16 import mozlog 17 import mozprocess 18 import mozrunner.utils 19 20 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) 21 22 # Export directory js/src for tests that need it. 23 env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..")) 24 25 26 class CPPUnitTests: 27 # Time (seconds) to wait for test process to complete 28 TEST_PROC_TIMEOUT = 900 29 # Time (seconds) in which process will be killed if it produces no output. 30 TEST_PROC_NO_OUTPUT_TIMEOUT = 300 31 32 def run_one_test( 33 self, 34 prog, 35 env, 36 symbols_path=None, 37 utility_path=None, 38 interactive=False, 39 timeout_factor=1, 40 ): 41 """ 42 Run a single C++ unit test program. 43 44 Arguments: 45 * prog: The path to the test program to run. 46 * env: The environment to use for running the program. 47 * symbols_path: A path to a directory containing Breakpad-formatted 48 symbol files for producing stack traces on crash. 49 * timeout_factor: An optional test-specific timeout multiplier. 50 51 Return True if the program exits with a zero status, False otherwise. 52 """ 53 CPPUnitTests.run_one_test.timed_out = False 54 output = [] 55 56 def timeout_handler(proc): 57 CPPUnitTests.run_one_test.timed_out = True 58 message = f"timed out after {CPPUnitTests.TEST_PROC_TIMEOUT} seconds" 59 self.log.test_end( 60 basename, status="TIMEOUT", expected="PASS", message=message 61 ) 62 mozcrash.kill_and_get_minidump(proc.pid, tempdir, utility_path) 63 64 def output_timeout_handler(proc): 65 CPPUnitTests.run_one_test.timed_out = True 66 message = f"timed out after {CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT} seconds without output" 67 self.log.test_end( 68 basename, status="TIMEOUT", expected="PASS", message=message 69 ) 70 mozcrash.kill_and_get_minidump(proc.pid, tempdir, utility_path) 71 72 def output_line_handler(_, line): 73 fixed_line = self.fix_stack(line) if self.fix_stack else line 74 if interactive: 75 print(fixed_line) 76 else: 77 output.append(fixed_line) 78 79 basename = os.path.basename(prog) 80 self.log.test_start(basename) 81 with mozfile.TemporaryDirectory() as tempdir: 82 test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor 83 proc = mozprocess.run_and_wait( 84 [prog], 85 cwd=tempdir, 86 env=env, 87 output_line_handler=output_line_handler, 88 timeout=test_timeout, 89 timeout_handler=timeout_handler, 90 output_timeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT, 91 output_timeout_handler=output_timeout_handler, 92 ) 93 94 if output: 95 output = "\n" + "\n".join(output) 96 self.log.process_output(proc.pid, output, command=[prog]) 97 if CPPUnitTests.run_one_test.timed_out: 98 return False 99 if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename): 100 self.log.test_end(basename, status="CRASH", expected="PASS") 101 return False 102 result = proc.returncode == 0 103 if not result: 104 self.log.test_end( 105 basename, 106 status="FAIL", 107 expected="PASS", 108 message=(f"test failed with return code {proc.returncode}"), 109 ) 110 else: 111 self.log.test_end(basename, status="PASS", expected="PASS") 112 return result 113 114 def build_core_environment(self, env): 115 """ 116 Add environment variables likely to be used across all platforms, including remote systems. 117 """ 118 env["MOZ_XRE_DIR"] = self.xre_path 119 # TODO: switch this to just abort once all C++ unit tests have 120 # been fixed to enable crash reporting 121 env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" 122 env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" 123 env["MOZ_CRASHREPORTER"] = "1" 124 return env 125 126 def build_environment(self): 127 """ 128 Create and return a dictionary of all the appropriate env variables and values. 129 On a remote system, we overload this to set different values and are missing things 130 like os.environ and PATH. 131 """ 132 if not os.path.isdir(self.xre_path): 133 raise Exception("xre_path does not exist: %s", self.xre_path) 134 env = dict(os.environ) 135 env = self.build_core_environment(env) 136 pathvar = "" 137 libpath = self.xre_path 138 if mozinfo.os == "linux": 139 pathvar = "LD_LIBRARY_PATH" 140 elif mozinfo.os == "mac": 141 applibpath = os.path.join(os.path.dirname(libpath), "MacOS") 142 if os.path.exists(applibpath): 143 # Set the library load path to Contents/MacOS if we're run from 144 # the app bundle. 145 libpath = applibpath 146 pathvar = "DYLD_LIBRARY_PATH" 147 elif mozinfo.os == "win": 148 pathvar = "PATH" 149 if pathvar: 150 if pathvar in env: 151 env[pathvar] = f"{libpath}{os.pathsep}{env[pathvar]}" 152 else: 153 env[pathvar] = libpath 154 155 symbolizer_path = None 156 if mozinfo.info["asan"]: 157 symbolizer_path = "ASAN_SYMBOLIZER_PATH" 158 elif mozinfo.info["tsan"]: 159 symbolizer_path = "TSAN_SYMBOLIZER_PATH" 160 161 if symbolizer_path is not None: 162 # Use llvm-symbolizer for ASan/TSan if available/required 163 if symbolizer_path in env and os.path.isfile(env[symbolizer_path]): 164 llvmsym = env[symbolizer_path] 165 else: 166 llvmsym = os.path.join( 167 self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"] 168 ) 169 if os.path.isfile(llvmsym): 170 env[symbolizer_path] = llvmsym 171 self.log.info(f"Using LLVM symbolizer at {llvmsym}") 172 else: 173 self.log.info(f"Failed to find LLVM symbolizer at {llvmsym}") 174 175 return env 176 177 def run_tests( 178 self, 179 programs, 180 xre_path, 181 symbols_path=None, 182 utility_path=None, 183 interactive=False, 184 ): 185 """ 186 Run a set of C++ unit test programs. 187 188 Arguments: 189 * programs: An iterable containing (test path, test timeout factor) tuples 190 * xre_path: A path to a directory containing a XUL Runtime Environment. 191 * symbols_path: A path to a directory containing Breakpad-formatted 192 symbol files for producing stack traces on crash. 193 * utility_path: A path to a directory containing utility programs 194 (xpcshell et al) 195 196 Returns True if all test programs exited with a zero status, False 197 otherwise. 198 """ 199 self.xre_path = xre_path 200 self.log = mozlog.get_default_logger() 201 if utility_path: 202 self.fix_stack = mozrunner.utils.get_stack_fixer_function( 203 utility_path, symbols_path 204 ) 205 self.log.suite_start(programs, name="cppunittest") 206 env = self.build_environment() 207 pass_count = 0 208 fail_count = 0 209 for prog in programs: 210 test_path = prog[0] 211 timeout_factor = prog[1] 212 single_result = self.run_one_test( 213 test_path, env, symbols_path, utility_path, interactive, timeout_factor 214 ) 215 if single_result: 216 pass_count += 1 217 else: 218 fail_count += 1 219 self.log.suite_end() 220 221 # Mozharness-parseable summary formatting. 222 self.log.info("Result summary:") 223 self.log.info(f"cppunittests INFO | Passed: {pass_count}") 224 self.log.info(f"cppunittests INFO | Failed: {fail_count}") 225 return fail_count == 0 226 227 228 class CPPUnittestOptions(OptionParser): 229 def __init__(self): 230 OptionParser.__init__(self) 231 self.add_option( 232 "--xre-path", 233 action="store", 234 type="string", 235 dest="xre_path", 236 default=None, 237 help="absolute path to directory containing XRE (probably xulrunner)", 238 ) 239 self.add_option( 240 "--symbols-path", 241 action="store", 242 type="string", 243 dest="symbols_path", 244 default=None, 245 help="absolute path to directory containing breakpad symbols, or " 246 "the URL of a zip file containing symbols", 247 ) 248 self.add_option( 249 "--manifest-path", 250 action="store", 251 type="string", 252 dest="manifest_path", 253 default=None, 254 help="path to test manifest, if different from the path to test binaries", 255 ) 256 self.add_option( 257 "--utility-path", 258 action="store", 259 type="string", 260 dest="utility_path", 261 default=None, 262 help="path to directory containing utility programs", 263 ) 264 265 266 def extract_unittests_from_args(args, environ, manifest_path): 267 """Extract unittests from args, expanding directories as needed""" 268 mp = manifestparser.TestManifest(strict=True) 269 tests = [] 270 binary_path = None 271 272 if manifest_path: 273 mp.read(manifest_path) 274 binary_path = os.path.abspath(args[0]) 275 else: 276 for p in args: 277 if os.path.isdir(p): 278 try: 279 mp.read(os.path.join(p, "cppunittest.toml")) 280 except OSError: 281 files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)] 282 tests.extend( 283 (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK) 284 ) 285 else: 286 tests.append((os.path.abspath(p), 1)) 287 288 # We don't use the manifest parser's existence-check only because it will 289 # fail on Windows due to the `.exe` suffix. 290 active_tests = mp.active_tests(exists=False, disabled=False, **environ) 291 suffix = ".exe" if mozinfo.isWin else "" 292 if binary_path: 293 tests.extend([ 294 ( 295 os.path.join(binary_path, test["relpath"] + suffix), 296 int(test.get("requesttimeoutfactor", 1)), 297 ) 298 for test in active_tests 299 ]) 300 else: 301 tests.extend([ 302 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1))) 303 for test in active_tests 304 ]) 305 306 # Manually confirm that all tests named in the manifest exist. 307 errors = False 308 log = mozlog.get_default_logger() 309 for test in tests: 310 if not os.path.isfile(test[0]): 311 errors = True 312 log.error(f"test file not found: {test[0]}") 313 314 if errors: 315 raise RuntimeError("One or more cppunittests not found; aborting.") 316 317 return tests 318 319 320 def update_mozinfo(): 321 """walk up directories to find mozinfo.json update the info""" 322 path = SCRIPT_DIR 323 dirs = set() 324 while path != os.path.expanduser("~"): 325 if path in dirs: 326 break 327 dirs.add(path) 328 path = os.path.split(path)[0] 329 mozinfo.find_and_update_from_json(*dirs) 330 print( 331 "These variables are available in the mozinfo environment and can be used to skip tests conditionally:" 332 ) 333 for k in sorted(mozinfo.info.keys()): 334 print(f" {k}: {mozinfo.info[k]}") 335 336 337 def run_test_harness(options, args): 338 update_mozinfo() 339 progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path) 340 options.xre_path = os.path.abspath(options.xre_path) 341 options.utility_path = os.path.abspath(options.utility_path) 342 tester = CPPUnitTests() 343 result = tester.run_tests( 344 progs, 345 options.xre_path, 346 options.symbols_path, 347 options.utility_path, 348 ) 349 350 return result 351 352 353 def main(): 354 parser = CPPUnittestOptions() 355 mozlog.commandline.add_logging_group(parser) 356 options, args = parser.parse_args() 357 if not args: 358 print( 359 f"Usage: {sys.argv[0]} <test binary> [<test binary>...]", 360 file=sys.stderr, 361 ) 362 sys.exit(1) 363 if not options.xre_path: 364 print("""Error: --xre-path is required""", file=sys.stderr) 365 sys.exit(1) 366 if options.manifest_path and len(args) > 1: 367 print( 368 "Error: multiple arguments not supported with --test-manifest", 369 file=sys.stderr, 370 ) 371 sys.exit(1) 372 log = mozlog.commandline.setup_logging( 373 "cppunittests", options, {"tbpl": sys.stdout} 374 ) 375 try: 376 result = run_test_harness(options, args) 377 except Exception as e: 378 log.error(str(e)) 379 result = False 380 381 sys.exit(0 if result else 1) 382 383 384 if __name__ == "__main__": 385 main()