gtest_test_instance.py (21451B)
1 # Copyright 2014 The Chromium Authors 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 7 import html.parser 8 import json 9 import logging 10 import os 11 import re 12 import tempfile 13 import threading 14 import xml.etree.ElementTree 15 16 from devil.android import apk_helper 17 from pylib import constants 18 from pylib.constants import host_paths 19 from pylib.base import base_test_result 20 from pylib.base import test_instance 21 from pylib.symbols import stack_symbolizer 22 from pylib.utils import test_filter 23 24 with host_paths.SysPath(host_paths.BUILD_UTIL_PATH): 25 from lib.common import unittest_util 26 27 BROWSER_TEST_SUITES = [ 28 'android_browsertests', 29 'android_sync_integration_tests', 30 'components_browsertests', 31 'content_browsertests', 32 'weblayer_browsertests', 33 ] 34 35 # The max number of tests to run on a shard during the test run. 36 MAX_SHARDS = 256 37 38 RUN_IN_SUB_THREAD_TEST_SUITES = [ 39 # Multiprocess tests should be run outside of the main thread. 40 'base_unittests', # file_locking_unittest.cc uses a child process. 41 'gwp_asan_unittests', 42 'ipc_perftests', 43 'ipc_tests', 44 'mojo_perftests', 45 'mojo_unittests', 46 'net_unittests' 47 ] 48 49 50 # Used for filtering large data deps at a finer grain than what's allowed in 51 # isolate files since pushing deps to devices is expensive. 52 # Wildcards are allowed. 53 _DEPS_EXCLUSION_LIST = [ 54 'chrome/test/data/extensions/api_test', 55 'chrome/test/data/extensions/secure_shell', 56 'chrome/test/data/firefox*', 57 'chrome/test/data/gpu', 58 'chrome/test/data/image_decoding', 59 'chrome/test/data/import', 60 'chrome/test/data/page_cycler', 61 'chrome/test/data/perf', 62 'chrome/test/data/pyauto_private', 63 'chrome/test/data/safari_import', 64 'chrome/test/data/scroll', 65 'chrome/test/data/third_party', 66 'third_party/hunspell_dictionaries/*.dic', 67 # crbug.com/258690 68 'webkit/data/bmp_decoder', 69 'webkit/data/ico_decoder', 70 ] 71 72 73 _EXTRA_NATIVE_TEST_ACTIVITY = ( 74 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 75 'NativeTestActivity') 76 _EXTRA_RUN_IN_SUB_THREAD = ( 77 'org.chromium.native_test.NativeTest.RunInSubThread') 78 EXTRA_SHARD_NANO_TIMEOUT = ( 79 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 80 'ShardNanoTimeout') 81 _EXTRA_SHARD_SIZE_LIMIT = ( 82 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' 83 'ShardSizeLimit') 84 85 # TODO(jbudorick): Remove these once we're no longer parsing stdout to generate 86 # results. 87 _RE_TEST_STATUS = re.compile( 88 # Test state. 89 r'\[ +((?:RUN)|(?:FAILED)|(?:OK)|(?:CRASHED)|(?:SKIPPED)) +\] ?' 90 # Test name. 91 r'([^ ]+)?' 92 # Optional parameters. 93 r'(?:, where' 94 # Type parameter 95 r'(?: TypeParam = [^()]*(?: and)?)?' 96 # Value parameter 97 r'(?: GetParam\(\) = [^()]*)?' 98 # End of optional parameters. 99 ')?' 100 # Optional test execution time. 101 r'(?: \((\d+) ms\))?$') 102 # Crash detection constants. 103 _RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,' 104 r' Failures: \d+, Errors: 1') 105 _RE_TEST_CURRENTLY_RUNNING = re.compile( 106 r'\[.*ERROR:.*?\] Currently running: (.*)') 107 _RE_TEST_DCHECK_FATAL = re.compile(r'\[.*:FATAL:.*\] (.*)') 108 _RE_DISABLED = re.compile(r'DISABLED_') 109 _RE_FLAKY = re.compile(r'FLAKY_') 110 111 # Detect a new launcher invocation. When encountered, the output parser will 112 # stop recording logs for a suddenly crashed test (if one was running) in the 113 # previous invocation. 114 _RE_LAUNCHER_MAIN_START = re.compile(r'>>ScopedMainEntryLogger') 115 116 # Regex that matches the printout when there are test failures. 117 # matches "[ FAILED ] 1 test, listed below:" 118 _RE_ANY_TESTS_FAILED = re.compile(r'\[ +FAILED +\].*listed below') 119 120 # Detect stack line in stdout. 121 _STACK_LINE_RE = re.compile(r'\s*#\d+') 122 123 def ParseGTestListTests(raw_list): 124 """Parses a raw test list as provided by --gtest_list_tests. 125 126 Args: 127 raw_list: The raw test listing with the following format: 128 129 IPCChannelTest. 130 SendMessageInChannelConnected 131 IPCSyncChannelTest. 132 Simple 133 DISABLED_SendWithTimeoutMixedOKAndTimeout 134 135 Returns: 136 A list of all tests. For the above raw listing: 137 138 [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple, 139 IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout] 140 """ 141 ret = [] 142 current = '' 143 for test in raw_list: 144 if not test: 145 continue 146 if not test.startswith(' '): 147 test_case = test.split()[0] 148 if test_case.endswith('.'): 149 current = test_case 150 else: 151 test = test.strip() 152 if test and not 'YOU HAVE' in test: 153 test_name = test.split()[0] 154 ret += [current + test_name] 155 return ret 156 157 158 def ParseGTestOutput(output, symbolizer, device_abi): 159 """Parses raw gtest output and returns a list of results. 160 161 Args: 162 output: A list of output lines. 163 symbolizer: The symbolizer used to symbolize stack. 164 device_abi: Device abi that is needed for symbolization. 165 Returns: 166 A list of base_test_result.BaseTestResults. 167 """ 168 duration = 0 169 fallback_result_type = None 170 log = [] 171 stack = [] 172 result_type = None 173 results = [] 174 test_name = None 175 176 def symbolize_stack_and_merge_with_log(): 177 log_string = '\n'.join(log or []) 178 if not stack: 179 stack_string = '' 180 else: 181 stack_string = '\n'.join( 182 symbolizer.ExtractAndResolveNativeStackTraces( 183 stack, device_abi)) 184 return '%s\n%s' % (log_string, stack_string) 185 186 def handle_possibly_unknown_test(): 187 if test_name is not None: 188 results.append( 189 base_test_result.BaseTestResult( 190 TestNameWithoutDisabledPrefix(test_name), 191 # If we get here, that means we started a test, but it did not 192 # produce a definitive test status output, so assume it crashed. 193 # crbug/1191716 194 fallback_result_type or base_test_result.ResultType.CRASH, 195 duration, 196 log=symbolize_stack_and_merge_with_log())) 197 198 for l in output: 199 matcher = _RE_TEST_STATUS.match(l) 200 launcher_main_start_match = _RE_LAUNCHER_MAIN_START.match(l) 201 if matcher: 202 if matcher.group(1) == 'RUN': 203 handle_possibly_unknown_test() 204 duration = 0 205 fallback_result_type = None 206 log = [] 207 stack = [] 208 result_type = None 209 elif matcher.group(1) == 'OK': 210 result_type = base_test_result.ResultType.PASS 211 elif matcher.group(1) == 'SKIPPED': 212 result_type = base_test_result.ResultType.SKIP 213 elif matcher.group(1) == 'FAILED': 214 result_type = base_test_result.ResultType.FAIL 215 elif matcher.group(1) == 'CRASHED': 216 fallback_result_type = base_test_result.ResultType.CRASH 217 # Be aware that test name and status might not appear on same line. 218 test_name = matcher.group(2) if matcher.group(2) else test_name 219 duration = int(matcher.group(3)) if matcher.group(3) else 0 220 221 else: 222 # Can possibly add more matchers, such as different results from DCHECK. 223 currently_running_matcher = _RE_TEST_CURRENTLY_RUNNING.match(l) 224 dcheck_matcher = _RE_TEST_DCHECK_FATAL.match(l) 225 226 if currently_running_matcher: 227 test_name = currently_running_matcher.group(1) 228 result_type = base_test_result.ResultType.CRASH 229 duration = None # Don't know. Not using 0 as this is unknown vs 0. 230 elif dcheck_matcher or launcher_main_start_match: 231 result_type = base_test_result.ResultType.CRASH 232 duration = None # Don't know. Not using 0 as this is unknown vs 0. 233 234 if not launcher_main_start_match: 235 log.append(l) 236 if not matcher and _STACK_LINE_RE.match(l): 237 stack.append(l) 238 239 if _RE_ANY_TESTS_FAILED.match(l): 240 break 241 242 if result_type and test_name: 243 # Don't bother symbolizing output if the test passed. 244 if result_type == base_test_result.ResultType.PASS: 245 stack = [] 246 results.append(base_test_result.BaseTestResult( 247 TestNameWithoutDisabledPrefix(test_name), result_type, duration, 248 log=symbolize_stack_and_merge_with_log())) 249 test_name = None 250 251 else: 252 # Executing this after tests have finished with a failure causes a 253 # duplicate test entry to be added to results. crbug/1380825 254 handle_possibly_unknown_test() 255 256 return results 257 258 259 def ParseGTestXML(xml_content): 260 """Parse gtest XML result.""" 261 results = [] 262 if not xml_content: 263 return results 264 265 html_parser = html.parser.HTMLParser() 266 267 testsuites = xml.etree.ElementTree.fromstring(xml_content) 268 for testsuite in testsuites: 269 suite_name = testsuite.attrib['name'] 270 for testcase in testsuite: 271 case_name = testcase.attrib['name'] 272 result_type = base_test_result.ResultType.PASS 273 log = [] 274 for failure in testcase: 275 result_type = base_test_result.ResultType.FAIL 276 log.append(html_parser.unescape(failure.attrib['message'])) 277 278 results.append(base_test_result.BaseTestResult( 279 '%s.%s' % (suite_name, TestNameWithoutDisabledPrefix(case_name)), 280 result_type, 281 int(float(testcase.attrib['time']) * 1000), 282 log=('\n'.join(log) if log else ''))) 283 284 return results 285 286 287 def ParseGTestJSON(json_content): 288 """Parse results in the JSON Test Results format.""" 289 results = [] 290 if not json_content: 291 return results 292 293 json_data = json.loads(json_content) 294 295 openstack = list(json_data['tests'].items()) 296 297 while openstack: 298 name, value = openstack.pop() 299 300 if 'expected' in value and 'actual' in value: 301 if value['actual'] == 'PASS': 302 result_type = base_test_result.ResultType.PASS 303 elif value['actual'] == 'SKIP': 304 result_type = base_test_result.ResultType.SKIP 305 elif value['actual'] == 'CRASH': 306 result_type = base_test_result.ResultType.CRASH 307 elif value['actual'] == 'TIMEOUT': 308 result_type = base_test_result.ResultType.TIMEOUT 309 else: 310 result_type = base_test_result.ResultType.FAIL 311 results.append(base_test_result.BaseTestResult(name, result_type)) 312 else: 313 openstack += [("%s.%s" % (name, k), v) for k, v in value.items()] 314 315 return results 316 317 318 def TestNameWithoutDisabledPrefix(test_name): 319 """Modify the test name without disabled prefix if prefix 'DISABLED_' or 320 'FLAKY_' presents. 321 322 Args: 323 test_name: The name of a test. 324 Returns: 325 A test name without prefix 'DISABLED_' or 'FLAKY_'. 326 """ 327 disabled_prefixes = [_RE_DISABLED, _RE_FLAKY] 328 for dp in disabled_prefixes: 329 test_name = dp.sub('', test_name) 330 return test_name 331 332 class GtestTestInstance(test_instance.TestInstance): 333 334 def __init__(self, args, data_deps_delegate, error_func): 335 super().__init__() 336 # TODO(jbudorick): Support multiple test suites. 337 if len(args.suite_name) > 1: 338 raise ValueError('Platform mode currently supports only 1 gtest suite') 339 self._additional_apks = [] 340 self._coverage_dir = args.coverage_dir 341 self._exe_dist_dir = None 342 self._external_shard_index = args.test_launcher_shard_index 343 self._extract_test_list_from_filter = args.extract_test_list_from_filter 344 self._filter_tests_lock = threading.Lock() 345 self._gs_test_artifacts_bucket = args.gs_test_artifacts_bucket 346 self._isolated_script_test_output = args.isolated_script_test_output 347 self._isolated_script_test_perf_output = ( 348 args.isolated_script_test_perf_output) 349 self._render_test_output_dir = args.render_test_output_dir 350 self._shard_timeout = args.shard_timeout 351 self._store_tombstones = args.store_tombstones 352 self._suite = args.suite_name[0] 353 self._symbolizer = stack_symbolizer.Symbolizer(None) 354 self._total_external_shards = args.test_launcher_total_shards 355 self._wait_for_java_debugger = args.wait_for_java_debugger 356 self._use_existing_test_data = args.use_existing_test_data 357 self._deploy_mock_openxr_runtime = args.deploy_mock_openxr_runtime 358 359 # GYP: 360 if args.executable_dist_dir: 361 self._exe_dist_dir = os.path.abspath(args.executable_dist_dir) 362 else: 363 # TODO(agrieve): Remove auto-detection once recipes pass flag explicitly. 364 exe_dist_dir = os.path.join(constants.GetOutDirectory(), 365 '%s__dist' % self._suite) 366 367 if os.path.exists(exe_dist_dir): 368 self._exe_dist_dir = exe_dist_dir 369 370 incremental_part = '' 371 if args.test_apk_incremental_install_json: 372 incremental_part = '_incremental' 373 374 self._test_launcher_batch_limit = MAX_SHARDS 375 if (args.test_launcher_batch_limit 376 and 0 < args.test_launcher_batch_limit < MAX_SHARDS): 377 self._test_launcher_batch_limit = args.test_launcher_batch_limit 378 379 apk_path = os.path.join( 380 constants.GetOutDirectory(), '%s_apk' % self._suite, 381 '%s-debug%s.apk' % (self._suite, incremental_part)) 382 self._test_apk_incremental_install_json = ( 383 args.test_apk_incremental_install_json) 384 if not os.path.exists(apk_path): 385 self._apk_helper = None 386 else: 387 self._apk_helper = apk_helper.ApkHelper(apk_path) 388 self._extras = { 389 _EXTRA_NATIVE_TEST_ACTIVITY: self._apk_helper.GetActivityName(), 390 } 391 if args.timeout_scale and args.timeout_scale != 1: 392 self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1 393 394 if self._suite in RUN_IN_SUB_THREAD_TEST_SUITES: 395 self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1 396 if self._suite in BROWSER_TEST_SUITES: 397 self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1 398 self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e9 * self._shard_timeout) 399 self._shard_timeout = 10 * self._shard_timeout 400 if args.wait_for_java_debugger: 401 self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e15) # Forever 402 403 if not self._apk_helper and not self._exe_dist_dir: 404 error_func('Could not find apk or executable for %s' % self._suite) 405 406 for x in args.additional_apks: 407 if not os.path.exists(x): 408 error_func('Could not find additional APK: %s' % x) 409 410 apk = apk_helper.ToHelper(x) 411 self._additional_apks.append(apk) 412 413 self._data_deps = [] 414 self._gtest_filters = test_filter.InitializeFiltersFromArgs(args) 415 self._run_disabled = args.run_disabled 416 self._run_pre_tests = args.run_pre_tests 417 418 self._data_deps_delegate = data_deps_delegate 419 self._runtime_deps_path = args.runtime_deps_path 420 if not self._runtime_deps_path: 421 logging.warning('No data dependencies will be pushed.') 422 423 if args.app_data_files: 424 self._app_data_files = args.app_data_files 425 if args.app_data_file_dir: 426 self._app_data_file_dir = args.app_data_file_dir 427 else: 428 self._app_data_file_dir = tempfile.mkdtemp() 429 logging.critical('Saving app files to %s', self._app_data_file_dir) 430 else: 431 self._app_data_files = None 432 self._app_data_file_dir = None 433 434 self._flags = None 435 self._initializeCommandLineFlags(args) 436 437 # TODO(jbudorick): Remove this once it's deployed. 438 self._enable_xml_result_parsing = args.enable_xml_result_parsing 439 440 def _initializeCommandLineFlags(self, args): 441 self._flags = [] 442 if args.command_line_flags: 443 self._flags.extend(args.command_line_flags) 444 if args.device_flags_file: 445 with open(args.device_flags_file) as f: 446 stripped_lines = (l.strip() for l in f) 447 self._flags.extend(flag for flag in stripped_lines if flag) 448 if args.run_disabled: 449 self._flags.append('--gtest_also_run_disabled_tests') 450 451 @property 452 def activity(self): 453 return self._apk_helper and self._apk_helper.GetActivityName() 454 455 @property 456 def additional_apks(self): 457 return self._additional_apks 458 459 @property 460 def apk(self): 461 return self._apk_helper and self._apk_helper.path 462 463 @property 464 def apk_helper(self): 465 return self._apk_helper 466 467 @property 468 def app_file_dir(self): 469 return self._app_data_file_dir 470 471 @property 472 def app_files(self): 473 return self._app_data_files 474 475 @property 476 def coverage_dir(self): 477 return self._coverage_dir 478 479 @property 480 def deploy_mock_openxr_runtime(self): 481 return self._deploy_mock_openxr_runtime 482 483 @property 484 def enable_xml_result_parsing(self): 485 return self._enable_xml_result_parsing 486 487 @property 488 def exe_dist_dir(self): 489 return self._exe_dist_dir 490 491 @property 492 def external_shard_index(self): 493 return self._external_shard_index 494 495 @property 496 def extract_test_list_from_filter(self): 497 return self._extract_test_list_from_filter 498 499 @property 500 def extras(self): 501 return self._extras 502 503 @property 504 def flags(self): 505 return self._flags 506 507 @property 508 def gs_test_artifacts_bucket(self): 509 return self._gs_test_artifacts_bucket 510 511 @property 512 def gtest_filters(self): 513 return self._gtest_filters 514 515 @property 516 def isolated_script_test_output(self): 517 return self._isolated_script_test_output 518 519 @property 520 def isolated_script_test_perf_output(self): 521 return self._isolated_script_test_perf_output 522 523 @property 524 def render_test_output_dir(self): 525 return self._render_test_output_dir 526 527 @property 528 def package(self): 529 return self._apk_helper and self._apk_helper.GetPackageName() 530 531 @property 532 def permissions(self): 533 return self._apk_helper and self._apk_helper.GetPermissions() 534 535 @property 536 def runner(self): 537 return self._apk_helper and self._apk_helper.GetInstrumentationName() 538 539 @property 540 def shard_timeout(self): 541 return self._shard_timeout 542 543 @property 544 def store_tombstones(self): 545 return self._store_tombstones 546 547 @property 548 def suite(self): 549 return self._suite 550 551 @property 552 def symbolizer(self): 553 return self._symbolizer 554 555 @property 556 def test_apk_incremental_install_json(self): 557 return self._test_apk_incremental_install_json 558 559 @property 560 def test_launcher_batch_limit(self): 561 return self._test_launcher_batch_limit 562 563 @property 564 def total_external_shards(self): 565 return self._total_external_shards 566 567 @property 568 def wait_for_java_debugger(self): 569 return self._wait_for_java_debugger 570 571 @property 572 def use_existing_test_data(self): 573 return self._use_existing_test_data 574 575 @property 576 def run_pre_tests(self): 577 return self._run_pre_tests 578 579 #override 580 def TestType(self): 581 return 'gtest' 582 583 #override 584 def GetPreferredAbis(self): 585 if not self._apk_helper: 586 return None 587 return self._apk_helper.GetAbis() 588 589 #override 590 def SetUp(self): 591 """Map data dependencies via isolate.""" 592 self._data_deps.extend( 593 self._data_deps_delegate(self._runtime_deps_path)) 594 595 def GetDataDependencies(self): 596 """Returns the test suite's data dependencies. 597 598 Returns: 599 A list of (host_path, device_path) tuples to push. If device_path is 600 None, the client is responsible for determining where to push the file. 601 """ 602 return self._data_deps 603 604 def FilterTests(self, test_list, disabled_prefixes=None): 605 """Filters |test_list| based on prefixes and, if present, a filter string. 606 607 Args: 608 test_list: The list of tests to filter. 609 disabled_prefixes: A list of test prefixes to filter. Defaults to 610 DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_ 611 Returns: 612 A filtered list of tests to run. 613 """ 614 gtest_filter_strings = [ 615 self._GenerateDisabledFilterString(disabled_prefixes)] 616 if self._gtest_filters: 617 gtest_filter_strings.extend(self._gtest_filters) 618 619 filtered_test_list = test_list 620 # This lock is required because on older versions of Python 621 # |unittest_util.FilterTestNames| use of |fnmatch| is not threadsafe. 622 with self._filter_tests_lock: 623 for gtest_filter_string in gtest_filter_strings: 624 logging.debug('Filtering tests using: %s', gtest_filter_string) 625 filtered_test_list = unittest_util.FilterTestNames( 626 filtered_test_list, gtest_filter_string) 627 628 if self._run_disabled and self._gtest_filters: 629 out_filtered_test_list = list(set(test_list)-set(filtered_test_list)) 630 for test in out_filtered_test_list: 631 test_name_no_disabled = TestNameWithoutDisabledPrefix(test) 632 if test_name_no_disabled == test: 633 continue 634 if all( 635 unittest_util.FilterTestNames([test_name_no_disabled], 636 gtest_filter) 637 for gtest_filter in self._gtest_filters): 638 filtered_test_list.append(test) 639 return filtered_test_list 640 641 def _GenerateDisabledFilterString(self, disabled_prefixes): 642 disabled_filter_items = [] 643 644 if disabled_prefixes is None: 645 disabled_prefixes = ['FAILS_'] 646 if '--run-manual' not in self._flags: 647 disabled_prefixes += ['MANUAL_'] 648 if not self._run_disabled: 649 disabled_prefixes += ['DISABLED_', 'FLAKY_'] 650 if not self._run_pre_tests: 651 disabled_prefixes += ['PRE_'] 652 653 disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes] 654 disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes] 655 656 disabled_tests_file_path = os.path.join( 657 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest', 658 'filter', '%s_disabled' % self._suite) 659 if disabled_tests_file_path and os.path.exists(disabled_tests_file_path): 660 with open(disabled_tests_file_path) as disabled_tests_file: 661 disabled_filter_items += [ 662 '%s' % l for l in (line.strip() for line in disabled_tests_file) 663 if l and not l.startswith('#')] 664 665 return '*-%s' % ':'.join(disabled_filter_items) 666 667 #override 668 def TearDown(self): 669 """Do nothing."""