googletest-failfast-unittest.py (14894B)
1 #!/usr/bin/env python 2 # 3 # Copyright 2020 Google Inc. All Rights Reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Unit test for Google Test fail_fast. 32 33 A user can specify if a Google Test program should continue test execution 34 after a test failure via the GTEST_FAIL_FAST environment variable or the 35 --gtest_fail_fast flag. The default value of the flag can also be changed 36 by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST. 37 38 This script tests such functionality by invoking googletest-failfast-unittest_ 39 (a program written with Google Test) with different environments and command 40 line flags. 41 """ 42 43 import os 44 import gtest_test_utils 45 46 # Constants. 47 48 # Bazel testbridge environment variable for fail fast 49 BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST' 50 51 # The environment variable for specifying fail fast. 52 FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST' 53 54 # The command line flag for specifying fail fast. 55 FAIL_FAST_FLAG = 'gtest_fail_fast' 56 57 # The command line flag to run disabled tests. 58 RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests' 59 60 # The command line flag for specifying a filter. 61 FILTER_FLAG = 'gtest_filter' 62 63 # Command to run the googletest-failfast-unittest_ program. 64 COMMAND = gtest_test_utils.GetTestExecutablePath( 65 'googletest-failfast-unittest_') 66 67 # The command line flag to tell Google Test to output the list of tests it 68 # will run. 69 LIST_TESTS_FLAG = '--gtest_list_tests' 70 71 # Indicates whether Google Test supports death tests. 72 SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess( 73 [COMMAND, LIST_TESTS_FLAG]).output 74 75 # Utilities. 76 77 environ = os.environ.copy() 78 79 80 def SetEnvVar(env_var, value): 81 """Sets the env variable to 'value'; unsets it when 'value' is None.""" 82 83 if value is not None: 84 environ[env_var] = value 85 elif env_var in environ: 86 del environ[env_var] 87 88 89 def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False): 90 """Runs the test program and returns its output.""" 91 92 args = [] 93 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 94 '.GTestFailFastUnitTest.xml') 95 args += ['--gtest_output=xml:' + xml_path] 96 if fail_fast is not None: 97 if isinstance(fail_fast, str): 98 args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)] 99 elif fail_fast: 100 args += ['--%s' % FAIL_FAST_FLAG] 101 else: 102 args += ['--no%s' % FAIL_FAST_FLAG] 103 if test_suite: 104 args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)] 105 if run_disabled: 106 args += ['--%s' % RUN_DISABLED_FLAG] 107 txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output 108 with open(xml_path) as xml_file: 109 return txt_out, xml_file.read() 110 111 112 # The unit test. 113 class GTestFailFastUnitTest(gtest_test_utils.TestCase): 114 """Tests the env variable or the command line flag for fail_fast.""" 115 116 def testDefaultBehavior(self): 117 """Tests the behavior of not specifying the fail_fast.""" 118 119 txt, _ = RunAndReturnOutput() 120 self.assertIn('22 FAILED TEST', txt) 121 122 def testGoogletestFlag(self): 123 txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True) 124 self.assertIn('1 FAILED TEST', txt) 125 self.assertIn('[ SKIPPED ] 3 tests', txt) 126 127 txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False) 128 self.assertIn('4 FAILED TEST', txt) 129 self.assertNotIn('[ SKIPPED ]', txt) 130 131 def testGoogletestEnvVar(self): 132 """Tests the behavior of specifying fail_fast via Googletest env var.""" 133 134 try: 135 SetEnvVar(FAIL_FAST_ENV_VAR, '1') 136 txt, _ = RunAndReturnOutput('HasSimpleTest') 137 self.assertIn('1 FAILED TEST', txt) 138 self.assertIn('[ SKIPPED ] 3 tests', txt) 139 140 SetEnvVar(FAIL_FAST_ENV_VAR, '0') 141 txt, _ = RunAndReturnOutput('HasSimpleTest') 142 self.assertIn('4 FAILED TEST', txt) 143 self.assertNotIn('[ SKIPPED ]', txt) 144 finally: 145 SetEnvVar(FAIL_FAST_ENV_VAR, None) 146 147 def testBazelEnvVar(self): 148 """Tests the behavior of specifying fail_fast via Bazel testbridge.""" 149 150 try: 151 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1') 152 txt, _ = RunAndReturnOutput('HasSimpleTest') 153 self.assertIn('1 FAILED TEST', txt) 154 self.assertIn('[ SKIPPED ] 3 tests', txt) 155 156 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0') 157 txt, _ = RunAndReturnOutput('HasSimpleTest') 158 self.assertIn('4 FAILED TEST', txt) 159 self.assertNotIn('[ SKIPPED ]', txt) 160 finally: 161 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None) 162 163 def testFlagOverridesEnvVar(self): 164 """Tests precedence of flag over env var.""" 165 166 try: 167 SetEnvVar(FAIL_FAST_ENV_VAR, '0') 168 txt, _ = RunAndReturnOutput('HasSimpleTest', True) 169 self.assertIn('1 FAILED TEST', txt) 170 self.assertIn('[ SKIPPED ] 3 tests', txt) 171 finally: 172 SetEnvVar(FAIL_FAST_ENV_VAR, None) 173 174 def testGoogletestEnvVarOverridesBazelEnvVar(self): 175 """Tests that the Googletest native env var over Bazel testbridge.""" 176 177 try: 178 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0') 179 SetEnvVar(FAIL_FAST_ENV_VAR, '1') 180 txt, _ = RunAndReturnOutput('HasSimpleTest') 181 self.assertIn('1 FAILED TEST', txt) 182 self.assertIn('[ SKIPPED ] 3 tests', txt) 183 finally: 184 SetEnvVar(FAIL_FAST_ENV_VAR, None) 185 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None) 186 187 def testEventListener(self): 188 txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True) 189 self.assertIn('1 FAILED TEST', txt) 190 self.assertIn('[ SKIPPED ] 3 tests', txt) 191 for expected_count, callback in [(1, 'OnTestSuiteStart'), 192 (5, 'OnTestStart'), 193 (5, 'OnTestEnd'), 194 (5, 'OnTestPartResult'), 195 (1, 'OnTestSuiteEnd')]: 196 self.assertEqual( 197 expected_count, txt.count(callback), 198 'Expected %d calls to callback %s match count on output: %s ' % 199 (expected_count, callback, txt)) 200 201 txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False) 202 self.assertIn('3 FAILED TEST', txt) 203 self.assertIn('[ SKIPPED ] 1 test', txt) 204 for expected_count, callback in [(1, 'OnTestSuiteStart'), 205 (5, 'OnTestStart'), 206 (5, 'OnTestEnd'), 207 (5, 'OnTestPartResult'), 208 (1, 'OnTestSuiteEnd')]: 209 self.assertEqual( 210 expected_count, txt.count(callback), 211 'Expected %d calls to callback %s match count on output: %s ' % 212 (expected_count, callback, txt)) 213 214 def assertXmlResultCount(self, result, count, xml): 215 self.assertEqual( 216 count, xml.count('result="%s"' % result), 217 'Expected \'result="%s"\' match count of %s: %s ' % 218 (result, count, xml)) 219 220 def assertXmlStatusCount(self, status, count, xml): 221 self.assertEqual( 222 count, xml.count('status="%s"' % status), 223 'Expected \'status="%s"\' match count of %s: %s ' % 224 (status, count, xml)) 225 226 def assertFailFastXmlAndTxtOutput(self, 227 fail_fast, 228 test_suite, 229 passed_count, 230 failure_count, 231 skipped_count, 232 suppressed_count, 233 run_disabled=False): 234 """Assert XML and text output of a test execution.""" 235 236 txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled) 237 if failure_count > 0: 238 self.assertIn('%s FAILED TEST' % failure_count, txt) 239 if suppressed_count > 0: 240 self.assertIn('%s DISABLED TEST' % suppressed_count, txt) 241 if skipped_count > 0: 242 self.assertIn('[ SKIPPED ] %s tests' % skipped_count, txt) 243 self.assertXmlStatusCount('run', 244 passed_count + failure_count + skipped_count, xml) 245 self.assertXmlStatusCount('notrun', suppressed_count, xml) 246 self.assertXmlResultCount('completed', passed_count + failure_count, xml) 247 self.assertXmlResultCount('skipped', skipped_count, xml) 248 self.assertXmlResultCount('suppressed', suppressed_count, xml) 249 250 def assertFailFastBehavior(self, 251 test_suite, 252 passed_count, 253 failure_count, 254 skipped_count, 255 suppressed_count, 256 run_disabled=False): 257 """Assert --fail_fast via flag.""" 258 259 for fail_fast in ('true', '1', 't', True): 260 self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count, 261 failure_count, skipped_count, 262 suppressed_count, run_disabled) 263 264 def assertNotFailFastBehavior(self, 265 test_suite, 266 passed_count, 267 failure_count, 268 skipped_count, 269 suppressed_count, 270 run_disabled=False): 271 """Assert --nofail_fast via flag.""" 272 273 for fail_fast in ('false', '0', 'f', False): 274 self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count, 275 failure_count, skipped_count, 276 suppressed_count, run_disabled) 277 278 def testFlag_HasFixtureTest(self): 279 """Tests the behavior of fail_fast and TEST_F.""" 280 self.assertFailFastBehavior( 281 test_suite='HasFixtureTest', 282 passed_count=1, 283 failure_count=1, 284 skipped_count=3, 285 suppressed_count=0) 286 self.assertNotFailFastBehavior( 287 test_suite='HasFixtureTest', 288 passed_count=1, 289 failure_count=4, 290 skipped_count=0, 291 suppressed_count=0) 292 293 def testFlag_HasSimpleTest(self): 294 """Tests the behavior of fail_fast and TEST.""" 295 self.assertFailFastBehavior( 296 test_suite='HasSimpleTest', 297 passed_count=1, 298 failure_count=1, 299 skipped_count=3, 300 suppressed_count=0) 301 self.assertNotFailFastBehavior( 302 test_suite='HasSimpleTest', 303 passed_count=1, 304 failure_count=4, 305 skipped_count=0, 306 suppressed_count=0) 307 308 def testFlag_HasParametersTest(self): 309 """Tests the behavior of fail_fast and TEST_P.""" 310 self.assertFailFastBehavior( 311 test_suite='HasParametersSuite/HasParametersTest', 312 passed_count=0, 313 failure_count=1, 314 skipped_count=3, 315 suppressed_count=0) 316 self.assertNotFailFastBehavior( 317 test_suite='HasParametersSuite/HasParametersTest', 318 passed_count=0, 319 failure_count=4, 320 skipped_count=0, 321 suppressed_count=0) 322 323 def testFlag_HasDisabledTest(self): 324 """Tests the behavior of fail_fast and Disabled test cases.""" 325 self.assertFailFastBehavior( 326 test_suite='HasDisabledTest', 327 passed_count=1, 328 failure_count=1, 329 skipped_count=2, 330 suppressed_count=1, 331 run_disabled=False) 332 self.assertNotFailFastBehavior( 333 test_suite='HasDisabledTest', 334 passed_count=1, 335 failure_count=3, 336 skipped_count=0, 337 suppressed_count=1, 338 run_disabled=False) 339 340 def testFlag_HasDisabledRunDisabledTest(self): 341 """Tests the behavior of fail_fast and Disabled test cases enabled.""" 342 self.assertFailFastBehavior( 343 test_suite='HasDisabledTest', 344 passed_count=1, 345 failure_count=1, 346 skipped_count=3, 347 suppressed_count=0, 348 run_disabled=True) 349 self.assertNotFailFastBehavior( 350 test_suite='HasDisabledTest', 351 passed_count=1, 352 failure_count=4, 353 skipped_count=0, 354 suppressed_count=0, 355 run_disabled=True) 356 357 def testFlag_HasDisabledSuiteTest(self): 358 """Tests the behavior of fail_fast and Disabled test suites.""" 359 self.assertFailFastBehavior( 360 test_suite='DISABLED_HasDisabledSuite', 361 passed_count=0, 362 failure_count=0, 363 skipped_count=0, 364 suppressed_count=5, 365 run_disabled=False) 366 self.assertNotFailFastBehavior( 367 test_suite='DISABLED_HasDisabledSuite', 368 passed_count=0, 369 failure_count=0, 370 skipped_count=0, 371 suppressed_count=5, 372 run_disabled=False) 373 374 def testFlag_HasDisabledSuiteRunDisabledTest(self): 375 """Tests the behavior of fail_fast and Disabled test suites enabled.""" 376 self.assertFailFastBehavior( 377 test_suite='DISABLED_HasDisabledSuite', 378 passed_count=1, 379 failure_count=1, 380 skipped_count=3, 381 suppressed_count=0, 382 run_disabled=True) 383 self.assertNotFailFastBehavior( 384 test_suite='DISABLED_HasDisabledSuite', 385 passed_count=1, 386 failure_count=4, 387 skipped_count=0, 388 suppressed_count=0, 389 run_disabled=True) 390 391 if SUPPORTS_DEATH_TESTS: 392 393 def testFlag_HasDeathTest(self): 394 """Tests the behavior of fail_fast and death tests.""" 395 self.assertFailFastBehavior( 396 test_suite='HasDeathTest', 397 passed_count=1, 398 failure_count=1, 399 skipped_count=3, 400 suppressed_count=0) 401 self.assertNotFailFastBehavior( 402 test_suite='HasDeathTest', 403 passed_count=1, 404 failure_count=4, 405 skipped_count=0, 406 suppressed_count=0) 407 408 409 if __name__ == '__main__': 410 gtest_test_utils.Main()