googletest-shuffle-test.py (12518B)
1 #!/usr/bin/env python 2 # 3 # Copyright 2009 Google Inc. All Rights Reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Verifies that test shuffling works.""" 32 33 import os 34 import gtest_test_utils 35 36 # Command to run the googletest-shuffle-test_ program. 37 COMMAND = gtest_test_utils.GetTestExecutablePath('googletest-shuffle-test_') 38 39 # The environment variables for test sharding. 40 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 41 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 42 43 TEST_FILTER = 'A*.A:A*.B:C*' 44 45 ALL_TESTS = [] 46 ACTIVE_TESTS = [] 47 FILTERED_TESTS = [] 48 SHARDED_TESTS = [] 49 50 SHUFFLED_ALL_TESTS = [] 51 SHUFFLED_ACTIVE_TESTS = [] 52 SHUFFLED_FILTERED_TESTS = [] 53 SHUFFLED_SHARDED_TESTS = [] 54 55 56 def AlsoRunDisabledTestsFlag(): 57 return '--gtest_also_run_disabled_tests' 58 59 60 def FilterFlag(test_filter): 61 return '--gtest_filter=%s' % (test_filter,) 62 63 64 def RepeatFlag(n): 65 return '--gtest_repeat=%s' % (n,) 66 67 68 def ShuffleFlag(): 69 return '--gtest_shuffle' 70 71 72 def RandomSeedFlag(n): 73 return '--gtest_random_seed=%s' % (n,) 74 75 76 def RunAndReturnOutput(extra_env, args): 77 """Runs the test program and returns its output.""" 78 79 environ_copy = os.environ.copy() 80 environ_copy.update(extra_env) 81 82 return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output 83 84 85 def GetTestsForAllIterations(extra_env, args): 86 """Runs the test program and returns a list of test lists. 87 88 Args: 89 extra_env: a map from environment variables to their values 90 args: command line flags to pass to googletest-shuffle-test_ 91 92 Returns: 93 A list where the i-th element is the list of tests run in the i-th 94 test iteration. 95 """ 96 97 test_iterations = [] 98 for line in RunAndReturnOutput(extra_env, args).split('\n'): 99 if line.startswith('----'): 100 tests = [] 101 test_iterations.append(tests) 102 elif line.strip(): 103 tests.append(line.strip()) # 'TestCaseName.TestName' 104 105 return test_iterations 106 107 108 def GetTestCases(tests): 109 """Returns a list of test cases in the given full test names. 110 111 Args: 112 tests: a list of full test names 113 114 Returns: 115 A list of test cases from 'tests', in their original order. 116 Consecutive duplicates are removed. 117 """ 118 119 test_cases = [] 120 for test in tests: 121 test_case = test.split('.')[0] 122 if not test_case in test_cases: 123 test_cases.append(test_case) 124 125 return test_cases 126 127 128 def CalculateTestLists(): 129 """Calculates the list of tests run under different flags.""" 130 131 if not ALL_TESTS: 132 ALL_TESTS.extend( 133 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) 134 135 if not ACTIVE_TESTS: 136 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 137 138 if not FILTERED_TESTS: 139 FILTERED_TESTS.extend( 140 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) 141 142 if not SHARDED_TESTS: 143 SHARDED_TESTS.extend( 144 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 145 SHARD_INDEX_ENV_VAR: '1'}, 146 [])[0]) 147 148 if not SHUFFLED_ALL_TESTS: 149 SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( 150 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) 151 152 if not SHUFFLED_ACTIVE_TESTS: 153 SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( 154 {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) 155 156 if not SHUFFLED_FILTERED_TESTS: 157 SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( 158 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) 159 160 if not SHUFFLED_SHARDED_TESTS: 161 SHUFFLED_SHARDED_TESTS.extend( 162 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 163 SHARD_INDEX_ENV_VAR: '1'}, 164 [ShuffleFlag(), RandomSeedFlag(1)])[0]) 165 166 167 class GTestShuffleUnitTest(gtest_test_utils.TestCase): 168 """Tests test shuffling.""" 169 170 def setUp(self): 171 CalculateTestLists() 172 173 def testShufflePreservesNumberOfTests(self): 174 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 175 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 176 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 177 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 178 179 def testShuffleChangesTestOrder(self): 180 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 181 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) 182 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, 183 SHUFFLED_FILTERED_TESTS) 184 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, 185 SHUFFLED_SHARDED_TESTS) 186 187 def testShuffleChangesTestCaseOrder(self): 188 self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 189 GetTestCases(SHUFFLED_ALL_TESTS)) 190 self.assert_( 191 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 192 GetTestCases(SHUFFLED_ACTIVE_TESTS)) 193 self.assert_( 194 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 195 GetTestCases(SHUFFLED_FILTERED_TESTS)) 196 self.assert_( 197 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 198 GetTestCases(SHUFFLED_SHARDED_TESTS)) 199 200 def testShuffleDoesNotRepeatTest(self): 201 for test in SHUFFLED_ALL_TESTS: 202 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), 203 '%s appears more than once' % (test,)) 204 for test in SHUFFLED_ACTIVE_TESTS: 205 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), 206 '%s appears more than once' % (test,)) 207 for test in SHUFFLED_FILTERED_TESTS: 208 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), 209 '%s appears more than once' % (test,)) 210 for test in SHUFFLED_SHARDED_TESTS: 211 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), 212 '%s appears more than once' % (test,)) 213 214 def testShuffleDoesNotCreateNewTest(self): 215 for test in SHUFFLED_ALL_TESTS: 216 self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) 217 for test in SHUFFLED_ACTIVE_TESTS: 218 self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 219 for test in SHUFFLED_FILTERED_TESTS: 220 self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 221 for test in SHUFFLED_SHARDED_TESTS: 222 self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 223 224 def testShuffleIncludesAllTests(self): 225 for test in ALL_TESTS: 226 self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 227 for test in ACTIVE_TESTS: 228 self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 229 for test in FILTERED_TESTS: 230 self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) 231 for test in SHARDED_TESTS: 232 self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 233 234 def testShuffleLeavesDeathTestsAtFront(self): 235 non_death_test_found = False 236 for test in SHUFFLED_ACTIVE_TESTS: 237 if 'DeathTest.' in test: 238 self.assert_(not non_death_test_found, 239 '%s appears after a non-death test' % (test,)) 240 else: 241 non_death_test_found = True 242 243 def _VerifyTestCasesDoNotInterleave(self, tests): 244 test_cases = [] 245 for test in tests: 246 [test_case, _] = test.split('.') 247 if test_cases and test_cases[-1] != test_case: 248 test_cases.append(test_case) 249 self.assertEqual(1, test_cases.count(test_case), 250 'Test case %s is not grouped together in %s' % 251 (test_case, tests)) 252 253 def testShuffleDoesNotInterleaveTestCases(self): 254 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 255 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 256 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 257 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 258 259 def testShuffleRestoresOrderAfterEachIteration(self): 260 # Get the test lists in all 3 iterations, using random seed 1, 2, 261 # and 3 respectively. Google Test picks a different seed in each 262 # iteration, and this test depends on the current implementation 263 # picking successive numbers. This dependency is not ideal, but 264 # makes the test much easier to write. 265 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 266 GetTestsForAllIterations( 267 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 268 269 # Make sure running the tests with random seed 1 gets the same 270 # order as in iteration 1 above. 271 [tests_with_seed1] = GetTestsForAllIterations( 272 {}, [ShuffleFlag(), RandomSeedFlag(1)]) 273 self.assertEqual(tests_in_iteration1, tests_with_seed1) 274 275 # Make sure running the tests with random seed 2 gets the same 276 # order as in iteration 2 above. Success means that Google Test 277 # correctly restores the test order before re-shuffling at the 278 # beginning of iteration 2. 279 [tests_with_seed2] = GetTestsForAllIterations( 280 {}, [ShuffleFlag(), RandomSeedFlag(2)]) 281 self.assertEqual(tests_in_iteration2, tests_with_seed2) 282 283 # Make sure running the tests with random seed 3 gets the same 284 # order as in iteration 3 above. Success means that Google Test 285 # correctly restores the test order before re-shuffling at the 286 # beginning of iteration 3. 287 [tests_with_seed3] = GetTestsForAllIterations( 288 {}, [ShuffleFlag(), RandomSeedFlag(3)]) 289 self.assertEqual(tests_in_iteration3, tests_with_seed3) 290 291 def testShuffleGeneratesNewOrderInEachIteration(self): 292 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 293 GetTestsForAllIterations( 294 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 295 296 self.assert_(tests_in_iteration1 != tests_in_iteration2, 297 tests_in_iteration1) 298 self.assert_(tests_in_iteration1 != tests_in_iteration3, 299 tests_in_iteration1) 300 self.assert_(tests_in_iteration2 != tests_in_iteration3, 301 tests_in_iteration2) 302 303 def testShuffleShardedTestsPreservesPartition(self): 304 # If we run M tests on N shards, the same M tests should be run in 305 # total, regardless of the random seeds used by the shards. 306 [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 307 SHARD_INDEX_ENV_VAR: '0'}, 308 [ShuffleFlag(), RandomSeedFlag(1)]) 309 [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 310 SHARD_INDEX_ENV_VAR: '1'}, 311 [ShuffleFlag(), RandomSeedFlag(20)]) 312 [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 313 SHARD_INDEX_ENV_VAR: '2'}, 314 [ShuffleFlag(), RandomSeedFlag(25)]) 315 sorted_sharded_tests = tests1 + tests2 + tests3 316 sorted_sharded_tests.sort() 317 sorted_active_tests = [] 318 sorted_active_tests.extend(ACTIVE_TESTS) 319 sorted_active_tests.sort() 320 self.assertEqual(sorted_active_tests, sorted_sharded_tests) 321 322 if __name__ == '__main__': 323 gtest_test_utils.Main()