json_results_test.py (13577B)
1 #!/usr/bin/env vpython3 2 # Copyright 2014 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 7 import unittest 8 9 from pylib.base import base_test_result 10 from pylib.results import json_results 11 12 13 class JsonResultsTest(unittest.TestCase): 14 15 def testGenerateResultsDict_passedResult(self): 16 result = base_test_result.BaseTestResult( 17 'test.package.TestName', base_test_result.ResultType.PASS) 18 19 all_results = base_test_result.TestRunResults() 20 all_results.AddResult(result) 21 22 results_dict = json_results.GenerateResultsDict([all_results]) 23 self.assertEqual(['test.package.TestName'], results_dict['all_tests']) 24 self.assertEqual(1, len(results_dict['per_iteration_data'])) 25 26 iteration_result = results_dict['per_iteration_data'][0] 27 self.assertTrue('test.package.TestName' in iteration_result) 28 self.assertEqual(1, len(iteration_result['test.package.TestName'])) 29 30 test_iteration_result = iteration_result['test.package.TestName'][0] 31 self.assertTrue('status' in test_iteration_result) 32 self.assertEqual('SUCCESS', test_iteration_result['status']) 33 34 def testGenerateResultsDict_skippedResult(self): 35 result = base_test_result.BaseTestResult( 36 'test.package.TestName', base_test_result.ResultType.SKIP) 37 38 all_results = base_test_result.TestRunResults() 39 all_results.AddResult(result) 40 41 results_dict = json_results.GenerateResultsDict([all_results]) 42 self.assertEqual(['test.package.TestName'], results_dict['all_tests']) 43 self.assertEqual(1, len(results_dict['per_iteration_data'])) 44 45 iteration_result = results_dict['per_iteration_data'][0] 46 self.assertTrue('test.package.TestName' in iteration_result) 47 self.assertEqual(1, len(iteration_result['test.package.TestName'])) 48 49 test_iteration_result = iteration_result['test.package.TestName'][0] 50 self.assertTrue('status' in test_iteration_result) 51 self.assertEqual('SKIPPED', test_iteration_result['status']) 52 53 def testGenerateResultsDict_failedResult(self): 54 result = base_test_result.BaseTestResult( 55 'test.package.TestName', base_test_result.ResultType.FAIL) 56 57 all_results = base_test_result.TestRunResults() 58 all_results.AddResult(result) 59 60 results_dict = json_results.GenerateResultsDict([all_results]) 61 self.assertEqual(['test.package.TestName'], results_dict['all_tests']) 62 self.assertEqual(1, len(results_dict['per_iteration_data'])) 63 64 iteration_result = results_dict['per_iteration_data'][0] 65 self.assertTrue('test.package.TestName' in iteration_result) 66 self.assertEqual(1, len(iteration_result['test.package.TestName'])) 67 68 test_iteration_result = iteration_result['test.package.TestName'][0] 69 self.assertTrue('status' in test_iteration_result) 70 self.assertEqual('FAILURE', test_iteration_result['status']) 71 72 def testGenerateResultsDict_duration(self): 73 result = base_test_result.BaseTestResult( 74 'test.package.TestName', base_test_result.ResultType.PASS, duration=123) 75 76 all_results = base_test_result.TestRunResults() 77 all_results.AddResult(result) 78 79 results_dict = json_results.GenerateResultsDict([all_results]) 80 self.assertEqual(['test.package.TestName'], results_dict['all_tests']) 81 self.assertEqual(1, len(results_dict['per_iteration_data'])) 82 83 iteration_result = results_dict['per_iteration_data'][0] 84 self.assertTrue('test.package.TestName' in iteration_result) 85 self.assertEqual(1, len(iteration_result['test.package.TestName'])) 86 87 test_iteration_result = iteration_result['test.package.TestName'][0] 88 self.assertTrue('elapsed_time_ms' in test_iteration_result) 89 self.assertEqual(123, test_iteration_result['elapsed_time_ms']) 90 91 def testGenerateResultsDict_multipleResults(self): 92 result1 = base_test_result.BaseTestResult( 93 'test.package.TestName1', base_test_result.ResultType.PASS) 94 result2 = base_test_result.BaseTestResult( 95 'test.package.TestName2', base_test_result.ResultType.PASS) 96 97 all_results = base_test_result.TestRunResults() 98 all_results.AddResult(result1) 99 all_results.AddResult(result2) 100 101 results_dict = json_results.GenerateResultsDict([all_results]) 102 self.assertEqual(['test.package.TestName1', 'test.package.TestName2'], 103 results_dict['all_tests']) 104 105 self.assertTrue('per_iteration_data' in results_dict) 106 iterations = results_dict['per_iteration_data'] 107 self.assertEqual(1, len(iterations)) 108 109 expected_tests = set([ 110 'test.package.TestName1', 111 'test.package.TestName2', 112 ]) 113 114 for test_name, iteration_result in iterations[0].items(): 115 self.assertTrue(test_name in expected_tests) 116 expected_tests.remove(test_name) 117 self.assertEqual(1, len(iteration_result)) 118 119 test_iteration_result = iteration_result[0] 120 self.assertTrue('status' in test_iteration_result) 121 self.assertEqual('SUCCESS', test_iteration_result['status']) 122 123 def testGenerateResultsDict_passOnRetry(self): 124 raw_results = [] 125 126 result1 = base_test_result.BaseTestResult( 127 'test.package.TestName1', base_test_result.ResultType.FAIL) 128 run_results1 = base_test_result.TestRunResults() 129 run_results1.AddResult(result1) 130 raw_results.append(run_results1) 131 132 result2 = base_test_result.BaseTestResult( 133 'test.package.TestName1', base_test_result.ResultType.PASS) 134 run_results2 = base_test_result.TestRunResults() 135 run_results2.AddResult(result2) 136 raw_results.append(run_results2) 137 138 results_dict = json_results.GenerateResultsDict([raw_results]) 139 self.assertEqual(['test.package.TestName1'], results_dict['all_tests']) 140 141 # Check that there's only one iteration. 142 self.assertIn('per_iteration_data', results_dict) 143 iterations = results_dict['per_iteration_data'] 144 self.assertEqual(1, len(iterations)) 145 146 # Check that test.package.TestName1 is the only test in the iteration. 147 self.assertEqual(1, len(iterations[0])) 148 self.assertIn('test.package.TestName1', iterations[0]) 149 150 # Check that there are two results for test.package.TestName1. 151 actual_test_results = iterations[0]['test.package.TestName1'] 152 self.assertEqual(2, len(actual_test_results)) 153 154 # Check that the first result is a failure. 155 self.assertIn('status', actual_test_results[0]) 156 self.assertEqual('FAILURE', actual_test_results[0]['status']) 157 158 # Check that the second result is a success. 159 self.assertIn('status', actual_test_results[1]) 160 self.assertEqual('SUCCESS', actual_test_results[1]['status']) 161 162 def testGenerateResultsDict_globalTags(self): 163 raw_results = [] 164 global_tags = ['UNRELIABLE_RESULTS'] 165 166 results_dict = json_results.GenerateResultsDict( 167 [raw_results], global_tags=global_tags) 168 self.assertEqual(['UNRELIABLE_RESULTS'], results_dict['global_tags']) 169 170 def testGenerateResultsDict_loslessSnippet(self): 171 result = base_test_result.BaseTestResult( 172 'test.package.TestName', base_test_result.ResultType.FAIL) 173 log = 'blah-blah' 174 result.SetLog(log) 175 176 all_results = base_test_result.TestRunResults() 177 all_results.AddResult(result) 178 179 results_dict = json_results.GenerateResultsDict([all_results]) 180 self.assertEqual(['test.package.TestName'], results_dict['all_tests']) 181 self.assertEqual(1, len(results_dict['per_iteration_data'])) 182 183 iteration_result = results_dict['per_iteration_data'][0] 184 self.assertTrue('test.package.TestName' in iteration_result) 185 self.assertEqual(1, len(iteration_result['test.package.TestName'])) 186 187 test_iteration_result = iteration_result['test.package.TestName'][0] 188 self.assertTrue('losless_snippet' in test_iteration_result) 189 self.assertTrue(test_iteration_result['losless_snippet']) 190 self.assertTrue('output_snippet' in test_iteration_result) 191 self.assertEqual(log, test_iteration_result['output_snippet']) 192 self.assertTrue('output_snippet_base64' in test_iteration_result) 193 self.assertEqual('', test_iteration_result['output_snippet_base64']) 194 195 def testGenerateJsonTestResultFormatDict_passedResult(self): 196 result = base_test_result.BaseTestResult('test.package.TestName', 197 base_test_result.ResultType.PASS) 198 199 all_results = base_test_result.TestRunResults() 200 all_results.AddResult(result) 201 202 results_dict = json_results.GenerateJsonTestResultFormatDict([all_results], 203 False) 204 self.assertEqual(1, len(results_dict['tests'])) 205 self.assertEqual(1, len(results_dict['tests']['test'])) 206 self.assertEqual(1, len(results_dict['tests']['test']['package'])) 207 self.assertEqual( 208 'PASS', 209 results_dict['tests']['test']['package']['TestName']['expected']) 210 self.assertEqual( 211 'PASS', results_dict['tests']['test']['package']['TestName']['actual']) 212 213 self.assertTrue('FAIL' not in results_dict['num_failures_by_type'] 214 or results_dict['num_failures_by_type']['FAIL'] == 0) 215 self.assertIn('PASS', results_dict['num_failures_by_type']) 216 self.assertEqual(1, results_dict['num_failures_by_type']['PASS']) 217 218 def testGenerateJsonTestResultFormatDict_failedResult(self): 219 result = base_test_result.BaseTestResult('test.package.TestName', 220 base_test_result.ResultType.FAIL) 221 222 all_results = base_test_result.TestRunResults() 223 all_results.AddResult(result) 224 225 results_dict = json_results.GenerateJsonTestResultFormatDict([all_results], 226 False) 227 self.assertEqual(1, len(results_dict['tests'])) 228 self.assertEqual(1, len(results_dict['tests']['test'])) 229 self.assertEqual(1, len(results_dict['tests']['test']['package'])) 230 self.assertEqual( 231 'PASS', 232 results_dict['tests']['test']['package']['TestName']['expected']) 233 self.assertEqual( 234 'FAIL', results_dict['tests']['test']['package']['TestName']['actual']) 235 self.assertEqual( 236 True, 237 results_dict['tests']['test']['package']['TestName']['is_unexpected']) 238 239 self.assertTrue('PASS' not in results_dict['num_failures_by_type'] 240 or results_dict['num_failures_by_type']['PASS'] == 0) 241 self.assertIn('FAIL', results_dict['num_failures_by_type']) 242 self.assertEqual(1, results_dict['num_failures_by_type']['FAIL']) 243 244 def testGenerateJsonTestResultFormatDict_skippedResult(self): 245 result = base_test_result.BaseTestResult('test.package.TestName', 246 base_test_result.ResultType.SKIP) 247 248 all_results = base_test_result.TestRunResults() 249 all_results.AddResult(result) 250 251 results_dict = json_results.GenerateJsonTestResultFormatDict([all_results], 252 False) 253 self.assertEqual(1, len(results_dict['tests'])) 254 self.assertEqual(1, len(results_dict['tests']['test'])) 255 self.assertEqual(1, len(results_dict['tests']['test']['package'])) 256 self.assertEqual( 257 'PASS', 258 results_dict['tests']['test']['package']['TestName']['expected']) 259 self.assertEqual( 260 'SKIP', results_dict['tests']['test']['package']['TestName']['actual']) 261 # Should only be set if the test fails. 262 self.assertNotIn('is_unexpected', 263 results_dict['tests']['test']['package']['TestName']) 264 265 self.assertTrue('FAIL' not in results_dict['num_failures_by_type'] 266 or results_dict['num_failures_by_type']['FAIL'] == 0) 267 self.assertTrue('PASS' not in results_dict['num_failures_by_type'] 268 or results_dict['num_failures_by_type']['PASS'] == 0) 269 self.assertIn('SKIP', results_dict['num_failures_by_type']) 270 self.assertEqual(1, results_dict['num_failures_by_type']['SKIP']) 271 272 def testGenerateJsonTestResultFormatDict_failedResultWithRetry(self): 273 result_1 = base_test_result.BaseTestResult('test.package.TestName', 274 base_test_result.ResultType.FAIL) 275 run_results_1 = base_test_result.TestRunResults() 276 run_results_1.AddResult(result_1) 277 278 # Simulate a second retry with failure. 279 result_2 = base_test_result.BaseTestResult('test.package.TestName', 280 base_test_result.ResultType.FAIL) 281 run_results_2 = base_test_result.TestRunResults() 282 run_results_2.AddResult(result_2) 283 284 all_results = [run_results_1, run_results_2] 285 286 results_dict = json_results.GenerateJsonTestResultFormatDict( 287 all_results, False) 288 self.assertEqual(1, len(results_dict['tests'])) 289 self.assertEqual(1, len(results_dict['tests']['test'])) 290 self.assertEqual(1, len(results_dict['tests']['test']['package'])) 291 self.assertEqual( 292 'PASS', 293 results_dict['tests']['test']['package']['TestName']['expected']) 294 self.assertEqual( 295 'FAIL FAIL', 296 results_dict['tests']['test']['package']['TestName']['actual']) 297 self.assertEqual( 298 True, 299 results_dict['tests']['test']['package']['TestName']['is_unexpected']) 300 301 self.assertTrue('PASS' not in results_dict['num_failures_by_type'] 302 or results_dict['num_failures_by_type']['PASS'] == 0) 303 # According to the spec: If a test was run more than once, only the first 304 # invocation's result is included in the totals. 305 self.assertIn('FAIL', results_dict['num_failures_by_type']) 306 self.assertEqual(1, results_dict['num_failures_by_type']['FAIL']) 307 308 309 if __name__ == '__main__': 310 unittest.main(verbosity=2)