gtest_xml_output_unittest.py (20597B)
1 #!/usr/bin/env python 2 # 3 # Copyright 2006, Google Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions are 8 # met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following disclaimer 14 # in the documentation and/or other materials provided with the 15 # distribution. 16 # * Neither the name of Google Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived from 18 # this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32 """Unit test for the gtest_xml_output module""" 33 34 import datetime 35 import errno 36 import os 37 import re 38 import sys 39 from xml.dom import minidom, Node 40 41 import gtest_test_utils 42 import gtest_xml_test_utils 43 44 GTEST_FILTER_FLAG = '--gtest_filter' 45 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' 46 GTEST_OUTPUT_FLAG = '--gtest_output' 47 GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.xml' 48 GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_' 49 50 # The flag indicating stacktraces are not supported 51 NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support' 52 53 # The environment variables for test sharding. 54 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 55 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 56 SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE' 57 58 SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv 59 60 if SUPPORTS_STACK_TRACES: 61 STACK_TRACE_TEMPLATE = '\nStack trace:\n*' 62 else: 63 STACK_TRACE_TEMPLATE = '' 64 # unittest.main() can't handle unknown flags 65 sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) 66 67 EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 68 <testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42"> 69 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 70 <testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/> 71 </testsuite> 72 <testsuite name="FailedTest" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 73 <testcase name="Fails" status="run" result="completed" time="*" timestamp="*" classname="FailedTest"> 74 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 1
 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 75 Expected equality of these values: 76 1 77 2%(stack)s]]></failure> 78 </testcase> 79 </testsuite> 80 <testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" skipped="0" errors="0" time="*" timestamp="*"> 81 <testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="MixedResultTest"/> 82 <testcase name="Fails" status="run" result="completed" time="*" timestamp="*" classname="MixedResultTest"> 83 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 1
 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 84 Expected equality of these values: 85 1 86 2%(stack)s]]></failure> 87 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 2
 3" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 88 Expected equality of these values: 89 2 90 3%(stack)s]]></failure> 91 </testcase> 92 <testcase name="DISABLED_test" status="notrun" result="suppressed" time="*" timestamp="*" classname="MixedResultTest"/> 93 </testsuite> 94 <testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 95 <testcase name="OutputsCData" status="run" result="completed" time="*" timestamp="*" classname="XmlQuotingTest"> 96 <failure message="gtest_xml_output_unittest_.cc:*
Failed
XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]></top>" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 97 Failed 98 XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]>]]><![CDATA[</top>%(stack)s]]></failure> 99 </testcase> 100 </testsuite> 101 <testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 102 <testcase name="InvalidCharactersInMessage" status="run" result="completed" time="*" timestamp="*" classname="InvalidCharactersTest"> 103 <failure message="gtest_xml_output_unittest_.cc:*
Failed
Invalid characters in brackets []" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 104 Failed 105 Invalid characters in brackets []%(stack)s]]></failure> 106 </testcase> 107 </testsuite> 108 <testsuite name="DisabledTest" tests="1" failures="0" disabled="1" skipped="0" errors="0" time="*" timestamp="*"> 109 <testcase name="DISABLED_test_not_run" status="notrun" result="suppressed" time="*" timestamp="*" classname="DisabledTest"/> 110 </testsuite> 111 <testsuite name="SkippedTest" tests="3" failures="1" disabled="0" skipped="2" errors="0" time="*" timestamp="*"> 112 <testcase name="Skipped" status="run" result="skipped" time="*" timestamp="*" classname="SkippedTest"> 113 <skipped message="gtest_xml_output_unittest_.cc:*
"><![CDATA[gtest_xml_output_unittest_.cc:* 114 %(stack)s]]></skipped> 115 </testcase> 116 <testcase name="SkippedWithMessage" status="run" result="skipped" time="*" timestamp="*" classname="SkippedTest"> 117 <skipped message="gtest_xml_output_unittest_.cc:*
It is good practice to tell why you skip a test."><![CDATA[gtest_xml_output_unittest_.cc:* 118 It is good practice to tell why you skip a test.%(stack)s]]></skipped> 119 </testcase> 120 <testcase name="SkippedAfterFailure" status="run" result="completed" time="*" timestamp="*" classname="SkippedTest"> 121 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 1
 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 122 Expected equality of these values: 123 1 124 2%(stack)s]]></failure> 125 <skipped message="gtest_xml_output_unittest_.cc:*
It is good practice to tell why you skip a test."><![CDATA[gtest_xml_output_unittest_.cc:* 126 It is good practice to tell why you skip a test.%(stack)s]]></skipped> 127 </testcase> 128 129 </testsuite> 130 <testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye"> 131 <testcase name="OneProperty" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest"> 132 <properties> 133 <property name="key_1" value="1"/> 134 </properties> 135 </testcase> 136 <testcase name="IntValuedProperty" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest"> 137 <properties> 138 <property name="key_int" value="1"/> 139 </properties> 140 </testcase> 141 <testcase name="ThreeProperties" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest"> 142 <properties> 143 <property name="key_1" value="1"/> 144 <property name="key_2" value="2"/> 145 <property name="key_3" value="3"/> 146 </properties> 147 </testcase> 148 <testcase name="TwoValuesForOneKeyUsesLastValue" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest"> 149 <properties> 150 <property name="key_1" value="2"/> 151 </properties> 152 </testcase> 153 </testsuite> 154 <testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 155 <testcase name="RecordProperty" status="run" result="completed" time="*" timestamp="*" classname="NoFixtureTest"> 156 <properties> 157 <property name="key" value="1"/> 158 </properties> 159 </testcase> 160 <testcase name="ExternalUtilityThatCallsRecordIntValuedProperty" status="run" result="completed" time="*" timestamp="*" classname="NoFixtureTest"> 161 <properties> 162 <property name="key_for_utility_int" value="1"/> 163 </properties> 164 </testcase> 165 <testcase name="ExternalUtilityThatCallsRecordStringValuedProperty" status="run" result="completed" time="*" timestamp="*" classname="NoFixtureTest"> 166 <properties> 167 <property name="key_for_utility_string" value="1"/> 168 </properties> 169 </testcase> 170 </testsuite> 171 <testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 172 <testcase name="HasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" /> 173 <testcase name="HasValueParamAttribute/1" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" /> 174 <testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" /> 175 <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" /> 176 </testsuite> 177 <testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 178 <testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" /> 179 </testsuite> 180 <testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 181 <testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" /> 182 </testsuite> 183 <testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 184 <testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" /> 185 </testsuite> 186 <testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 187 <testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" /> 188 </testsuite> 189 </testsuites>""" % { 190 'stack': STACK_TRACE_TEMPLATE 191 } 192 193 EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?> 194 <testsuites tests="1" failures="0" disabled="0" errors="0" time="*" 195 timestamp="*" name="AllTests" ad_hoc_property="42"> 196 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" 197 errors="0" time="*" timestamp="*"> 198 <testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/> 199 </testsuite> 200 </testsuites>""" 201 202 EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?> 203 <testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42"> 204 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 205 <testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/> 206 </testsuite> 207 <testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye"> 208 <testcase name="IntValuedProperty" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest"> 209 <properties> 210 <property name="key_int" value="1"/> 211 </properties> 212 </testcase> 213 </testsuite> 214 <testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 215 <testcase name="HasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" /> 216 </testsuite> 217 </testsuites>""" 218 219 EXPECTED_NO_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?> 220 <testsuites tests="0" failures="0" disabled="0" errors="0" time="*" 221 timestamp="*" name="AllTests"> 222 <testsuite name="NonTestSuiteFailure" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*"> 223 <testcase name="" status="run" result="completed" time="*" timestamp="*" classname=""> 224 <failure message="gtest_no_test_unittest.cc:*
Expected equality of these values:
 1
 2" type=""><![CDATA[gtest_no_test_unittest.cc:* 225 Expected equality of these values: 226 1 227 2%(stack)s]]></failure> 228 </testcase> 229 </testsuite> 230 </testsuites>""" % { 231 'stack': STACK_TRACE_TEMPLATE 232 } 233 234 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) 235 236 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess( 237 [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output 238 239 240 class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): 241 """ 242 Unit test for Google Test's XML output functionality. 243 """ 244 245 # This test currently breaks on platforms that do not support typed and 246 # type-parameterized tests, so we don't run it under them. 247 if SUPPORTS_TYPED_TESTS: 248 def testNonEmptyXmlOutput(self): 249 """ 250 Runs a test program that generates a non-empty XML output, and 251 tests that the XML output is expected. 252 """ 253 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY_XML, 1) 254 255 def testNoTestXmlOutput(self): 256 """Verifies XML output for a Google Test binary without actual tests. 257 258 Runs a test program that generates an XML output for a binary without tests, 259 and tests that the XML output is expected. 260 """ 261 262 self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_NO_TEST_XML, 0) 263 264 def testTimestampValue(self): 265 """Checks whether the timestamp attribute in the XML output is valid. 266 267 Runs a test program that generates an empty XML output, and checks if 268 the timestamp attribute in the testsuites tag is valid. 269 """ 270 actual = self._GetXmlOutput('gtest_no_test_unittest', [], {}, 0) 271 date_time_str = actual.documentElement.getAttributeNode('timestamp').value 272 # datetime.strptime() is only available in Python 2.5+ so we have to 273 # parse the expected datetime manually. 274 match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str) 275 self.assertTrue( 276 re.match, 277 'XML datettime string %s has incorrect format' % date_time_str) 278 date_time_from_xml = datetime.datetime( 279 year=int(match.group(1)), month=int(match.group(2)), 280 day=int(match.group(3)), hour=int(match.group(4)), 281 minute=int(match.group(5)), second=int(match.group(6))) 282 283 time_delta = abs(datetime.datetime.now() - date_time_from_xml) 284 # timestamp value should be near the current local time 285 self.assertTrue(time_delta < datetime.timedelta(seconds=600), 286 'time_delta is %s' % time_delta) 287 actual.unlink() 288 289 def testDefaultOutputFile(self): 290 """ 291 Confirms that Google Test produces an XML output file with the expected 292 default name if no name is explicitly specified. 293 """ 294 output_file = os.path.join(gtest_test_utils.GetTempDir(), 295 GTEST_DEFAULT_OUTPUT_FILE) 296 gtest_prog_path = gtest_test_utils.GetTestExecutablePath( 297 'gtest_no_test_unittest') 298 try: 299 os.remove(output_file) 300 except OSError: 301 e = sys.exc_info()[1] 302 if e.errno != errno.ENOENT: 303 raise 304 305 p = gtest_test_utils.Subprocess( 306 [gtest_prog_path, '%s=xml' % GTEST_OUTPUT_FLAG], 307 working_dir=gtest_test_utils.GetTempDir()) 308 self.assert_(p.exited) 309 self.assertEquals(0, p.exit_code) 310 self.assert_(os.path.isfile(output_file)) 311 312 def testSuppressedXmlOutput(self): 313 """ 314 Tests that no XML file is generated if the default XML listener is 315 shut down before RUN_ALL_TESTS is invoked. 316 """ 317 318 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 319 GTEST_PROGRAM_NAME + 'out.xml') 320 if os.path.isfile(xml_path): 321 os.remove(xml_path) 322 323 command = [GTEST_PROGRAM_PATH, 324 '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path), 325 '--shut_down_xml'] 326 p = gtest_test_utils.Subprocess(command) 327 if p.terminated_by_signal: 328 # p.signal is available only if p.terminated_by_signal is True. 329 self.assertFalse( 330 p.terminated_by_signal, 331 '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal)) 332 else: 333 self.assert_(p.exited) 334 self.assertEquals(1, p.exit_code, 335 "'%s' exited with code %s, which doesn't match " 336 'the expected exit code %s.' 337 % (command, p.exit_code, 1)) 338 339 self.assert_(not os.path.isfile(xml_path)) 340 341 def testFilteredTestXmlOutput(self): 342 """Verifies XML output when a filter is applied. 343 344 Runs a test program that executes only some tests and verifies that 345 non-selected tests do not show up in the XML output. 346 """ 347 348 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED_TEST_XML, 0, 349 extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG]) 350 351 def testShardedTestXmlOutput(self): 352 """Verifies XML output when run using multiple shards. 353 354 Runs a test program that executes only one shard and verifies that tests 355 from other shards do not show up in the XML output. 356 """ 357 358 self._TestXmlOutput( 359 GTEST_PROGRAM_NAME, 360 EXPECTED_SHARDED_TEST_XML, 361 0, 362 extra_env={SHARD_INDEX_ENV_VAR: '0', 363 TOTAL_SHARDS_ENV_VAR: '10'}) 364 365 def _GetXmlOutput(self, gtest_prog_name, extra_args, extra_env, 366 expected_exit_code): 367 """ 368 Returns the xml output generated by running the program gtest_prog_name. 369 Furthermore, the program's exit code must be expected_exit_code. 370 """ 371 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 372 gtest_prog_name + 'out.xml') 373 gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) 374 375 command = ([gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] + 376 extra_args) 377 environ_copy = os.environ.copy() 378 if extra_env: 379 environ_copy.update(extra_env) 380 p = gtest_test_utils.Subprocess(command, env=environ_copy) 381 382 if p.terminated_by_signal: 383 self.assert_(False, 384 '%s was killed by signal %d' % (gtest_prog_name, p.signal)) 385 else: 386 self.assert_(p.exited) 387 self.assertEquals(expected_exit_code, p.exit_code, 388 "'%s' exited with code %s, which doesn't match " 389 'the expected exit code %s.' 390 % (command, p.exit_code, expected_exit_code)) 391 actual = minidom.parse(xml_path) 392 return actual 393 394 def _TestXmlOutput(self, gtest_prog_name, expected_xml, 395 expected_exit_code, extra_args=None, extra_env=None): 396 """ 397 Asserts that the XML document generated by running the program 398 gtest_prog_name matches expected_xml, a string containing another 399 XML document. Furthermore, the program's exit code must be 400 expected_exit_code. 401 """ 402 403 actual = self._GetXmlOutput(gtest_prog_name, extra_args or [], 404 extra_env or {}, expected_exit_code) 405 expected = minidom.parseString(expected_xml) 406 self.NormalizeXml(actual.documentElement) 407 self.AssertEquivalentNodes(expected.documentElement, 408 actual.documentElement) 409 expected.unlink() 410 actual.unlink() 411 412 413 if __name__ == '__main__': 414 os.environ['GTEST_STACK_TRACE_DEPTH'] = '1' 415 gtest_test_utils.Main()