instrumentation_test_instance_test.py (40768B)
1 #!/usr/bin/env vpython3 2 # Copyright 2014 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Unit tests for instrumentation_test_instance.""" 7 8 # pylint: disable=protected-access 9 10 11 import collections 12 import tempfile 13 import unittest 14 15 from pylib.base import base_test_result 16 from pylib.instrumentation import instrumentation_test_instance 17 18 import mock # pylint: disable=import-error 19 20 _INSTRUMENTATION_TEST_INSTANCE_PATH = ( 21 'pylib.instrumentation.instrumentation_test_instance.%s') 22 23 class InstrumentationTestInstanceTest(unittest.TestCase): 24 25 @staticmethod 26 def createTestInstance(): 27 c = _INSTRUMENTATION_TEST_INSTANCE_PATH % 'InstrumentationTestInstance' 28 # yapf: disable 29 with mock.patch('%s._initializeApkAttributes' % c), ( 30 mock.patch('%s._initializeDataDependencyAttributes' % c)), ( 31 mock.patch('%s._initializeTestFilterAttributes' %c)), ( 32 mock.patch('%s._initializeFlagAttributes' % c)), ( 33 mock.patch('%s._initializeTestControlAttributes' % c)), ( 34 mock.patch('%s._initializeTestCoverageAttributes' % c)), ( 35 mock.patch('%s._initializeSkiaGoldAttributes' % c)): 36 # yapf: enable 37 return instrumentation_test_instance.InstrumentationTestInstance( 38 mock.MagicMock(), mock.MagicMock(), lambda s: None) 39 40 _FlagAttributesArgs = collections.namedtuple('_FlagAttributesArgs', [ 41 'command_line_flags', 'device_flags_file', 'strict_mode', 42 'use_apk_under_test_flags_file', 'coverage_dir' 43 ]) 44 45 def createFlagAttributesArgs(self, 46 command_line_flags=None, 47 device_flags_file=None, 48 strict_mode=None, 49 use_apk_under_test_flags_file=False, 50 coverage_dir=None): 51 return self._FlagAttributesArgs(command_line_flags, device_flags_file, 52 strict_mode, use_apk_under_test_flags_file, 53 coverage_dir) 54 55 def test_initializeFlagAttributes_commandLineFlags(self): 56 o = self.createTestInstance() 57 args = self.createFlagAttributesArgs(command_line_flags=['--foo', '--bar']) 58 o._initializeFlagAttributes(args) 59 self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) 60 61 def test_initializeFlagAttributes_deviceFlagsFile(self): 62 o = self.createTestInstance() 63 with tempfile.NamedTemporaryFile(mode='w') as flags_file: 64 flags_file.write('\n'.join(['--foo', '--bar'])) 65 flags_file.flush() 66 67 args = self.createFlagAttributesArgs(device_flags_file=flags_file.name) 68 o._initializeFlagAttributes(args) 69 self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) 70 71 def test_initializeFlagAttributes_strictModeOn(self): 72 o = self.createTestInstance() 73 args = self.createFlagAttributesArgs(strict_mode='on') 74 o._initializeFlagAttributes(args) 75 self.assertEqual(o._flags, ['--enable-test-intents', '--strict-mode=on']) 76 77 def test_initializeFlagAttributes_strictModeOn_coverageOn(self): 78 o = self.createTestInstance() 79 args = self.createFlagAttributesArgs( 80 strict_mode='on', coverage_dir='/coverage/dir') 81 o._initializeFlagAttributes(args) 82 self.assertEqual(o._flags, ['--enable-test-intents']) 83 84 def test_initializeFlagAttributes_strictModeOff(self): 85 o = self.createTestInstance() 86 args = self.createFlagAttributesArgs(strict_mode='off') 87 o._initializeFlagAttributes(args) 88 self.assertEqual(o._flags, ['--enable-test-intents']) 89 90 def testGetTests_noFilter(self): 91 o = self.createTestInstance() 92 raw_tests = [ 93 { 94 'annotations': {'Feature': {'value': ['Foo']}}, 95 'class': 'org.chromium.test.SampleTest', 96 'superclass': 'java.lang.Object', 97 'methods': [ 98 { 99 'annotations': {'SmallTest': None}, 100 'method': 'testMethod1', 101 }, 102 { 103 'annotations': {'MediumTest': None}, 104 'method': 'testMethod2', 105 }, 106 ], 107 }, 108 { 109 'annotations': {'Feature': {'value': ['Bar']}}, 110 'class': 'org.chromium.test.SampleTest2', 111 'superclass': 'java.lang.Object', 112 'methods': [ 113 { 114 'annotations': {'SmallTest': None}, 115 'method': 'testMethod1', 116 }, 117 ], 118 } 119 ] 120 121 expected_tests = [ 122 { 123 'annotations': { 124 'Feature': {'value': ['Foo']}, 125 'SmallTest': None, 126 }, 127 'class': 'org.chromium.test.SampleTest', 128 'method': 'testMethod1', 129 }, 130 { 131 'annotations': { 132 'Feature': {'value': ['Foo']}, 133 'MediumTest': None, 134 }, 135 'class': 'org.chromium.test.SampleTest', 136 'method': 'testMethod2', 137 }, 138 { 139 'annotations': { 140 'Feature': {'value': ['Bar']}, 141 'SmallTest': None, 142 }, 143 'class': 'org.chromium.test.SampleTest2', 144 'method': 'testMethod1', 145 }, 146 ] 147 148 o._junit4_runner_class = 'J4Runner' 149 actual_tests = o.ProcessRawTests(raw_tests) 150 151 self.assertEqual(actual_tests, expected_tests) 152 153 def testGetTests_simpleGtestFilter(self): 154 o = self.createTestInstance() 155 raw_tests = [ 156 { 157 'annotations': {'Feature': {'value': ['Foo']}}, 158 'class': 'org.chromium.test.SampleTest', 159 'superclass': 'java.lang.Object', 160 'methods': [ 161 { 162 'annotations': {'SmallTest': None}, 163 'method': 'testMethod1', 164 }, 165 { 166 'annotations': {'MediumTest': None}, 167 'method': 'testMethod2', 168 }, 169 ], 170 } 171 ] 172 173 expected_tests = [ 174 { 175 'annotations': { 176 'Feature': {'value': ['Foo']}, 177 'SmallTest': None, 178 }, 179 'class': 'org.chromium.test.SampleTest', 180 'method': 'testMethod1', 181 }, 182 ] 183 184 o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] 185 o._junit4_runner_class = 'J4Runner' 186 actual_tests = o.ProcessRawTests(raw_tests) 187 188 self.assertEqual(actual_tests, expected_tests) 189 190 def testGetTests_simpleGtestPositiveAndNegativeFilter(self): 191 o = self.createTestInstance() 192 raw_tests = [{ 193 'annotations': { 194 'Feature': { 195 'value': ['Foo'] 196 } 197 }, 198 'class': 199 'org.chromium.test.SampleTest', 200 'superclass': 201 'java.lang.Object', 202 'methods': [ 203 { 204 'annotations': { 205 'SmallTest': None 206 }, 207 'method': 'testMethod1', 208 }, 209 { 210 'annotations': { 211 'MediumTest': None 212 }, 213 'method': 'testMethod2', 214 }, 215 ], 216 }, { 217 'annotations': { 218 'Feature': { 219 'value': ['Foo'] 220 } 221 }, 222 'class': 223 'org.chromium.test.SampleTest2', 224 'superclass': 225 'java.lang.Object', 226 'methods': [{ 227 'annotations': { 228 'SmallTest': None 229 }, 230 'method': 'testMethod1', 231 }], 232 }] 233 234 expected_tests = [ 235 { 236 'annotations': { 237 'Feature': { 238 'value': ['Foo'] 239 }, 240 'SmallTest': None, 241 }, 242 'class': 'org.chromium.test.SampleTest', 243 'method': 'testMethod1', 244 }, 245 ] 246 247 o._test_filters = [ 248 'org.chromium.test.SampleTest.*'\ 249 '-org.chromium.test.SampleTest.testMethod2' 250 ] 251 o._junit4_runner_class = 'J4Runner' 252 actual_tests = o.ProcessRawTests(raw_tests) 253 254 self.assertEqual(actual_tests, expected_tests) 255 256 def testGetTests_multipleGtestPositiveAndNegativeFilter(self): 257 o = self.createTestInstance() 258 raw_tests = [{ 259 'annotations': { 260 'Feature': { 261 'value': ['Foo'] 262 } 263 }, 264 'class': 265 'org.chromium.test.SampleTest', 266 'superclass': 267 'java.lang.Object', 268 'methods': [ 269 { 270 'annotations': { 271 'SmallTest': None 272 }, 273 'method': 'testMethod1', 274 }, 275 { 276 'annotations': { 277 'MediumTest': None 278 }, 279 'method': 'testMethod2', 280 }, 281 ], 282 }, { 283 'annotations': { 284 'Feature': { 285 'value': ['Foo'] 286 } 287 }, 288 'class': 289 'org.chromium.test.SampleTest2', 290 'superclass': 291 'java.lang.Object', 292 'methods': [{ 293 'annotations': { 294 'SmallTest': None 295 }, 296 'method': 'testMethod1', 297 }], 298 }] 299 300 expected_tests = [ 301 { 302 'annotations': { 303 'Feature': { 304 'value': ['Foo'] 305 }, 306 'SmallTest': None, 307 }, 308 'class': 'org.chromium.test.SampleTest', 309 'method': 'testMethod1', 310 }, 311 ] 312 313 o._test_filters = [ 314 'org.chromium.test.SampleTest*testMethod1', 315 'org.chromium.test.SampleTest.*'\ 316 '-org.chromium.test.SampleTest.testMethod2' 317 ] 318 o._junit4_runner_class = 'J4Runner' 319 actual_tests = o.ProcessRawTests(raw_tests) 320 321 self.assertEqual(actual_tests, expected_tests) 322 323 def testGetTests_simpleGtestUnqualifiedNameFilter(self): 324 o = self.createTestInstance() 325 raw_tests = [ 326 { 327 'annotations': {'Feature': {'value': ['Foo']}}, 328 'class': 'org.chromium.test.SampleTest', 329 'superclass': 'java.lang.Object', 330 'methods': [ 331 { 332 'annotations': {'SmallTest': None}, 333 'method': 'testMethod1', 334 }, 335 { 336 'annotations': {'MediumTest': None}, 337 'method': 'testMethod2', 338 }, 339 ], 340 } 341 ] 342 343 expected_tests = [ 344 { 345 'annotations': { 346 'Feature': {'value': ['Foo']}, 347 'SmallTest': None, 348 }, 349 'class': 'org.chromium.test.SampleTest', 350 'method': 'testMethod1', 351 }, 352 ] 353 354 o._test_filters = ['SampleTest.testMethod1'] 355 o._junit4_runner_class = 'J4Runner' 356 actual_tests = o.ProcessRawTests(raw_tests) 357 358 self.assertEqual(actual_tests, expected_tests) 359 360 def testGetTests_parameterizedTestGtestFilter(self): 361 o = self.createTestInstance() 362 raw_tests = [ 363 { 364 'annotations': {'Feature': {'value': ['Foo']}}, 365 'class': 'org.chromium.test.SampleTest', 366 'superclass': 'java.lang.Object', 367 'methods': [ 368 { 369 'annotations': {'SmallTest': None}, 370 'method': 'testMethod1', 371 }, 372 { 373 'annotations': {'SmallTest': None}, 374 'method': 'testMethod1__sandboxed_mode', 375 }, 376 ], 377 }, 378 { 379 'annotations': {'Feature': {'value': ['Bar']}}, 380 'class': 'org.chromium.test.SampleTest2', 381 'superclass': 'java.lang.Object', 382 'methods': [ 383 { 384 'annotations': {'SmallTest': None}, 385 'method': 'testMethod1', 386 }, 387 ], 388 } 389 ] 390 391 expected_tests = [ 392 { 393 'annotations': { 394 'Feature': {'value': ['Foo']}, 395 'SmallTest': None, 396 }, 397 'class': 'org.chromium.test.SampleTest', 398 'method': 'testMethod1', 399 }, 400 { 401 'annotations': { 402 'Feature': {'value': ['Foo']}, 403 'SmallTest': None, 404 }, 405 'class': 'org.chromium.test.SampleTest', 406 'method': 'testMethod1__sandboxed_mode', 407 }, 408 ] 409 410 o._junit4_runner_class = 'J4Runner' 411 o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] 412 actual_tests = o.ProcessRawTests(raw_tests) 413 414 self.assertEqual(actual_tests, expected_tests) 415 416 def testGetTests_wildcardGtestFilter(self): 417 o = self.createTestInstance() 418 raw_tests = [ 419 { 420 'annotations': {'Feature': {'value': ['Foo']}}, 421 'class': 'org.chromium.test.SampleTest', 422 'superclass': 'java.lang.Object', 423 'methods': [ 424 { 425 'annotations': {'SmallTest': None}, 426 'method': 'testMethod1', 427 }, 428 { 429 'annotations': {'MediumTest': None}, 430 'method': 'testMethod2', 431 }, 432 ], 433 }, 434 { 435 'annotations': {'Feature': {'value': ['Bar']}}, 436 'class': 'org.chromium.test.SampleTest2', 437 'superclass': 'java.lang.Object', 438 'methods': [ 439 { 440 'annotations': {'SmallTest': None}, 441 'method': 'testMethod1', 442 }, 443 ], 444 } 445 ] 446 447 expected_tests = [ 448 { 449 'annotations': { 450 'Feature': {'value': ['Bar']}, 451 'SmallTest': None, 452 }, 453 'class': 'org.chromium.test.SampleTest2', 454 'method': 'testMethod1', 455 }, 456 ] 457 458 o._test_filters = ['org.chromium.test.SampleTest2.*'] 459 o._junit4_runner_class = 'J4Runner' 460 actual_tests = o.ProcessRawTests(raw_tests) 461 462 self.assertEqual(actual_tests, expected_tests) 463 464 def testGetTests_negativeGtestFilter(self): 465 o = self.createTestInstance() 466 raw_tests = [ 467 { 468 'annotations': {'Feature': {'value': ['Foo']}}, 469 'class': 'org.chromium.test.SampleTest', 470 'superclass': 'java.lang.Object', 471 'methods': [ 472 { 473 'annotations': {'SmallTest': None}, 474 'method': 'testMethod1', 475 }, 476 { 477 'annotations': {'MediumTest': None}, 478 'method': 'testMethod2', 479 }, 480 ], 481 }, 482 { 483 'annotations': {'Feature': {'value': ['Bar']}}, 484 'class': 'org.chromium.test.SampleTest2', 485 'superclass': 'java.lang.Object', 486 'methods': [ 487 { 488 'annotations': {'SmallTest': None}, 489 'method': 'testMethod1', 490 }, 491 ], 492 } 493 ] 494 495 expected_tests = [ 496 { 497 'annotations': { 498 'Feature': {'value': ['Foo']}, 499 'MediumTest': None, 500 }, 501 'class': 'org.chromium.test.SampleTest', 502 'method': 'testMethod2', 503 }, 504 { 505 'annotations': { 506 'Feature': {'value': ['Bar']}, 507 'SmallTest': None, 508 }, 509 'class': 'org.chromium.test.SampleTest2', 510 'method': 'testMethod1', 511 }, 512 ] 513 514 o._test_filters = ['*-org.chromium.test.SampleTest.testMethod1'] 515 o._junit4_runner_class = 'J4Runner' 516 actual_tests = o.ProcessRawTests(raw_tests) 517 518 self.assertEqual(actual_tests, expected_tests) 519 520 def testGetTests_annotationFilter(self): 521 o = self.createTestInstance() 522 raw_tests = [ 523 { 524 'annotations': {'Feature': {'value': ['Foo']}}, 525 'class': 'org.chromium.test.SampleTest', 526 'superclass': 'java.lang.Object', 527 'methods': [ 528 { 529 'annotations': {'SmallTest': None}, 530 'method': 'testMethod1', 531 }, 532 { 533 'annotations': {'MediumTest': None}, 534 'method': 'testMethod2', 535 }, 536 ], 537 }, 538 { 539 'annotations': {'Feature': {'value': ['Bar']}}, 540 'class': 'org.chromium.test.SampleTest2', 541 'superclass': 'java.lang.Object', 542 'methods': [ 543 { 544 'annotations': {'SmallTest': None}, 545 'method': 'testMethod1', 546 }, 547 ], 548 } 549 ] 550 551 expected_tests = [ 552 { 553 'annotations': { 554 'Feature': {'value': ['Foo']}, 555 'SmallTest': None, 556 }, 557 'class': 'org.chromium.test.SampleTest', 558 'method': 'testMethod1', 559 }, 560 { 561 'annotations': { 562 'Feature': {'value': ['Bar']}, 563 'SmallTest': None, 564 }, 565 'class': 'org.chromium.test.SampleTest2', 566 'method': 'testMethod1', 567 }, 568 ] 569 570 o._annotations = [('SmallTest', None)] 571 o._junit4_runner_class = 'J4Runner' 572 actual_tests = o.ProcessRawTests(raw_tests) 573 574 self.assertEqual(actual_tests, expected_tests) 575 576 def testGetTests_excludedAnnotationFilter(self): 577 o = self.createTestInstance() 578 raw_tests = [ 579 { 580 'annotations': {'Feature': {'value': ['Foo']}}, 581 'class': 'org.chromium.test.SampleTest', 582 'superclass': 'junit.framework.TestCase', 583 'methods': [ 584 { 585 'annotations': {'SmallTest': None}, 586 'method': 'testMethod1', 587 }, 588 { 589 'annotations': {'MediumTest': None}, 590 'method': 'testMethod2', 591 }, 592 ], 593 }, 594 { 595 'annotations': {'Feature': {'value': ['Bar']}}, 596 'class': 'org.chromium.test.SampleTest2', 597 'superclass': 'junit.framework.TestCase', 598 'methods': [ 599 { 600 'annotations': {'SmallTest': None}, 601 'method': 'testMethod1', 602 }, 603 ], 604 } 605 ] 606 607 expected_tests = [ 608 { 609 'annotations': { 610 'Feature': { 611 'value': ['Foo'] 612 }, 613 'MediumTest': None, 614 }, 615 'class': 'org.chromium.test.SampleTest', 616 'method': 'testMethod2', 617 }, 618 ] 619 620 o._excluded_annotations = [('SmallTest', None)] 621 o._junit4_runner_class = 'J4Runner' 622 actual_tests = o.ProcessRawTests(raw_tests) 623 624 self.assertEqual(actual_tests, expected_tests) 625 626 def testGetTests_excludedDoNotReviveAnnotation(self): 627 o = self.createTestInstance() 628 raw_tests = [{ 629 'annotations': { 630 'Feature': { 631 'value': ['Foo'] 632 } 633 }, 634 'class': 635 'org.chromium.test.SampleTest', 636 'superclass': 637 'junit.framework.TestCase', 638 'methods': [ 639 { 640 'annotations': { 641 'DisabledTest': None, 642 'DoNotRevive': { 643 'reason': 'sample reason' 644 }, 645 }, 646 'method': 'testMethod1', 647 }, 648 { 649 'annotations': { 650 'FlakyTest': None, 651 }, 652 'method': 'testMethod2', 653 }, 654 ], 655 }, { 656 'annotations': { 657 'Feature': { 658 'value': ['Bar'] 659 } 660 }, 661 'class': 662 'org.chromium.test.SampleTest2', 663 'superclass': 664 'junit.framework.TestCase', 665 'methods': [ 666 { 667 'annotations': { 668 'FlakyTest': None, 669 'DoNotRevive': { 670 'reason': 'sample reason' 671 }, 672 }, 673 'method': 'testMethod1', 674 }, 675 ], 676 }, { 677 'annotations': { 678 'Feature': { 679 'value': ['Baz'] 680 } 681 }, 682 'class': 683 'org.chromium.test.SampleTest3', 684 'superclass': 685 'junit.framework.TestCase', 686 'methods': [ 687 { 688 'annotations': { 689 'FlakyTest': None, 690 'Manual': { 691 'message': 'sample message' 692 }, 693 }, 694 'method': 'testMethod1', 695 }, 696 ], 697 }] 698 699 expected_tests = [ 700 { 701 'annotations': { 702 'Feature': { 703 'value': ['Foo'] 704 }, 705 'FlakyTest': None, 706 }, 707 'class': 'org.chromium.test.SampleTest', 708 'method': 'testMethod2', 709 }, 710 ] 711 712 o._excluded_annotations = [('DoNotRevive', None), ('Manual', None)] 713 o._junit4_runner_class = 'J4Runner' 714 actual_tests = o.ProcessRawTests(raw_tests) 715 716 self.assertEqual(actual_tests, expected_tests) 717 718 def testGetTests_annotationSimpleValueFilter(self): 719 o = self.createTestInstance() 720 raw_tests = [ 721 { 722 'annotations': {'Feature': {'value': ['Foo']}}, 723 'class': 'org.chromium.test.SampleTest', 724 'superclass': 'junit.framework.TestCase', 725 'methods': [ 726 { 727 'annotations': { 728 'SmallTest': None, 729 'TestValue': '1', 730 }, 731 'method': 'testMethod1', 732 }, 733 { 734 'annotations': { 735 'MediumTest': None, 736 'TestValue': '2', 737 }, 738 'method': 'testMethod2', 739 }, 740 ], 741 }, 742 { 743 'annotations': {'Feature': {'value': ['Bar']}}, 744 'class': 'org.chromium.test.SampleTest2', 745 'superclass': 'junit.framework.TestCase', 746 'methods': [ 747 { 748 'annotations': { 749 'SmallTest': None, 750 'TestValue': '3', 751 }, 752 'method': 'testMethod1', 753 }, 754 ], 755 } 756 ] 757 758 expected_tests = [ 759 { 760 'annotations': { 761 'Feature': { 762 'value': ['Foo'] 763 }, 764 'SmallTest': None, 765 'TestValue': '1', 766 }, 767 'class': 'org.chromium.test.SampleTest', 768 'method': 'testMethod1', 769 }, 770 ] 771 772 o._annotations = [('TestValue', '1')] 773 o._junit4_runner_class = 'J4Runner' 774 actual_tests = o.ProcessRawTests(raw_tests) 775 776 self.assertEqual(actual_tests, expected_tests) 777 778 def testGetTests_annotationDictValueFilter(self): 779 o = self.createTestInstance() 780 raw_tests = [ 781 { 782 'annotations': {'Feature': {'value': ['Foo']}}, 783 'class': 'org.chromium.test.SampleTest', 784 'superclass': 'java.lang.Object', 785 'methods': [ 786 { 787 'annotations': {'SmallTest': None}, 788 'method': 'testMethod1', 789 }, 790 { 791 'annotations': {'MediumTest': None}, 792 'method': 'testMethod2', 793 }, 794 ], 795 }, 796 { 797 'annotations': {'Feature': {'value': ['Bar']}}, 798 'class': 'org.chromium.test.SampleTest2', 799 'superclass': 'java.lang.Object', 800 'methods': [ 801 { 802 'annotations': {'SmallTest': None}, 803 'method': 'testMethod1', 804 }, 805 ], 806 } 807 ] 808 809 expected_tests = [ 810 { 811 'annotations': { 812 'Feature': {'value': ['Bar']}, 813 'SmallTest': None, 814 }, 815 'class': 'org.chromium.test.SampleTest2', 816 'method': 'testMethod1', 817 }, 818 ] 819 820 o._annotations = [('Feature', 'Bar')] 821 o._junit4_runner_class = 'J4Runner' 822 actual_tests = o.ProcessRawTests(raw_tests) 823 824 self.assertEqual(actual_tests, expected_tests) 825 826 def testGetTestName(self): 827 test = { 828 'annotations': { 829 'RunWith': {'value': 'class J4Runner'}, 830 'SmallTest': {}, 831 'Test': {'expected': 'class org.junit.Test$None', 832 'timeout': '0'}, 833 'UiThreadTest': {}}, 834 'class': 'org.chromium.TestA', 835 'method': 'testSimple'} 836 unqualified_class_test = { 837 'class': test['class'].split('.')[-1], 838 'method': test['method'] 839 } 840 841 self.assertEqual(instrumentation_test_instance.GetTestName(test, sep='.'), 842 'org.chromium.TestA.testSimple') 843 self.assertEqual( 844 instrumentation_test_instance.GetTestName(unqualified_class_test, 845 sep='.'), 'TestA.testSimple') 846 847 def testGetUniqueTestName(self): 848 test = { 849 'annotations': { 850 'RunWith': {'value': 'class J4Runner'}, 851 'SmallTest': {}, 852 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 853 'UiThreadTest': {}}, 854 'class': 'org.chromium.TestA', 855 'flags': ['enable_features=abc'], 856 'method': 'testSimple'} 857 self.assertEqual( 858 instrumentation_test_instance.GetUniqueTestName(test, sep='.'), 859 'org.chromium.TestA.testSimple_with_enable_features=abc') 860 861 def testGetTestNameWithoutParameterSuffix(self): 862 test = { 863 'annotations': { 864 'RunWith': {'value': 'class J4Runner'}, 865 'SmallTest': {}, 866 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 867 'UiThreadTest': {}}, 868 'class': 'org.chromium.TestA__sandbox_mode', 869 'flags': 'enable_features=abc', 870 'method': 'testSimple'} 871 unqualified_class_test = { 872 'class': test['class'].split('.')[-1], 873 'method': test['method'] 874 } 875 self.assertEqual( 876 instrumentation_test_instance.GetTestNameWithoutParameterSuffix( 877 test, sep='.'), 'org.chromium.TestA') 878 self.assertEqual( 879 instrumentation_test_instance.GetTestNameWithoutParameterSuffix( 880 unqualified_class_test, sep='.'), 'TestA') 881 882 def testGetTests_multipleAnnotationValuesRequested(self): 883 o = self.createTestInstance() 884 raw_tests = [ 885 { 886 'annotations': {'Feature': {'value': ['Foo']}}, 887 'class': 'org.chromium.test.SampleTest', 888 'superclass': 'junit.framework.TestCase', 889 'methods': [ 890 { 891 'annotations': {'SmallTest': None}, 892 'method': 'testMethod1', 893 }, 894 { 895 'annotations': { 896 'Feature': {'value': ['Baz']}, 897 'MediumTest': None, 898 }, 899 'method': 'testMethod2', 900 }, 901 ], 902 }, 903 { 904 'annotations': {'Feature': {'value': ['Bar']}}, 905 'class': 'org.chromium.test.SampleTest2', 906 'superclass': 'junit.framework.TestCase', 907 'methods': [ 908 { 909 'annotations': {'SmallTest': None}, 910 'method': 'testMethod1', 911 }, 912 ], 913 } 914 ] 915 916 expected_tests = [ 917 { 918 'annotations': { 919 'Feature': { 920 'value': ['Baz'] 921 }, 922 'MediumTest': None, 923 }, 924 'class': 'org.chromium.test.SampleTest', 925 'method': 'testMethod2', 926 }, 927 { 928 'annotations': { 929 'Feature': { 930 'value': ['Bar'] 931 }, 932 'SmallTest': None, 933 }, 934 'class': 'org.chromium.test.SampleTest2', 935 'method': 'testMethod1', 936 }, 937 ] 938 939 o._annotations = [('Feature', 'Bar'), ('Feature', 'Baz')] 940 o._junit4_runner_class = 'J4Runner' 941 actual_tests = o.ProcessRawTests(raw_tests) 942 943 self.assertEqual(actual_tests, expected_tests) 944 945 def testGenerateTestResults_noStatus(self): 946 results = instrumentation_test_instance.GenerateTestResults( 947 None, None, [], 1000, None, None) 948 self.assertEqual([], results) 949 950 def testGenerateTestResults_testPassed(self): 951 statuses = [ 952 (1, { 953 'class': 'test.package.TestClass', 954 'test': 'testMethod', 955 }), 956 (0, { 957 'class': 'test.package.TestClass', 958 'test': 'testMethod', 959 }), 960 ] 961 results = instrumentation_test_instance.GenerateTestResults( 962 None, None, statuses, 1000, None, None) 963 self.assertEqual(1, len(results)) 964 self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) 965 966 def testGenerateTestResults_testFailed(self): 967 statuses = [ 968 (1, { 969 'class': 'test.package.TestClass', 970 'test': 'testMethod', 971 }), 972 (-2, { 973 'class': 'test.package.TestClass', 974 'test': 'testMethod', 975 }), 976 ] 977 results = instrumentation_test_instance.GenerateTestResults( 978 None, None, statuses, 1000, None, None) 979 self.assertEqual(1, len(results)) 980 self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) 981 982 def testGenerateTestResults_testUnknownException(self): 983 stacktrace = 'long\nstacktrace' 984 statuses = [ 985 (1, { 986 'class': 'test.package.TestClass', 987 'test': 'testMethod', 988 }), 989 (-1, { 990 'class': 'test.package.TestClass', 991 'test': 'testMethod', 992 'stack': stacktrace, 993 }), 994 ] 995 results = instrumentation_test_instance.GenerateTestResults( 996 None, None, statuses, 1000, None, None) 997 self.assertEqual(1, len(results)) 998 self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) 999 self.assertEqual(stacktrace, results[0].GetLog()) 1000 1001 def testGenerateTestResults_testSkipped_true(self): 1002 statuses = [ 1003 (1, { 1004 'class': 'test.package.TestClass', 1005 'test': 'testMethod', 1006 }), 1007 (-3, { 1008 'class': 'test.package.TestClass', 1009 'test': 'testMethod', 1010 }), 1011 ] 1012 results = instrumentation_test_instance.GenerateTestResults( 1013 None, None, statuses, 1000, None, None) 1014 self.assertEqual(1, len(results)) 1015 self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) 1016 1017 def testGenerateTestResults_beforeClassFailure(self): 1018 stacktrace = 'long\nstacktrace' 1019 statuses = [ 1020 (1, { 1021 'class': 'test.package.TestClass', 1022 'test': 'null', 1023 }), 1024 (-2, { 1025 'class': 'test.package.TestClass', 1026 'test': 'null', 1027 'stack': stacktrace, 1028 }), 1029 (1, { 1030 'class': 'test.package.TestClass', 1031 'test': 'testMethod1', 1032 }), 1033 (0, { 1034 'class': 'test.package.TestClass', 1035 'test': 'testMethod1', 1036 }), 1037 (1, { 1038 'class': 'test.package.TestClass', 1039 'test': 'testMethod2', 1040 }), 1041 (0, { 1042 'class': 'test.package.TestClass', 1043 'test': 'testMethod2', 1044 }), 1045 ] 1046 results = instrumentation_test_instance.GenerateTestResults( 1047 None, None, statuses, 1000, None, None) 1048 self.assertEqual(2, len(results)) 1049 self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) 1050 self.assertEqual(base_test_result.ResultType.FAIL, results[1].GetType()) 1051 self.assertEqual(stacktrace, results[0].GetLog()) 1052 self.assertEqual(stacktrace, results[1].GetLog()) 1053 1054 def testGenerateTestResults_afterClassFailure(self): 1055 stacktrace = 'long\nstacktrace' 1056 statuses = [ 1057 (1, { 1058 'class': 'test.package.TestClass', 1059 'test': 'testMethod1', 1060 }), 1061 (0, { 1062 'class': 'test.package.TestClass', 1063 'test': 'testMethod1', 1064 }), 1065 (1, { 1066 'class': 'test.package.TestClass', 1067 'test': 'testMethod2', 1068 }), 1069 (-3, { 1070 'class': 'test.package.TestClass', 1071 'test': 'testMethod2', 1072 }), 1073 (1, { 1074 'class': 'test.package.TestClass', 1075 'test': 'null', 1076 }), 1077 (-2, { 1078 'class': 'test.package.TestClass', 1079 'test': 'null', 1080 'stack': stacktrace, 1081 }), 1082 ] 1083 results = instrumentation_test_instance.GenerateTestResults( 1084 None, None, statuses, 1000, None, None) 1085 self.assertEqual(2, len(results)) 1086 self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) 1087 self.assertEqual(base_test_result.ResultType.SKIP, results[1].GetType()) 1088 self.assertEqual(stacktrace, results[0].GetLog()) 1089 1090 def testParameterizedCommandLineFlagsSwitches(self): 1091 o = self.createTestInstance() 1092 raw_tests = [{ 1093 'annotations': { 1094 'ParameterizedCommandLineFlags$Switches': { 1095 'value': ['enable-features=abc', 'enable-features=def'] 1096 } 1097 }, 1098 'class': 1099 'org.chromium.test.SampleTest', 1100 'superclass': 1101 'java.lang.Object', 1102 'methods': [ 1103 { 1104 'annotations': { 1105 'SmallTest': None 1106 }, 1107 'method': 'testMethod1', 1108 }, 1109 { 1110 'annotations': { 1111 'MediumTest': None, 1112 'ParameterizedCommandLineFlags$Switches': { 1113 'value': ['enable-features=ghi', 'enable-features=jkl'] 1114 }, 1115 }, 1116 'method': 'testMethod2', 1117 }, 1118 { 1119 'annotations': { 1120 'MediumTest': None, 1121 'ParameterizedCommandLineFlags$Switches': { 1122 'value': [] 1123 }, 1124 }, 1125 'method': 'testMethod3', 1126 }, 1127 { 1128 'annotations': { 1129 'MediumTest': None, 1130 'SkipCommandLineParameterization': None, 1131 }, 1132 'method': 'testMethod4', 1133 }, 1134 ], 1135 }] 1136 1137 expected_tests = [ 1138 { 1139 'annotations': {}, 1140 'class': 'org.chromium.test.SampleTest', 1141 'flags': ['--enable-features=abc', '--enable-features=def'], 1142 'method': 'testMethod1' 1143 }, 1144 { 1145 'annotations': {}, 1146 'class': 'org.chromium.test.SampleTest', 1147 'flags': ['--enable-features=ghi', '--enable-features=jkl'], 1148 'method': 'testMethod2' 1149 }, 1150 { 1151 'annotations': {}, 1152 'class': 'org.chromium.test.SampleTest', 1153 'method': 'testMethod3' 1154 }, 1155 { 1156 'annotations': {}, 1157 'class': 'org.chromium.test.SampleTest', 1158 'method': 'testMethod4' 1159 }, 1160 ] 1161 for i in range(4): 1162 expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) 1163 expected_tests[i]['annotations'].update( 1164 raw_tests[0]['methods'][i]['annotations']) 1165 1166 o._junit4_runner_class = 'J4Runner' 1167 actual_tests = o.ProcessRawTests(raw_tests) 1168 self.assertEqual(actual_tests, expected_tests) 1169 1170 def testParameterizedCommandLineFlags(self): 1171 o = self.createTestInstance() 1172 raw_tests = [{ 1173 'annotations': { 1174 'ParameterizedCommandLineFlags': { 1175 'value': [ 1176 { 1177 'ParameterizedCommandLineFlags$Switches': { 1178 'value': [ 1179 'enable-features=abc', 1180 'force-fieldtrials=trial/group' 1181 ], 1182 } 1183 }, 1184 { 1185 'ParameterizedCommandLineFlags$Switches': { 1186 'value': [ 1187 'enable-features=abc2', 1188 'force-fieldtrials=trial/group2' 1189 ], 1190 } 1191 }, 1192 ], 1193 }, 1194 }, 1195 'class': 1196 'org.chromium.test.SampleTest', 1197 'superclass': 1198 'java.lang.Object', 1199 'methods': [ 1200 { 1201 'annotations': { 1202 'SmallTest': None 1203 }, 1204 'method': 'testMethod1', 1205 }, 1206 { 1207 'annotations': { 1208 'MediumTest': None, 1209 'ParameterizedCommandLineFlags': { 1210 'value': [{ 1211 'ParameterizedCommandLineFlags$Switches': { 1212 'value': ['enable-features=def'] 1213 } 1214 }], 1215 }, 1216 }, 1217 'method': 'testMethod2', 1218 }, 1219 { 1220 'annotations': { 1221 'MediumTest': None, 1222 'ParameterizedCommandLineFlags': { 1223 'value': [], 1224 }, 1225 }, 1226 'method': 'testMethod3', 1227 }, 1228 { 1229 'annotations': { 1230 'MediumTest': None, 1231 'SkipCommandLineParameterization': None, 1232 }, 1233 'method': 'testMethod4', 1234 }, 1235 ], 1236 }] 1237 1238 expected_tests = [ 1239 { 1240 'annotations': {}, 1241 'class': 'org.chromium.test.SampleTest', 1242 'flags': 1243 ['--enable-features=abc', '--force-fieldtrials=trial/group'], 1244 'method': 'testMethod1' 1245 }, 1246 { 1247 'annotations': {}, 1248 'class': 'org.chromium.test.SampleTest', 1249 'flags': ['--enable-features=def'], 1250 'method': 'testMethod2' 1251 }, 1252 { 1253 'annotations': {}, 1254 'class': 'org.chromium.test.SampleTest', 1255 'method': 'testMethod3' 1256 }, 1257 { 1258 'annotations': {}, 1259 'class': 'org.chromium.test.SampleTest', 1260 'method': 'testMethod4' 1261 }, 1262 { 1263 'annotations': {}, 1264 'class': 1265 'org.chromium.test.SampleTest', 1266 'flags': [ 1267 '--enable-features=abc2', 1268 '--force-fieldtrials=trial/group2', 1269 ], 1270 'method': 1271 'testMethod1' 1272 }, 1273 ] 1274 for i in range(4): 1275 expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) 1276 expected_tests[i]['annotations'].update( 1277 raw_tests[0]['methods'][i]['annotations']) 1278 expected_tests[4]['annotations'].update(raw_tests[0]['annotations']) 1279 expected_tests[4]['annotations'].update( 1280 raw_tests[0]['methods'][0]['annotations']) 1281 1282 o._junit4_runner_class = 'J4Runner' 1283 actual_tests = o.ProcessRawTests(raw_tests) 1284 self.assertEqual(actual_tests, expected_tests) 1285 1286 def testDifferentCommandLineParameterizations(self): 1287 o = self.createTestInstance() 1288 raw_tests = [{ 1289 'annotations': {}, 1290 'class': 1291 'org.chromium.test.SampleTest', 1292 'superclass': 1293 'java.lang.Object', 1294 'methods': [ 1295 { 1296 'annotations': { 1297 'SmallTest': None, 1298 'ParameterizedCommandLineFlags': { 1299 'value': [ 1300 { 1301 'ParameterizedCommandLineFlags$Switches': { 1302 'value': ['a1', 'a2'], 1303 } 1304 }, 1305 ], 1306 }, 1307 }, 1308 'method': 'testMethod2', 1309 }, 1310 { 1311 'annotations': { 1312 'SmallTest': None, 1313 'ParameterizedCommandLineFlags$Switches': { 1314 'value': ['b1', 'b2'], 1315 }, 1316 }, 1317 'method': 'testMethod3', 1318 }, 1319 ], 1320 }] 1321 1322 expected_tests = [ 1323 { 1324 'annotations': {}, 1325 'class': 'org.chromium.test.SampleTest', 1326 'flags': ['--a1', '--a2'], 1327 'method': 'testMethod2' 1328 }, 1329 { 1330 'annotations': {}, 1331 'class': 'org.chromium.test.SampleTest', 1332 'flags': ['--b1', '--b2'], 1333 'method': 'testMethod3' 1334 }, 1335 ] 1336 for i in range(2): 1337 expected_tests[i]['annotations'].update( 1338 raw_tests[0]['methods'][i]['annotations']) 1339 1340 o._junit4_runner_class = 'J4Runner' 1341 actual_tests = o.ProcessRawTests(raw_tests) 1342 self.assertEqual(actual_tests, expected_tests) 1343 1344 def testMultipleCommandLineParameterizations_raises(self): 1345 o = self.createTestInstance() 1346 raw_tests = [ 1347 { 1348 'annotations': { 1349 'ParameterizedCommandLineFlags': { 1350 'value': [ 1351 { 1352 'ParameterizedCommandLineFlags$Switches': { 1353 'value': [ 1354 'enable-features=abc', 1355 'force-fieldtrials=trial/group', 1356 ], 1357 } 1358 }, 1359 ], 1360 }, 1361 }, 1362 'class': 1363 'org.chromium.test.SampleTest', 1364 'superclass': 1365 'java.lang.Object', 1366 'methods': [ 1367 { 1368 'annotations': { 1369 'SmallTest': None, 1370 'ParameterizedCommandLineFlags$Switches': { 1371 'value': [ 1372 'enable-features=abc', 1373 'force-fieldtrials=trial/group', 1374 ], 1375 }, 1376 }, 1377 'method': 'testMethod1', 1378 }, 1379 ], 1380 }, 1381 ] 1382 1383 o._junit4_runner_class = 'J4Runner' 1384 self.assertRaises( 1385 instrumentation_test_instance.CommandLineParameterizationException, 1386 o.ProcessRawTests, [raw_tests[0]]) 1387 1388 1389 if __name__ == '__main__': 1390 unittest.main(verbosity=2)