test_runner_test.py (14630B)
1 #!/usr/bin/env vpython3 2 # Copyright 2020 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import json 7 import os 8 import shutil 9 import sys 10 import tempfile 11 from textwrap import dedent 12 import unittest 13 14 # The following non-std imports are fetched via vpython. See the list at 15 # //.vpython3 16 import mock # pylint: disable=import-error 17 from parameterized import parameterized # pylint: disable=import-error 18 19 import test_runner 20 21 _TAST_TEST_RESULTS_JSON = { 22 "name": "login.Chrome", 23 "errors": None, 24 "start": "2020-01-01T15:41:30.799228462-08:00", 25 "end": "2020-01-01T15:41:53.318914698-08:00", 26 "skipReason": "" 27 } 28 29 30 class TestRunnerTest(unittest.TestCase): 31 32 def setUp(self): 33 self._tmp_dir = tempfile.mkdtemp() 34 self.mock_rdb = mock.patch.object( 35 test_runner.result_sink, 'TryInitClient', return_value=None) 36 self.mock_rdb.start() 37 self.mock_env = mock.patch.dict( 38 os.environ, {'SWARMING_BOT_ID': 'cros-chrome-chromeos8-row29'}) 39 self.mock_env.start() 40 41 def tearDown(self): 42 shutil.rmtree(self._tmp_dir, ignore_errors=True) 43 self.mock_rdb.stop() 44 self.mock_env.stop() 45 46 def safeAssertItemsEqual(self, list1, list2): 47 """A Py3 safe version of assertItemsEqual. 48 49 See https://bugs.python.org/issue17866. 50 """ 51 self.assertSetEqual(set(list1), set(list2)) 52 53 54 class TastTests(TestRunnerTest): 55 56 def get_common_tast_args(self, use_vm, fetch_cros_hostname): 57 return [ 58 'script_name', 59 'tast', 60 '--suite-name=chrome_all_tast_tests', 61 '--board=eve', 62 '--flash', 63 '--path-to-outdir=out_eve/Release', 64 '--logs-dir=%s' % self._tmp_dir, 65 '--use-vm' if use_vm else 66 ('--fetch-cros-hostname' 67 if fetch_cros_hostname else '--device=localhost:2222'), 68 ] 69 70 def get_common_tast_expectations(self, use_vm, fetch_cros_hostname): 71 expectation = [ 72 test_runner.CROS_RUN_TEST_PATH, 73 '--board', 74 'eve', 75 '--cache-dir', 76 test_runner.DEFAULT_CROS_CACHE, 77 '--results-dest-dir', 78 '%s/system_logs' % self._tmp_dir, 79 '--flash', 80 '--build-dir', 81 'out_eve/Release', 82 '--results-dir', 83 self._tmp_dir, 84 '--tast-total-shards=1', 85 '--tast-shard-index=0', 86 ] 87 expectation.extend(['--start', '--copy-on-write'] if use_vm else ( 88 ['--device', 'chrome-chromeos8-row29'] 89 if fetch_cros_hostname else ['--device', 'localhost:2222'])) 90 for p in test_runner.SYSTEM_LOG_LOCATIONS: 91 expectation.extend(['--results-src', p]) 92 93 expectation += [ 94 '--mount', 95 '--deploy', 96 '--nostrip', 97 ] 98 return expectation 99 100 def test_tast_gtest_filter(self): 101 """Tests running tast tests with a gtest-style filter.""" 102 with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: 103 json.dump(_TAST_TEST_RESULTS_JSON, f) 104 105 args = self.get_common_tast_args(False, False) + [ 106 '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)', 107 '--gtest_filter=login.Chrome:ui.WindowControl', 108 ] 109 with mock.patch.object(sys, 'argv', args),\ 110 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: 111 mock_popen.return_value.returncode = 0 112 113 test_runner.main() 114 # The gtest filter should cause the Tast expr to be replaced with a list 115 # of the tests in the filter. 116 expected_cmd = self.get_common_tast_expectations(False, False) + [ 117 '--tast=("name:login.Chrome" || "name:ui.WindowControl")' 118 ] 119 120 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 121 122 @parameterized.expand([ 123 [True, False], 124 [False, True], 125 [False, False], 126 ]) 127 def test_tast_attr_expr(self, use_vm, fetch_cros_hostname): 128 """Tests running a tast tests specified by an attribute expression.""" 129 with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: 130 json.dump(_TAST_TEST_RESULTS_JSON, f) 131 132 args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [ 133 '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)', 134 ] 135 with mock.patch.object(sys, 'argv', args),\ 136 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: 137 mock_popen.return_value.returncode = 0 138 139 test_runner.main() 140 expected_cmd = self.get_common_tast_expectations( 141 use_vm, fetch_cros_hostname) + [ 142 '--tast=( "group:mainline" && "dep:chrome" && !informational)', 143 ] 144 145 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 146 147 @parameterized.expand([ 148 [True, False], 149 [False, True], 150 [False, False], 151 ]) 152 def test_tast_with_vars(self, use_vm, fetch_cros_hostname): 153 """Tests running a tast tests with runtime variables.""" 154 with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: 155 json.dump(_TAST_TEST_RESULTS_JSON, f) 156 157 args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [ 158 '-t=login.Chrome', 159 '--tast-var=key=value', 160 ] 161 with mock.patch.object(sys, 'argv', args),\ 162 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: 163 mock_popen.return_value.returncode = 0 164 test_runner.main() 165 expected_cmd = self.get_common_tast_expectations( 166 use_vm, fetch_cros_hostname) + [ 167 '--tast', 'login.Chrome', '--tast-var', 'key=value' 168 ] 169 170 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 171 172 @parameterized.expand([ 173 [True, False], 174 [False, True], 175 [False, False], 176 ]) 177 def test_tast_retries(self, use_vm, fetch_cros_hostname): 178 """Tests running a tast tests with retries.""" 179 with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: 180 json.dump(_TAST_TEST_RESULTS_JSON, f) 181 182 args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [ 183 '-t=login.Chrome', 184 '--tast-retries=1', 185 ] 186 with mock.patch.object(sys, 'argv', args),\ 187 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: 188 mock_popen.return_value.returncode = 0 189 test_runner.main() 190 expected_cmd = self.get_common_tast_expectations( 191 use_vm, 192 fetch_cros_hostname) + ['--tast', 'login.Chrome', '--tast-retries=1'] 193 194 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 195 196 @parameterized.expand([ 197 [True, False], 198 [False, True], 199 [False, False], 200 ]) 201 def test_tast(self, use_vm, fetch_cros_hostname): 202 """Tests running a tast tests.""" 203 with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: 204 json.dump(_TAST_TEST_RESULTS_JSON, f) 205 206 args = self.get_common_tast_args(use_vm, fetch_cros_hostname) + [ 207 '-t=login.Chrome', 208 ] 209 with mock.patch.object(sys, 'argv', args),\ 210 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: 211 mock_popen.return_value.returncode = 0 212 213 test_runner.main() 214 expected_cmd = self.get_common_tast_expectations( 215 use_vm, fetch_cros_hostname) + ['--tast', 'login.Chrome'] 216 217 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 218 219 220 class GTestTest(TestRunnerTest): 221 222 @parameterized.expand([ 223 [True, True, True, False, True], 224 [True, False, False, False, False], 225 [False, True, True, True, True], 226 [False, False, False, True, False], 227 [False, True, True, False, True], 228 [False, False, False, False, False], 229 ]) 230 def test_gtest(self, use_vm, stop_ui, use_test_sudo_helper, 231 fetch_cros_hostname, use_deployed_dbus_configs): 232 """Tests running a gtest.""" 233 fd_mock = mock.mock_open() 234 235 args = [ 236 'script_name', 237 'gtest', 238 '--test-exe=out_eve/Release/base_unittests', 239 '--board=eve', 240 '--path-to-outdir=out_eve/Release', 241 '--use-vm' if use_vm else 242 ('--fetch-cros-hostname' 243 if fetch_cros_hostname else '--device=localhost:2222'), 244 ] 245 if stop_ui: 246 args.append('--stop-ui') 247 if use_test_sudo_helper: 248 args.append('--run-test-sudo-helper') 249 if use_deployed_dbus_configs: 250 args.append('--use-deployed-dbus-configs') 251 252 with mock.patch.object(sys, 'argv', args),\ 253 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen,\ 254 mock.patch.object(os, 'fdopen', fd_mock),\ 255 mock.patch.object(os, 'remove') as mock_remove,\ 256 mock.patch.object(tempfile, 'mkstemp', 257 side_effect=[(3, 'out_eve/Release/device_script.sh'),\ 258 (4, 'out_eve/Release/runtime_files.txt')]),\ 259 mock.patch.object(os, 'fchmod'): 260 mock_popen.return_value.returncode = 0 261 262 test_runner.main() 263 self.assertEqual(1, mock_popen.call_count) 264 expected_cmd = [ 265 'vpython3', test_runner.CROS_RUN_TEST_PATH, '--board', 'eve', 266 '--cache-dir', test_runner.DEFAULT_CROS_CACHE, '--remote-cmd', 267 '--cwd', 'out_eve/Release', '--files-from', 268 'out_eve/Release/runtime_files.txt' 269 ] 270 if not stop_ui: 271 expected_cmd.append('--as-chronos') 272 expected_cmd.extend(['--start', '--copy-on-write'] if use_vm else ( 273 ['--device', 'chrome-chromeos8-row29'] 274 if fetch_cros_hostname else ['--device', 'localhost:2222'])) 275 expected_cmd.extend(['--', './device_script.sh']) 276 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 277 278 expected_device_script = dedent("""\ 279 #!/bin/sh 280 export HOME=/usr/local/tmp 281 export TMPDIR=/usr/local/tmp 282 """) 283 284 core_cmd = 'LD_LIBRARY_PATH=./ ./out_eve/Release/base_unittests'\ 285 ' --test-launcher-shard-index=0 --test-launcher-total-shards=1' 286 287 if use_test_sudo_helper: 288 expected_device_script += dedent("""\ 289 TEST_SUDO_HELPER_PATH=$(mktemp) 290 ./test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} & 291 TEST_SUDO_HELPER_PID=$! 292 """) 293 core_cmd += ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}' 294 295 if use_deployed_dbus_configs: 296 expected_device_script += dedent("""\ 297 mount --bind ./dbus /opt/google/chrome/dbus 298 kill -s HUP $(pgrep dbus) 299 """) 300 301 if stop_ui: 302 dbus_cmd = 'dbus-send --system --type=method_call'\ 303 ' --dest=org.chromium.PowerManager'\ 304 ' /org/chromium/PowerManager'\ 305 ' org.chromium.PowerManager.HandleUserActivity int32:0' 306 expected_device_script += dedent("""\ 307 stop ui 308 {0} 309 chown -R chronos: ../.. 310 sudo -E -u chronos -- /bin/bash -c \"{1}\" 311 TEST_RETURN_CODE=$? 312 start ui 313 """).format(dbus_cmd, core_cmd) 314 else: 315 expected_device_script += dedent("""\ 316 {0} 317 TEST_RETURN_CODE=$? 318 """).format(core_cmd) 319 320 if use_test_sudo_helper: 321 expected_device_script += dedent("""\ 322 pkill -P $TEST_SUDO_HELPER_PID 323 kill $TEST_SUDO_HELPER_PID 324 unlink ${TEST_SUDO_HELPER_PATH} 325 """) 326 327 if use_deployed_dbus_configs: 328 expected_device_script += dedent("""\ 329 umount /opt/google/chrome/dbus 330 kill -s HUP $(pgrep dbus) 331 """) 332 333 expected_device_script += dedent("""\ 334 exit $TEST_RETURN_CODE 335 """) 336 337 self.assertEqual(2, fd_mock().write.call_count) 338 write_calls = fd_mock().write.call_args_list 339 340 # Split the strings to make failure messages easier to read. 341 # Verify the first write of device script. 342 self.assertListEqual( 343 expected_device_script.split('\n'), 344 str(write_calls[0][0][0]).split('\n')) 345 346 # Verify the 2nd write of runtime files. 347 expected_runtime_files = ['out_eve/Release/device_script.sh'] 348 self.assertListEqual(expected_runtime_files, 349 str(write_calls[1][0][0]).strip().split('\n')) 350 351 mock_remove.assert_called_once_with('out_eve/Release/device_script.sh') 352 353 def test_gtest_with_vpython(self): 354 """Tests building a gtest with --vpython-dir.""" 355 args = mock.MagicMock() 356 args.test_exe = 'base_unittests' 357 args.test_launcher_summary_output = None 358 args.trace_dir = None 359 args.runtime_deps_path = None 360 args.path_to_outdir = self._tmp_dir 361 args.vpython_dir = self._tmp_dir 362 args.logs_dir = self._tmp_dir 363 364 # With vpython_dir initially empty, the test_runner should error out 365 # due to missing vpython binaries. 366 gtest = test_runner.GTestTest(args, None) 367 with self.assertRaises(test_runner.TestFormatError): 368 gtest.build_test_command() 369 370 # Create the two expected tools, and the test should be ready to run. 371 with open(os.path.join(args.vpython_dir, 'vpython3'), 'w'): 372 pass # Just touch the file. 373 os.mkdir(os.path.join(args.vpython_dir, 'bin')) 374 with open(os.path.join(args.vpython_dir, 'bin', 'python3'), 'w'): 375 pass 376 gtest = test_runner.GTestTest(args, None) 377 gtest.build_test_command() 378 379 380 class HostCmdTests(TestRunnerTest): 381 382 @parameterized.expand([ 383 [False, False], 384 [False, True], 385 [True, False], 386 [True, True], 387 ]) 388 def test_host_cmd(self, deploy_chrome, strip_chrome): 389 args = [ 390 'script_name', 391 'host-cmd', 392 '--board=eve', 393 '--flash', 394 '--path-to-outdir=out/Release', 395 '--device=localhost:2222', 396 ] 397 if deploy_chrome: 398 args += ['--deploy-chrome'] 399 if strip_chrome: 400 args += ['--strip-chrome'] 401 args += [ 402 '--', 403 'fake_cmd', 404 ] 405 with mock.patch.object(sys, 'argv', args),\ 406 mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: 407 mock_popen.return_value.returncode = 0 408 409 test_runner.main() 410 expected_cmd = [ 411 test_runner.CROS_RUN_TEST_PATH, 412 '--board', 413 'eve', 414 '--cache-dir', 415 test_runner.DEFAULT_CROS_CACHE, 416 '--flash', 417 '--device', 418 'localhost:2222', 419 '--host-cmd', 420 ] 421 if deploy_chrome: 422 expected_cmd += [ 423 '--mount', 424 '--deploy', 425 '--build-dir', 426 os.path.join(test_runner.CHROMIUM_SRC_PATH, 'out/Release'), 427 ] 428 if not strip_chrome: 429 expected_cmd += ['--nostrip'] 430 431 expected_cmd += [ 432 '--', 433 'fake_cmd', 434 ] 435 436 self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) 437 438 439 if __name__ == '__main__': 440 unittest.main()