test_runner.py (42798B)
1 #!/usr/bin/env vpython3 2 # 3 # Copyright 2018 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 import argparse 8 import collections 9 import json 10 import logging 11 import os 12 import re 13 import shutil 14 import signal 15 import socket 16 import sys 17 import tempfile 18 19 # The following non-std imports are fetched via vpython. See the list at 20 # //.vpython3 21 import dateutil.parser # pylint: disable=import-error 22 import jsonlines # pylint: disable=import-error 23 import psutil # pylint: disable=import-error 24 25 CHROMIUM_SRC_PATH = os.path.abspath( 26 os.path.join(os.path.dirname(__file__), '..', '..')) 27 28 # Use the android test-runner's gtest results support library for generating 29 # output json ourselves. 30 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android')) 31 from pylib.base import base_test_result # pylint: disable=import-error 32 from pylib.results import json_results # pylint: disable=import-error 33 34 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util')) 35 # TODO(crbug.com/40259280): Re-enable the 'no-name-in-module' check. 36 from lib.results import result_sink # pylint: disable=import-error,no-name-in-module 37 38 import subprocess # pylint: disable=import-error,wrong-import-order 39 40 DEFAULT_CROS_CACHE = os.path.abspath( 41 os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache')) 42 CHROMITE_PATH = os.path.abspath( 43 os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite')) 44 CROS_RUN_TEST_PATH = os.path.abspath( 45 os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test')) 46 47 # This is a special hostname that resolves to a different DUT in the lab 48 # depending on which lab machine you're on. 49 LAB_DUT_HOSTNAME = 'variable_chromeos_device_hostname' 50 51 SYSTEM_LOG_LOCATIONS = [ 52 '/home/chronos/crash/', 53 '/var/log/chrome/', 54 '/var/log/messages', 55 '/var/log/ui/', 56 ] 57 58 TAST_DEBUG_DOC = 'https://bit.ly/2LgvIXz' 59 60 61 class TestFormatError(Exception): 62 pass 63 64 65 class RemoteTest: 66 67 # This is a basic shell script that can be appended to in order to invoke the 68 # test on the device. 69 BASIC_SHELL_SCRIPT = [ 70 '#!/bin/sh', 71 72 # /home and /tmp are mounted with "noexec" in the device, but some of our 73 # tools and tests use those dirs as a workspace (eg: vpython downloads 74 # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap). 75 # /usr/local/tmp doesn't have this restriction, so change the location of 76 # the home and temp dirs for the duration of the test. 77 'export HOME=/usr/local/tmp', 78 'export TMPDIR=/usr/local/tmp', 79 ] 80 81 def __init__(self, args, unknown_args): 82 self._additional_args = unknown_args 83 self._path_to_outdir = args.path_to_outdir 84 self._test_launcher_summary_output = args.test_launcher_summary_output 85 self._logs_dir = args.logs_dir 86 self._use_vm = args.use_vm 87 self._rdb_client = result_sink.TryInitClient() 88 89 self._retries = 0 90 self._timeout = None 91 self._test_launcher_shard_index = args.test_launcher_shard_index 92 self._test_launcher_total_shards = args.test_launcher_total_shards 93 94 # The location on disk of a shell script that can be optionally used to 95 # invoke the test on the device. If it's not set, we assume self._test_cmd 96 # contains the test invocation. 97 self._on_device_script = None 98 99 self._test_cmd = [ 100 CROS_RUN_TEST_PATH, 101 '--board', 102 args.board, 103 '--cache-dir', 104 args.cros_cache, 105 ] 106 if args.use_vm: 107 self._test_cmd += [ 108 '--start', 109 # Don't persist any filesystem changes after the VM shutsdown. 110 '--copy-on-write', 111 ] 112 else: 113 if args.fetch_cros_hostname: 114 self._test_cmd += ['--device', get_cros_hostname()] 115 else: 116 self._test_cmd += [ 117 '--device', args.device if args.device else LAB_DUT_HOSTNAME 118 ] 119 120 if args.logs_dir: 121 for log in SYSTEM_LOG_LOCATIONS: 122 self._test_cmd += ['--results-src', log] 123 self._test_cmd += [ 124 '--results-dest-dir', 125 os.path.join(args.logs_dir, 'system_logs') 126 ] 127 if args.flash: 128 self._test_cmd += ['--flash'] 129 if args.public_image: 130 self._test_cmd += ['--public-image'] 131 132 self._test_env = setup_env() 133 134 @property 135 def suite_name(self): 136 raise NotImplementedError('Child classes need to define suite name.') 137 138 @property 139 def test_cmd(self): 140 return self._test_cmd 141 142 def write_test_script_to_disk(self, script_contents): 143 # Since we're using an on_device_script to invoke the test, we'll need to 144 # set cwd. 145 self._test_cmd += [ 146 '--remote-cmd', 147 '--cwd', 148 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH), 149 ] 150 logging.info('Running the following command on the device:') 151 logging.info('\n%s', '\n'.join(script_contents)) 152 fd, tmp_path = tempfile.mkstemp(suffix='.sh', dir=self._path_to_outdir) 153 os.fchmod(fd, 0o755) 154 with os.fdopen(fd, 'w') as f: 155 f.write('\n'.join(script_contents) + '\n') 156 return tmp_path 157 158 def write_runtime_files_to_disk(self, runtime_files): 159 logging.info('Writing runtime files to disk.') 160 fd, tmp_path = tempfile.mkstemp(suffix='.txt', dir=self._path_to_outdir) 161 os.fchmod(fd, 0o755) 162 with os.fdopen(fd, 'w') as f: 163 f.write('\n'.join(runtime_files) + '\n') 164 return tmp_path 165 166 def run_test(self): 167 # Traps SIGTERM and kills all child processes of cros_run_test when it's 168 # caught. This will allow us to capture logs from the device if a test hangs 169 # and gets timeout-killed by swarming. See also: 170 # https://chromium.googlesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance 171 test_proc = None 172 173 def _kill_child_procs(trapped_signal, _): 174 logging.warning('Received signal %d. Killing child processes of test.', 175 trapped_signal) 176 if not test_proc or not test_proc.pid: 177 # This shouldn't happen? 178 logging.error('Test process not running.') 179 return 180 for child in psutil.Process(test_proc.pid).children(): 181 logging.warning('Killing process %s', child) 182 child.kill() 183 184 signal.signal(signal.SIGTERM, _kill_child_procs) 185 186 for i in range(self._retries + 1): 187 logging.info('########################################') 188 logging.info('Test attempt #%d', i) 189 logging.info('########################################') 190 test_proc = subprocess.Popen( 191 self._test_cmd, 192 stdout=sys.stdout, 193 stderr=sys.stderr, 194 env=self._test_env) 195 try: 196 test_proc.wait(timeout=self._timeout) 197 except subprocess.TimeoutExpired: # pylint: disable=no-member 198 logging.error('Test timed out. Sending SIGTERM.') 199 # SIGTERM the proc and wait 10s for it to close. 200 test_proc.terminate() 201 try: 202 test_proc.wait(timeout=10) 203 except subprocess.TimeoutExpired: # pylint: disable=no-member 204 # If it hasn't closed in 10s, SIGKILL it. 205 logging.error('Test did not exit in time. Sending SIGKILL.') 206 test_proc.kill() 207 test_proc.wait() 208 logging.info('Test exited with %d.', test_proc.returncode) 209 if test_proc.returncode == 0: 210 break 211 212 self.post_run(test_proc.returncode) 213 # Allow post_run to override test proc return code. (Useful when the host 214 # side Tast bin returns 0 even for failed tests.) 215 return test_proc.returncode 216 217 def post_run(self, _): 218 if self._on_device_script: 219 os.remove(self._on_device_script) 220 221 @staticmethod 222 def get_artifacts(path): 223 """Crawls a given directory for file artifacts to attach to a test. 224 225 Args: 226 path: Path to a directory to search for artifacts. 227 Returns: 228 A dict mapping name of the artifact to its absolute filepath. 229 """ 230 artifacts = {} 231 for dirpath, _, filenames in os.walk(path): 232 for f in filenames: 233 artifact_path = os.path.join(dirpath, f) 234 artifact_id = os.path.relpath(artifact_path, path) 235 # Some artifacts will have non-Latin characters in the filename, eg: 236 # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such 237 # characters as an artifact ID, so force the file name down into ascii. 238 # For more info, see: 239 # https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/proto/v1/artifact.proto;drc=3bff13b8037ca76ec19f9810033d914af7ec67cb;l=46 240 artifact_id = artifact_id.encode('ascii', 'replace').decode() 241 artifact_id = artifact_id.replace('\\', '?') 242 artifacts[artifact_id] = { 243 'filePath': artifact_path, 244 } 245 return artifacts 246 247 248 class TastTest(RemoteTest): 249 250 def __init__(self, args, unknown_args): 251 super().__init__(args, unknown_args) 252 253 self._suite_name = args.suite_name 254 self._tast_vars = args.tast_vars 255 self._tast_retries = args.tast_retries 256 self._tests = args.tests 257 # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it 258 # here and parse it later to integrate it into Tast executions. 259 self._gtest_style_filter = args.gtest_filter 260 self._attr_expr = args.attr_expr 261 self._should_strip = args.strip_chrome 262 self._deploy_chrome = args.deploy_chrome 263 264 if not self._logs_dir: 265 # The host-side Tast bin returns 0 when tests fail, so we need to capture 266 # and parse its json results to reliably determine if tests fail. 267 raise TestFormatError( 268 'When using the host-side Tast bin, "--logs-dir" must be passed in ' 269 'order to parse its results.') 270 271 # If the first test filter is negative, it should be safe to assume all of 272 # them are, so just test the first filter. 273 if self._gtest_style_filter and self._gtest_style_filter[0] == '-': 274 raise TestFormatError('Negative test filters not supported for Tast.') 275 276 @property 277 def suite_name(self): 278 return self._suite_name 279 280 def build_test_command(self): 281 unsupported_args = [ 282 '--test-launcher-retry-limit', 283 '--test-launcher-batch-limit', 284 '--gtest_repeat', 285 ] 286 for unsupported_arg in unsupported_args: 287 if any(arg.startswith(unsupported_arg) for arg in self._additional_args): 288 logging.info( 289 '%s not supported for Tast tests. The arg will be ignored.', 290 unsupported_arg) 291 self._additional_args = [ 292 arg for arg in self._additional_args 293 if not arg.startswith(unsupported_arg) 294 ] 295 296 self._test_cmd.extend(['--deploy', '--mount']) 297 self._test_cmd += [ 298 '--build-dir', 299 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH) 300 ] + self._additional_args 301 302 # Capture tast's results in the logs dir as well. 303 if self._logs_dir: 304 self._test_cmd += [ 305 '--results-dir', 306 self._logs_dir, 307 ] 308 self._test_cmd += [ 309 '--tast-total-shards=%d' % self._test_launcher_total_shards, 310 '--tast-shard-index=%d' % self._test_launcher_shard_index, 311 ] 312 # If we're using a test filter, replace the contents of the Tast 313 # conditional with a long list of "name:test" expressions, one for each 314 # test in the filter. 315 if self._gtest_style_filter: 316 if self._attr_expr or self._tests: 317 logging.warning( 318 'Presence of --gtest_filter will cause the specified Tast expr' 319 ' or test list to be ignored.') 320 names = [] 321 for test in self._gtest_style_filter.split(':'): 322 names.append('"name:%s"' % test) 323 self._attr_expr = '(' + ' || '.join(names) + ')' 324 325 if self._attr_expr: 326 # Don't use shlex.quote() here. Something funky happens with the arg 327 # as it gets passed down from cros_run_test to tast. (Tast picks up the 328 # escaping single quotes and complains that the attribute expression 329 # "must be within parentheses".) 330 self._test_cmd.append('--tast=%s' % self._attr_expr) 331 else: 332 self._test_cmd.append('--tast') 333 self._test_cmd.extend(self._tests) 334 335 for v in self._tast_vars or []: 336 self._test_cmd.extend(['--tast-var', v]) 337 338 if self._tast_retries: 339 self._test_cmd.append('--tast-retries=%d' % self._tast_retries) 340 341 # Mounting ash-chrome gives it enough disk space to not need stripping, 342 # but only for one not instrumented with code coverage. 343 if not self._should_strip: 344 self._test_cmd.append('--nostrip') 345 346 def post_run(self, return_code): 347 tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') 348 if not os.path.exists(tast_results_path): 349 logging.error( 350 'Tast results not found at %s. Falling back to generic result ' 351 'reporting.', tast_results_path) 352 return super().post_run(return_code) 353 354 # See the link below for the format of the results: 355 # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult 356 with jsonlines.open(tast_results_path) as reader: 357 tast_results = collections.deque(reader) 358 359 suite_results = base_test_result.TestRunResults() 360 for test in tast_results: 361 errors = test['errors'] 362 start, end = test['start'], test['end'] 363 # Use dateutil to parse the timestamps since datetime can't handle 364 # nanosecond precision. 365 duration = dateutil.parser.parse(end) - dateutil.parser.parse(start) 366 # If the duration is negative, Tast has likely reported an incorrect 367 # duration. See https://issuetracker.google.com/issues/187973541. Round 368 # up to 0 in that case to avoid confusing RDB. 369 duration_ms = max(duration.total_seconds() * 1000, 0) 370 if bool(test['skipReason']): 371 result = base_test_result.ResultType.SKIP 372 elif errors: 373 result = base_test_result.ResultType.FAIL 374 else: 375 result = base_test_result.ResultType.PASS 376 primary_error_message = None 377 error_log = '' 378 if errors: 379 # See the link below for the format of these errors: 380 # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go 381 primary_error_message = errors[0]['reason'] 382 for err in errors: 383 error_log += err['stack'] + '\n' 384 debug_link = ("If you're unsure why this test failed, consult the steps " 385 'outlined <a href="%s">here</a>.' % TAST_DEBUG_DOC) 386 base_result = base_test_result.BaseTestResult( 387 test['name'], result, duration=duration_ms, log=error_log) 388 suite_results.AddResult(base_result) 389 self._maybe_handle_perf_results(test['name']) 390 391 if self._rdb_client: 392 # Walk the contents of the test's "outDir" and atttach any file found 393 # inside as an RDB 'artifact'. (This could include system logs, screen 394 # shots, etc.) 395 artifacts = self.get_artifacts(test['outDir']) 396 html_artifact = debug_link 397 if result == base_test_result.ResultType.SKIP: 398 html_artifact = 'Test was skipped because: ' + test['skipReason'] 399 self._rdb_client.Post( 400 test['name'], 401 result, 402 duration_ms, 403 error_log, 404 None, 405 artifacts=artifacts, 406 failure_reason=primary_error_message, 407 html_artifact=html_artifact) 408 409 if self._rdb_client and self._logs_dir: 410 # Attach artifacts from the device that don't apply to a single test. 411 artifacts = self.get_artifacts( 412 os.path.join(self._logs_dir, 'system_logs')) 413 artifacts.update( 414 self.get_artifacts(os.path.join(self._logs_dir, 'crashes'))) 415 self._rdb_client.ReportInvocationLevelArtifacts(artifacts) 416 417 if self._test_launcher_summary_output: 418 with open(self._test_launcher_summary_output, 'w') as f: 419 json.dump(json_results.GenerateResultsDict([suite_results]), f) 420 421 if not suite_results.DidRunPass(): 422 return 1 423 if return_code: 424 logging.warning( 425 'No failed tests found, but exit code of %d was returned from ' 426 'cros_run_test.', return_code) 427 return return_code 428 return 0 429 430 def _maybe_handle_perf_results(self, test_name): 431 """Prepares any perf results from |test_name| for process_perf_results. 432 433 - process_perf_results looks for top level directories containing a 434 perf_results.json file and a test_results.json file. The directory names 435 are used as the benchmark names. 436 - If a perf_results.json or results-chart.json file exists in the 437 |test_name| results directory, a top level directory is created and the 438 perf results file is copied to perf_results.json. 439 - A trivial test_results.json file is also created to indicate that the test 440 succeeded (this function would not be called otherwise). 441 - When process_perf_results is run, it will find the expected files in the 442 named directory and upload the benchmark results. 443 """ 444 445 perf_results = os.path.join(self._logs_dir, 'tests', test_name, 446 'perf_results.json') 447 # TODO(stevenjb): Remove check for crosbolt results-chart.json file. 448 if not os.path.exists(perf_results): 449 perf_results = os.path.join(self._logs_dir, 'tests', test_name, 450 'results-chart.json') 451 if os.path.exists(perf_results): 452 benchmark_dir = os.path.join(self._logs_dir, test_name) 453 if not os.path.isdir(benchmark_dir): 454 os.makedirs(benchmark_dir) 455 shutil.copyfile(perf_results, 456 os.path.join(benchmark_dir, 'perf_results.json')) 457 # process_perf_results.py expects a test_results.json file. 458 test_results = {'valid': True, 'failures': []} 459 with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out: 460 json.dump(test_results, out) 461 462 463 class GTestTest(RemoteTest): 464 465 # The following list corresponds to paths that should not be copied over to 466 # the device during tests. In other words, these files are only ever used on 467 # the host. 468 _FILE_IGNORELIST = [ 469 re.compile(r'.*build/android.*'), 470 re.compile(r'.*build/chromeos.*'), 471 re.compile(r'.*build/cros_cache.*'), 472 # The following matches anything under //testing/ that isn't under 473 # //testing/buildbot/filters/. 474 re.compile(r'.*testing/(?!buildbot/filters).*'), 475 re.compile(r'.*third_party/chromite.*'), 476 ] 477 478 def __init__(self, args, unknown_args): 479 super().__init__(args, unknown_args) 480 481 self._test_cmd = ['vpython3'] + self._test_cmd 482 if not args.clean: 483 self._test_cmd += ['--no-clean'] 484 485 self._test_exe = args.test_exe 486 self._runtime_deps_path = args.runtime_deps_path 487 self._vpython_dir = args.vpython_dir 488 489 self._on_device_script = None 490 self._env_vars = args.env_var 491 self._stop_ui = args.stop_ui 492 self._as_root = args.as_root 493 self._trace_dir = args.trace_dir 494 self._run_test_sudo_helper = args.run_test_sudo_helper 495 self._set_selinux_label = args.set_selinux_label 496 self._use_deployed_dbus_configs = args.use_deployed_dbus_configs 497 498 @property 499 def suite_name(self): 500 return self._test_exe 501 502 def build_test_command(self): 503 # To keep things easy for us, ensure both types of output locations are 504 # the same. 505 if self._test_launcher_summary_output and self._logs_dir: 506 json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.' 507 if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir): 508 raise TestFormatError( 509 '--test-launcher-summary-output and --logs-dir must point to ' 510 'the same directory.') 511 512 if self._test_launcher_summary_output: 513 result_dir, result_file = os.path.split( 514 self._test_launcher_summary_output) 515 # If args.test_launcher_summary_output is a file in cwd, result_dir will 516 # be an empty string, so replace it with '.' when this is the case so 517 # cros_run_test can correctly handle it. 518 if not result_dir: 519 result_dir = '.' 520 device_result_file = '/tmp/%s' % result_file 521 self._test_cmd += [ 522 '--results-src', 523 device_result_file, 524 '--results-dest-dir', 525 result_dir, 526 ] 527 528 if self._trace_dir and self._logs_dir: 529 trace_path = os.path.dirname(self._trace_dir) or '.' 530 if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir): 531 raise TestFormatError( 532 '--trace-dir and --logs-dir must point to the same directory.') 533 534 if self._trace_dir: 535 trace_path, trace_dirname = os.path.split(self._trace_dir) 536 device_trace_dir = '/tmp/%s' % trace_dirname 537 self._test_cmd += [ 538 '--results-src', 539 device_trace_dir, 540 '--results-dest-dir', 541 trace_path, 542 ] 543 544 # Build the shell script that will be used on the device to invoke the test. 545 # Stored here as a list of lines. 546 device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] 547 for var_name, var_val in self._env_vars: 548 device_test_script_contents += ['export %s=%s' % (var_name, var_val)] 549 550 if self._vpython_dir: 551 vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, 552 'vpython3') 553 cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, 554 'bin', 'python3') 555 if not os.path.exists(vpython_path) or not os.path.exists(cpython_path): 556 raise TestFormatError( 557 '--vpython-dir must point to a dir with both ' 558 'infra/3pp/tools/cpython3 and infra/tools/luci/vpython3 ' 559 'installed.') 560 vpython_spec_path = os.path.relpath( 561 os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir) 562 # Initialize the vpython cache. This can take 10-20s, and some tests 563 # can't afford to wait that long on the first invocation. 564 device_test_script_contents.extend([ 565 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' % 566 (self._vpython_dir, self._vpython_dir), 567 'vpython3 -vpython-spec %s -vpython-tool install' % 568 (vpython_spec_path), 569 ]) 570 571 test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d ' 572 '--test-launcher-total-shards=%d' % 573 (self._test_exe, self._test_launcher_shard_index, 574 self._test_launcher_total_shards)) 575 if self._test_launcher_summary_output: 576 test_invocation += ' --test-launcher-summary-output=%s' % ( 577 device_result_file) 578 579 if self._trace_dir: 580 device_test_script_contents.extend([ 581 'rm -rf %s' % device_trace_dir, 582 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir, 583 ]) 584 test_invocation += ' --trace-dir=%s' % device_trace_dir 585 586 if self._run_test_sudo_helper: 587 device_test_script_contents.extend([ 588 'TEST_SUDO_HELPER_PATH=$(mktemp)', 589 './test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &', 590 'TEST_SUDO_HELPER_PID=$!' 591 ]) 592 test_invocation += ( 593 ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}') 594 595 # Append the selinux labels. The 'setfiles' command takes a file with each 596 # line consisting of "<file-regex> <file-type> <new-label>", where '--' is 597 # the type of a regular file. 598 if self._set_selinux_label: 599 for label_pair in self._set_selinux_label: 600 filename, label = label_pair.split('=', 1) 601 specfile = filename + '.specfile' 602 device_test_script_contents.extend([ 603 'echo %s -- %s > %s' % (filename, label, specfile), 604 'setfiles -F %s %s' % (specfile, filename), 605 ]) 606 607 # Mount the deploy dbus config dir on top of chrome's dbus dir. Send SIGHUP 608 # to dbus daemon to reload config from the newly mounted dir. 609 if self._use_deployed_dbus_configs: 610 device_test_script_contents.extend([ 611 'mount --bind ./dbus /opt/google/chrome/dbus', 612 'kill -s HUP $(pgrep dbus)', 613 ]) 614 615 if self._additional_args: 616 test_invocation += ' %s' % ' '.join(self._additional_args) 617 618 if self._stop_ui: 619 device_test_script_contents += [ 620 'stop ui', 621 ] 622 # Send a user activity ping to powerd to ensure the display is on. 623 device_test_script_contents += [ 624 'dbus-send --system --type=method_call' 625 ' --dest=org.chromium.PowerManager /org/chromium/PowerManager' 626 ' org.chromium.PowerManager.HandleUserActivity int32:0' 627 ] 628 # The UI service on the device owns the chronos user session, so shutting 629 # it down as chronos kills the entire execution of the test. So we'll have 630 # to run as root up until the test invocation. 631 test_invocation = ( 632 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation) 633 # And we'll need to chown everything since cros_run_test's "--as-chronos" 634 # option normally does that for us. 635 device_test_script_contents.append('chown -R chronos: ../..') 636 elif not self._as_root: 637 self._test_cmd += [ 638 # Some tests fail as root, so run as the less privileged user 639 # 'chronos'. 640 '--as-chronos', 641 ] 642 643 device_test_script_contents.append(test_invocation) 644 device_test_script_contents.append('TEST_RETURN_CODE=$?') 645 646 # (Re)start ui after all tests are done. This is for developer convenienve. 647 # Without this, the device would remain in a black screen which looks like 648 # powered off. 649 if self._stop_ui: 650 device_test_script_contents += [ 651 'start ui', 652 ] 653 654 # Stop the crosier helper. 655 if self._run_test_sudo_helper: 656 device_test_script_contents.extend([ 657 'pkill -P $TEST_SUDO_HELPER_PID', 658 'kill $TEST_SUDO_HELPER_PID', 659 'unlink ${TEST_SUDO_HELPER_PATH}', 660 ]) 661 662 # Undo the dbus config mount and reload dbus config. 663 if self._use_deployed_dbus_configs: 664 device_test_script_contents.extend([ 665 'umount /opt/google/chrome/dbus', 666 'kill -s HUP $(pgrep dbus)', 667 ]) 668 669 # This command should always be the last bash commandline so infra can 670 # correctly get the error code from test invocations. 671 device_test_script_contents.append('exit $TEST_RETURN_CODE') 672 673 self._on_device_script = self.write_test_script_to_disk( 674 device_test_script_contents) 675 676 runtime_files = [os.path.relpath(self._on_device_script)] 677 runtime_files += self._read_runtime_files() 678 if self._vpython_dir: 679 # --vpython-dir is relative to the out dir, but --files-from expects paths 680 # relative to src dir, so fix the path up a bit. 681 runtime_files.append( 682 os.path.relpath( 683 os.path.abspath( 684 os.path.join(self._path_to_outdir, self._vpython_dir)), 685 CHROMIUM_SRC_PATH)) 686 687 self._test_cmd.extend( 688 ['--files-from', 689 self.write_runtime_files_to_disk(runtime_files)]) 690 691 self._test_cmd += [ 692 '--', 693 './' + os.path.relpath(self._on_device_script, self._path_to_outdir) 694 ] 695 696 def _read_runtime_files(self): 697 if not self._runtime_deps_path: 698 return [] 699 700 abs_runtime_deps_path = os.path.abspath( 701 os.path.join(self._path_to_outdir, self._runtime_deps_path)) 702 with open(abs_runtime_deps_path) as runtime_deps_file: 703 files = [l.strip() for l in runtime_deps_file if l] 704 rel_file_paths = [] 705 for f in files: 706 rel_file_path = os.path.relpath( 707 os.path.abspath(os.path.join(self._path_to_outdir, f))) 708 if not any(regex.match(rel_file_path) for regex in self._FILE_IGNORELIST): 709 rel_file_paths.append(rel_file_path) 710 return rel_file_paths 711 712 def post_run(self, _): 713 if self._on_device_script: 714 os.remove(self._on_device_script) 715 716 if self._test_launcher_summary_output and self._rdb_client: 717 logging.error('Native ResultDB integration is not supported for GTests. ' 718 'Upload results via result_adapter instead. ' 719 'See crbug.com/1330441.') 720 721 722 def device_test(args, unknown_args): 723 # cros_run_test has trouble with relative paths that go up directories, 724 # so cd to src/, which should be the root of all data deps. 725 os.chdir(CHROMIUM_SRC_PATH) 726 727 # TODO: Remove the above when depot_tool's pylint is updated to include the 728 # fix to https://github.com/PyCQA/pylint/issues/710. 729 if args.test_type == 'tast': 730 test = TastTest(args, unknown_args) 731 else: 732 test = GTestTest(args, unknown_args) 733 734 test.build_test_command() 735 logging.info('Running the following command on the device:') 736 logging.info(' '.join(test.test_cmd)) 737 738 return test.run_test() 739 740 741 def host_cmd(args, cmd_args): 742 if not cmd_args: 743 raise TestFormatError('Must specify command to run on the host.') 744 if args.deploy_chrome and not args.path_to_outdir: 745 raise TestFormatError( 746 '--path-to-outdir must be specified if --deploy-chrome is passed.') 747 748 cros_run_test_cmd = [ 749 CROS_RUN_TEST_PATH, 750 '--board', 751 args.board, 752 '--cache-dir', 753 os.path.join(CHROMIUM_SRC_PATH, args.cros_cache), 754 ] 755 if args.use_vm: 756 cros_run_test_cmd += [ 757 '--start', 758 # Don't persist any filesystem changes after the VM shutsdown. 759 '--copy-on-write', 760 ] 761 else: 762 if args.fetch_cros_hostname: 763 cros_run_test_cmd += ['--device', get_cros_hostname()] 764 else: 765 cros_run_test_cmd += [ 766 '--device', args.device if args.device else LAB_DUT_HOSTNAME 767 ] 768 if args.verbose: 769 cros_run_test_cmd.append('--debug') 770 if args.flash: 771 cros_run_test_cmd.append('--flash') 772 if args.public_image: 773 cros_run_test_cmd += ['--public-image'] 774 775 if args.logs_dir: 776 for log in SYSTEM_LOG_LOCATIONS: 777 cros_run_test_cmd += ['--results-src', log] 778 cros_run_test_cmd += [ 779 '--results-dest-dir', 780 os.path.join(args.logs_dir, 'system_logs') 781 ] 782 783 test_env = setup_env() 784 if args.deploy_chrome: 785 # Mounting ash-chrome gives it enough disk space to not need stripping 786 # most of the time. 787 cros_run_test_cmd.extend(['--deploy', '--mount']) 788 789 if not args.strip_chrome: 790 cros_run_test_cmd.append('--nostrip') 791 792 cros_run_test_cmd += [ 793 '--build-dir', 794 os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir) 795 ] 796 797 cros_run_test_cmd += [ 798 '--host-cmd', 799 '--', 800 ] + cmd_args 801 802 logging.info('Running the following command:') 803 logging.info(' '.join(cros_run_test_cmd)) 804 805 return subprocess.call( 806 cros_run_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=test_env) 807 808 809 def get_cros_hostname_from_bot_id(bot_id): 810 """Parse hostname from a chromeos-swarming bot id.""" 811 for prefix in ['cros-', 'crossk-']: 812 if bot_id.startswith(prefix): 813 return bot_id[len(prefix):] 814 return bot_id 815 816 817 def get_cros_hostname(): 818 """Fetch bot_id from env var and parse hostname.""" 819 820 # In chromeos-swarming, we can extract hostname from bot ID, since 821 # bot ID is formatted as "{prefix}{hostname}". 822 bot_id = os.environ.get('SWARMING_BOT_ID') 823 if bot_id: 824 return get_cros_hostname_from_bot_id(bot_id) 825 826 logging.warning( 827 'Attempted to read from SWARMING_BOT_ID env var and it was' 828 ' not defined. Will set %s as device instead.', LAB_DUT_HOSTNAME) 829 return LAB_DUT_HOSTNAME 830 831 832 def setup_env(): 833 """Returns a copy of the current env with some needed vars added.""" 834 env = os.environ.copy() 835 # Some chromite scripts expect chromite/bin to be on PATH. 836 env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin') 837 # deploy_chrome needs a set of GN args used to build chrome to determine if 838 # certain libraries need to be pushed to the device. It looks for the args via 839 # an env var. To trigger the default deploying behavior, give it a dummy set 840 # of args. 841 # TODO(crbug.com/40567963): Make the GN-dependent deps controllable via cmd 842 # line args. 843 if not env.get('GN_ARGS'): 844 env['GN_ARGS'] = 'enable_nacl = true' 845 if not env.get('USE'): 846 env['USE'] = 'highdpi' 847 return env 848 849 850 def add_common_args(*parsers): 851 for parser in parsers: 852 parser.add_argument('--verbose', '-v', action='store_true') 853 parser.add_argument( 854 '--board', type=str, required=True, help='Type of CrOS device.') 855 parser.add_argument( 856 '--deploy-chrome', 857 action='store_true', 858 help='Will deploy a locally built ash-chrome binary to the device ' 859 'before running the host-cmd.') 860 parser.add_argument( 861 '--cros-cache', 862 type=str, 863 default=DEFAULT_CROS_CACHE, 864 help='Path to cros cache.') 865 parser.add_argument( 866 '--path-to-outdir', 867 type=str, 868 required=True, 869 help='Path to output directory, all of whose contents will be ' 870 'deployed to the device.') 871 parser.add_argument( 872 '--runtime-deps-path', 873 type=str, 874 help='Runtime data dependency file from GN.') 875 parser.add_argument( 876 '--vpython-dir', 877 type=str, 878 help='Location on host of a directory containing a vpython binary to ' 879 'deploy to the device before the test starts. The location of ' 880 'this dir will be added onto PATH in the device. WARNING: The ' 881 'arch of the device might not match the arch of the host, so ' 882 'avoid using "${platform}" when downloading vpython via CIPD.') 883 parser.add_argument( 884 '--logs-dir', 885 type=str, 886 dest='logs_dir', 887 help='Will copy everything under /var/log/ from the device after the ' 888 'test into the specified dir.') 889 # Shard args are parsed here since we might also specify them via env vars. 890 parser.add_argument( 891 '--test-launcher-shard-index', 892 type=int, 893 default=os.environ.get('GTEST_SHARD_INDEX', 0), 894 help='Index of the external shard to run.') 895 parser.add_argument( 896 '--test-launcher-total-shards', 897 type=int, 898 default=os.environ.get('GTEST_TOTAL_SHARDS', 1), 899 help='Total number of external shards.') 900 parser.add_argument( 901 '--flash', 902 action='store_true', 903 help='Will flash the device to the current SDK version before running ' 904 'the test.') 905 parser.add_argument( 906 '--no-flash', 907 action='store_false', 908 dest='flash', 909 help='Will not flash the device before running the test.') 910 parser.add_argument( 911 '--public-image', 912 action='store_true', 913 help='Will flash a public "full" image to the device.') 914 parser.add_argument( 915 '--magic-vm-cache', 916 help='Path to the magic CrOS VM cache dir. See the comment above ' 917 '"magic_cros_vm_cache" in mixins.pyl for more info.') 918 919 vm_or_device_group = parser.add_mutually_exclusive_group() 920 vm_or_device_group.add_argument( 921 '--use-vm', 922 action='store_true', 923 help='Will run the test in the VM instead of a device.') 924 vm_or_device_group.add_argument( 925 '--device', 926 type=str, 927 help='Hostname (or IP) of device to run the test on. This arg is not ' 928 'required if --use-vm is set.') 929 vm_or_device_group.add_argument( 930 '--fetch-cros-hostname', 931 action='store_true', 932 help='Will extract device hostname from the SWARMING_BOT_ID env var if ' 933 'running on ChromeOS Swarming.') 934 935 def main(): 936 parser = argparse.ArgumentParser() 937 subparsers = parser.add_subparsers(dest='test_type') 938 # Host-side test args. 939 host_cmd_parser = subparsers.add_parser( 940 'host-cmd', 941 help='Runs a host-side test. Pass the host-side command to run after ' 942 '"--". If --use-vm is passed, hostname and port for the device ' 943 'will be 127.0.0.1:9222.') 944 host_cmd_parser.set_defaults(func=host_cmd) 945 host_cmd_parser.add_argument( 946 '--strip-chrome', 947 action='store_true', 948 help='Strips symbols from ash-chrome before deploying to the device.') 949 950 gtest_parser = subparsers.add_parser( 951 'gtest', help='Runs a device-side gtest.') 952 gtest_parser.set_defaults(func=device_test) 953 gtest_parser.add_argument( 954 '--test-exe', 955 type=str, 956 required=True, 957 help='Path to test executable to run inside the device.') 958 959 # GTest args. Some are passed down to the test binary in the device. Others 960 # are parsed here since they might need tweaking or special handling. 961 gtest_parser.add_argument( 962 '--test-launcher-summary-output', 963 type=str, 964 help='When set, will pass the same option down to the test and retrieve ' 965 'its result file at the specified location.') 966 gtest_parser.add_argument( 967 '--stop-ui', 968 action='store_true', 969 help='Will stop the UI service in the device before running the test. ' 970 'Also start the UI service after all tests are done.') 971 gtest_parser.add_argument( 972 '--as-root', 973 action='store_true', 974 help='Will run the test as root on the device. Runs as user=chronos ' 975 'otherwise. This is mutually exclusive with "--stop-ui" above due to ' 976 'setup issues.') 977 gtest_parser.add_argument( 978 '--trace-dir', 979 type=str, 980 help='When set, will pass down to the test to generate the trace and ' 981 'retrieve the trace files to the specified location.') 982 gtest_parser.add_argument( 983 '--env-var', 984 nargs=2, 985 action='append', 986 default=[], 987 help='Env var to set on the device for the duration of the test. ' 988 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify ' 989 'multiple times for more than one var.') 990 gtest_parser.add_argument( 991 '--run-test-sudo-helper', 992 action='store_true', 993 help='When set, will run test_sudo_helper before the test and stop it ' 994 'after test finishes.') 995 gtest_parser.add_argument( 996 "--no-clean", 997 action="store_false", 998 dest="clean", 999 default=True, 1000 help="Do not clean up the deployed files after running the test. " 1001 "Only supported for --remote-cmd tests") 1002 gtest_parser.add_argument( 1003 '--set-selinux-label', 1004 action='append', 1005 default=[], 1006 help='Set the selinux label for a file before running. The format is:\n' 1007 ' --set-selinux-label=<filename>=<label>\n' 1008 'So:\n' 1009 ' --set-selinux-label=my_test=u:r:cros_foo_label:s0\n' 1010 'You can specify it more than one time to set multiple files tags.') 1011 gtest_parser.add_argument( 1012 '--use-deployed-dbus-configs', 1013 action='store_true', 1014 help='When set, will bind mount deployed dbus config to chrome dbus dir ' 1015 'and ask dbus daemon to reload config before running tests.') 1016 1017 # Tast test args. 1018 # pylint: disable=line-too-long 1019 tast_test_parser = subparsers.add_parser( 1020 'tast', 1021 help='Runs a device-side set of Tast tests. For more details, see: ' 1022 'https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/running_tests.md' 1023 ) 1024 tast_test_parser.set_defaults(func=device_test) 1025 tast_test_parser.add_argument( 1026 '--suite-name', 1027 type=str, 1028 required=True, 1029 help='Name to apply to the set of Tast tests to run. This has no effect ' 1030 'on what is executed, but is used mainly for test results reporting ' 1031 'and tracking (eg: flakiness dashboard).') 1032 tast_test_parser.add_argument( 1033 '--test-launcher-summary-output', 1034 type=str, 1035 help='Generates a simple GTest-style JSON result file for the test run.') 1036 tast_test_parser.add_argument( 1037 '--attr-expr', 1038 type=str, 1039 help='A boolean expression whose matching tests will run ' 1040 '(eg: ("dep:chrome")).') 1041 tast_test_parser.add_argument( 1042 '--strip-chrome', 1043 action='store_true', 1044 help='Strips symbols from ash-chrome before deploying to the device.') 1045 tast_test_parser.add_argument( 1046 '--tast-var', 1047 action='append', 1048 dest='tast_vars', 1049 help='Runtime variables for Tast tests, and the format are expected to ' 1050 'be "key=value" pairs.') 1051 tast_test_parser.add_argument( 1052 '--tast-retries', 1053 type=int, 1054 dest='tast_retries', 1055 help='Number of retries for failed Tast tests on the same DUT.') 1056 tast_test_parser.add_argument( 1057 '--test', 1058 '-t', 1059 action='append', 1060 dest='tests', 1061 help='A Tast test to run in the device (eg: "login.Chrome").') 1062 tast_test_parser.add_argument( 1063 '--gtest_filter', 1064 type=str, 1065 help="Similar to GTest's arg of the same name, this will filter out the " 1066 "specified tests from the Tast run. However, due to the nature of Tast's " 1067 'cmd-line API, this will overwrite the value(s) of "--test" above.') 1068 1069 add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) 1070 args, unknown_args = parser.parse_known_args() 1071 1072 if args.test_type == 'gtest' and args.stop_ui and args.as_root: 1073 parser.error('Unable to run gtests with both --stop-ui and --as-root') 1074 1075 # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are 1076 # running. The assumption is that only one verbosity incrase would be meant 1077 # for this script since it's a boolean value instead of increasing verbosity 1078 # with more instances. 1079 verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')] 1080 if verbose_flags: 1081 unknown_args += verbose_flags[1:] 1082 1083 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) 1084 1085 if not args.use_vm and not args.device and not args.fetch_cros_hostname: 1086 logging.warning( 1087 'The test runner is now assuming running in the lab environment, if ' 1088 'this is unintentional, please re-invoke the test runner with the ' 1089 '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg ' 1090 'to specify a DUT.') 1091 1092 # If we're not running on a VM, but haven't specified a hostname, assume 1093 # we're on a lab bot and are trying to run a test on a lab DUT. See if the 1094 # magic lab DUT hostname resolves to anything. (It will in the lab and will 1095 # not on dev machines.) 1096 try: 1097 socket.getaddrinfo(LAB_DUT_HOSTNAME, None) 1098 except socket.gaierror: 1099 logging.error('The default lab DUT hostname of %s is unreachable.', 1100 LAB_DUT_HOSTNAME) 1101 return 1 1102 1103 if args.flash and args.public_image: 1104 # The flashing tools depend on being unauthenticated with GS when flashing 1105 # public images, so make sure the env var GS uses to locate its creds is 1106 # unset in that case. 1107 os.environ.pop('BOTO_CONFIG', None) 1108 1109 if args.magic_vm_cache: 1110 full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache) 1111 if os.path.exists(full_vm_cache_path): 1112 with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f: 1113 f.write('non-empty file to make swarming persist this cache') 1114 1115 return args.func(args, unknown_args) 1116 1117 1118 if __name__ == '__main__': 1119 sys.exit(main())