tasks_adb_remote.py (10283B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 import os 5 import posixpath 6 import sys 7 import tempfile 8 from datetime import timedelta 9 10 from mozdevice import ADBDevice, ADBError, ADBProcessError, ADBTimeoutError 11 12 from .adaptor import xdr_annotate 13 from .remote import init_device 14 from .results import TestOutput, escape_cmdline 15 16 TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__)) 17 JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR)) 18 JS_TESTS_DIR = posixpath.join(JS_DIR, "tests") 19 TEST_DIR = os.path.join(JS_DIR, "jit-test", "tests") 20 21 22 def aggregate_script_stdout(stdout_lines, prefix, tempdir, uniq_tag, tests, options): 23 test = None 24 tStart = None 25 cmd = "" 26 stdout = "" 27 28 # Use to debug this script in case of assertion failure. 29 meta_history = [] 30 last_line = "" 31 32 # Assert that the streamed content is not interrupted. 33 ended = False 34 35 # Check if the tag is present, if so, this is controlled output 36 # produced by the test runner, otherwise this is stdout content. 37 try: 38 for line in stdout_lines: 39 last_line = line 40 if line.startswith(uniq_tag): 41 meta = line[len(uniq_tag) :].strip() 42 meta_history.append(meta) 43 if meta.startswith("START="): 44 assert test is None 45 params = meta[len("START=") :].split(",") 46 test_idx = int(params[0]) 47 test = tests[test_idx] 48 tStart = timedelta(seconds=float(params[1])) 49 cmd = test.command( 50 prefix, 51 posixpath.join(options.remote_test_root, "lib/"), 52 posixpath.join(options.remote_test_root, "modules/"), 53 tempdir, 54 posixpath.join(options.remote_test_root, "tests"), 55 ) 56 stdout = "" 57 if options.show_cmd: 58 print(escape_cmdline(cmd)) 59 elif meta.startswith("STOP="): 60 assert test is not None 61 params = meta[len("STOP=") :].split(",") 62 exitcode = int(params[0]) 63 dt = timedelta(seconds=float(params[1])) - tStart 64 yield TestOutput( 65 test, 66 cmd, 67 stdout, 68 # NOTE: mozdevice fuse stdout and stderr. Thus, we are 69 # using stdout for both stdout and stderr. So far, 70 # doing so did not cause any issues. 71 stdout, 72 exitcode, 73 dt.total_seconds(), 74 dt > timedelta(seconds=int(options.timeout)), 75 ) 76 stdout = "" 77 cmd = "" 78 test = None 79 elif meta.startswith("RETRY="): 80 # On timeout, we discard the first timeout to avoid a 81 # random hang on pthread_join. 82 assert test is not None 83 stdout = "" 84 cmd = "" 85 test = None 86 else: 87 assert meta.startswith("THE_END") 88 ended = True 89 else: 90 assert uniq_tag not in line 91 stdout += line 92 93 # This assertion fails if the streamed content is interrupted, either 94 # by unplugging the phone or some adb failures. 95 assert ended 96 except AssertionError as e: 97 sys.stderr.write("Metadata history:\n{}\n".format("\n".join(meta_history))) 98 sys.stderr.write(f"Last line: {last_line}\n") 99 raise e 100 101 102 def setup_device(prefix, options): 103 try: 104 device = init_device(options) 105 106 def replace_lib_file(path, name): 107 localfile = os.path.join(JS_TESTS_DIR, *path) 108 remotefile = posixpath.join(options.remote_test_root, "lib", name) 109 device.push(localfile, remotefile, timeout=10) 110 111 prefix[0] = posixpath.join(options.remote_test_root, "bin", "js") 112 tempdir = posixpath.join(options.remote_test_root, "tmp") 113 114 print("tasks_adb_remote.py : Transfering test files") 115 116 # Push tests & lib directories. 117 device.push(os.path.dirname(TEST_DIR), options.remote_test_root, timeout=600) 118 119 # Substitute lib files which are aliasing non262 files. 120 replace_lib_file(["non262", "shell.js"], "non262.js") 121 replace_lib_file(["non262", "reflect-parse", "Match.js"], "match.js") 122 replace_lib_file(["non262", "Math", "shell.js"], "math.js") 123 device.chmod(options.remote_test_root, recursive=True) 124 125 print("tasks_adb_remote.py : Device initialization completed") 126 return device, tempdir 127 except (ADBError, ADBTimeoutError): 128 print( 129 "TEST-UNEXPECTED-FAIL | tasks_adb_remote.py : " 130 + "Device initialization failed" 131 ) 132 raise 133 134 135 def script_preamble(tag, prefix, options): 136 timeout = int(options.timeout) 137 retry = int(options.timeout_retry) 138 lib_path = os.path.dirname(prefix[0]) 139 return f""" 140 export LD_LIBRARY_PATH={lib_path} 141 142 do_test() 143 {{ 144 local idx=$1; shift; 145 local attempt=$1; shift; 146 147 # Read 10ms timestamp in seconds using shell builtins and /proc/uptime. 148 local time; 149 local unused; 150 151 # When printing the tag, we prefix by a new line, in case the 152 # previous command output did not contain any new line. 153 read time unused < /proc/uptime 154 echo '\\n{tag}START='$idx,$time 155 timeout {timeout}s "$@" 156 local rc=$? 157 read time unused < /proc/uptime 158 159 # Retry on timeout, to mute unlikely pthread_join hang issue. 160 # 161 # The timeout command send a SIGTERM signal, which should return 143 162 # (=128+15). However, due to a bug in tinybox, it returns 142. 163 if test \\( $rc -eq 143 -o $rc -eq 142 \\) -a $attempt -lt {retry}; then 164 echo '\\n{tag}RETRY='$rc,$time 165 attempt=$((attempt + 1)) 166 do_test $idx $attempt "$@" 167 else 168 echo '\\n{tag}STOP='$rc,$time 169 fi 170 }} 171 172 do_end() 173 {{ 174 echo '\\n{tag}THE_END' 175 }} 176 """ 177 178 179 def setup_script(device, prefix, tempdir, options, uniq_tag, tests): 180 timeout = int(options.timeout) 181 script_timeout = 0 182 try: 183 tmpf = tempfile.NamedTemporaryFile(mode="w", delete=False) 184 tmpf.write(script_preamble(uniq_tag, prefix, options)) 185 for i, test in enumerate(tests): 186 # This test is common to all tasks_*.py files, however, jit-test do 187 # not provide the `run_skipped` option, and all tests are always 188 # enabled. 189 assert test.enable # and not options.run_skipped 190 if options.test_reflect_stringify: 191 raise ValueError("can't run Reflect.stringify tests remotely") 192 193 cmd = test.command( 194 prefix, 195 posixpath.join(options.remote_test_root, "lib/"), 196 posixpath.join(options.remote_test_root, "modules/"), 197 tempdir, 198 posixpath.join(options.remote_test_root, "tests"), 199 ) 200 201 # replace with shlex.join when move to Python 3.8+ 202 cmd = ADBDevice._escape_command_line(cmd) 203 204 env = {} 205 if test.tz_pacific: 206 env["TZ"] = "PST8PDT" 207 envStr = "".join(key + "='" + val + "' " for key, val in env.items()) 208 209 tmpf.write(f"{envStr}do_test {i} 0 {cmd};\n") 210 script_timeout += timeout 211 tmpf.write("do_end;\n") 212 tmpf.close() 213 script = posixpath.join(options.remote_test_root, "test_manifest.sh") 214 device.push(tmpf.name, script) 215 device.chmod(script) 216 print("tasks_adb_remote.py : Batch script created") 217 except Exception as e: 218 print("tasks_adb_remote.py : Batch script failed") 219 raise e 220 finally: 221 if tmpf: 222 os.unlink(tmpf.name) 223 return script, script_timeout 224 225 226 def start_script( 227 device, prefix, tempdir, script, uniq_tag, script_timeout, tests, options 228 ): 229 env = {} 230 231 # Allow ADBError or ADBTimeoutError to terminate the test run, but handle 232 # ADBProcessError in order to support the use of non-zero exit codes in the 233 # JavaScript shell tests. 234 # 235 # The stdout_callback will aggregate each output line, and reconstruct the 236 # output produced by each test, and queue TestOutput in the qResult queue. 237 try: 238 adb_process = device.shell( 239 f"sh {script}", 240 env=env, 241 cwd=options.remote_test_root, 242 timeout=script_timeout, 243 yield_stdout=True, 244 ) 245 yield from aggregate_script_stdout( 246 adb_process, prefix, tempdir, uniq_tag, tests, options 247 ) 248 except ADBProcessError as e: 249 # After a device error, the device is typically in a 250 # state where all further tests will fail so there is no point in 251 # continuing here. 252 sys.stderr.write(f"Error running remote tests: {repr(e)}") 253 254 255 def get_remote_results(tests, prefix, pb, options): 256 """Create a script which batches the run of all tests, and spawn a thread to 257 reconstruct the TestOutput for each test. This is made to avoid multiple 258 `adb.shell` commands which has a high latency. 259 """ 260 device, tempdir = setup_device(prefix, options) 261 262 # Tests are sequentially executed in a batch. The first test executed is in 263 # charge of creating the xdr file for the self-hosted code. 264 if options.use_xdr: 265 tests = xdr_annotate(tests, options) 266 267 # We need tests to be subscriptable to find the test structure matching the 268 # index within the generated script. 269 tests = list(tests) 270 271 # Create a script which spawn each test one after the other, and upload the 272 # script 273 uniq_tag = "@@@TASKS_ADB_REMOTE@@@" 274 script, script_timeout = setup_script( 275 device, prefix, tempdir, options, uniq_tag, tests 276 ) 277 278 yield from start_script( 279 device, prefix, tempdir, script, uniq_tag, script_timeout, tests, options 280 )