remotexpcshelltests.py (31129B)
1 #!/usr/bin/env python 2 # 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 import datetime 8 import os 9 import posixpath 10 import shutil 11 import sys 12 import tempfile 13 import time 14 import uuid 15 from argparse import Namespace 16 from zipfile import ZipFile 17 18 import mozcrash 19 import mozdevice 20 import mozfile 21 import mozinfo 22 import runxpcshelltests as xpcshell 23 from mozdevice import ADBDevice, ADBDeviceFactory, ADBTimeoutError 24 from mozinfo.platforminfo import android_api_to_os_version 25 from mozlog import commandline 26 from xpcshellcommandline import parser_remote 27 28 here = os.path.dirname(os.path.abspath(__file__)) 29 30 31 class RemoteProcessMonitor: 32 processStatus = [] 33 34 def __init__(self, package, device, log, remoteLogFile): 35 self.package = package 36 self.device = device 37 self.log = log 38 self.remoteLogFile = remoteLogFile 39 self.selectedProcess = -1 40 41 @classmethod 42 def pickUnusedProcess(cls): 43 for i in range(len(cls.processStatus)): 44 if not cls.processStatus[i]: 45 cls.processStatus[i] = True 46 return i 47 # No more free processes :( 48 return -1 49 50 @classmethod 51 def freeProcess(cls, processId): 52 cls.processStatus[processId] = False 53 54 def kill(self): 55 self.device.pkill(self.process_name, sig=9, attempts=1) 56 57 def launch_service(self, extra_args, env, selectedProcess, test_name=None): 58 if not self.device.process_exist(self.package): 59 # Make sure the main app is running, this should help making the 60 # tests get foreground priority scheduling. 61 self.device.launch_activity( 62 self.package, 63 intent="org.mozilla.geckoview.test_runner.XPCSHELL_TEST_MAIN", 64 activity_name="TestRunnerActivity", 65 ) 66 # Newer Androids require that background services originate from 67 # active apps, so wait here until the test runner is the top 68 # activity. 69 retries = 20 70 top = self.device.get_top_activity(timeout=60) 71 while top != self.package and retries > 0: 72 self.log.info( 73 "%s | Checking that %s is the top activity." 74 % (test_name, self.package) 75 ) 76 top = self.device.get_top_activity(timeout=60) 77 time.sleep(1) 78 retries -= 1 79 80 self.process_name = self.package + (":xpcshell%d" % selectedProcess) 81 82 retries = 20 83 while retries > 0 and self.device.process_exist(self.process_name): 84 self.log.info( 85 "%s | %s | Killing left-over process %s" 86 % (test_name, self.pid, self.process_name) 87 ) 88 self.kill() 89 time.sleep(1) 90 retries -= 1 91 92 if self.device.process_exist(self.process_name): 93 raise Exception( 94 "%s | %s | Could not kill left-over process" % (test_name, self.pid) 95 ) 96 97 self.device.launch_service( 98 self.package, 99 activity_name=("XpcshellTestRunnerService$i%d" % selectedProcess), 100 moz_env=env, 101 grant_runtime_permissions=False, 102 extra_args=extra_args, 103 out_file=self.remoteLogFile, 104 ) 105 return self.pid 106 107 def wait(self, timeout, interval=0.1, test_name=None): 108 timer = 0 109 status = True 110 111 # wait for log creation on startup 112 retries = 0 113 while retries < 20 / interval and not self.device.is_file(self.remoteLogFile): 114 retries += 1 115 time.sleep(interval) 116 if not self.device.is_file(self.remoteLogFile): 117 self.log.warning( 118 "%s | Failed wait for remote log: %s missing?" 119 % (test_name, self.remoteLogFile) 120 ) 121 122 while self.device.process_exist(self.process_name): 123 time.sleep(interval) 124 timer += interval 125 interval *= 1.5 126 # We're using exponential back-off. To avoid unnecessarily waiting 127 # for too long, cap the maximum sleep interval to 15 seconds. 128 interval = min(15, interval) 129 if timeout and timer > timeout: 130 status = False 131 self.log.info( 132 "remotexpcshelltests.py | %s | %s | Timing out" 133 % (test_name, str(self.pid)) 134 ) 135 self.kill() 136 break 137 return status 138 139 @property 140 def pid(self): 141 """ 142 Determine the pid of the remote process (or the first process with 143 the same name). 144 """ 145 procs = self.device.get_process_list() 146 # limit the comparison to the first 75 characters due to a 147 # limitation in processname length in android. 148 pids = [proc[0] for proc in procs if proc[1] == self.process_name[:75]] 149 if pids is None or len(pids) < 1: 150 return 0 151 return pids[0] 152 153 154 class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread): 155 def __init__(self, *args, **kwargs): 156 xpcshell.XPCShellTestThread.__init__(self, *args, **kwargs) 157 158 self.shellReturnCode = None 159 # embed the mobile params from the harness into the TestThread 160 mobileArgs = kwargs.get("mobileArgs") 161 for key in mobileArgs: 162 setattr(self, key, mobileArgs[key]) 163 self.remoteLogFile = posixpath.join( 164 mobileArgs["remoteLogFolder"], "xpcshell-%s.log" % str(uuid.uuid4()) 165 ) 166 167 def initDir(self, path, mask="777", timeout=None): 168 """Initialize a directory by removing it if it exists, creating it 169 and changing the permissions.""" 170 self.device.rm(path, recursive=True, force=True, timeout=timeout) 171 self.device.mkdir(path, parents=True, timeout=timeout) 172 173 def updateTestPrefsFile(self): 174 # The base method will either be no-op (and return the existing 175 # remote path), or return a path to a new local file. 176 testPrefsFile = xpcshell.XPCShellTestThread.updateTestPrefsFile(self) 177 if testPrefsFile == self.rootPrefsFile: 178 # The pref file is the shared one, which has been already pushed on the 179 # device, and so there is nothing more to do here. 180 return self.rootPrefsFile 181 182 # Push the per-test prefs file in the remote temp dir. 183 remoteTestPrefsFile = posixpath.join(self.remoteTmpDir, "user.js") 184 self.device.push(testPrefsFile, remoteTestPrefsFile) 185 self.device.chmod(remoteTestPrefsFile) 186 os.remove(testPrefsFile) 187 return remoteTestPrefsFile 188 189 def buildCmdTestFile(self, name): 190 remoteDir = self.remoteForLocal(os.path.dirname(name)) 191 if remoteDir == self.remoteHere: 192 remoteName = os.path.basename(name) 193 else: 194 remoteName = posixpath.join(remoteDir, os.path.basename(name)) 195 return [ 196 "-e", 197 'const _TEST_CWD = "%s";' % self.remoteHere, 198 "-e", 199 'const _TEST_FILE = ["%s"];' % remoteName.replace("\\", "/"), 200 ] 201 202 def remoteForLocal(self, local): 203 for mapping in self.pathMapping: 204 if os.path.abspath(mapping.local) == os.path.abspath(local): 205 return mapping.remote 206 return local 207 208 def setupTempDir(self): 209 self.remoteTmpDir = posixpath.join(self.remoteTmpDir, str(uuid.uuid4())) 210 # make sure the temp dir exists 211 self.initDir(self.remoteTmpDir) 212 # env var is set in buildEnvironment 213 self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir 214 return self.remoteTmpDir 215 216 def setupProfileDir(self): 217 profileId = str(uuid.uuid4()) 218 self.profileDir = posixpath.join(self.profileDir, profileId) 219 self.initDir(self.profileDir) 220 if self.interactive or self.singleFile: 221 self.log.info("profile dir is %s" % self.profileDir) 222 self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir 223 self.env["TMPDIR"] = self.profileDir 224 self.remoteMinidumpDir = posixpath.join(self.remoteMinidumpRootDir, profileId) 225 self.initDir(self.remoteMinidumpDir) 226 self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir 227 return self.profileDir 228 229 def clean_temp_dirs(self, name): 230 self.log.info("Cleaning up profile for %s folder: %s" % (name, self.profileDir)) 231 self.device.rm(self.profileDir, force=True, recursive=True) 232 self.device.rm(self.remoteTmpDir, force=True, recursive=True) 233 self.device.rm(self.remoteMinidumpDir, force=True, recursive=True) 234 235 def setupMozinfoJS(self): 236 local = tempfile.mktemp() 237 mozinfo.output_to_file(local) 238 mozInfoJSPath = posixpath.join(self.profileDir, "mozinfo.json") 239 self.device.push(local, mozInfoJSPath) 240 self.device.chmod(mozInfoJSPath) 241 os.remove(local) 242 return mozInfoJSPath 243 244 def logCommand(self, name, completeCmd, testdir): 245 self.log.info("%s | full command: %r" % (name, completeCmd)) 246 self.log.info("%s | current directory: %r" % (name, self.remoteHere)) 247 self.log.info("%s | environment: %s" % (name, self.env)) 248 249 def getHeadFiles(self, test): 250 """Override parent method to find files on remote device. 251 252 Obtains lists of head- files. Returns a list of head files. 253 """ 254 255 def sanitize_list(s, kind): 256 for f in s.strip().split(" "): 257 f = f.strip() 258 if len(f) < 1: 259 continue 260 261 path = posixpath.join(self.remoteHere, f) 262 263 # skip check for file existence: the convenience of discovering 264 # a missing file does not justify the time cost of the round trip 265 # to the device 266 yield path 267 268 self.remoteHere = self.remoteForLocal(test["here"]) 269 270 headlist = test.get("head", "") 271 return list(sanitize_list(headlist, "head")) 272 273 def buildXpcsCmd(self): 274 # change base class' paths to remote paths and use base class to build command 275 self.xpcshell = posixpath.join(self.remoteBinDir, "xpcw") 276 self.headJSPath = posixpath.join(self.remoteScriptsDir, "head.js") 277 self.testingModulesDir = self.remoteModulesDir 278 self.testharnessdir = self.remoteScriptsDir 279 xpcsCmd = xpcshell.XPCShellTestThread.buildXpcsCmd(self) 280 # remove "-g <dir> -a <dir>" and replace with remote alternatives 281 del xpcsCmd[1:5] 282 if self.options["localAPK"]: 283 xpcsCmd.insert(1, "--greomni") 284 xpcsCmd.insert(2, self.remoteAPK) 285 xpcsCmd.insert(1, "-g") 286 xpcsCmd.insert(2, self.remoteBinDir) 287 288 if self.remoteDebugger: 289 # for example, "/data/local/gdbserver" "localhost:12345" 290 xpcsCmd = [self.remoteDebugger, self.remoteDebuggerArgs] + xpcsCmd 291 return xpcsCmd 292 293 def killTimeout(self, proc): 294 self.kill(proc) 295 296 def launchProcess( 297 self, cmd, stdout, stderr, env, cwd, timeout=None, test_name=None 298 ): 299 rpm = RemoteProcessMonitor( 300 "org.mozilla.geckoview.test_runner", 301 self.device, 302 self.log, 303 self.remoteLogFile, 304 ) 305 306 startTime = datetime.datetime.now() 307 308 try: 309 pid = rpm.launch_service( 310 cmd[1:], self.env, self.selectedProcess, test_name=test_name 311 ) 312 except Exception as e: 313 self.log.info( 314 "remotexpcshelltests.py | Failed to start process: %s" % str(e) 315 ) 316 self.shellReturnCode = 1 317 return "" 318 319 self.log.info( 320 "remotexpcshelltests.py | %s | %s | Launched Test App" 321 % (test_name, str(pid)) 322 ) 323 324 if rpm.wait(timeout, test_name=test_name): 325 self.shellReturnCode = 0 326 else: 327 self.shellReturnCode = 1 328 self.log.info( 329 "remotexpcshelltests.py | %s | %s | Application ran for: %s" 330 % (test_name, str(pid), str(datetime.datetime.now() - startTime)) 331 ) 332 333 try: 334 return self.device.get_file(self.remoteLogFile) 335 except mozdevice.ADBTimeoutError: 336 raise 337 except Exception as e: 338 self.log.info( 339 "remotexpcshelltests.py | %s | %s | Could not read log file: %s" 340 % (test_name, str(pid), str(e)) 341 ) 342 self.shellReturnCode = 1 343 return "" 344 345 def checkForCrashes(self, dump_directory, symbols_path, test_name=None): 346 with mozfile.TemporaryDirectory() as dumpDir: 347 self.device.pull(self.remoteMinidumpDir, dumpDir) 348 crashed = mozcrash.log_crashes( 349 self.log, dumpDir, symbols_path, test=test_name 350 ) 351 return crashed 352 353 def communicate(self, proc): 354 return proc, "" 355 356 def poll(self, proc): 357 if not self.device.process_exist("xpcshell"): 358 return self.getReturnCode(proc) 359 # Process is still running 360 return None 361 362 def kill(self, proc): 363 return self.device.pkill("xpcshell") 364 365 def getReturnCode(self, proc): 366 if self.shellReturnCode is not None: 367 return self.shellReturnCode 368 else: 369 return -1 370 371 def removeDir(self, dirname): 372 try: 373 self.device.rm(dirname, recursive=True) 374 except ADBTimeoutError: 375 raise 376 except Exception as e: 377 self.log.warning(str(e)) 378 379 def createLogFile(self, test, stdout): 380 filename = test.replace("\\", "/").split("/")[-1] + ".log" 381 with open(filename, "wb") as f: 382 f.write(stdout) 383 384 385 # A specialization of XPCShellTests that runs tests on an Android device. 386 class XPCShellRemote(xpcshell.XPCShellTests): 387 def __init__(self, options, log): 388 xpcshell.XPCShellTests.__init__(self, log) 389 390 self.options = options 391 verbose = False 392 if options["log_tbpl_level"] == "debug" or options["log_mach_level"] == "debug": 393 verbose = True 394 self.device = ADBDeviceFactory( 395 adb=options["adbPath"] or "adb", 396 device=options["deviceSerial"], 397 test_root=options["remoteTestRoot"], 398 verbose=verbose, 399 ) 400 self.remoteTestRoot = posixpath.join(self.device.test_root, "xpc") 401 self.remoteLogFolder = posixpath.join(self.remoteTestRoot, "logs") 402 # Use Android version (SDK level) to get os_version for mozinfo 403 # so that manifest entries can be conditional on os_version. 404 android_version = str(self.device.version) 405 os_version = android_api_to_os_version(android_version) 406 self.log.info( 407 f"Android sdk version '{android_version}' corresponds to os_version '{os_version}'; use os_version to filter manifests" 408 ) 409 mozinfo.info["os_version"] = os_version 410 mozinfo.info["is_emulator"] = self.device._device_serial.startswith("emulator-") 411 412 self.localBin = options["localBin"] 413 self.pathMapping = [] 414 # remoteBinDir contains xpcshell and its wrapper script, both of which must 415 # be executable. Since +x permissions cannot usually be set on /mnt/sdcard, 416 # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on 417 # /data/local, always. 418 self.remoteBinDir = posixpath.join(self.device.test_root, "xpcb") 419 # Terse directory names are used here ("c" for the components directory) 420 # to minimize the length of the command line used to execute 421 # xpcshell on the remote device. adb has a limit to the number 422 # of characters used in a shell command, and the xpcshell command 423 # line can be quite complex. 424 self.remoteTmpDir = posixpath.join(self.remoteTestRoot, "tmp") 425 self.remoteScriptsDir = self.remoteTestRoot 426 self.remoteComponentsDir = posixpath.join(self.remoteTestRoot, "c") 427 self.remoteModulesDir = posixpath.join(self.remoteTestRoot, "m") 428 self.remoteMinidumpRootDir = posixpath.join(self.remoteTestRoot, "minidumps") 429 self.profileDir = posixpath.join(self.remoteTestRoot, "p") 430 self.remoteDebugger = options["debugger"] 431 self.remoteDebuggerArgs = options["debuggerArgs"] 432 self.testingModulesDir = options["testingModulesDir"] 433 434 self.initDir(self.remoteTmpDir) 435 self.initDir(self.profileDir) 436 437 # Make sure we get a fresh start 438 self.device.stop_application("org.mozilla.geckoview.test_runner") 439 440 for i in range(options["threadCount"]): 441 RemoteProcessMonitor.processStatus += [False] 442 443 self.env = {} 444 445 if options["objdir"]: 446 self.xpcDir = os.path.join(options["objdir"], "_tests/xpcshell") 447 elif os.path.isdir(os.path.join(here, "tests")): 448 self.xpcDir = os.path.join(here, "tests") 449 else: 450 print("Couldn't find local xpcshell test directory", file=sys.stderr) 451 sys.exit(1) 452 453 self.remoteAPK = None 454 if options["localAPK"]: 455 self.localAPKContents = ZipFile(options["localAPK"]) 456 self.remoteAPK = posixpath.join( 457 self.remoteBinDir, os.path.basename(options["localAPK"]) 458 ) 459 else: 460 self.localAPKContents = None 461 if options["setup"]: 462 self.setupTestDir() 463 self.setupUtilities() 464 self.setupModules() 465 self.initDir(self.remoteMinidumpRootDir) 466 self.initDir(self.remoteLogFolder) 467 468 eprefs = options.get("extraPrefs") or [] 469 if options.get("disableFission"): 470 eprefs.append("fission.autostart=false") 471 else: 472 # should be by default, just in case 473 eprefs.append("fission.autostart=true") 474 options["extraPrefs"] = eprefs 475 476 # data that needs to be passed to the RemoteXPCShellTestThread 477 self.mobileArgs = { 478 "device": self.device, 479 "remoteBinDir": self.remoteBinDir, 480 "remoteScriptsDir": self.remoteScriptsDir, 481 "remoteComponentsDir": self.remoteComponentsDir, 482 "remoteModulesDir": self.remoteModulesDir, 483 "options": self.options, 484 "remoteDebugger": self.remoteDebugger, 485 "remoteDebuggerArgs": self.remoteDebuggerArgs, 486 "pathMapping": self.pathMapping, 487 "profileDir": self.profileDir, 488 "remoteLogFolder": self.remoteLogFolder, 489 "remoteTmpDir": self.remoteTmpDir, 490 "remoteMinidumpRootDir": self.remoteMinidumpRootDir, 491 } 492 if self.remoteAPK: 493 self.mobileArgs["remoteAPK"] = self.remoteAPK 494 495 def initDir(self, path, mask="777", timeout=None): 496 """Initialize a directory by removing it if it exists, creating it 497 and changing the permissions.""" 498 self.device.rm(path, recursive=True, force=True, timeout=timeout) 499 self.device.mkdir(path, parents=True, timeout=timeout) 500 501 def setLD_LIBRARY_PATH(self): 502 self.env["LD_LIBRARY_PATH"] = self.remoteBinDir 503 504 def pushWrapper(self): 505 # Rather than executing xpcshell directly, this wrapper script is 506 # used. By setting environment variables and the cwd in the script, 507 # the length of the per-test command line is shortened. This is 508 # often important when using ADB, as there is a limit to the length 509 # of the ADB command line. 510 localWrapper = tempfile.mktemp() 511 with open(localWrapper, "w") as f: 512 f.write("#!/system/bin/sh\n") 513 for envkey, envval in self.env.items(): 514 f.write("export %s=%s\n" % (envkey, envval)) 515 f.writelines([ 516 "cd $1\n", 517 "echo xpcw: cd $1\n", 518 "shift\n", 519 'echo xpcw: xpcshell "$@"\n', 520 '%s/xpcshell "$@"\n' % self.remoteBinDir, 521 ]) 522 remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw") 523 self.device.push(localWrapper, remoteWrapper) 524 self.device.chmod(remoteWrapper) 525 os.remove(localWrapper) 526 527 def start_test(self, test): 528 test.selectedProcess = RemoteProcessMonitor.pickUnusedProcess() 529 if test.selectedProcess == -1: 530 self.log.error( 531 "TEST-UNEXPECTED-FAIL | remotexpcshelltests.py | no more free processes" 532 ) 533 test.start() 534 535 def test_ended(self, test): 536 RemoteProcessMonitor.freeProcess(test.selectedProcess) 537 538 def buildPrefsFile(self, extraPrefs): 539 prefs = super().buildPrefsFile(extraPrefs) 540 remotePrefsFile = posixpath.join(self.remoteTestRoot, "user.js") 541 self.device.push(self.prefsFile, remotePrefsFile) 542 self.device.chmod(remotePrefsFile) 543 # os.remove(self.prefsFile) is not called despite having pushed the 544 # file to the device, because the local file is relied upon by the 545 # updateTestPrefsFile method 546 self.prefsFile = remotePrefsFile 547 return prefs 548 549 def buildEnvironment(self): 550 self.buildCoreEnvironment() 551 self.setLD_LIBRARY_PATH() 552 self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir 553 self.env["GRE_HOME"] = self.remoteBinDir 554 self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir 555 self.env["HOME"] = self.profileDir 556 self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir 557 self.env["MOZ_ANDROID_DATA_DIR"] = self.remoteBinDir 558 self.env["MOZ_IN_AUTOMATION"] = "1" 559 560 # Guard against intermittent failures to retrieve abi property; 561 # without an abi, xpcshell cannot find greprefs.js and crashes. 562 abilistprop = None 563 abi = None 564 retries = 0 565 while not abi and retries < 3: 566 abi = self.device.get_prop("ro.product.cpu.abi") 567 retries += 1 568 if not abi: 569 raise Exception("failed to get ro.product.cpu.abi from device") 570 self.log.info("ro.product.cpu.abi %s" % abi) 571 if self.localAPKContents: 572 abilist = [abi] 573 retries = 0 574 while not abilistprop and retries < 3: 575 abilistprop = self.device.get_prop("ro.product.cpu.abilist") 576 retries += 1 577 self.log.info("ro.product.cpu.abilist %s" % abilistprop) 578 abi_found = False 579 names = [ 580 n for n in self.localAPKContents.namelist() if n.startswith("lib/") 581 ] 582 self.log.debug("apk names: %s" % names) 583 if abilistprop and len(abilistprop) > 0: 584 abilist.extend(abilistprop.split(",")) 585 for abicand in abilist: 586 abi_found = ( 587 len([n for n in names if n.startswith("lib/%s" % abicand)]) > 0 588 ) 589 if abi_found: 590 abi = abicand 591 break 592 if not abi_found: 593 self.log.info("failed to get matching abi from apk.") 594 if len(names) > 0: 595 self.log.info( 596 "device cpu abi not found in apk. Using abi from apk." 597 ) 598 abi = names[0].split("/")[1] 599 self.log.info("Using abi %s." % abi) 600 self.env["MOZ_ANDROID_CPU_ABI"] = abi 601 self.log.info("Using env %r" % (self.env,)) 602 603 def setupUtilities(self): 604 self.initDir(self.remoteTmpDir) 605 self.initDir(self.remoteBinDir) 606 remotePrefDir = posixpath.join(self.remoteBinDir, "defaults", "pref") 607 self.initDir(posixpath.join(remotePrefDir, "extra")) 608 self.initDir(self.remoteComponentsDir) 609 610 local = os.path.join(os.path.dirname(os.path.abspath(__file__)), "head.js") 611 remoteFile = posixpath.join(self.remoteScriptsDir, "head.js") 612 self.device.push(local, remoteFile) 613 self.device.chmod(remoteFile) 614 615 # Additional binaries are required for some tests. This list should be 616 # similar to TEST_HARNESS_BINS in testing/mochitest/Makefile.in. 617 binaries = [ 618 "ssltunnel", 619 "certutil", 620 "pk12util", 621 "BadCertAndPinningServer", 622 "DelegatedCredentialsServer", 623 "EncryptedClientHelloServer", 624 "FaultyServer", 625 "OCSPStaplingServer", 626 "GenerateOCSPResponse", 627 "SanctionsTestServer", 628 ] 629 for fname in binaries: 630 local = os.path.join(self.localBin, fname) 631 if os.path.isfile(local): 632 print("Pushing %s.." % fname, file=sys.stderr) 633 remoteFile = posixpath.join(self.remoteBinDir, fname) 634 self.device.push(local, remoteFile) 635 self.device.chmod(remoteFile) 636 else: 637 print( 638 "*** Expected binary %s not found in %s!" % (fname, self.localBin), 639 file=sys.stderr, 640 ) 641 642 local = os.path.join(self.localBin, "components/httpd.sys.mjs") 643 remoteFile = posixpath.join(self.remoteComponentsDir, "httpd.sys.mjs") 644 self.device.push(local, remoteFile) 645 self.device.chmod(remoteFile) 646 647 if self.options["localAPK"]: 648 remoteFile = posixpath.join( 649 self.remoteBinDir, os.path.basename(self.options["localAPK"]) 650 ) 651 self.device.push(self.options["localAPK"], remoteFile) 652 self.device.chmod(remoteFile) 653 654 self.pushLibs() 655 else: 656 raise Exception("unable to install gre: no APK") 657 658 def pushLibs(self): 659 pushed_libs_count = 0 660 try: 661 dir = tempfile.mkdtemp() 662 for info in self.localAPKContents.infolist(): 663 if info.filename.endswith(".so"): 664 print("Pushing %s.." % info.filename, file=sys.stderr) 665 remoteFile = posixpath.join( 666 self.remoteBinDir, os.path.basename(info.filename) 667 ) 668 self.localAPKContents.extract(info, dir) 669 localFile = os.path.join(dir, info.filename) 670 self.device.push(localFile, remoteFile) 671 pushed_libs_count += 1 672 self.device.chmod(remoteFile) 673 finally: 674 shutil.rmtree(dir) 675 return pushed_libs_count 676 677 def setupModules(self): 678 if self.testingModulesDir: 679 self.device.push(self.testingModulesDir, self.remoteModulesDir) 680 self.device.chmod(self.remoteModulesDir) 681 682 def setupTestDir(self): 683 print("pushing %s" % self.xpcDir) 684 # The tests directory can be quite large: 5000 files and growing! 685 # Sometimes - like on a low-end aws instance running an emulator - the push 686 # may exceed the default 5 minute timeout, so we increase it here to 10 minutes. 687 self.device.rm(self.remoteScriptsDir, recursive=True, force=True, timeout=None) 688 self.device.push(self.xpcDir, self.remoteScriptsDir, timeout=600) 689 self.device.chmod(self.remoteScriptsDir, recursive=True) 690 691 def trySetupNode(self): 692 super().trySetupNode() 693 # make node host ports visible to device 694 if "MOZHTTP2_PORT" in self.env: 695 port = "tcp:{}".format(self.env["MOZHTTP2_PORT"]) 696 self.device.create_socket_connection( 697 ADBDevice.SOCKET_DIRECTION_REVERSE, port, port 698 ) 699 self.log.info("reversed MOZHTTP2_PORT connection for port " + port) 700 if "MOZNODE_EXEC_PORT" in self.env: 701 port = "tcp:{}".format(self.env["MOZNODE_EXEC_PORT"]) 702 self.device.create_socket_connection( 703 ADBDevice.SOCKET_DIRECTION_REVERSE, port, port 704 ) 705 self.log.info("reversed MOZNODE_EXEC_PORT connection for port " + port) 706 707 def shutdownNode(self): 708 super().shutdownNode() 709 710 if "MOZHTTP2_PORT" in self.env: 711 port = "tcp:{}".format(self.env["MOZHTTP2_PORT"]) 712 self.device.remove_socket_connections( 713 ADBDevice.SOCKET_DIRECTION_REVERSE, port 714 ) 715 self.log.info("cleared MOZHTTP2_PORT connection for port " + port) 716 if "MOZNODE_EXEC_PORT" in self.env: 717 port = "tcp:{}".format(self.env["MOZNODE_EXEC_PORT"]) 718 self.device.remove_socket_connections( 719 ADBDevice.SOCKET_DIRECTION_REVERSE, port 720 ) 721 self.log.info("cleared MOZNODE_EXEC_PORT connection for port " + port) 722 723 def buildTestList(self, test_tags=None, test_paths=None, verify=False): 724 xpcshell.XPCShellTests.buildTestList( 725 self, test_tags=test_tags, test_paths=test_paths, verify=verify 726 ) 727 uniqueTestPaths = set([]) 728 for test in self.alltests: 729 uniqueTestPaths.add(test["here"]) 730 for testdir in uniqueTestPaths: 731 abbrevTestDir = os.path.relpath(testdir, self.xpcDir) 732 remoteScriptDir = posixpath.join(self.remoteScriptsDir, abbrevTestDir) 733 self.pathMapping.append(PathMapping(testdir, remoteScriptDir)) 734 if self.options["setup"]: 735 self.pushWrapper() 736 737 738 def verifyRemoteOptions(parser, options): 739 if isinstance(options, Namespace): 740 options = vars(options) 741 742 if options["localBin"] is None: 743 if options["objdir"]: 744 options["localBin"] = os.path.join(options["objdir"], "dist", "bin") 745 if not os.path.isdir(options["localBin"]): 746 parser.error("Couldn't find local binary dir, specify --local-bin-dir") 747 elif os.path.isfile(os.path.join(here, "..", "bin", "xpcshell")): 748 # assume tests are being run from a tests archive 749 options["localBin"] = os.path.abspath(os.path.join(here, "..", "bin")) 750 else: 751 parser.error("Couldn't find local binary dir, specify --local-bin-dir") 752 return options 753 754 755 class PathMapping: 756 def __init__(self, localDir, remoteDir): 757 self.local = localDir 758 self.remote = remoteDir 759 760 761 def main(): 762 parser = parser_remote() 763 options = parser.parse_args() 764 765 options = verifyRemoteOptions(parser, options) 766 log = commandline.setup_logging("Remote XPCShell", options, {"raw": sys.stdout}) 767 768 if options["interactive"] and not options["testPath"]: 769 print( 770 "Error: You must specify a test filename in interactive mode!", 771 file=sys.stderr, 772 ) 773 sys.exit(1) 774 775 if options["xpcshell"] is None: 776 options["xpcshell"] = "xpcshell" 777 778 # The threadCount depends on the emulator rather than the host machine and 779 # empirically 10 seems to yield the best performance. 780 options["threadCount"] = min(options["threadCount"], 10) 781 782 xpcsh = XPCShellRemote(options, log) 783 784 if not xpcsh.runTests( 785 options, testClass=RemoteXPCShellTestThread, mobileArgs=xpcsh.mobileArgs 786 ): 787 sys.exit(1) 788 789 790 if __name__ == "__main__": 791 main()