runtestsremote.py (17131B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import datetime 6 import os 7 import posixpath 8 import shutil 9 import sys 10 import tempfile 11 import traceback 12 import uuid 13 14 sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(__file__)))) 15 16 import mozcrash 17 import mozinfo 18 from mochitest_options import MochitestArgumentParser, build_obj 19 from mozdevice import ADBDeviceFactory, ADBTimeoutError, RemoteProcessMonitor 20 from mozinfo.platforminfo import android_api_to_os_version 21 from mozscreenshot import dump_device_screen, dump_screen 22 from runtests import MessageLogger, MochitestDesktop 23 24 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) 25 26 27 class MochiRemote(MochitestDesktop): 28 localProfile = None 29 logMessages = [] 30 31 def __init__(self, options): 32 MochitestDesktop.__init__(self, options.flavor, vars(options)) 33 34 verbose = False 35 if ( 36 options.log_mach_verbose 37 or options.log_tbpl_level == "debug" 38 or options.log_mach_level == "debug" 39 or options.log_raw_level == "debug" 40 ): 41 verbose = True 42 if hasattr(options, "log"): 43 delattr(options, "log") 44 45 self.certdbNew = True 46 self.chromePushed = False 47 48 expected = options.app.split("/")[-1] 49 self.device = ADBDeviceFactory( 50 adb=options.adbPath or "adb", 51 device=options.deviceSerial, 52 test_root=options.remoteTestRoot, 53 verbose=verbose, 54 run_as_package=expected, 55 ) 56 57 if options.remoteTestRoot is None: 58 options.remoteTestRoot = self.device.test_root 59 options.dumpOutputDirectory = options.remoteTestRoot 60 self.remoteLogFile = posixpath.join( 61 options.remoteTestRoot, "logs", "mochitest.log" 62 ) 63 logParent = posixpath.dirname(self.remoteLogFile) 64 self.device.rm(logParent, force=True, recursive=True) 65 self.device.mkdir(logParent, parents=True) 66 67 self.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") 68 self.device.rm(self.remoteProfile, force=True, recursive=True) 69 70 self.message_logger = MessageLogger(logger=None) 71 self.message_logger.logger = self.log 72 73 # Check that Firefox is installed 74 expected = options.app.split("/")[-1] 75 if not self.device.is_app_installed(expected): 76 raise Exception("%s is not installed on this device" % expected) 77 78 self.device.clear_logcat() 79 80 self.remoteModulesDir = posixpath.join(options.remoteTestRoot, "modules/") 81 82 self.remoteCache = posixpath.join(options.remoteTestRoot, "cache/") 83 self.device.rm(self.remoteCache, force=True, recursive=True) 84 85 # move necko cache to a location that can be cleaned up 86 options.extraPrefs += [ 87 "browser.cache.disk.parent_directory=%s" % self.remoteCache 88 ] 89 90 self.remoteMozLog = posixpath.join(options.remoteTestRoot, "mozlog") 91 self.device.rm(self.remoteMozLog, force=True, recursive=True) 92 self.device.mkdir(self.remoteMozLog, parents=True) 93 94 self.remoteChromeTestDir = posixpath.join(options.remoteTestRoot, "chrome") 95 self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) 96 self.device.mkdir(self.remoteChromeTestDir, parents=True) 97 98 self.appName = options.remoteappname 99 self.appActivity = options.appActivity 100 self.device.stop_application(self.appName) 101 if self.device.process_exist(self.appName): 102 self.log.warning("unable to kill %s before running tests!" % self.appName) 103 104 android_version = str(self.device.version) 105 os_version = android_api_to_os_version(android_version) 106 # Add Android version (SDK level) to mozinfo so that manifest entries 107 # can be conditional on android_version. 108 self.log.info( 109 f"Android sdk version '{android_version}' corresponds to os_version '{os_version}'; use os_version to filter manifests" 110 ) 111 mozinfo.info["os_version"] = os_version 112 mozinfo.info["is_emulator"] = self.device._device_serial.startswith("emulator-") 113 114 def cleanup(self, options, final=False): 115 if final: 116 self.device.rm(self.remoteChromeTestDir, force=True, recursive=True) 117 self.chromePushed = False 118 uploadDir = os.environ.get("MOZ_UPLOAD_DIR", None) 119 if uploadDir and self.device.is_dir(self.remoteMozLog): 120 self.device.pull(self.remoteMozLog, uploadDir) 121 self.device.rm(self.remoteLogFile, force=True) 122 self.device.rm(self.remoteProfile, force=True, recursive=True) 123 self.device.rm(self.remoteCache, force=True, recursive=True) 124 MochitestDesktop.cleanup(self, options, final) 125 self.localProfile = None 126 127 def dumpScreen(self, utilityPath): 128 if self.haveDumpedScreen: 129 self.log.info( 130 "Not taking screenshot here: see the one that was previously logged" 131 ) 132 return 133 self.haveDumpedScreen = True 134 if self.device._device_serial.startswith("emulator-"): 135 dump_screen(utilityPath, self.log) 136 else: 137 dump_device_screen(self.device, self.log) 138 139 def findPath(self, paths, filename=None): 140 for path in paths: 141 p = path 142 if filename: 143 p = os.path.join(p, filename) 144 if os.path.exists(self.getFullPath(p)): 145 return path 146 return None 147 148 # This seems kludgy, but this class uses paths from the remote host in the 149 # options, except when calling up to the base class, which doesn't 150 # understand the distinction. This switches out the remote values for local 151 # ones that the base class understands. This is necessary for the web 152 # server, SSL tunnel and profile building functions. 153 def switchToLocalPaths(self, options): 154 """Set local paths in the options, return a function that will restore remote values""" 155 remoteXrePath = options.xrePath 156 remoteProfilePath = options.profilePath 157 remoteUtilityPath = options.utilityPath 158 159 paths = [ 160 options.xrePath, 161 ] 162 if build_obj: 163 paths.append(os.path.join(build_obj.topobjdir, "dist", "bin")) 164 options.xrePath = self.findPath(paths) 165 if options.xrePath is None: 166 self.log.error( 167 "unable to find xulrunner path for %s, please specify with --xre-path" 168 % os.name 169 ) 170 sys.exit(1) 171 172 xpcshell = "xpcshell" 173 if os.name == "nt": 174 xpcshell += ".exe" 175 176 if options.utilityPath: 177 paths = [options.utilityPath, options.xrePath] 178 else: 179 paths = [options.xrePath] 180 options.utilityPath = self.findPath(paths, xpcshell) 181 182 if options.utilityPath is None: 183 self.log.error( 184 "unable to find utility path for %s, please specify with --utility-path" 185 % os.name 186 ) 187 sys.exit(1) 188 189 xpcshell_path = os.path.join(options.utilityPath, xpcshell) 190 if RemoteProcessMonitor.elf_arm(xpcshell_path): 191 self.log.error( 192 "xpcshell at %s is an ARM binary; please use " 193 "the --utility-path argument to specify the path " 194 "to a desktop version." % xpcshell_path 195 ) 196 sys.exit(1) 197 198 if self.localProfile: 199 options.profilePath = self.localProfile 200 else: 201 options.profilePath = None 202 203 def fixup(): 204 options.xrePath = remoteXrePath 205 options.utilityPath = remoteUtilityPath 206 options.profilePath = remoteProfilePath 207 208 return fixup 209 210 def startServers(self, options, debuggerInfo, public=None): 211 """Create the servers on the host and start them up""" 212 restoreRemotePaths = self.switchToLocalPaths(options) 213 MochitestDesktop.startServers(self, options, debuggerInfo, public=True) 214 restoreRemotePaths() 215 216 def buildProfile(self, options): 217 restoreRemotePaths = self.switchToLocalPaths(options) 218 if options.testingModulesDir: 219 try: 220 self.device.push(options.testingModulesDir, self.remoteModulesDir) 221 self.device.chmod(self.remoteModulesDir, recursive=True) 222 except Exception: 223 self.log.error( 224 "Automation Error: Unable to copy test modules to device." 225 ) 226 raise 227 savedTestingModulesDir = options.testingModulesDir 228 options.testingModulesDir = self.remoteModulesDir 229 else: 230 savedTestingModulesDir = None 231 manifest = MochitestDesktop.buildProfile(self, options) 232 if savedTestingModulesDir: 233 options.testingModulesDir = savedTestingModulesDir 234 self.localProfile = options.profilePath 235 236 restoreRemotePaths() 237 options.profilePath = self.remoteProfile 238 return manifest 239 240 def buildURLOptions(self, options, env): 241 saveLogFile = options.logFile 242 options.logFile = self.remoteLogFile 243 options.profilePath = self.localProfile 244 env["MOZ_HIDE_RESULTS_TABLE"] = "1" 245 retVal = MochitestDesktop.buildURLOptions(self, options, env) 246 247 # we really need testConfig.js (for browser chrome) 248 try: 249 self.device.push(options.profilePath, self.remoteProfile) 250 self.device.chmod(self.remoteProfile, recursive=True) 251 except Exception: 252 self.log.error("Automation Error: Unable to copy profile to device.") 253 raise 254 255 options.profilePath = self.remoteProfile 256 options.logFile = saveLogFile 257 return retVal 258 259 def getChromeTestDir(self, options): 260 local = super().getChromeTestDir(options) 261 remote = self.remoteChromeTestDir 262 if options.flavor == "chrome" and not self.chromePushed: 263 self.log.info("pushing %s to %s on device..." % (local, remote)) 264 local = os.path.join(local, "chrome") 265 self.device.push(local, remote) 266 self.chromePushed = True 267 return remote 268 269 def getLogFilePath(self, logFile): 270 return logFile 271 272 def getGMPPluginPath(self, options): 273 # TODO: bug 1149374 274 return None 275 276 def environment(self, env=None, crashreporter=True, **kwargs): 277 # Since running remote, do not mimic the local env: do not copy os.environ 278 if env is None: 279 env = {} 280 281 if crashreporter: 282 env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" 283 env["MOZ_CRASHREPORTER"] = "1" 284 env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" 285 else: 286 env["MOZ_CRASHREPORTER_DISABLE"] = "1" 287 288 # Crash on non-local network connections by default. 289 # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily 290 # enable non-local connections for the purposes of local testing. 291 # Don't override the user's choice here. See bug 1049688. 292 env.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1") 293 294 # Send an env var noting that we are in automation. Passing any 295 # value except the empty string will declare the value to exist. 296 # 297 # This may be used to disabled network connections during testing, e.g. 298 # Switchboard & telemetry uploads. 299 env.setdefault("MOZ_IN_AUTOMATION", "1") 300 301 # Set WebRTC logging in case it is not set yet. 302 env.setdefault("R_LOG_LEVEL", "6") 303 env.setdefault("R_LOG_DESTINATION", "stderr") 304 env.setdefault("R_LOG_VERBOSE", "1") 305 306 return env 307 308 def buildBrowserEnv(self, options, debugger=False): 309 browserEnv = MochitestDesktop.buildBrowserEnv(self, options, debugger=debugger) 310 # remove desktop environment not used on device 311 if "XPCOM_MEM_BLOAT_LOG" in browserEnv: 312 del browserEnv["XPCOM_MEM_BLOAT_LOG"] 313 if self.mozLogs: 314 browserEnv["MOZ_LOG_FILE"] = os.path.join( 315 self.remoteMozLog, f"moz-pid=%PID-uid={str(uuid.uuid4())}.log" 316 ) 317 if options.dmd: 318 browserEnv["DMD"] = "1" 319 # Contents of remoteMozLog will be pulled from device and copied to the 320 # host MOZ_UPLOAD_DIR, to be made available as test artifacts. Make 321 # MOZ_UPLOAD_DIR available to the browser environment so that tests 322 # can use it as though they were running on the host. 323 browserEnv["MOZ_UPLOAD_DIR"] = self.remoteMozLog 324 return browserEnv 325 326 def runApp( 327 self, 328 testUrl, 329 env, 330 app, 331 profile, 332 extraArgs, 333 utilityPath, 334 debuggerInfo=None, 335 valgrindPath=None, 336 valgrindArgs=None, 337 valgrindSuppFiles=None, 338 symbolsPath=None, 339 timeout=-1, 340 detectShutdownLeaks=False, 341 screenshotOnFail=False, 342 bisectChunk=None, 343 restartAfterFailure=False, 344 marionette_args=None, 345 e10s=True, 346 runFailures=False, 347 crashAsPass=False, 348 currentManifest=None, 349 ): 350 """ 351 Run the app, log the duration it took to execute, return the status code. 352 Kill the app if it outputs nothing for |timeout| seconds. 353 """ 354 355 if timeout == -1: 356 timeout = self.DEFAULT_TIMEOUT 357 358 rpm = RemoteProcessMonitor( 359 self.appName, 360 self.device, 361 self.log, 362 self.message_logger, 363 self.remoteLogFile, 364 self.remoteProfile, 365 ) 366 startTime = datetime.datetime.now() 367 status = 0 368 profileDirectory = self.remoteProfile + "/" 369 args = [] 370 args.extend(extraArgs) 371 args.extend(("-profile", profileDirectory)) 372 373 pid = rpm.launch( 374 app, 375 debuggerInfo, 376 testUrl, 377 args, 378 env=self.environment(env=env, crashreporter=not debuggerInfo), 379 e10s=e10s, 380 activity=self.appActivity, 381 ) 382 383 # TODO: not using runFailures or crashAsPass, if we choose to use them 384 # we need to adjust status and check_for_crashes 385 self.log.info("runtestsremote.py | Application pid: %d" % pid) 386 if not rpm.wait(timeout): 387 status = 1 388 self.log.info( 389 "runtestsremote.py | Application ran for: %s" 390 % str(datetime.datetime.now() - startTime) 391 ) 392 393 lastTestSeen = currentManifest or "Main app process exited normally" 394 395 crashed = self.check_for_crashes(symbolsPath, lastTestSeen) 396 if crashed: 397 status = 1 398 399 self.countpass += rpm.counts["pass"] 400 self.countfail += rpm.counts["fail"] 401 self.counttodo += rpm.counts["todo"] 402 403 return status, lastTestSeen 404 405 def check_for_crashes(self, symbols_path, last_test_seen): 406 """ 407 Pull any minidumps from remote profile and log any associated crashes. 408 """ 409 try: 410 dump_dir = tempfile.mkdtemp() 411 remote_crash_dir = posixpath.join(self.remoteProfile, "minidumps") 412 if not self.device.is_dir(remote_crash_dir): 413 return False 414 self.device.pull(remote_crash_dir, dump_dir) 415 crashed = mozcrash.log_crashes( 416 self.log, dump_dir, symbols_path, test=last_test_seen 417 ) 418 finally: 419 try: 420 shutil.rmtree(dump_dir) 421 except Exception as e: 422 self.log.warning( 423 "unable to remove directory %s: %s" % (dump_dir, str(e)) 424 ) 425 return crashed 426 427 428 def run_test_harness(parser, options): 429 parser.validate(options) 430 431 if options is None: 432 raise ValueError( 433 "Invalid options specified, use --help for a list of valid options" 434 ) 435 436 options.runByManifest = True 437 438 mochitest = MochiRemote(options) 439 440 try: 441 if options.verify: 442 retVal = mochitest.verifyTests(options) 443 else: 444 retVal = mochitest.runTests(options) 445 except Exception as e: 446 mochitest.log.error("Automation Error: Exception caught while running tests") 447 traceback.print_exc() 448 if isinstance(e, ADBTimeoutError): 449 mochitest.log.info("Device disconnected. Will not run mochitest.cleanup().") 450 else: 451 try: 452 mochitest.cleanup(options) 453 except Exception: 454 # device error cleaning up... oh well! 455 traceback.print_exc() 456 retVal = 1 457 458 mochitest.archiveMozLogs() 459 mochitest.message_logger.finish() 460 461 return retVal 462 463 464 def main(args=sys.argv[1:]): 465 parser = MochitestArgumentParser(app="android") 466 options = parser.parse_args(args) 467 468 return run_test_harness(parser, options) 469 470 471 if __name__ == "__main__": 472 sys.exit(main())