remotereftest.py (20120B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import datetime 6 import os 7 import posixpath 8 import shutil 9 import signal 10 import subprocess 11 import sys 12 import tempfile 13 import time 14 import traceback 15 from contextlib import closing 16 from urllib.request import urlopen 17 18 import mozcrash 19 import reftestcommandline 20 from mozdevice import ADBDeviceFactory, RemoteProcessMonitor 21 from output import OutputHandler 22 from runreftest import RefTest, ReftestResolver, build_obj 23 24 # We need to know our current directory so that we can serve our test files from it. 25 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) 26 27 28 class RemoteReftestResolver(ReftestResolver): 29 def absManifestPath(self, path): 30 script_abs_path = os.path.join(SCRIPT_DIRECTORY, path) 31 if os.path.exists(script_abs_path): 32 rv = script_abs_path 33 elif os.path.exists(os.path.abspath(path)): 34 rv = os.path.abspath(path) 35 else: 36 print("Could not find manifest %s" % script_abs_path, file=sys.stderr) 37 sys.exit(1) 38 return os.path.normpath(rv) 39 40 def manifestURL(self, options, path): 41 # Dynamically build the reftest URL if possible, beware that 42 # args[0] should exist 'inside' webroot. It's possible for 43 # this url to have a leading "..", but reftest.js will fix 44 # that. Use the httpdPath to determine if we are running in 45 # production or locally. If we are running the jsreftests 46 # locally, strip text up to jsreftest. We want the docroot of 47 # the server to include a link jsreftest that points to the 48 # test-stage location of the test files. The desktop oriented 49 # setup has already created a link for tests which points 50 # directly into the source tree. For the remote tests we need 51 # a separate symbolic link to point to the staged test files. 52 if "jsreftest" not in path or os.environ.get("MOZ_AUTOMATION"): 53 relPath = os.path.relpath(path, SCRIPT_DIRECTORY) 54 else: 55 relPath = "jsreftest/" + path.split("jsreftest/")[-1] 56 return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath) 57 58 59 class ReftestServer: 60 """Web server used to serve Reftests, for closer fidelity to the real web. 61 It is virtually identical to the server used in mochitest and will only 62 be used for running reftests remotely. 63 Bug 581257 has been filed to refactor this wrapper around httpd.js into 64 it's own class and use it in both remote and non-remote testing.""" 65 66 def __init__(self, options, scriptDir, log): 67 self.log = log 68 self.utilityPath = options.utilityPath 69 self.xrePath = options.xrePath 70 self.profileDir = options.serverProfilePath 71 self.webServer = options.remoteWebServer 72 self.httpPort = options.httpPort 73 self.scriptDir = scriptDir 74 self.httpdPath = os.path.abspath(options.httpdPath) 75 if options.remoteWebServer == "10.0.2.2": 76 # probably running an Android emulator and 10.0.2.2 will 77 # not be visible from host 78 shutdownServer = "127.0.0.1" 79 else: 80 shutdownServer = self.webServer 81 self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { 82 "server": shutdownServer, 83 "port": self.httpPort, 84 } 85 86 def start(self): 87 "Run the Refest server, returning the process ID of the server." 88 89 env = dict(os.environ) 90 env["XPCOM_DEBUG_BREAK"] = "warn" 91 bin_suffix = "" 92 if sys.platform in ("win32", "msys", "cygwin"): 93 env["PATH"] = env["PATH"] + ";" + self.xrePath 94 bin_suffix = ".exe" 95 elif "LD_LIBRARY_PATH" not in env or env["LD_LIBRARY_PATH"] is None: 96 env["LD_LIBRARY_PATH"] = self.xrePath 97 else: 98 env["LD_LIBRARY_PATH"] = ":".join([self.xrePath, env["LD_LIBRARY_PATH"]]) 99 100 args = [ 101 "-g", 102 self.xrePath, 103 "-e", 104 "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = " 105 "'%(port)s'; const _SERVER_ADDR ='%(server)s'; " 106 "const _HTTPD_PATH = '%(httpdPath)s';" 107 % { 108 "httpdPath": self.httpdPath.replace("\\", "\\\\"), 109 "profile": self.profileDir.replace("\\", "\\\\"), 110 "port": self.httpPort, 111 "server": self.webServer, 112 }, 113 "-f", 114 os.path.join(self.scriptDir, "server.js"), 115 ] 116 117 xpcshell = os.path.join(self.utilityPath, "xpcshell" + bin_suffix) 118 119 if not os.access(xpcshell, os.F_OK): 120 raise Exception("xpcshell not found at %s" % xpcshell) 121 if RemoteProcessMonitor.elf_arm(xpcshell): 122 raise Exception( 123 "xpcshell at %s is an ARM binary; please use " 124 "the --utility-path argument to specify the path " 125 "to a desktop version." % xpcshell 126 ) 127 128 self._process = subprocess.Popen([xpcshell] + args, env=env) 129 pid = self._process.pid 130 if pid < 0: 131 self.log.error( 132 "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server." 133 ) 134 return 2 135 self.log.info("INFO | remotereftests.py | Server pid: %d" % pid) 136 137 def ensureReady(self, timeout): 138 assert timeout >= 0 139 140 aliveFile = os.path.join(self.profileDir, "server_alive.txt") 141 i = 0 142 while i < timeout: 143 if os.path.exists(aliveFile): 144 break 145 time.sleep(1) 146 i += 1 147 else: 148 self.log.error( 149 "TEST-UNEXPECTED-FAIL | remotereftests.py | " 150 "Timed out while waiting for server startup." 151 ) 152 self.stop() 153 return 1 154 155 def stop(self): 156 if hasattr(self, "_process"): 157 try: 158 with closing(urlopen(self.shutdownURL)) as c: 159 c.read() 160 161 rtncode = self._process.poll() 162 if rtncode is None: 163 self._process.terminate() 164 except Exception: 165 self.log.info("Failed to shutdown server at %s" % self.shutdownURL) 166 traceback.print_exc() 167 self._process.kill() 168 169 170 class RemoteReftest(RefTest): 171 use_marionette = False 172 resolver_cls = RemoteReftestResolver 173 174 def __init__(self, options, scriptDir): 175 RefTest.__init__(self, options.suite) 176 self.run_by_manifest = False 177 self.scriptDir = scriptDir 178 self.localLogName = options.localLogName 179 180 verbose = False 181 if ( 182 options.log_mach_verbose 183 or options.log_tbpl_level == "debug" 184 or options.log_mach_level == "debug" 185 or options.log_raw_level == "debug" 186 ): 187 verbose = True 188 print("set verbose!") 189 expected = options.app.split("/")[-1] 190 self.device = ADBDeviceFactory( 191 adb=options.adb_path or "adb", 192 device=options.deviceSerial, 193 test_root=options.remoteTestRoot, 194 verbose=verbose, 195 run_as_package=expected, 196 ) 197 if options.remoteTestRoot is None: 198 options.remoteTestRoot = posixpath.join(self.device.test_root, "reftest") 199 options.remoteProfile = posixpath.join(options.remoteTestRoot, "profile") 200 options.remoteLogFile = posixpath.join(options.remoteTestRoot, "reftest.log") 201 options.logFile = options.remoteLogFile 202 self.remoteProfile = options.remoteProfile 203 self.remoteTestRoot = options.remoteTestRoot 204 205 if not options.ignoreWindowSize: 206 parts = self.device.get_info("screen")["screen"][0].split() 207 width = int(parts[0].split(":")[1]) 208 height = int(parts[1].split(":")[1]) 209 if width < 1366 or height < 1050: 210 self.error( 211 "ERROR: Invalid screen resolution %sx%s, " 212 "please adjust to 1366x1050 or higher" % (width, height) 213 ) 214 215 self._populate_logger(options) 216 self.outputHandler = OutputHandler( 217 self.log, options.utilityPath, options.symbolsPath 218 ) 219 220 self.SERVER_STARTUP_TIMEOUT = 90 221 222 self.remoteCache = os.path.join(options.remoteTestRoot, "cache/") 223 224 # Check that Firefox is installed 225 expected = options.app.split("/")[-1] 226 if not self.device.is_app_installed(expected): 227 raise Exception("%s is not installed on this device" % expected) 228 self.device.run_as_package = expected 229 self.device.clear_logcat() 230 231 self.device.rm(self.remoteCache, force=True, recursive=True) 232 233 procName = options.app.split("/")[-1] 234 self.device.stop_application(procName) 235 if self.device.process_exist(procName): 236 self.log.error("unable to kill %s before starting tests!" % procName) 237 238 def findPath(self, paths, filename=None): 239 for path in paths: 240 p = path 241 if filename: 242 p = os.path.join(p, filename) 243 if os.path.exists(self.getFullPath(p)): 244 return path 245 return None 246 247 def startWebServer(self, options): 248 """Create the webserver on the host and start it up""" 249 remoteXrePath = options.xrePath 250 remoteUtilityPath = options.utilityPath 251 252 paths = [options.xrePath] 253 if build_obj: 254 paths.append(os.path.join(build_obj.topobjdir, "dist", "bin")) 255 options.xrePath = self.findPath(paths) 256 if options.xrePath is None: 257 print( 258 "ERROR: unable to find xulrunner path for %s, " 259 "please specify with --xre-path" % (os.name) 260 ) 261 return 1 262 paths.append("bin") 263 paths.append(os.path.join("..", "bin")) 264 265 xpcshell = "xpcshell" 266 if os.name == "nt": 267 xpcshell += ".exe" 268 269 if options.utilityPath: 270 paths.insert(0, options.utilityPath) 271 options.utilityPath = self.findPath(paths, xpcshell) 272 if options.utilityPath is None: 273 print( 274 "ERROR: unable to find utility path for %s, " 275 "please specify with --utility-path" % (os.name) 276 ) 277 return 1 278 279 options.serverProfilePath = tempfile.mkdtemp() 280 self.server = ReftestServer(options, self.scriptDir, self.log) 281 retVal = self.server.start() 282 if retVal: 283 return retVal 284 retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT) 285 if retVal: 286 return retVal 287 288 options.xrePath = remoteXrePath 289 options.utilityPath = remoteUtilityPath 290 return 0 291 292 def stopWebServer(self, options): 293 self.server.stop() 294 295 def killNamedProc(self, pname, orphans=True): 296 """Kill processes matching the given command name""" 297 try: 298 import psutil 299 except ImportError as e: 300 self.log.warning("Unable to import psutil: %s" % str(e)) 301 self.log.warning("Unable to verify that %s is not already running." % pname) 302 return 303 304 self.log.info("Checking for %s processes..." % pname) 305 306 for proc in psutil.process_iter(): 307 try: 308 if proc.name() == pname: 309 procd = proc.as_dict(attrs=["pid", "ppid", "name", "username"]) 310 if proc.ppid() == 1 or not orphans: 311 self.log.info("killing %s" % procd) 312 try: 313 os.kill( 314 proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM) 315 ) 316 except Exception as e: 317 self.log.info( 318 "Failed to kill process %d: %s" % (proc.pid, str(e)) 319 ) 320 else: 321 self.log.info("NOT killing %s (not an orphan?)" % procd) 322 except Exception: 323 # may not be able to access process info for all processes 324 continue 325 326 def createReftestProfile(self, options, **kwargs): 327 profile = RefTest.createReftestProfile( 328 self, 329 options, 330 server=options.remoteWebServer, 331 port=options.httpPort, 332 **kwargs, 333 ) 334 profileDir = profile.profile 335 prefs = {} 336 prefs["reftest.remote"] = True 337 prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True 338 # move necko cache to a location that can be cleaned up 339 prefs["browser.cache.disk.parent_directory"] = self.remoteCache 340 341 prefs["layout.css.devPixelsPerPx"] = "1.0" 342 # Because Fennec is a little wacky (see bug 1156817) we need to load the 343 # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport. 344 prefs["apz.allow_zooming"] = False 345 346 # Set the extra prefs. 347 profile.set_preferences(prefs) 348 349 try: 350 self.device.push(profileDir, options.remoteProfile) 351 # make sure the parent directories of the profile which 352 # may have been created by the push, also have their 353 # permissions set to allow access. 354 self.device.chmod(options.remoteTestRoot, recursive=True) 355 except Exception: 356 print("Automation Error: Failed to copy profiledir to device") 357 raise 358 359 return profile 360 361 def environment(self, env=None, crashreporter=True, **kwargs): 362 # Since running remote, do not mimic the local env: do not copy os.environ 363 if env is None: 364 env = {} 365 366 if crashreporter: 367 env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" 368 env["MOZ_CRASHREPORTER"] = "1" 369 env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" 370 else: 371 env["MOZ_CRASHREPORTER_DISABLE"] = "1" 372 373 # Crash on non-local network connections by default. 374 # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily 375 # enable non-local connections for the purposes of local testing. 376 # Don't override the user's choice here. See bug 1049688. 377 env.setdefault("MOZ_DISABLE_NONLOCAL_CONNECTIONS", "1") 378 379 # Send an env var noting that we are in automation. Passing any 380 # value except the empty string will declare the value to exist. 381 # 382 # This may be used to disabled network connections during testing, e.g. 383 # Switchboard & telemetry uploads. 384 env.setdefault("MOZ_IN_AUTOMATION", "1") 385 386 # Set WebRTC logging in case it is not set yet. 387 env.setdefault("R_LOG_LEVEL", "6") 388 env.setdefault("R_LOG_DESTINATION", "stderr") 389 env.setdefault("R_LOG_VERBOSE", "1") 390 391 return env 392 393 def buildBrowserEnv(self, options, profileDir): 394 browserEnv = RefTest.buildBrowserEnv(self, options, profileDir) 395 # remove desktop environment not used on device 396 if "XPCOM_MEM_BLOAT_LOG" in browserEnv: 397 del browserEnv["XPCOM_MEM_BLOAT_LOG"] 398 return browserEnv 399 400 def runApp( 401 self, 402 options, 403 cmdargs=None, 404 timeout=None, 405 debuggerInfo=None, 406 symbolsPath=None, 407 valgrindPath=None, 408 valgrindArgs=None, 409 valgrindSuppFiles=None, 410 **profileArgs, 411 ): 412 if cmdargs is None: 413 cmdargs = [] 414 415 if self.use_marionette: 416 cmdargs.append("-marionette") 417 418 binary = options.app 419 profile = self.createReftestProfile(options, **profileArgs) 420 421 # browser environment 422 env = self.buildBrowserEnv(options, profile.profile) 423 424 rpm = RemoteProcessMonitor( 425 binary, 426 self.device, 427 self.log, 428 self.outputHandler, 429 options.remoteLogFile, 430 self.remoteProfile, 431 ) 432 startTime = datetime.datetime.now() 433 status = 0 434 profileDirectory = self.remoteProfile + "/" 435 cmdargs.extend(("-profile", profileDirectory)) 436 437 pid = rpm.launch( 438 binary, 439 debuggerInfo, 440 None, 441 cmdargs, 442 env=env, 443 e10s=options.e10s, 444 ) 445 self.log.info("remotereftest.py | Application pid: %d" % pid) 446 if not rpm.wait(timeout): 447 status = 1 448 self.log.info( 449 "remotereftest.py | Application ran for: %s" 450 % str(datetime.datetime.now() - startTime) 451 ) 452 crashed = self.check_for_crashes(symbolsPath, rpm.last_test_seen) 453 if crashed: 454 status = 1 455 456 self.cleanup(profile.profile) 457 return status 458 459 def check_for_crashes(self, symbols_path, last_test_seen): 460 """ 461 Pull any minidumps from remote profile and log any associated crashes. 462 """ 463 try: 464 dump_dir = tempfile.mkdtemp() 465 remote_crash_dir = posixpath.join(self.remoteProfile, "minidumps") 466 if not self.device.is_dir(remote_crash_dir): 467 return False 468 self.device.pull(remote_crash_dir, dump_dir) 469 crashed = mozcrash.log_crashes( 470 self.log, dump_dir, symbols_path, test=last_test_seen 471 ) 472 finally: 473 try: 474 shutil.rmtree(dump_dir) 475 except Exception as e: 476 self.log.warning( 477 "unable to remove directory %s: %s" % (dump_dir, str(e)) 478 ) 479 return crashed 480 481 def cleanup(self, profileDir): 482 self.device.rm(self.remoteTestRoot, force=True, recursive=True) 483 self.device.rm(self.remoteProfile, force=True, recursive=True) 484 self.device.rm(self.remoteCache, force=True, recursive=True) 485 RefTest.cleanup(self, profileDir) 486 487 488 def run_test_harness(parser, options): 489 reftest = RemoteReftest(options, SCRIPT_DIRECTORY) 490 parser.validate_remote(options) 491 parser.validate(options, reftest) 492 493 # Hack in a symbolic link for jsreftest in the SCRIPT_DIRECTORY 494 # which is the document root for the reftest web server. This 495 # allows a separate redirection for the jsreftests which must 496 # run through the web server using the staged tests files and 497 # the desktop which will use the tests symbolic link to find 498 # the JavaScript tests. 499 jsreftest_target = str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")) 500 if os.environ.get("MOZ_AUTOMATION"): 501 os.system("ln -s ../jsreftest " + jsreftest_target) 502 else: 503 jsreftest_source = os.path.join( 504 build_obj.topobjdir, "dist", "test-stage", "jsreftest" 505 ) 506 if not os.path.islink(jsreftest_target): 507 os.symlink(jsreftest_source, jsreftest_target) 508 509 # Despite our efforts to clean up servers started by this script, in practice 510 # we still see infrequent cases where a process is orphaned and interferes 511 # with future tests, typically because the old server is keeping the port in use. 512 # Try to avoid those failures by checking for and killing servers before 513 # trying to start new ones. 514 reftest.killNamedProc("ssltunnel") 515 reftest.killNamedProc("xpcshell") 516 517 # Start the webserver 518 retVal = reftest.startWebServer(options) 519 if retVal: 520 return retVal 521 522 retVal = 0 523 try: 524 if options.verify: 525 retVal = reftest.verifyTests(options.tests, options) 526 else: 527 retVal = reftest.runTests(options.tests, options) 528 except Exception: 529 print("Automation Error: Exception caught while running tests") 530 traceback.print_exc() 531 retVal = 1 532 533 reftest.stopWebServer(options) 534 535 return retVal 536 537 538 if __name__ == "__main__": 539 parser = reftestcommandline.RemoteArgumentsParser() 540 options = parser.parse_args() 541 sys.exit(run_test_harness(parser, options))