android_wrench.py (11519B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import datetime 7 import enum 8 import os 9 import subprocess 10 import sys 11 import tempfile 12 import time 13 14 # load modules from parent dir 15 sys.path.insert(1, os.path.dirname(sys.path[0])) 16 17 from mozharness.base.script import BaseScript 18 from mozharness.mozilla.automation import EXIT_STATUS_DICT, TBPL_FAILURE 19 from mozharness.mozilla.mozbase import MozbaseMixin 20 from mozharness.mozilla.testing.android import AndroidMixin 21 from mozharness.mozilla.testing.testbase import TestingMixin 22 23 24 class TestMode(enum.Enum): 25 OPTIMIZED_SHADER_COMPILATION = 0 26 UNOPTIMIZED_SHADER_COMPILATION = 1 27 SHADER_TEST = 2 28 REFTEST = 3 29 30 31 class AndroidWrench(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin): 32 def __init__(self, require_config_file=False): 33 # code in BaseScript.__init__ iterates all the properties to attach 34 # pre- and post-flight listeners, so we need _is_emulator be defined 35 # before that happens. Doesn't need to be a real value though. 36 self._is_emulator = None 37 38 # Directory for wrench input and output files. Note that we hard-code 39 # the path here, rather than using something like self.device.test_root, 40 # because it needs to be kept in sync with the path hard-coded inside 41 # the wrench source code. 42 self.wrench_dir = "/data/data/org.mozilla.wrench/files/wrench" 43 44 super().__init__() 45 46 # Override AndroidMixin's use_root to ensure we use run-as instead of 47 # root to push and pull files from the device, as the latter fails due 48 # to permission errors on recent Android versions. 49 self.use_root = False 50 51 if self.device_serial is None: 52 # Running on an emulator. 53 self._is_emulator = True 54 self.device_serial = "emulator-5554" 55 self.use_gles3 = True 56 else: 57 # Running on a device, ensure self.is_emulator returns False. 58 # The adb binary is preinstalled on the bitbar image and is 59 # already on the $PATH. 60 self._is_emulator = False 61 self._adb_path = "adb" 62 self._errored = False 63 64 @property 65 def is_emulator(self): 66 """Overrides the is_emulator property on AndroidMixin.""" 67 if self._is_emulator is None: 68 self._is_emulator = self.device_serial is None 69 return self._is_emulator 70 71 def activate_virtualenv(self): 72 """Overrides the method on AndroidMixin to be a no-op, because the 73 setup for wrench doesn't require a special virtualenv.""" 74 pass 75 76 def query_abs_dirs(self): 77 if self.abs_dirs: 78 return self.abs_dirs 79 80 abs_dirs = {} 81 82 abs_dirs["abs_work_dir"] = os.path.expanduser("~/.wrench") 83 if os.environ.get("MOZ_AUTOMATION", "0") == "1": 84 # In automation use the standard work dir if there is one 85 parent_abs_dirs = super().query_abs_dirs() 86 if "abs_work_dir" in parent_abs_dirs: 87 abs_dirs["abs_work_dir"] = parent_abs_dirs["abs_work_dir"] 88 89 abs_dirs["abs_blob_upload_dir"] = os.path.join(abs_dirs["abs_work_dir"], "logs") 90 abs_dirs["abs_apk_path"] = os.environ.get( 91 "WRENCH_APK", "gfx/wr/target/debug/apk/wrench.apk" 92 ) 93 abs_dirs["abs_reftests_path"] = os.environ.get( 94 "WRENCH_REFTESTS", "gfx/wr/wrench/reftests" 95 ) 96 if os.environ.get("MOZ_AUTOMATION", "0") == "1": 97 fetches_dir = os.environ.get("MOZ_FETCHES_DIR") 98 work_dir = ( 99 fetches_dir 100 if fetches_dir and self.is_emulator 101 else abs_dirs["abs_work_dir"] 102 ) 103 abs_dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux") 104 abs_dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device") 105 abs_dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar") 106 else: 107 mozbuild_path = os.environ.get( 108 "MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild") 109 ) 110 mozbuild_sdk = os.environ.get( 111 "ANDROID_SDK_HOME", os.path.join(mozbuild_path, "android-sdk-linux") 112 ) 113 abs_dirs["abs_sdk_dir"] = mozbuild_sdk 114 avds_dir = os.environ.get( 115 "ANDROID_EMULATOR_HOME", os.path.join(mozbuild_path, "android-device") 116 ) 117 abs_dirs["abs_avds_dir"] = avds_dir 118 abs_dirs["abs_bundletool_path"] = os.path.join( 119 mozbuild_path, "bundletool.jar" 120 ) 121 122 self.abs_dirs = abs_dirs 123 return self.abs_dirs 124 125 def logcat_start(self): 126 """Ensures any pre-existing logcat is cleared before starting to record 127 the new logcat. This is helpful when running multiple times in a local 128 emulator.""" 129 logcat_cmd = [self.adb_path, "-s", self.device_serial, "logcat", "-c"] 130 self.info(" ".join(logcat_cmd)) 131 subprocess.check_call(logcat_cmd) 132 super().logcat_start() 133 134 def wait_until_process_done(self, process_name, timeout): 135 """Waits until the specified process has exited. Polls the process list 136 every 5 seconds until the process disappears. 137 138 :param process_name: string containing the package name of the 139 application. 140 :param timeout: integer specifying the maximum time in seconds 141 to wait for the application to finish. 142 :returns: boolean - True if the process exited within the indicated 143 timeout, False if the process had not exited by the timeout. 144 """ 145 end_time = datetime.datetime.now() + datetime.timedelta(seconds=timeout) 146 while self.device.process_exist(process_name, timeout=timeout): 147 if datetime.datetime.now() > end_time: 148 stop_cmd = [ 149 self.adb_path, 150 "-s", 151 self.device_serial, 152 "shell", 153 "am", 154 "force-stop", 155 process_name, 156 ] 157 subprocess.check_call(stop_cmd) 158 return False 159 time.sleep(5) 160 161 return True 162 163 def setup_sdcard(self, test_mode): 164 self.device.rm(self.wrench_dir, recursive=True, force=True) 165 self.device.mkdir(self.wrench_dir, parents=True) 166 if test_mode == TestMode.REFTEST: 167 self.device.push( 168 self.query_abs_dirs()["abs_reftests_path"], 169 self.wrench_dir + "/reftests", 170 ) 171 args_file = os.path.join(self.query_abs_dirs()["abs_work_dir"], "wrench_args") 172 with open(args_file, "w") as argfile: 173 if self.is_emulator: 174 argfile.write("env: WRENCH_REFTEST_CONDITION_EMULATOR=1\n") 175 else: 176 argfile.write("env: WRENCH_REFTEST_CONDITION_DEVICE=1\n") 177 if test_mode == TestMode.OPTIMIZED_SHADER_COMPILATION: 178 argfile.write("--precache test_init") 179 elif test_mode == TestMode.UNOPTIMIZED_SHADER_COMPILATION: 180 argfile.write("--precache --use-unoptimized-shaders test_init") 181 elif test_mode == TestMode.SHADER_TEST: 182 argfile.write("--precache test_shaders") 183 elif test_mode == TestMode.REFTEST: 184 argfile.write("reftest") 185 self.device.push(args_file, self.wrench_dir + "/args") 186 187 def run_tests(self, timeout): 188 self.timed_screenshots(None) 189 self.device.launch_application( 190 app_name="org.mozilla.wrench", 191 activity_name="android.app.NativeActivity", 192 intent=None, 193 grant_runtime_permissions=False, 194 ) 195 self.info("App launched") 196 done = self.wait_until_process_done("org.mozilla.wrench", timeout=timeout) 197 if not done: 198 self._errored = True 199 self.error("Wrench still running after timeout") 200 201 def scrape_log(self): 202 """Wrench dumps stdout to a file rather than logcat because logcat 203 truncates long lines, and the base64 reftest images therefore get 204 truncated. In the past we split long lines and stitched them together 205 again, but this was unreliable. This scrapes the output file and dumps 206 it into our main log. 207 """ 208 logfile = tempfile.NamedTemporaryFile() 209 self.device.pull(self.wrench_dir + "/stdout", logfile.name) 210 with open(logfile.name, encoding="utf-8") as f: 211 self.info("=== scraped log output ===") 212 for line in f: 213 if "UNEXPECTED-FAIL" in line or "panicked" in line: 214 self._errored = True 215 self.error(line) 216 else: 217 self.info(line) 218 self.info("=== end scraped log output ===") 219 220 def setup_emulator(self): 221 avds_dir = self.query_abs_dirs()["abs_avds_dir"] 222 if not os.path.exists(avds_dir): 223 self.error("Unable to find android AVDs at %s" % avds_dir) 224 return 225 226 sdk_path = self.query_abs_dirs()["abs_sdk_dir"] 227 if not os.path.exists(sdk_path): 228 self.error("Unable to find android SDK at %s" % sdk_path) 229 return 230 self.start_emulator() 231 232 def do_test(self): 233 if self.is_emulator: 234 self.setup_emulator() 235 236 self.verify_device() 237 self.info("Logging device properties...") 238 self.info(self.shell_output("getprop", attempts=3)) 239 self.info("Uninstalling APK...") 240 self.device.uninstall_app("org.mozilla.wrench") 241 self.info("Installing APK...") 242 self.install_android_app(self.query_abs_dirs()["abs_apk_path"], replace=True) 243 244 if not self._errored: 245 self.info("Setting up SD card...") 246 self.setup_sdcard(TestMode.OPTIMIZED_SHADER_COMPILATION) 247 self.info("Running optimized shader compilation tests...") 248 self.run_tests(60) 249 self.info("Tests done; parsing log...") 250 self.scrape_log() 251 252 if not self._errored: 253 self.info("Setting up SD card...") 254 self.setup_sdcard(TestMode.UNOPTIMIZED_SHADER_COMPILATION) 255 self.info("Running unoptimized shader compilation tests...") 256 self.run_tests(60) 257 self.info("Tests done; parsing log...") 258 self.scrape_log() 259 260 if not self._errored: 261 self.info("Setting up SD card...") 262 self.setup_sdcard(TestMode.SHADER_TEST) 263 self.info("Running shader tests...") 264 self.run_tests(60 * 5) 265 self.info("Tests done; parsing log...") 266 self.scrape_log() 267 268 if not self._errored: 269 self.info("Setting up SD card...") 270 self.setup_sdcard(TestMode.REFTEST) 271 self.info("Running reftests...") 272 self.run_tests(60 * 30) 273 self.info("Tests done; parsing log...") 274 self.scrape_log() 275 276 self.logcat_stop() 277 self.info("All done!") 278 279 def check_errors(self): 280 if self._errored: 281 self.info("Errors encountered, terminating with error code...") 282 sys.exit(EXIT_STATUS_DICT[TBPL_FAILURE]) 283 284 285 if __name__ == "__main__": 286 test = AndroidWrench() 287 test.do_test() 288 test.check_errors()