android_emulator_pgo.py (10507B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import copy 7 import glob 8 import json 9 import os 10 import posixpath 11 import subprocess 12 import sys 13 import time 14 15 # load modules from parent dir 16 sys.path.insert(1, os.path.dirname(sys.path[0])) 17 18 from mozharness.base.script import BaseScript, PreScriptAction 19 from mozharness.mozilla.automation import EXIT_STATUS_DICT, TBPL_RETRY 20 from mozharness.mozilla.mozbase import MozbaseMixin 21 from mozharness.mozilla.testing.android import AndroidMixin 22 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options 23 24 PAGES = [ 25 "js-input/webkit/PerformanceTests/Speedometer/index.html", 26 "js-input/webkit/PerformanceTests/Speedometer3/index.html?startAutomatically=true", 27 "blueprint/sample.html", 28 "blueprint/forms.html", 29 "blueprint/grid.html", 30 "blueprint/elements.html", 31 "js-input/3d-thingy.html", 32 "js-input/crypto-otp.html", 33 "js-input/sunspider/3d-cube.html", 34 "js-input/sunspider/3d-morph.html", 35 "js-input/sunspider/3d-raytrace.html", 36 "js-input/sunspider/access-binary-trees.html", 37 "js-input/sunspider/access-fannkuch.html", 38 "js-input/sunspider/access-nbody.html", 39 "js-input/sunspider/access-nsieve.html", 40 "js-input/sunspider/bitops-3bit-bits-in-byte.html", 41 "js-input/sunspider/bitops-bits-in-byte.html", 42 "js-input/sunspider/bitops-bitwise-and.html", 43 "js-input/sunspider/bitops-nsieve-bits.html", 44 "js-input/sunspider/controlflow-recursive.html", 45 "js-input/sunspider/crypto-aes.html", 46 "js-input/sunspider/crypto-md5.html", 47 "js-input/sunspider/crypto-sha1.html", 48 "js-input/sunspider/date-format-tofte.html", 49 "js-input/sunspider/date-format-xparb.html", 50 "js-input/sunspider/math-cordic.html", 51 "js-input/sunspider/math-partial-sums.html", 52 "js-input/sunspider/math-spectral-norm.html", 53 "js-input/sunspider/regexp-dna.html", 54 "js-input/sunspider/string-base64.html", 55 "js-input/sunspider/string-fasta.html", 56 "js-input/sunspider/string-tagcloud.html", 57 "js-input/sunspider/string-unpack-code.html", 58 "js-input/sunspider/string-validate-input.html", 59 ] 60 61 62 class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin): 63 """ 64 Mozharness script to generate an android PGO profile using the emulator 65 """ 66 67 config_options = copy.deepcopy(testing_config_options) 68 69 def __init__(self, require_config_file=False): 70 super().__init__( 71 config_options=self.config_options, 72 all_actions=[ 73 "download", 74 "create-virtualenv", 75 "start-emulator", 76 "verify-device", 77 "install", 78 "run-tests", 79 ], 80 require_config_file=require_config_file, 81 config={ 82 "virtualenv_modules": [], 83 "virtualenv_requirements": [], 84 "require_test_zip": True, 85 "mozbase_requirements": "mozbase_source_requirements.txt", 86 }, 87 ) 88 89 # these are necessary since self.config is read only 90 c = self.config 91 self.installer_path = c.get("installer_path") 92 self.device_serial = "emulator-5554" 93 94 def query_abs_dirs(self): 95 if self.abs_dirs: 96 return self.abs_dirs 97 abs_dirs = super().query_abs_dirs() 98 dirs = {} 99 100 dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_src_dir"], "testing") 101 dirs["abs_blob_upload_dir"] = "/builds/worker/artifacts/blobber_upload_dir" 102 work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"] 103 dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils") 104 dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux") 105 dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device") 106 dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar") 107 108 for key in dirs.keys(): 109 if key not in abs_dirs: 110 abs_dirs[key] = dirs[key] 111 self.abs_dirs = abs_dirs 112 return self.abs_dirs 113 114 ########################################## 115 # Actions for AndroidProfileRun # 116 ########################################## 117 118 def preflight_install(self): 119 # in the base class, this checks for mozinstall, but we don't use it 120 pass 121 122 @PreScriptAction("create-virtualenv") 123 def pre_create_virtualenv(self, action): 124 dirs = self.query_abs_dirs() 125 self.register_virtualenv_module( 126 "marionette", 127 os.path.join(dirs["abs_test_install_dir"], "marionette", "client"), 128 ) 129 130 def download(self): 131 """ 132 Download host utilities 133 """ 134 dirs = self.query_abs_dirs() 135 self.xre_path = dirs["abs_xre_dir"] 136 137 def install(self): 138 """ 139 Install APKs on the device. 140 """ 141 assert self.installer_path is not None, ( 142 "Either add installer_path to the config or use --installer-path." 143 ) 144 self.install_android_app(self.installer_path) 145 self.info("Finished installing apps for %s" % self.device_serial) 146 147 def run_tests(self): 148 """ 149 Generate the PGO profile data 150 """ 151 from marionette_driver.marionette import Marionette 152 from mozdevice import ADBDeviceFactory, ADBTimeoutError 153 from mozhttpd import MozHttpd 154 from mozprofile import Preferences 155 from six import string_types 156 157 app = self.query_package_name() 158 159 IP = "10.0.2.2" 160 PORT = 8888 161 162 PATH_MAPPINGS = { 163 "/js-input/webkit/PerformanceTests": "third_party/webkit/PerformanceTests", 164 } 165 166 dirs = self.query_abs_dirs() 167 topsrcdir = dirs["abs_src_dir"] 168 adb = self.query_exe("adb") 169 170 path_mappings = { 171 k: os.path.join(topsrcdir, v) for k, v in PATH_MAPPINGS.items() 172 } 173 httpd = MozHttpd( 174 port=PORT, 175 docroot=os.path.join(topsrcdir, "build", "pgo"), 176 path_mappings=path_mappings, 177 ) 178 httpd.start(block=False) 179 180 profile_data_dir = os.path.join(topsrcdir, "testing", "profiles") 181 with open(os.path.join(profile_data_dir, "profiles.json")) as fh: 182 base_profiles = json.load(fh)["profileserver"] 183 184 prefpaths = [ 185 os.path.join(profile_data_dir, profile, "user.js") 186 for profile in base_profiles 187 ] 188 189 prefs = {} 190 for path in prefpaths: 191 prefs.update(Preferences.read_prefs(path)) 192 193 interpolation = {"server": "%s:%d" % httpd.httpd.server_address, "OOP": "false"} 194 for k, v in prefs.items(): 195 if isinstance(v, string_types): 196 v = v.format(**interpolation) 197 prefs[k] = Preferences.cast(v) 198 199 adbdevice = ADBDeviceFactory(adb=adb, device="emulator-5554") 200 201 outputdir = posixpath.join(adbdevice.test_root, "pgo_profile") 202 jarlog = posixpath.join(outputdir, "en-US.log") 203 profdata = posixpath.join(outputdir, "default_%p_random_%m.profraw") 204 205 env = {} 206 env["XPCOM_DEBUG_BREAK"] = "warn" 207 env["MOZ_IN_AUTOMATION"] = "1" 208 env["MOZ_JAR_LOG_FILE"] = jarlog 209 env["LLVM_PROFILE_FILE"] = profdata 210 211 if self.query_minidump_stackwalk(): 212 os.environ["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path 213 os.environ["MINIDUMP_SAVE_PATH"] = self.query_abs_dirs()["abs_blob_upload_dir"] 214 if not self.symbols_path: 215 self.symbols_path = os.environ.get("MOZ_FETCHES_DIR") 216 217 adbdevice.mkdir(outputdir, parents=True) 218 219 try: 220 # Run Fennec a first time to initialize its profile 221 driver = Marionette( 222 app="fennec", 223 package_name=app, 224 adb_path=adb, 225 bin="geckoview-androidTest.apk", 226 prefs=prefs, 227 connect_to_running_emulator=True, 228 startup_timeout=1000, 229 env=env, 230 symbols_path=self.symbols_path, 231 ) 232 driver.start_session() 233 234 # Now generate the profile and wait for it to complete 235 for page in PAGES: 236 driver.navigate("http://%s:%d/%s" % (IP, PORT, page)) 237 timeout = 2 238 if "Speedometer" in page: 239 # The Speedometer[23] test actually runs many tests internally in 240 # javascript, so it needs extra time to run through them. The 241 # emulator doesn't get very far through the whole suite, but 242 # this extra time at least lets some of them process. 243 timeout = 360 244 time.sleep(timeout) 245 246 driver.quit(in_app=True) 247 248 # Pull all the profraw files and en-US.log 249 adbdevice.pull(outputdir, "/builds/worker/workspace/") 250 except ADBTimeoutError: 251 self.fatal( 252 "INFRA-ERROR: Failed with an ADBTimeoutError", 253 EXIT_STATUS_DICT[TBPL_RETRY], 254 ) 255 256 profraw_files = glob.glob("/builds/worker/workspace/*.profraw") 257 if not profraw_files: 258 self.fatal("Could not find any profraw files in /builds/worker/workspace") 259 elif len(profraw_files) == 1: 260 self.fatal( 261 "Only found 1 profraw file. Did child processes terminate early?" 262 ) 263 merge_cmd = [ 264 os.path.join(os.environ["MOZ_FETCHES_DIR"], "clang/bin/llvm-profdata"), 265 "merge", 266 "-o", 267 "/builds/worker/workspace/merged.profdata", 268 ] + profraw_files 269 rc = subprocess.call(merge_cmd) 270 if rc != 0: 271 self.fatal( 272 "INFRA-ERROR: Failed to merge profile data. Corrupt profile?", 273 EXIT_STATUS_DICT[TBPL_RETRY], 274 ) 275 276 # tarfile doesn't support xz in this version of Python 277 tar_cmd = [ 278 "tar", 279 "-acvf", 280 "/builds/worker/artifacts/profdata.tar.xz", 281 "-C", 282 "/builds/worker/workspace", 283 "merged.profdata", 284 "en-US.log", 285 ] 286 subprocess.check_call(tar_cmd) 287 288 httpd.stop() 289 290 291 if __name__ == "__main__": 292 test = AndroidProfileRun() 293 test.run_and_exit()