android_startup_cmff_cvns.py (12323B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 import os 5 import re 6 import sys 7 import time 8 from datetime import datetime 9 10 import mozdevice 11 12 ITERATIONS = 50 13 DATETIME_FORMAT = "%Y.%m.%d" 14 PAGE_START_MOZ = re.compile("GeckoSession: handleMessage GeckoView:PageStart uri=") 15 16 PROD_FENIX = "fenix" 17 PROD_FOCUS = "focus" 18 PROD_GVEX = "geckoview" 19 PROD_CHRM = "chrome-m" 20 MOZILLA_PRODUCTS = [PROD_FENIX, PROD_FOCUS, PROD_GVEX] 21 22 OLD_VERSION_FOCUS_PAGE_START_LINE_COUNT = 3 23 NEW_VERSION_FOCUS_PAGE_START_LINE_COUNT = 2 24 STDOUT_LINE_COUNT = 2 25 26 TEST_COLD_MAIN_FF = "cold_main_first_frame" 27 TEST_COLD_MAIN_RESTORE = "cold_main_session_restore" 28 TEST_COLD_VIEW_FF = "cold_view_first_frame" 29 TEST_COLD_VIEW_NAV_START = "cold_view_nav_start" 30 TEST_URI = "https://example.com" 31 32 PROD_TO_CHANNEL_TO_PKGID = { 33 PROD_FENIX: { 34 "nightly": "org.mozilla.fenix", 35 "beta": "org.mozilla.firefox.beta", 36 "release": "org.mozilla.firefox", 37 "debug": "org.mozilla.fenix.debug", 38 }, 39 PROD_FOCUS: { 40 "nightly": "org.mozilla.focus.nightly", 41 "beta": "org.mozilla.focus.beta", # only present since post-fenix update. 42 "release": "org.mozilla.focus.nightly", 43 "debug": "org.mozilla.focus.debug", 44 }, 45 PROD_GVEX: { 46 "nightly": "org.mozilla.geckoview_example", 47 "release": "org.mozilla.geckoview_example", 48 }, 49 PROD_CHRM: { 50 "nightly": "com.android.chrome", 51 "release": "com.android.chrome", 52 }, 53 } 54 TEST_LIST = [ 55 "cold_main_first_frame", 56 "cold_view_nav_start", 57 "cold_view_first_frame", 58 "cold_main_session_restore", 59 ] 60 # "cold_view_first_frame", "cold_main_session_restore" are 2 disabled tests(broken) 61 62 63 class AndroidStartUpUnknownTestError(Exception): 64 """ 65 Test name provided is not one avaiable to test, this is either because 66 the test is currently not being tested or a typo in the spelling 67 """ 68 69 pass 70 71 72 class AndroidStartUpMatchingError(Exception): 73 """ 74 We expected a certain number of matches but did not get them 75 """ 76 77 pass 78 79 80 class Startup_test: 81 def __init__(self, browser, startup_test): 82 self.test_name = startup_test 83 self.product = browser 84 85 def run(self): 86 self.device = mozdevice.ADBDevice(use_root=False) 87 self.release_channel = "nightly" 88 self.architecture = "arm64-v8a" 89 self.startup_cache = True 90 self.package_id = PROD_TO_CHANNEL_TO_PKGID[self.product][self.release_channel] 91 self.proc_start = re.compile( 92 rf"ActivityManager: Start proc \d+:{self.package_id}/" 93 ) 94 self.key_name = f"{self.product}_nightly_{self.architecture}.apk" 95 results = self.run_tests() 96 97 # Cleanup 98 if self.product in MOZILLA_PRODUCTS: 99 self.device.uninstall_app(self.package_id) 100 101 return results 102 103 def should_alert(self, key_name): 104 return True 105 106 def run_tests(self): 107 measurements = {} 108 # Iterate through the tests in the test list 109 print(f"Running {self.test_name} on {self.package_id}...") 110 self.device.shell("mkdir -p /sdcard/Download") 111 time.sleep(self.get_warmup_delay_seconds()) 112 self.skip_onboarding(self.test_name) 113 test_measurements = [] 114 115 for i in range(ITERATIONS): 116 start_cmd_args = self.get_start_cmd(self.test_name) 117 print(start_cmd_args) 118 self.device.stop_application(self.package_id) 119 time.sleep(1) 120 print(f"iteration {i + 1}") 121 self.device.shell("logcat -c") 122 process = self.device.shell_output(start_cmd_args).splitlines() 123 test_measurements.append(self.get_measurement(self.test_name, process)) 124 if i % 10 == 0: 125 screenshot_file = f"/sdcard/Download/{self.product}_iteration_{i}_startup_done_frame.png" 126 self.device.shell(f"screencap -p {screenshot_file}") 127 self.device.command_output([ 128 "pull", 129 "-a", 130 screenshot_file, 131 os.environ["TESTING_DIR"], 132 ]) 133 self.device.stop_application(self.package_id) 134 print(f"{self.test_name}: {str(test_measurements)}") 135 # Bug 1934023 - create way to pass median and still have replicates available 136 # Bug 1971336 Remove the .mean metric once we have a sufficient data redundancy 137 measurements[f"{self.test_name}.mean"] = test_measurements 138 return measurements 139 140 def get_measurement(self, test_name, stdout): 141 if test_name in [TEST_COLD_MAIN_FF, TEST_COLD_VIEW_FF]: 142 return self.get_measurement_from_am_start_log(stdout) 143 elif ( 144 test_name in [TEST_COLD_VIEW_NAV_START, TEST_COLD_MAIN_RESTORE] 145 and self.product in MOZILLA_PRODUCTS 146 ): 147 # We must sleep until the Navigation::Start event occurs. If we don't 148 # the script will fail. This can take up to 14s on the G5 149 time.sleep(17) 150 proc = self.device.shell_output("logcat -d") 151 return self.get_measurement_from_nav_start_logcat(proc) 152 else: 153 raise AndroidStartUpUnknownTestError( 154 "invalid test settings selected, please double check that " 155 "the test name is valid and that the test is supported for " 156 "the browser you are testing" 157 ) 158 159 def get_measurement_from_am_start_log(self, stdout): 160 total_time_prefix = "TotalTime: " 161 matching_lines = [line for line in stdout if line.startswith(total_time_prefix)] 162 if len(matching_lines) != 1: 163 raise AndroidStartUpMatchingError( 164 f"Each run should only have 1 {total_time_prefix}." 165 f"However, this run unexpectedly had {matching_lines} matching lines" 166 ) 167 duration = int(matching_lines[0][len(total_time_prefix) :]) 168 return duration 169 170 def get_measurement_from_nav_start_logcat(self, process_output): 171 def __line_to_datetime(line): 172 date_str = " ".join(line.split(" ")[:2]) # e.g. "05-18 14:32:47.366" 173 # strptime needs microseconds. logcat outputs millis so we append zeroes 174 date_str_with_micros = date_str + "000" 175 return datetime.strptime(date_str_with_micros, "%m-%d %H:%M:%S.%f") 176 177 def __get_proc_start_datetime(): 178 # This regex may not work on older versions of Android: we don't care 179 # yet because supporting older versions isn't in our requirements. 180 proc_start_lines = [line for line in lines if self.proc_start.search(line)] 181 if len(proc_start_lines) != 1: 182 raise AndroidStartUpMatchingError( 183 f"Expected to match 1 process start string but matched {len(proc_start_lines)}" 184 ) 185 return __line_to_datetime(proc_start_lines[0]) 186 187 def __get_page_start_datetime(): 188 page_start_lines = [line for line in lines if PAGE_START_MOZ.search(line)] 189 page_start_line_count = len(page_start_lines) 190 page_start_assert_msg = "found len=" + str(page_start_line_count) 191 192 # In focus versions <= v8.8.2, it logs 3 PageStart lines and these include actual uris. 193 # We need to handle our assertion differently due to the different line count. In focus 194 # versions >= v8.8.3, this measurement is broken because the logcat were removed. 195 is_old_version_of_focus = ( 196 "about:blank" in page_start_lines[0] and self.product == PROD_FOCUS 197 ) 198 if is_old_version_of_focus: 199 assert ( 200 page_start_line_count 201 == OLD_VERSION_FOCUS_PAGE_START_LINE_COUNT # should be 3 202 ), page_start_assert_msg # Lines: about:blank, target URL, target URL. 203 else: 204 assert ( 205 page_start_line_count 206 == NEW_VERSION_FOCUS_PAGE_START_LINE_COUNT # Should be 2 207 ), page_start_assert_msg # Lines: about:blank, target URL. 208 return __line_to_datetime( 209 page_start_lines[1] 210 ) # 2nd PageStart is for target URL. 211 212 lines = process_output.split("\n") 213 elapsed_seconds = ( 214 __get_page_start_datetime() - __get_proc_start_datetime() 215 ).total_seconds() 216 elapsed_millis = round(elapsed_seconds * 1000) 217 return elapsed_millis 218 219 def get_warmup_delay_seconds(self): 220 """ 221 We've been told the start up cache is populated ~60s after first start up. As such, 222 we should measure start up with the start up cache populated. If the 223 args say we shouldn't wait, we only wait a short duration ~= visual completeness. 224 """ 225 return 60 if self.startup_cache else 5 226 227 def get_start_cmd(self, test_name): 228 intent_action_prefix = "android.intent.action.{}" 229 if test_name in [TEST_COLD_MAIN_FF, TEST_COLD_MAIN_RESTORE]: 230 intent = ( 231 f"-a {intent_action_prefix.format('MAIN')} " 232 f"-c android.intent.category.LAUNCHER" 233 ) 234 elif test_name in [TEST_COLD_VIEW_FF, TEST_COLD_VIEW_NAV_START]: 235 intent = f"-a {intent_action_prefix.format('VIEW')} -d {TEST_URI}" 236 else: 237 raise AndroidStartUpUnknownTestError( 238 "Unknown test provided please double check the test name and spelling" 239 ) 240 241 # You can't launch an app without an pkg_id/activity pair 242 component_name = self.get_component_name_for_intent(intent) 243 cmd = f"am start-activity -W -n {component_name} {intent} " 244 245 # If focus skip onboarding: it is not stateful so must be sent for every cold start intent 246 if self.product == PROD_FOCUS: 247 cmd += "--ez performancetest true" 248 249 return cmd 250 251 def get_component_name_for_intent(self, intent): 252 resolve_component_args = ( 253 f"cmd package resolve-activity --brief {intent} {self.package_id}" 254 ) 255 result_output = self.device.shell_output(resolve_component_args) 256 stdout = result_output.splitlines() 257 if len(stdout) != STDOUT_LINE_COUNT: # Should be 2 258 if "No activity found" in stdout: 259 raise Exception("Please verify your apk is installed") 260 raise AndroidStartUpMatchingError(f"expected 2 lines. Got: {stdout}") 261 return stdout[1] 262 263 def skip_onboarding(self, test_name): 264 self.device.enable_notifications(self.package_id) 265 if self.product in MOZILLA_PRODUCTS: 266 self.skip_app_onboarding() 267 268 if self.product == PROD_FOCUS or test_name not in { 269 TEST_COLD_MAIN_FF, 270 TEST_COLD_MAIN_RESTORE, 271 }: 272 return 273 274 def skip_app_onboarding(self): 275 """ 276 We skip onboarding for focus in measure_start_up.py because it's stateful 277 and needs to be called for every cold start intent. 278 Onboarding only visibly gets in the way of our MAIN test results. 279 """ 280 # This sets mutable state we only need to pass this flag once, before we start the test 281 self.device.shell( 282 f"am start-activity -W -a android.intent.action.MAIN --ez " 283 f"performancetest true -n {self.package_id}/org.mozilla.fenix.App" 284 ) 285 time.sleep(4) # ensure skip onboarding call has time to propagate. 286 287 288 if __name__ == "__main__": 289 if len(sys.argv) < 2: 290 raise Exception("Didn't pass the arg properly :(") 291 print(len(sys.argv)) 292 browser = sys.argv[1] 293 test = sys.argv[2] 294 start_video_timestamp = [] 295 296 Startup = Startup_test(browser, test) 297 startup_data = Startup.run() 298 # Bug 1971336 Remove the .mean metric once we have a sufficient data redundancy 299 print( 300 'perfMetrics: {"values": ', 301 startup_data[f"{test}.mean"], 302 ', "name": "' + f"{test}.mean" + '", "shouldAlert": true', 303 "}", 304 ) 305 306 print( 307 'perfMetrics: {"values": ', 308 startup_data[test], 309 ', "name": "' + f"{test}" + '", "shouldAlert": true', 310 "}", 311 )