android_startup_videoapplink.py (12017B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 import os 5 import pathlib 6 import re 7 import subprocess 8 import sys 9 import time 10 11 from mozperftest.utils import ON_TRY 12 13 # Add the python packages installed by mozperftest 14 sys.path.insert(0, os.environ["PYTHON_PACKAGES"]) 15 16 import cv2 17 import numpy as np 18 from mozdevice import ADBDevice 19 from mozperftest.profiler import ProfilingMediator 20 21 """ 22 Homeview: 23 An error of greater than 0.0002 indicates we have 1 icon, any less than this startup is done 24 Else(newssite(cvne), shopify (cvne), tab-restore): 25 An error of greater than 0.001 indicates we have the loading bar present, any less than this startup is done 26 """ 27 ACCEPTABLE_THRESHOLD_ERROR = { 28 "homeview_startup": 0.0002, 29 "cold_view_nav_end": 0.001, 30 "mobile_restore": 0.001, 31 } 32 BACKGROUND_TABS = [ 33 "https://www.google.com/search?q=toronto+weather", 34 "https://en.m.wikipedia.org/wiki/Anemone_hepatica", 35 "https://www.temu.com", 36 "https://www.espn.com/nfl/game/_/gameId/401671793/chiefs-falcons", 37 ] 38 ERROR_THRESHOLD = 8 # This is the lower bound for the high pass filter to remove noise 39 ITERATIONS = 5 40 MAX_STARTUP_TIME = 25000 # 25000ms = 25 seconds 41 PROD_CHRM = "chrome-m" 42 PROD_FENIX = "fenix" 43 44 45 class ImageAnalzer: 46 def __init__(self, browser, test, test_url): 47 self.video = None 48 self.browser = browser 49 self.test = test 50 self.acceptable_error = ACCEPTABLE_THRESHOLD_ERROR[test] 51 self.test_url = test_url 52 self.width = 0 53 self.height = 0 54 self.video_name = "" 55 self.package_name = os.environ["BROWSER_BINARY"] 56 self.device = ADBDevice() 57 self.profiler = ProfilingMediator() 58 self.cpu_data = {"total": {"time": []}} 59 if self.browser == PROD_FENIX: 60 self.package_and_activity = ( 61 "org.mozilla.fenix/org.mozilla.fenix.IntentReceiverActivity" 62 ) 63 elif self.browser == PROD_CHRM: 64 self.package_and_activity = ( 65 "com.android.chrome/com.google.android.apps.chrome.IntentDispatcher" 66 ) 67 else: 68 raise Exception("Bad browser name") 69 self.nav_start_command = f"am start-activity -W -n {self.package_and_activity} -a android.intent.action.VIEW -d " 70 self.view_intent_command = ( 71 f"am start-activity -W -n {self.package_and_activity} -a " 72 f"android.intent.action.VIEW" 73 ) 74 self.device.shell("mkdir -p /sdcard/Download") 75 self.device.shell("settings put global window_animation_scale 1") 76 self.device.shell("settings put global transition_animation_scale 1") 77 self.device.shell("settings put global animator_duration_scale 1") 78 self.device.disable_notifications("com.topjohnwu.magisk") 79 80 def app_setup(self): 81 if ON_TRY: 82 self.device.shell(f"pm clear {self.package_name}") 83 time.sleep(3) 84 self.skip_onboarding() 85 self.device.enable_notifications( 86 self.package_name 87 ) # enabling notifications for android 88 if self.test != "homeview_startup": 89 self.create_background_tabs() 90 self.device.shell(f"am force-stop {self.package_name}") 91 92 def skip_onboarding(self): 93 # Skip onboarding for chrome and fenix 94 if self.browser == PROD_CHRM: 95 self.device.shell_output( 96 'echo "chrome --no-default-browser-check --no-first-run ' 97 '--disable-fre" > /data/local/tmp/chrome-command-line ' 98 ) 99 self.device.shell("am set-debug-app --persistent com.android.chrome") 100 elif self.browser == PROD_FENIX: 101 self.device.shell( 102 "am start-activity -W -a android.intent.action.MAIN --ez " 103 "performancetest true -n org.mozilla.fenix/org.mozilla.fenix.App" 104 ) 105 106 def create_background_tabs(self): 107 # Add background tabs that allow us to see the impact of having background tabs open 108 # when we do the cold applink startup test. This makes the test workload more realistic 109 # and will also help catch regressions that affect per-open-tab startup work. 110 for website in BACKGROUND_TABS: 111 self.device.shell(self.nav_start_command + website) 112 time.sleep(3) 113 if self.test == "mobile_restore": 114 self.load_page_to_test_startup() 115 116 def get_video(self, run): 117 self.video_name = f"vid{run}_{self.browser}.mp4" 118 video_location = f"/sdcard/Download/{self.video_name}" 119 120 # Bug 1927548 - Recording command doesn't use mozdevice shell because the mozdevice shell 121 # outputs an adbprocess obj whose adbprocess.proc.kill() does not work when called 122 recording = subprocess.Popen([ 123 "adb", 124 "shell", 125 "screenrecord", 126 "--bugreport", 127 video_location, 128 ]) 129 130 # Start Profilers if enabled. 131 self.profiler.start() 132 133 if self.test == "cold_view_nav_end": 134 self.load_page_to_test_startup() 135 elif self.test in ["mobile_restore", "homeview_startup"]: 136 self.open_browser_with_view_intent() 137 138 # Stop Profilers if enabled. 139 self.profiler.stop(os.environ["TESTING_DIR"], run) 140 141 self.process_cpu_info(run) 142 recording.kill() 143 time.sleep(5) 144 self.device.command_output([ 145 "pull", 146 "-a", 147 video_location, 148 os.environ["TESTING_DIR"], 149 ]) 150 151 time.sleep(4) 152 video_location = str(pathlib.Path(os.environ["TESTING_DIR"], self.video_name)) 153 self.video = cv2.VideoCapture(video_location) 154 self.width = self.video.get(cv2.CAP_PROP_FRAME_WIDTH) 155 self.height = self.video.get(cv2.CAP_PROP_FRAME_HEIGHT) 156 self.device.shell(f"am force-stop {self.package_name}") 157 158 def get_image(self, frame_position, cropped=True, bw=True): 159 self.video.set(cv2.CAP_PROP_POS_FRAMES, frame_position) 160 ret, frame = self.video.read() 161 if bw: 162 frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 163 if not ret: 164 raise Exception("Frame not read") 165 # We crop out the top 100 pixels in each image as when we have --bug-report in the 166 # screen-recording command it displays a timestamp which interferes with the image comparisons 167 # We crop out the bottom 100 pixels to remove the fading in of the OS navigation controls 168 # We crop out the right 20 pixels to remove the scroll bar as it interferes with startup accuracy 169 if cropped: 170 return frame[100 : int(self.height) - 100, 0 : int(self.width) - 20] 171 return frame 172 173 def error(self, img1, img2): 174 h = img1.shape[0] 175 w = img1.shape[1] 176 diff = cv2.absdiff(img1, img2) 177 threshold_diff = cv2.threshold(diff, ERROR_THRESHOLD, 255, cv2.THRESH_BINARY)[1] 178 err = np.sum(threshold_diff**2) 179 mse = err / (float(h * w)) 180 return mse 181 182 def get_page_loaded_time(self, iteration): 183 """ 184 Returns the index of the frame where the main image on the shopify demo page is displayed 185 for the first time. 186 Specifically, we find the index of the first frame whose image is within an error of 20 187 compared to the final frame, via binary search. The binary search assumes that the error 188 compared to the final frame decreases monotonically in the captured frames. 189 """ 190 final_frame_index = self.video.get(cv2.CAP_PROP_FRAME_COUNT) - 1 191 final_frame = self.get_image(final_frame_index) 192 compare_to_end_frame = final_frame_index 193 diff = 0 194 195 while diff <= self.acceptable_error: 196 compare_to_end_frame -= 1 197 if compare_to_end_frame < 0: 198 raise Exception( 199 "Could not find the initial pageload frame, all possible images compared" 200 ) 201 diff = self.error(self.get_image(compare_to_end_frame), final_frame) 202 203 compare_to_end_frame += 1 204 save_image_location = pathlib.Path( 205 os.environ["TESTING_DIR"], 206 f"iter_{iteration}_startup_done.png", 207 ) 208 cv2.imwrite( 209 save_image_location, 210 self.get_image(compare_to_end_frame, cropped=False, bw=False), 211 ) 212 return compare_to_end_frame 213 214 def get_time_from_frame_num(self, frame_num): 215 self.video.set(cv2.CAP_PROP_POS_FRAMES, frame_num) 216 self.video.read() 217 video_timestamp = self.video.get(cv2.CAP_PROP_POS_MSEC) 218 if video_timestamp > MAX_STARTUP_TIME: 219 raise ValueError( 220 f"Startup time of {video_timestamp / 1000}s exceeds max time of {MAX_STARTUP_TIME / 1000}s" 221 ) 222 return video_timestamp 223 224 def load_page_to_test_startup(self): 225 # Navigate to the page we want to use for testing startup 226 self.device.shell(self.nav_start_command + self.test_url) 227 time.sleep(5) 228 229 def open_browser_with_view_intent(self): 230 self.device.shell(self.view_intent_command) 231 time.sleep(5) 232 233 def process_cpu_info(self, run): 234 cpu_info = self.device.shell_output( 235 f"ps -A -o name=,cpu=,time+=,%cpu= | grep {self.package_name}" 236 ).split("\n") 237 total_time_seconds = tab_processes_time = 0 238 for process in cpu_info: 239 process_name = re.search(r"([\w\d_.:]+)\s", process).group(1) 240 time = re.search(r"\s(\d+):(\d+).(\d+)\s", process) 241 time_seconds = ( 242 10 * int(time.group(3)) 243 + 1000 * int(time.group(2)) 244 + 60 * 1000 * int(time.group(1)) 245 ) 246 total_time_seconds += time_seconds 247 if "org.mozilla.fenix:tab" in process_name: 248 process_name = "org.mozilla.fenix:tab" 249 if ( 250 "com.android.chrome" in process_name 251 and "sandboxed_process" in process_name 252 ): 253 process_name = "com.android.chrome:sandboxed_process" 254 255 if process_name not in self.cpu_data.keys(): 256 self.cpu_data[process_name] = {} 257 self.cpu_data[process_name]["time"] = [] 258 259 if "org.mozilla.fenix:tab" == process_name: 260 tab_processes_time += time_seconds 261 continue 262 self.cpu_data[process_name]["time"] += [time_seconds] 263 264 if tab_processes_time: 265 self.cpu_data["org.mozilla.fenix:tab"]["time"] += [tab_processes_time] 266 self.cpu_data["total"]["time"] += [total_time_seconds] 267 268 def perfmetrics_cpu_data_ingesting(self): 269 for process in self.cpu_data.keys(): 270 print( 271 'perfMetrics: {"values": ' 272 + str(self.cpu_data[process]["time"]) 273 + ', "name": "' 274 + process 275 + '-cpu-time", "shouldAlert": true }' 276 ) 277 278 279 if __name__ == "__main__": 280 if len(sys.argv) != 4: 281 raise Exception("Didn't pass the args properly :(") 282 start_video_timestamp = [] 283 browser = sys.argv[1] 284 test = sys.argv[2] 285 test_url = sys.argv[3] 286 287 perfherder_names = { 288 "cold_view_nav_end": "applink_startup", 289 "mobile_restore": "tab_restore", 290 "homeview_startup": "homeview_startup", 291 } 292 293 ImageObject = ImageAnalzer(browser, test, test_url) 294 for iteration in range(ITERATIONS): 295 ImageObject.app_setup() 296 ImageObject.get_video(iteration) 297 nav_done_frame = ImageObject.get_page_loaded_time(iteration) 298 start_video_timestamp += [ImageObject.get_time_from_frame_num(nav_done_frame)] 299 print( 300 'perfMetrics: {"values": ' 301 + str(start_video_timestamp) 302 + ', "name": "' 303 + perfherder_names[test] 304 + '", "shouldAlert": true}' 305 ) 306 ImageObject.perfmetrics_cpu_data_ingesting()