test_raptor.py (10701B)
1 import os 2 import sys 3 import threading 4 import traceback 5 from unittest import mock 6 from unittest.mock import patch 7 8 import mozunit 9 import pytest 10 from mozprofile import BaseProfile 11 12 # need this so the raptor unit tests can find output & filter classes 13 here = os.path.abspath(os.path.dirname(__file__)) 14 raptor_dir = os.path.join(os.path.dirname(here), "raptor") 15 sys.path.insert(0, raptor_dir) 16 17 18 from browsertime import BrowsertimeAndroid, BrowsertimeDesktop 19 20 DEFAULT_TIMEOUT = 125 21 22 23 class TestBrowserThread(threading.Thread): 24 def __init__(self, raptor_instance, tests, names): 25 super().__init__() 26 self.raptor_instance = raptor_instance 27 self.tests = tests 28 self.names = names 29 self.exc = None 30 31 def print_error(self): 32 if self.exc is None: 33 return 34 type, value, tb = self.exc 35 traceback.print_exception(type, value, tb, None, sys.stderr) 36 37 def run(self): 38 try: 39 self.raptor_instance.run_tests(self.tests, self.names) 40 except BaseException: 41 self.exc = sys.exc_info() 42 43 44 # Perftest tests 45 @patch("logger.logger.RaptorLogger.info") 46 @patch("logger.logger.RaptorLogger.critical") 47 @pytest.mark.parametrize( 48 "perftest_class, app_name", 49 [ 50 [BrowsertimeDesktop, "firefox"], 51 [BrowsertimeAndroid, "geckoview"], 52 ], 53 ) 54 def test_build_profile( 55 mock_info, mock_critical, options, perftest_class, app_name, get_prefs 56 ): 57 options["app"] = app_name 58 59 # We need to do the mock ourselves because of how the perftest_class 60 # is being defined 61 original_get = perftest_class.get_browser_meta 62 perftest_class.get_browser_meta = mock.MagicMock() 63 perftest_class.get_browser_meta.return_value = (app_name, "100") 64 65 perftest_instance = perftest_class(**options) 66 perftest_class.get_browser_meta = original_get 67 68 assert isinstance(perftest_instance.profile, BaseProfile) 69 if app_name != "firefox": 70 return 71 72 # These prefs are set in mozprofile 73 firefox_prefs = [ 74 'user_pref("app.update.checkInstallTime", false);', 75 'user_pref("app.update.disabledForTesting", true);', 76 'user_pref("' 77 'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);', 78 ] 79 # This pref is set in raptor 80 raptor_pref = 'user_pref("security.enable_java", false);' 81 82 prefs_file = os.path.join(perftest_instance.profile.profile, "user.js") 83 with open(prefs_file) as fh: 84 prefs = fh.read() 85 for firefox_pref in firefox_prefs: 86 assert firefox_pref in prefs 87 assert raptor_pref in prefs 88 89 90 @patch("logger.logger.RaptorLogger.info") 91 @patch("logger.logger.RaptorLogger.critical") 92 def test_perftest_host_ip( 93 mock_info, mock_critical, ConcretePerftest, options, get_prefs 94 ): 95 os.environ["HOST_IP"] = "some_dummy_host_ip" 96 options["host"] = "HOST_IP" 97 98 perftest = ConcretePerftest(**options) 99 100 assert perftest.config["host"] == os.environ["HOST_IP"] 101 102 103 @patch("logger.logger.RaptorLogger.info") 104 @patch("logger.logger.RaptorLogger.critical") 105 @pytest.mark.parametrize( 106 "app_name, expected_e10s_flag", 107 [["firefox", True], ["geckoview", True]], 108 ) 109 def test_e10s_enabling( 110 mock_info, mock_critical, ConcretePerftest, options, app_name, expected_e10s_flag 111 ): 112 options["app"] = app_name 113 perftest = ConcretePerftest(profile_class="firefox", **options) 114 assert perftest.config["e10s"] == expected_e10s_flag 115 116 117 @patch("logger.logger.RaptorLogger.info") 118 @patch("logger.logger.RaptorLogger.critical") 119 def test_profile_was_provided_locally( 120 mock_info, mock_critical, ConcretePerftest, options 121 ): 122 perftest = ConcretePerftest(**options) 123 assert os.path.isdir(perftest.config["local_profile_dir"]) 124 125 126 @patch("logger.logger.RaptorLogger.info") 127 @patch("logger.logger.RaptorLogger.critical") 128 @pytest.mark.parametrize( 129 "profile_class, app, expected_profile", 130 [ 131 ["firefox", "firefox", "firefox"], 132 [None, "firefox", "firefox"], 133 ["firefox", None, "firefox"], 134 ], 135 ) 136 def test_profile_class_assignation( 137 mock_info, 138 mock_critical, 139 ConcretePerftest, 140 options, 141 profile_class, 142 app, 143 expected_profile, 144 ): 145 options["app"] = app 146 perftest = ConcretePerftest(profile_class=profile_class, **options) 147 assert perftest.profile_class == expected_profile 148 149 150 @patch("logger.logger.RaptorLogger.info") 151 @patch("logger.logger.RaptorLogger.critical") 152 def test_raptor_venv(mock_info, mock_critical, ConcretePerftest, options): 153 perftest = ConcretePerftest(**options) 154 assert perftest.raptor_venv.endswith("raptor-venv") 155 156 157 @patch("logger.logger.RaptorLogger.info") 158 @patch("logger.logger.RaptorLogger.critical") 159 @mock.patch("perftest.Perftest.build_browser_profile", new=mock.MagicMock()) 160 @pytest.mark.parametrize( 161 "app," 162 "run_local," 163 "debug_mode," 164 "conditioned_profile," 165 "post_startup_delay," 166 "expected_post_startup_delay," 167 "expected_debug_mode", 168 [ 169 ["firefox", True, True, None, 1234, 1234, True], 170 ["firefox", True, True, None, None, 3000, True], 171 ["firefox", True, False, None, None, 30000, False], 172 ["firefox", True, False, "settled", None, 1000, False], 173 ["fenix", True, False, None, None, 20000, False], 174 ["fenix", True, False, "settled", None, 1000, False], 175 ["firefox", False, False, None, 1234, 1234, False], 176 ["firefox", False, False, None, 12345, 12345, False], 177 ["firefox", True, False, None, 1234, 1234, False], 178 ["firefox", True, False, None, 12345, 12345, False], 179 ["firefox", False, True, None, 1234, 1234, False], 180 ["firefox", False, True, None, 12345, 12345, False], 181 ], 182 ) 183 def test_post_startup_delay( 184 mock_info, 185 mock_critical, 186 ConcretePerftest, 187 options, 188 app, 189 run_local, 190 debug_mode, 191 conditioned_profile, 192 post_startup_delay, 193 expected_post_startup_delay, 194 expected_debug_mode, 195 ): 196 options["app"] = app 197 198 perftest = ConcretePerftest( 199 run_local=run_local, 200 debug_mode=debug_mode, 201 post_startup_delay=post_startup_delay, 202 conditioned_profile=conditioned_profile, 203 **options, 204 ) 205 206 assert perftest.post_startup_delay == expected_post_startup_delay 207 assert perftest.debug_mode == expected_debug_mode 208 209 210 @patch("logger.logger.RaptorLogger.info") 211 @patch("logger.logger.RaptorLogger.critical") 212 @pytest.mark.parametrize( 213 "alert, expected_alert", [["test_to_alert_on", "test_to_alert_on"], [None, None]] 214 ) 215 def test_perftest_run_test_setup( 216 mock_info, 217 mock_critical, 218 ConcretePerftest, 219 options, 220 mock_test, 221 alert, 222 expected_alert, 223 ): 224 perftest = ConcretePerftest(**options) 225 mock_test["alert_on"] = alert 226 227 perftest.run_test_setup(mock_test) 228 229 assert perftest.config["subtest_alert_on"] == expected_alert 230 231 232 # Browsertime tests 233 @patch("logger.logger.RaptorLogger.info") 234 @patch("logger.logger.RaptorLogger.critical") 235 def test_cmd_arguments( 236 mock_info, mock_critical, ConcreteBrowsertime, browsertime_options, mock_test 237 ): 238 expected_cmd = { 239 browsertime_options["browsertime_node"], 240 browsertime_options["browsertime_browsertimejs"], 241 "--firefox.geckodriverPath", 242 browsertime_options["browsertime_geckodriver"], 243 "--browsertime.page_cycles", 244 "1", 245 "--browsertime.url", 246 mock_test["test_url"], 247 "--browsertime.secondary_url", 248 mock_test["secondary_url"], 249 "--browsertime.page_cycle_delay", 250 "1000", 251 "--browsertime.post_startup_delay", 252 str(DEFAULT_TIMEOUT), 253 "--firefox.profileTemplate", 254 "--skipHar", 255 "--video", 256 "true", 257 "--visualMetrics", 258 "false", 259 "--timeouts.pageLoad", 260 str(DEFAULT_TIMEOUT), 261 "--timeouts.script", 262 str(DEFAULT_TIMEOUT), 263 "--resultDir", 264 "--iterations", 265 "1", 266 } 267 if browsertime_options.get("app") in ["chrome", "chrome-m"]: 268 expected_cmd.add( 269 "--chrome.chromedriverPath", browsertime_options["browsertime_chromedriver"] 270 ) 271 272 browsertime = ConcreteBrowsertime( 273 post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options 274 ) 275 browsertime.run_test_setup(mock_test) 276 cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT) 277 278 assert expected_cmd.issubset(set(cmd)) 279 280 281 def extract_arg_value(cmd, arg): 282 param_index = cmd.index(arg) + 1 283 return cmd[param_index] 284 285 286 @pytest.mark.parametrize( 287 "arg_to_test, expected, test_patch, options_patch", 288 [ 289 ["--iterations", "1", {}, {"browser_cycles": None}], 290 ["--iterations", "123", {"browser_cycles": 123}, {}], 291 ["--video", "false", {}, {"browsertime_video": None}], 292 ["--video", "true", {}, {"browsertime_video": "dummy_value"}], 293 ["--timeouts.script", str(DEFAULT_TIMEOUT), {}, {"page_cycles": None}], 294 ["--timeouts.script", str(123 * DEFAULT_TIMEOUT), {"page_cycles": 123}, {}], 295 ["--browsertime.page_cycles", "1", {}, {"page_cycles": None}], 296 ["--browsertime.page_cycles", "123", {"page_cycles": 123}, {}], 297 ], 298 ) 299 @patch("logger.logger.RaptorLogger.info") 300 @patch("logger.logger.RaptorLogger.critical") 301 def test_browsertime_arguments( 302 mock_info, 303 mock_critical, 304 ConcreteBrowsertime, 305 browsertime_options, 306 mock_test, 307 arg_to_test, 308 expected, 309 test_patch, 310 options_patch, 311 ): 312 mock_test.update(test_patch) 313 browsertime_options.update(options_patch) 314 browsertime = ConcreteBrowsertime( 315 post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options 316 ) 317 browsertime.run_test_setup(mock_test) 318 cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT) 319 320 param_value = extract_arg_value(cmd, arg_to_test) 321 assert expected == param_value 322 323 324 @pytest.mark.parametrize( 325 "timeout, expected_timeout, test_patch, options_patch", 326 [ 327 [0, 80, {}, {}], 328 [0, 80, {}, {"gecko_profile": False}], 329 [1000, 381, {}, {"gecko_profile": True}], 330 ], 331 ) 332 @patch("logger.logger.RaptorLogger.info") 333 @patch("logger.logger.RaptorLogger.critical") 334 def test_compute_process_timeout( 335 mock_info, 336 mock_critical, 337 ConcreteBrowsertime, 338 browsertime_options, 339 mock_test, 340 timeout, 341 expected_timeout, 342 test_patch, 343 options_patch, 344 ): 345 mock_test.update(test_patch) 346 browsertime_options.update(options_patch) 347 browsertime = ConcreteBrowsertime( 348 post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options 349 ) 350 bt_timeout = browsertime._compute_process_timeout(mock_test, timeout, []) 351 assert bt_timeout == expected_timeout 352 353 354 if __name__ == "__main__": 355 mozunit.main()