test_manifest.py (15443B)
1 import os 2 import sys 3 from unittest.mock import patch 4 from urllib.parse import parse_qs, urlsplit 5 6 import mozinfo 7 import mozunit 8 import pytest 9 10 # need this so raptor imports work both from /raptor and via mach 11 here = os.path.abspath(os.path.dirname(__file__)) 12 13 raptor_dir = os.path.join(os.path.dirname(here), "raptor") 14 sys.path.insert(0, raptor_dir) 15 16 from manifest import ( 17 add_test_url_params, 18 get_browser_test_list, 19 get_raptor_test_list, 20 validate_test_toml, 21 ) 22 23 # some test details (test TOMLs) 24 VALID_MANIFESTS = [ 25 { 26 # page load test with local playback 27 "alert_on": "fcp", 28 "alert_threshold": 2.0, 29 "apps": "firefox", 30 "lower_is_better": True, 31 "manifest": "valid_details_0", 32 "measure": ["fnbpaint", "fcp"], 33 "page_cycles": 25, 34 "playback": "mitmproxy", 35 "playback_pageset_manifest": "pageset.manifest", 36 "test_url": "http://www.test-url/goes/here", 37 "type": "pageload", 38 "unit": "ms", 39 }, 40 { 41 # test optional settings with None 42 "alert_threshold": 2.0, 43 "apps": "firefox", 44 "lower_is_better": True, 45 "manifest": "valid_details_1", 46 "measure": "fnbpaint, fcb", 47 "page_cycles": 25, 48 "test_url": "http://www.test-url/goes/here", 49 "type": "pageload", 50 "unit": "ms", 51 "alert_change_type": None, 52 "alert_on": None, 53 "playback": None, 54 }, 55 { 56 # page load test for geckoview 57 "alert_threshold": 2.0, 58 "apps": "geckoview", 59 "browser_cycles": 10, 60 "lower_is_better": False, 61 "manifest": "valid_details_2", 62 "measure": "fcp", 63 "page_cycles": 1, 64 "test_url": "http://www.test-url/goes/here", 65 "type": "pageload", 66 "unit": "score", 67 }, 68 { 69 # benchmark test for chrome 70 "alert_threshold": 2.0, 71 "apps": "chrome", 72 "lower_is_better": False, 73 "manifest": "valid_details_1", 74 "measure": "fcp", 75 "page_cycles": 5, 76 "test_url": "http://www.test-url/goes/here", 77 "type": "benchmark", 78 "unit": "score", 79 }, 80 ] 81 82 INVALID_MANIFESTS = [ 83 { 84 "alert_threshold": 2.0, 85 "apps": "firefox", 86 "lower_is_better": True, 87 "manifest": "invalid_details_0", 88 "page_cycles": 25, 89 "playback": "mitmproxy", 90 "playback_pageset_manifest": "pageset.manifest", 91 "test_url": "http://www.test-url/goes/here", 92 "type": "pageload", 93 "unit": "ms", 94 }, 95 { 96 "alert_threshold": 2.0, 97 "apps": "chrome", 98 "lower_is_better": True, 99 "manifest": "invalid_details_1", 100 "measure": "fnbpaint, fcp", 101 "page_cycles": 25, 102 "playback": "mitmproxy", 103 "test_url": "http://www.test-url/goes/here", 104 "type": "pageload", 105 "unit": "ms", 106 }, 107 { 108 "alert_on": "nope", 109 "alert_threshold": 2.0, 110 "apps": "firefox", 111 "lower_is_better": True, 112 "manifest": "invalid_details_2", 113 "measure": "fnbpaint, fcp", 114 "page_cycles": 25, 115 "playback": "mitmproxy", 116 "playback_pageset_manifest": "pageset.manifest", 117 "test_url": "http://www.test-url/goes/here", 118 "type": "pageload", 119 "unit": "ms", 120 }, 121 ] 122 123 124 @patch("logger.logger.RaptorLogger.info") 125 @patch("logger.logger.RaptorLogger.critical") 126 @pytest.mark.parametrize("app", ["firefox", "chrome", "geckoview", "refbrow", "fenix"]) 127 def test_get_browser_test_list(mock_info, mock_critical, app): 128 test_list = get_browser_test_list(app, run_local=True) 129 assert len(test_list) > 0 130 131 132 @pytest.mark.parametrize("test_details", VALID_MANIFESTS) 133 def test_validate_test_toml_valid(test_details): 134 assert validate_test_toml(test_details) 135 136 137 @patch("logger.logger.RaptorLogger.info") 138 @patch("logger.logger.RaptorLogger.critical") 139 @patch("logger.logger.RaptorLogger.error") 140 @pytest.mark.parametrize("test_details", INVALID_MANIFESTS) 141 def test_validate_test_toml_invalid(mock_info, mock_critical, mock_error, test_details): 142 assert not (validate_test_toml(test_details)) 143 144 145 @patch("logger.logger.RaptorLogger.info") 146 def test_get_raptor_test_list_firefox(mock_info, create_args): 147 args = create_args(browser_cycles=1) 148 149 test_list = get_raptor_test_list(args, mozinfo.os) 150 assert len(test_list) == 4 151 152 subtests = ["test-page-1", "test-page-2", "test-page-3", "test-page-4"] 153 154 for next_subtest in test_list: 155 assert next_subtest["name"] in subtests 156 157 158 @patch("logger.logger.RaptorLogger.info") 159 def test_get_raptor_test_list_chrome(mock_info, create_args): 160 args = create_args(app="chrome", test="speedometer", browser_cycles=1) 161 162 test_list = get_raptor_test_list(args, mozinfo.os) 163 assert len(test_list) == 1 164 assert test_list[0]["name"] == "speedometer" 165 166 167 @patch("logger.logger.RaptorLogger.info") 168 def test_get_raptor_test_list_geckoview(mock_info, create_args): 169 args = create_args(app="geckoview", test="unity-webgl", browser_cycles=1) 170 171 test_list = get_raptor_test_list(args, mozinfo.os) 172 assert len(test_list) == 1 173 assert test_list[0]["name"] == "unity-webgl" 174 175 176 @patch("logger.logger.RaptorLogger.info") 177 def test_get_raptor_test_list_gecko_profiling_enabled(mock_info, create_args): 178 args = create_args(test="amazon", gecko_profile=True, browser_cycles=1) 179 180 test_list = get_raptor_test_list(args, mozinfo.os) 181 assert len(test_list) == 1 182 assert test_list[0]["name"] == "amazon" 183 assert test_list[0]["gecko_profile"] is True 184 assert test_list[0].get("gecko_profile_interval") == "1" 185 assert test_list[0].get("gecko_profile_threads") is None 186 assert test_list[0].get("gecko_profile_features") is None 187 188 189 @patch("logger.logger.RaptorLogger.info") 190 def test_get_raptor_test_list_gecko_profiling_enabled_args_override( 191 mock_info, create_args 192 ): 193 args = create_args( 194 test="amazon", 195 gecko_profile=True, 196 gecko_profile_entries=42, 197 gecko_profile_interval=100, 198 gecko_profile_threads="Foo", 199 gecko_profile_features="Mood,UserNetWorth", 200 browser_cycles=1, 201 ) 202 203 test_list = get_raptor_test_list(args, mozinfo.os) 204 assert len(test_list) == 1 205 assert test_list[0]["name"] == "amazon" 206 assert test_list[0]["gecko_profile"] is True 207 assert test_list[0]["gecko_profile_entries"] == "42" 208 assert test_list[0]["gecko_profile_interval"] == "100" 209 assert test_list[0]["gecko_profile_threads"] == "Foo" 210 assert test_list[0]["gecko_profile_features"] == "Mood,UserNetWorth" 211 212 213 @patch("logger.logger.RaptorLogger.info") 214 def test_get_raptor_test_list_gecko_profiling_enabled_extra_args_override( 215 mock_info, create_args 216 ): 217 args = create_args( 218 test="amazon", 219 gecko_profile=True, 220 gecko_profile_entries=42, 221 gecko_profile_interval=100, 222 gecko_profile_extra_threads=["Foo", "Oof"], 223 gecko_profile_threads="String,Rope", 224 browser_cycles=1, 225 ) 226 227 test_list = get_raptor_test_list(args, mozinfo.os) 228 assert len(test_list) == 1 229 assert test_list[0]["name"] == "amazon" 230 assert test_list[0]["gecko_profile"] is True 231 assert test_list[0]["gecko_profile_entries"] == "42" 232 assert test_list[0]["gecko_profile_interval"] == "100" 233 assert test_list[0]["gecko_profile_threads"] == "String,Rope,Foo,Oof" 234 235 236 @patch("logger.logger.RaptorLogger.info") 237 def test_get_raptor_test_list_gecko_profiling_disabled(mock_info, create_args): 238 args = create_args( 239 test="amazon", 240 gecko_profile=False, 241 gecko_profile_entries=42, 242 gecko_profile_interval=100, 243 gecko_profile_threads=["Foo"], 244 gecko_profile_features=["Temperature"], 245 browser_cycles=1, 246 ) 247 248 test_list = get_raptor_test_list(args, mozinfo.os) 249 assert len(test_list) == 1 250 assert test_list[0]["name"] == "amazon" 251 assert test_list[0].get("gecko_profile") is None 252 assert test_list[0].get("gecko_profile_entries") is None 253 assert test_list[0].get("gecko_profile_interval") is None 254 assert test_list[0].get("gecko_profile_threads") is None 255 assert test_list[0].get("gecko_profile_features") is None 256 257 258 @patch("logger.logger.RaptorLogger.info") 259 def test_get_raptor_test_list_gecko_profiling_disabled_args_override( 260 mock_info, create_args 261 ): 262 args = create_args( 263 test="amazon", 264 gecko_profile=False, 265 gecko_profile_entries=42, 266 gecko_profile_interval=100, 267 gecko_profile_threads=["Foo"], 268 gecko_profile_features=["Temperature"], 269 browser_cycles=1, 270 ) 271 272 test_list = get_raptor_test_list(args, mozinfo.os) 273 assert len(test_list) == 1 274 assert test_list[0]["name"] == "amazon" 275 assert test_list[0].get("gecko_profile") is None 276 assert test_list[0].get("gecko_profile_entries") is None 277 assert test_list[0].get("gecko_profile_interval") is None 278 assert test_list[0].get("gecko_profile_threads") is None 279 assert test_list[0].get("gecko_profile_features") is None 280 281 282 @patch("logger.logger.RaptorLogger.info") 283 def test_get_raptor_test_list_extra_profiler_run_enabled(mock_info, create_args): 284 args = create_args(test="amazon", extra_profiler_run=True, browser_cycles=1) 285 286 test_list = get_raptor_test_list(args, mozinfo.os) 287 assert len(test_list) == 1 288 assert test_list[0]["name"] == "amazon" 289 assert test_list[0]["extra_profiler_run"] is True 290 291 292 @patch("logger.logger.RaptorLogger.info") 293 def test_get_raptor_test_list_extra_profiler_run_disabled(mock_info, create_args): 294 args = create_args(test="amazon", browser_cycles=1) 295 296 test_list = get_raptor_test_list(args, mozinfo.os) 297 assert len(test_list) == 1 298 assert test_list[0]["name"] == "amazon" 299 assert test_list[0].get("extra_profiler_run") is None 300 301 302 @patch("logger.logger.RaptorLogger.info") 303 def test_get_raptor_test_list_extra_profiler_run_enabled_chrome(mock_info, create_args): 304 args = create_args( 305 app="chrome", test="amazon", extra_profiler_run=True, browser_cycles=1 306 ) 307 308 test_list = get_raptor_test_list(args, mozinfo.os) 309 assert len(test_list) == 1 310 assert test_list[0]["name"] == "amazon" 311 assert test_list[0]["extra_profiler_run"] is True 312 313 314 @patch("logger.logger.RaptorLogger.info") 315 def test_get_raptor_test_list_extra_profiler_run_disabled_chrome( 316 mock_info, create_args 317 ): 318 args = create_args(app="chrome", test="amazon", browser_cycles=1) 319 320 test_list = get_raptor_test_list(args, mozinfo.os) 321 assert len(test_list) == 1 322 assert test_list[0]["name"] == "amazon" 323 assert test_list[0].get("extra_profiler_run") is None 324 325 326 @patch("logger.logger.RaptorLogger.info") 327 def test_get_raptor_test_list_debug_mode(mock_info, create_args): 328 args = create_args(test="amazon", debug_mode=True, browser_cycles=1) 329 330 test_list = get_raptor_test_list(args, mozinfo.os) 331 assert len(test_list) == 1 332 assert test_list[0]["name"] == "amazon" 333 assert test_list[0]["debug_mode"] is True 334 assert test_list[0]["page_cycles"] == 2 335 336 337 @patch("logger.logger.RaptorLogger.info") 338 def test_get_raptor_test_list_using_live_sites(mock_info, create_args): 339 args = create_args(test="amazon", live_sites=True, browser_cycles=1) 340 341 test_list = get_raptor_test_list(args, mozinfo.os) 342 assert len(test_list) == 1 343 assert test_list[0]["name"] == "amazon" 344 assert test_list[0]["use_live_sites"] == "true" 345 assert test_list[0]["playback"] is None 346 347 348 @patch("logger.logger.RaptorLogger.info") 349 def test_get_raptor_test_list_using_collect_perfstats(mock_info, create_args): 350 args = create_args(test="amazon", collect_perfstats=True, browser_cycles=1) 351 352 test_list = get_raptor_test_list(args, mozinfo.os) 353 assert len(test_list) == 1 354 assert test_list[0]["name"] == "amazon" 355 assert test_list[0]["perfstats"] == "true" 356 357 358 @patch("logger.logger.RaptorLogger.info") 359 def test_get_raptor_test_list_override_page_cycles(mock_info, create_args): 360 args = create_args(test="amazon", page_cycles=99, browser_cycles=1) 361 362 test_list = get_raptor_test_list(args, mozinfo.os) 363 assert len(test_list) == 1 364 assert test_list[0]["name"] == "amazon" 365 assert test_list[0]["page_cycles"] == 99 366 367 368 @patch("logger.logger.RaptorLogger.info") 369 def test_get_raptor_test_list_override_page_timeout(mock_info, create_args): 370 args = create_args(test="amazon", page_timeout=9999, browser_cycles=1) 371 372 test_list = get_raptor_test_list(args, mozinfo.os) 373 assert len(test_list) == 1 374 assert test_list[0]["name"] == "amazon" 375 assert test_list[0]["page_timeout"] == 9999 376 377 378 @patch("logger.logger.RaptorLogger.info") 379 def test_get_raptor_test_list_add_test_url_params(mock_info, create_args): 380 args = create_args(test="amazon", test_url_params="c=4", browser_cycles=1) 381 382 test_list = get_raptor_test_list(args, mozinfo.os) 383 assert len(test_list) == 1 384 assert test_list[0]["name"] == "amazon" 385 query_params = parse_qs(urlsplit(test_list[0]["test_url"]).query) 386 assert query_params.get("c") == ["4"] 387 388 389 @patch("logger.logger.RaptorLogger.info") 390 def test_get_raptor_test_list_refbrow(mock_info, create_args): 391 args = create_args(app="refbrow", test="speedometer", browser_cycles=1) 392 393 test_list = get_raptor_test_list(args, mozinfo.os) 394 assert len(test_list) == 1 395 assert test_list[0]["name"] == "speedometer" 396 397 398 @patch("logger.logger.RaptorLogger.info") 399 def test_get_raptor_test_list_fenix(mock_info, create_args): 400 args = create_args(app="fenix", test="speedometer", browser_cycles=1) 401 402 test_list = get_raptor_test_list(args, mozinfo.os) 403 assert len(test_list) == 1 404 assert test_list[0]["name"] == "speedometer" 405 406 407 def test_add_test_url_params_with_single_extra_param(): 408 initial_test_url = "http://test.com?a=1&b=2" 409 extra_params = "c=3" 410 411 result = add_test_url_params(initial_test_url, extra_params) 412 413 expected_params = {"a": ["1"], "b": ["2"], "c": ["3"]} 414 actual_params = parse_qs(urlsplit(result).query) 415 assert actual_params == expected_params 416 417 418 def test_add_test_url_params_with_multiple_extra_param(): 419 initial_test_url = "http://test.com?a=1&b=2" 420 extra_params = "c=3&d=4" 421 422 result = add_test_url_params(initial_test_url, extra_params) 423 424 expected_params = {"a": ["1"], "b": ["2"], "c": ["3"], "d": ["4"]} 425 actual_params = parse_qs(urlsplit(result).query) 426 assert actual_params == expected_params 427 428 429 def test_add_test_url_params_without_params_in_url(): 430 initial_test_url = "http://test.com" 431 extra_params = "c=3" 432 433 result = add_test_url_params(initial_test_url, extra_params) 434 435 expected_params = {"c": ["3"]} 436 actual_params = parse_qs(urlsplit(result).query) 437 assert actual_params == expected_params 438 439 440 def test_add_test_url_params_overwrites_single_param(): 441 initial_test_url = "http://test.com?a=1&b=2" 442 extra_params = "b=3" 443 444 result = add_test_url_params(initial_test_url, extra_params) 445 446 expected_params = {"a": ["1"], "b": ["3"]} 447 actual_params = parse_qs(urlsplit(result).query) 448 assert actual_params == expected_params 449 450 451 def test_add_test_url_params_overwrites_multiple_param(): 452 initial_test_url = "http://test.com?a=1&b=2&c=3" 453 extra_params = "c=4&b=5" 454 455 result = add_test_url_params(initial_test_url, extra_params) 456 457 expected_params = {"a": ["1"], "b": ["5"], "c": ["4"]} 458 actual_params = parse_qs(urlsplit(result).query) 459 assert actual_params == expected_params 460 461 462 if __name__ == "__main__": 463 mozunit.main()