test_util_chunking.py (12698B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import re 7 from itertools import combinations 8 9 import pytest 10 from mozunit import main 11 12 from gecko_taskgraph.util import chunking 13 14 pytestmark = pytest.mark.slow 15 16 17 @pytest.fixture(scope="module") 18 def mock_manifest_runtimes(): 19 """Deterministically produce a list of simulated manifest runtimes. 20 21 Args: 22 manifests (list): list of manifests against which simulated manifest 23 runtimes would be paired up to. 24 25 Returns: 26 dict of manifest data paired with a float value representing runtime. 27 """ 28 29 def inner(manifests): 30 manifests = sorted(manifests) 31 # Generate deterministic runtime data. 32 runtimes = [(i / 10) ** (i / 10) for i in range(len(manifests))] 33 return dict(zip(manifests, runtimes)) 34 35 return inner 36 37 38 @pytest.fixture(scope="module") 39 def unchunked_manifests(): 40 """Produce a list of unchunked manifests to be consumed by test method. 41 42 Args: 43 length (int, optional): number of path elements to keep. 44 cutoff (int, optional): number of generated test paths to remove 45 from the test set if user wants to limit the number of paths. 46 47 Returns: 48 list: list of test paths. 49 """ 50 data = ["blueberry", "nashi", "peach", "watermelon"] 51 52 def inner(suite, length=2, cutoff=0): 53 if "web-platform" in suite: 54 suffix = "" 55 prefix = "/" 56 elif "reftest" in suite: 57 suffix = ".list" 58 prefix = "" 59 else: 60 suffix = ".ini" 61 prefix = "" 62 return [prefix + "/".join(p) + suffix for p in combinations(data, length)][ 63 cutoff: 64 ] 65 66 return inner 67 68 69 @pytest.fixture(scope="module") 70 def mock_task_definition(): 71 """Builds a mock task definition for use in testing. 72 73 Args: 74 os_name (str): represents the os. 75 os_version (str): represents the os version 76 bits (int): software bits. 77 build_type (str): opt or debug. 78 build_attrs (list, optional): specify build attribute(s) 79 variants (list, optional): specify runtime variant(s) 80 81 Returns: 82 dict: mocked task definition. 83 """ 84 85 def inner(os_name, os_version, bits, build_type, build_attrs=None, variants=None): 86 setting = { 87 "platform": { 88 "arch": str(bits), 89 "os": { 90 "name": os_name, 91 "version": os_version, 92 }, 93 }, 94 "build": { 95 "type": build_type, 96 }, 97 "runtime": {}, 98 } 99 100 # Optionally set build attributes and runtime variants. 101 if build_attrs: 102 if isinstance(build_attrs, str): 103 build_attrs = [build_attrs] 104 for attr in build_attrs: 105 setting["build"][attr] = True 106 107 if variants: 108 if isinstance(variants, str): 109 variants = [variants] 110 for variant in variants: 111 setting["runtime"][variant] = True 112 return {"test-name": "foo", "test-setting": setting} 113 114 return inner 115 116 117 @pytest.fixture(scope="module") 118 def mock_mozinfo(): 119 """Returns a mocked mozinfo object, similar to guess_mozinfo_from_task(). 120 121 Args: 122 os (str): typically one of 'win, linux, mac, android'. 123 processor (str): processor architecture. 124 asan (bool, optional): addressanitizer build. 125 bits (int, optional): defaults to 64. 126 ccov (bool, optional): code coverage build. 127 debug (bool, optional): debug type build. 128 fission (bool, optional): process fission. 129 headless (bool, optional): headless browser testing without displays. 130 tsan (bool, optional): threadsanitizer build. 131 132 Returns: 133 dict: Dictionary mimickign the results from guess_mozinfo_from_task. 134 """ 135 136 def inner( 137 os, 138 processor, 139 asan=False, 140 bits=64, 141 ccov=False, 142 debug=False, 143 fission=False, 144 headless=False, 145 tsan=False, 146 tag="[]", 147 mingwclang=False, 148 nightly_build=False, 149 repo="try", 150 crashreporter=False, 151 ): 152 return { 153 "os": os, 154 "processor": processor, 155 "toolkit": "", 156 "asan": asan, 157 "bits": bits, 158 "ccov": ccov, 159 "debug": debug, 160 "e10s": True, 161 "fission": fission, 162 "headless": headless, 163 "tsan": tsan, 164 "appname": "firefox", 165 "condprof": False, 166 "canvas": False, 167 "webgpu": False, 168 "webcodecs": False, 169 "eme": False, 170 "privatebrowsing": False, 171 "tag": tag, 172 } 173 174 return inner 175 176 177 @pytest.mark.parametrize( 178 "params,exception", 179 [ 180 [("win", "7", 32, "opt"), None], 181 [("win", "10", 64, "opt"), None], 182 [("linux", "1804", 64, "debug"), None], 183 [("macosx", "1015", 64, "debug"), None], 184 [("macosx", "1100", 64, "opt"), None], 185 [("android", "13.0", 64, "debug"), None], 186 [("and", "", 64, "debug"), ValueError], 187 [("", "", 64, "opt"), ValueError], 188 [("linux", "1804", 64, "opt", ["ccov"]), None], 189 [("linux", "1804", 64, "opt", ["asan"]), None], 190 [("win", "10", 64, "opt", ["tsan"]), None], 191 [("mac", "1100", 64, "opt", ["ccov"]), None], 192 [("android", "13.0", 64, "opt", None, ["fission"]), None], 193 [("win", "10", "aarch64", "opt"), None], 194 ], 195 ) 196 def test_guess_mozinfo_from_task(params, exception, mock_task_definition): 197 """Tests the mozinfo guessing process.""" 198 # Set up a mocked task object. 199 task = mock_task_definition(*params) 200 201 if exception: 202 with pytest.raises(exception): 203 result = chunking.guess_mozinfo_from_task(task) 204 else: 205 expected_toolkits = { 206 "android": "android", 207 "linux": "gtk", 208 "mac": "cocoa", 209 "win": "windows", 210 } 211 result = chunking.guess_mozinfo_from_task(task) 212 setting = task["test-setting"] 213 214 assert str(result["bits"]) in setting["platform"]["arch"] 215 assert result["os"] in ("android", "linux", "mac", "win") 216 assert result["os"] in setting["platform"]["os"]["name"] 217 assert result["toolkit"] == expected_toolkits[result["os"]] 218 219 # Ensure the outcome of special build variants being present match what 220 # guess_mozinfo_from_task method returns for these attributes. 221 assert ("asan" in setting["build"]) == result["asan"] 222 assert ("tsan" in setting["build"]) == result["tsan"] 223 assert ("ccov" in setting["build"]) == result["ccov"] 224 225 # Ensure runtime variants match 226 assert ("fission" in setting["runtime"]) == result["fission"] 227 assert ("1proc" in setting["runtime"]) != result["e10s"] 228 229 230 @pytest.mark.parametrize("platform", ["unix", "windows", "android"]) 231 @pytest.mark.parametrize( 232 "suite", ["crashtest", "reftest", "web-platform-tests", "xpcshell"] 233 ) 234 def test_get_runtimes(platform, suite): 235 """Tests that runtime information is returned for known good configurations.""" 236 assert chunking.get_runtimes(platform, suite) 237 238 239 @pytest.mark.parametrize( 240 "platform,suite,exception", 241 [ 242 ("unix", "", TypeError), 243 ("", "", TypeError), 244 ("", "nonexistent_suite", TypeError), 245 ], 246 ) 247 def test_get_runtimes_invalid(platform, suite, exception): 248 """Ensure get_runtimes() method raises an exception if improper request is made.""" 249 with pytest.raises(exception): 250 chunking.get_runtimes(platform, suite) 251 252 253 @pytest.mark.parametrize( 254 "suite", 255 [ 256 "web-platform-tests", 257 "web-platform-tests-reftest", 258 "web-platform-tests-wdspec", 259 "web-platform-tests-crashtest", 260 ], 261 ) 262 @pytest.mark.parametrize("chunks", [1, 3, 6, 20]) 263 def test_mock_chunk_manifests_wpt(unchunked_manifests, suite, chunks): 264 """Tests web-platform-tests and its subsuites chunking process.""" 265 # Setup. 266 manifests = unchunked_manifests(suite) 267 268 # Generate the expected results, by generating list of indices that each 269 # manifest should go into and then appending each item to that index. 270 # This method is intentionally different from the way chunking.py performs 271 # chunking for cross-checking. 272 expected = [[] for _ in range(chunks)] 273 indexed = zip(manifests, list(range(0, chunks)) * len(manifests)) 274 for i in indexed: 275 expected[i[1]].append(i[0]) 276 277 # Call the method under test on unchunked manifests. 278 chunked_manifests = chunking.chunk_manifests(suite, "unix", chunks, manifests) 279 280 # Assertions and end test. 281 assert chunked_manifests 282 if chunks > len(manifests): 283 # If chunk count exceeds number of manifests, not all chunks will have 284 # manifests. 285 with pytest.raises(AssertionError): 286 assert all(chunked_manifests) 287 else: 288 assert all(chunked_manifests) 289 minimum = min(len(c) for c in chunked_manifests) 290 maximum = max(len(c) for c in chunked_manifests) 291 assert maximum - minimum <= 1 292 assert expected == chunked_manifests 293 294 295 @pytest.mark.parametrize( 296 "suite", 297 [ 298 "mochitest-devtools-chrome", 299 "mochitest-browser-chrome", 300 "mochitest-plain", 301 "mochitest-chrome", 302 "xpcshell", 303 ], 304 ) 305 @pytest.mark.parametrize("chunks", [1, 3, 6, 20]) 306 def test_mock_chunk_manifests( 307 mock_manifest_runtimes, unchunked_manifests, suite, chunks 308 ): 309 """Tests non-WPT tests and its subsuites chunking process.""" 310 # Setup. 311 manifests = unchunked_manifests(suite) 312 313 # Call the method under test on unchunked manifests. 314 chunked_manifests = chunking.chunk_manifests(suite, "unix", chunks, manifests) 315 316 # Assertions and end test. 317 assert chunked_manifests 318 if chunks > len(manifests): 319 # If chunk count exceeds number of manifests, not all chunks will have 320 # manifests. 321 with pytest.raises(AssertionError): 322 assert all(chunked_manifests) 323 else: 324 assert all(chunked_manifests) 325 326 327 @pytest.mark.parametrize( 328 "suite", 329 [ 330 "web-platform-tests", 331 "web-platform-tests-reftest", 332 "xpcshell", 333 "mochitest-plain", 334 "mochitest-devtools-chrome", 335 "mochitest-browser-chrome", 336 "mochitest-chrome", 337 ], 338 ) 339 @pytest.mark.parametrize( 340 "platform", 341 [ 342 ("mac", "x86_64"), 343 ("win", "x86_64"), 344 ("win", "x86"), 345 ("win", "aarch64"), 346 ("linux", "x86_64"), 347 ("linux", "x86"), 348 ], 349 ) 350 def test_get_manifests(suite, platform, mock_mozinfo): 351 """Tests the DefaultLoader class' ability to load manifests.""" 352 mozinfo = mock_mozinfo(*platform) 353 354 loader = chunking.DefaultLoader([]) 355 manifests = loader.get_manifests(suite, frozenset(mozinfo.items())) 356 357 assert manifests 358 assert manifests["active"] 359 if "web-platform" in suite: 360 assert manifests["skipped"] == [] 361 else: 362 assert manifests["skipped"] 363 364 items = manifests["active"] 365 if suite == "xpcshell": 366 assert all([re.search(r"xpcshell(.*)?(.ini|.toml)", m) for m in items]) 367 if "mochitest" in suite: 368 assert all([ 369 re.search(r"(perftest|mochitest|chrome|browser).*(.ini|.toml)", m) 370 for m in items 371 ]) 372 if "web-platform" in suite: 373 assert all([m.startswith("/") and m.count("/") <= 4 for m in items]) 374 375 376 @pytest.mark.parametrize( 377 "suite", 378 [ 379 "mochitest-devtools-chrome", 380 "mochitest-browser-chrome", 381 "mochitest-plain", 382 "mochitest-chrome", 383 "web-platform-tests", 384 "web-platform-tests-reftest", 385 "xpcshell", 386 ], 387 ) 388 @pytest.mark.parametrize( 389 "platform", 390 [ 391 ("mac", "x86_64"), 392 ("win", "x86_64"), 393 ("linux", "x86_64"), 394 ], 395 ) 396 @pytest.mark.parametrize("chunks", [1, 3, 6, 20]) 397 def test_chunk_manifests(suite, platform, chunks, mock_mozinfo): 398 """Tests chunking with real manifests.""" 399 mozinfo = mock_mozinfo(*platform) 400 401 loader = chunking.DefaultLoader([]) 402 manifests = loader.get_manifests(suite, frozenset(mozinfo.items())) 403 404 chunked_manifests = chunking.chunk_manifests( 405 suite, platform, chunks, manifests["active"] 406 ) 407 408 # Assertions and end test. 409 assert chunked_manifests 410 assert len(chunked_manifests) == chunks 411 assert all(chunked_manifests) 412 413 414 if __name__ == "__main__": 415 main()