test_resolve.py (19787B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 # flake8: noqa: E501 5 6 try: 7 import cPickle as pickle 8 except ImportError: 9 import pickle 10 11 import json 12 import os 13 import re 14 from collections import defaultdict 15 16 import manifestupdate 17 import mozpack.path as mozpath 18 import mozunit 19 import pytest 20 from mozbuild.frontend.reader import BuildReader 21 from mozbuild.test.common import MockConfig 22 from moztest.resolve import ( 23 TEST_SUITES, 24 BuildBackendLoader, 25 TestManifestLoader, 26 TestResolver, 27 ) 28 29 here = os.path.abspath(os.path.dirname(__file__)) 30 data_path = os.path.join(here, "data") 31 32 33 @pytest.fixture(scope="module") 34 def topsrcdir(): 35 return mozpath.join(data_path, "srcdir") 36 37 38 @pytest.fixture(scope="module") 39 def create_tests(topsrcdir): 40 def inner(*paths, **defaults): 41 tests = defaultdict(list) 42 for path in paths: 43 if isinstance(path, tuple): 44 path, kwargs = path 45 else: 46 kwargs = {} 47 48 path = mozpath.normpath(path) 49 manifest_name = kwargs.get("flavor", defaults.get("flavor", "manifest")) 50 manifest = kwargs.pop( 51 "manifest", 52 defaults.pop( 53 "manifest", 54 mozpath.join(mozpath.dirname(path), manifest_name + ".toml"), 55 ), 56 ) 57 58 manifest_abspath = mozpath.join(topsrcdir, manifest) 59 relpath = mozpath.relpath(path, mozpath.dirname(manifest)) 60 test = { 61 "name": relpath, 62 "path": mozpath.join(topsrcdir, path), 63 "relpath": relpath, 64 "file_relpath": path, 65 "flavor": "faketest", 66 "dir_relpath": mozpath.dirname(path), 67 "here": mozpath.dirname(manifest_abspath), 68 "manifest": manifest_abspath, 69 "manifest_relpath": manifest, 70 } 71 test.update(**defaults) 72 test.update(**kwargs) 73 74 # Normalize paths to ensure that the fixture matches reality. 75 for k in [ 76 "ancestor_manifest", 77 "manifest", 78 "manifest_relpath", 79 "path", 80 "relpath", 81 ]: 82 p = test.get(k) 83 if p: 84 test[k] = p.replace("/", os.path.sep) 85 86 tests[path].append(test) 87 88 # dump tests to stdout for easier debugging on failure 89 print("The 'create_tests' fixture returned:") 90 print(json.dumps(dict(tests), indent=2, sort_keys=True)) 91 return tests 92 93 return inner 94 95 96 @pytest.fixture(scope="module") 97 def all_tests(create_tests): 98 return create_tests(*[ 99 ( 100 "apple/test_a11y.html", 101 { 102 "expected": "pass", 103 "manifest": "apple/a11y.toml", 104 "flavor": "a11y", 105 }, 106 ), 107 ( 108 "banana/currant/test_xpcshell_A.js", 109 { 110 "firefox-appdir": "browser", 111 "flavor": "xpcshell", 112 "head": "head_global.js head_helpers.js head_http.js", 113 }, 114 ), 115 ( 116 "banana/currant/test_xpcshell_B.js", 117 { 118 "firefox-appdir": "browser", 119 "flavor": "xpcshell", 120 "head": "head_global.js head_helpers.js head_http.js", 121 }, 122 ), 123 ( 124 "carrot/test_included.js", 125 { 126 "ancestor_manifest": "carrot/xpcshell-one.toml", 127 "manifest": "carrot/xpcshell-shared.toml", 128 "flavor": "xpcshell", 129 "stick": "one", 130 }, 131 ), 132 ( 133 "carrot/test_included.js", 134 { 135 "ancestor_manifest": "carrot/xpcshell-two.toml", 136 "manifest": "carrot/xpcshell-shared.toml", 137 "flavor": "xpcshell", 138 "stick": "two", 139 }, 140 ), 141 ( 142 "dragonfruit/elderberry/test_xpcshell_C.js", 143 { 144 "flavor": "xpcshell", 145 "generated-files": "head_update.js", 146 "head": "head_update.js", 147 "manifest": "dragonfruit/xpcshell.toml", 148 "reason": "busted", 149 "run-sequentially": "Launches application.", 150 "skip-if": "os == 'android'", 151 }, 152 ), 153 ( 154 "dragonfruit/elderberry/test_xpcshell_C.js", 155 { 156 "flavor": "xpcshell", 157 "generated-files": "head_update.js", 158 "head": "head_update.js head2.js", 159 "manifest": "dragonfruit/elderberry/xpcshell_updater.toml", 160 "reason": "don't work", 161 "run-sequentially": "Launches application.", 162 "skip-if": "os == 'android'", 163 }, 164 ), 165 ( 166 "fig/grape/src/TestInstrumentationA.java", 167 { 168 "flavor": "instrumentation", 169 "manifest": "fig/grape/instrumentation.toml", 170 "subsuite": "background", 171 }, 172 ), 173 ( 174 "fig/huckleberry/src/TestInstrumentationB.java", 175 { 176 "flavor": "instrumentation", 177 "manifest": "fig/huckleberry/instrumentation.toml", 178 "subsuite": "browser", 179 }, 180 ), 181 ( 182 "juniper/browser_chrome.js", 183 { 184 "flavor": "browser-chrome", 185 "manifest": "juniper/browser.toml", 186 "skip-if": "e10s # broken", 187 }, 188 ), 189 ( 190 "kiwi/browser_devtools.js", 191 { 192 "flavor": "browser-chrome", 193 "manifest": "kiwi/browser.toml", 194 "subsuite": "devtools", 195 "tags": "devtools", 196 }, 197 ), 198 ]) 199 200 201 @pytest.fixture(scope="module") 202 def defaults(topsrcdir): 203 def to_abspath(relpath): 204 # test-defaults.pkl uses absolute paths with platform-specific path separators. 205 # Use platform-specific separators if needed to avoid regressing on bug 1644223. 206 return os.path.normpath(os.path.join(topsrcdir, relpath)) 207 208 def to_ancestor_manifest_path(relpath): 209 # ancestor_manifest uses relative paths with platform-specific path separators. 210 # This format must match the actual generation as defined in 211 # https://searchfox.org/mozilla-central/rev/1460e875c4189d975835af78fd06bd6daa9b553a/testing/mozbase/manifestparser/manifestparser/manifestparser.py#239-249 212 return os.path.normpath(relpath) 213 214 # This matches the format of "test-defaults.pkl", which is generated by 215 # testing/mozbase/manifestparser/manifestparser/manifestparser.py and 216 # test_test_defaults_metadata_with_include_and_ancestor_manifest in 217 # python/mozbuild/mozbuild/test/backend/test_test_manifest.py verifies that 218 # the format is as expected. 219 return { 220 (to_abspath("dragonfruit/elderberry/xpcshell_updater.toml")): { 221 "support-files": "data/**\nxpcshell_updater.toml" 222 }, 223 ( 224 to_ancestor_manifest_path("carrot/xpcshell-one.toml"), 225 to_abspath("carrot/xpcshell-shared.toml"), 226 ): { 227 "head": "head_one.js", 228 }, 229 ( 230 to_ancestor_manifest_path("carrot/xpcshell-two.toml"), 231 to_abspath("carrot/xpcshell-shared.toml"), 232 ): { 233 "head": "head_two.js", 234 }, 235 } 236 237 238 class WPTManifestNamespace: 239 """Stand-in object for various WPT classes.""" 240 241 def __init__(self, *args): 242 self.args = args 243 244 def __hash__(self): 245 return hash(str(self)) 246 247 def __eq__(self, other): 248 return self.args == other.args 249 250 def __iter__(self): 251 yield tuple(self.args) 252 253 254 def fake_wpt_manifestupdate(topsrcdir, *args, **kwargs): 255 with open(os.path.join(topsrcdir, "wpt_manifest_data.json")) as fh: 256 data = json.load(fh) 257 258 items = {} 259 for tests_root, test_data in data.items(): 260 kwargs = {"tests_path": os.path.join(topsrcdir, tests_root)} 261 262 for test_type, tests in test_data.items(): 263 for test in tests: 264 obj = WPTManifestNamespace() 265 if "mozilla" in tests_root: 266 obj.id = "/_mozilla/" + test 267 else: 268 obj.id = "/" + test 269 270 items[WPTManifestNamespace(test_type, test, {obj})] = kwargs 271 return items 272 273 274 @pytest.fixture(params=[BuildBackendLoader, TestManifestLoader]) 275 def resolver(request, tmpdir, monkeypatch, topsrcdir, all_tests, defaults): 276 topobjdir = tmpdir.mkdir("objdir").strpath 277 loader_cls = request.param 278 279 if loader_cls == BuildBackendLoader: 280 with open(os.path.join(topobjdir, "all-tests.pkl"), "wb") as fh: 281 pickle.dump(all_tests, fh) 282 with open(os.path.join(topobjdir, "test-defaults.pkl"), "wb") as fh: 283 pickle.dump(defaults, fh) 284 285 # The mock data already exists, so prevent BuildBackendLoader from regenerating 286 # the build information from the whole gecko tree... 287 class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader): 288 def backend_out_of_date(self, backend_file): 289 return False 290 291 loader_cls = BuildBackendLoaderNeverOutOfDate 292 293 # Patch WPT's manifestupdate.run to return tests based on the contents of 294 # 'data/srcdir/wpt_manifest_data.json'. 295 monkeypatch.setattr(manifestupdate, "run", fake_wpt_manifestupdate) 296 297 resolver = TestResolver( 298 topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls 299 ) 300 resolver._puppeteer_loaded = True 301 302 if loader_cls == TestManifestLoader: 303 config = MockConfig(topsrcdir) 304 resolver.load_tests.reader = BuildReader(config) 305 return resolver 306 307 308 def test_load(resolver): 309 assert len(resolver.tests_by_path) == 9 310 311 assert len(resolver.tests_by_flavor["mochitest-plain"]) == 0 312 assert len(resolver.tests_by_flavor["xpcshell"]) == 4 313 assert len(resolver.tests_by_flavor["web-platform-tests"]) == 0 314 315 assert len(resolver.tests_by_manifest) == 9 316 317 resolver.add_wpt_manifest_data() 318 assert len(resolver.tests_by_path) == 11 319 assert len(resolver.tests_by_flavor["web-platform-tests"]) == 2 320 assert len(resolver.tests_by_manifest) == 11 321 assert "/html" in resolver.tests_by_manifest 322 assert "/_mozilla/html" in resolver.tests_by_manifest 323 324 325 def test_resolve_all(resolver): 326 assert len(list(resolver._resolve())) == 13 327 328 329 def test_resolve_filter_flavor(resolver): 330 assert len(list(resolver._resolve(flavor="xpcshell"))) == 6 331 332 333 def test_resolve_by_dir(resolver): 334 assert len(list(resolver._resolve(paths=["banana/currant"]))) == 2 335 336 337 def test_resolve_under_path(resolver): 338 assert len(list(resolver._resolve(under_path="banana"))) == 2 339 assert len(list(resolver._resolve(flavor="xpcshell", under_path="banana"))) == 2 340 341 342 def test_resolve_multiple_paths(resolver): 343 result = list(resolver.resolve_tests(paths=["banana", "dragonfruit"])) 344 assert len(result) == 4 345 346 347 def test_resolve_support_files(resolver): 348 expected_support_files = "data/**\nxpcshell_updater.toml" 349 tests = list(resolver.resolve_tests(paths=["dragonfruit"])) 350 assert len(tests) == 2 351 352 for test in tests: 353 if test["manifest"].endswith("xpcshell_updater.toml"): 354 assert test["support-files"] == expected_support_files 355 else: 356 assert "support-files" not in test 357 358 359 def test_resolve_path_prefix(resolver): 360 tests = list(resolver._resolve(paths=["juniper"])) 361 assert len(tests) == 1 362 363 # relative manifest 364 tests = list(resolver._resolve(paths=["apple/a11y.toml"])) 365 assert len(tests) == 1 366 assert tests[0]["name"] == "test_a11y.html" 367 368 # absolute manifest 369 tests = list( 370 resolver._resolve(paths=[os.path.join(resolver.topsrcdir, "apple/a11y.toml")]) 371 ) 372 assert len(tests) == 1 373 assert tests[0]["name"] == "test_a11y.html" 374 375 376 def test_cwd_children_only(resolver): 377 """If cwd is defined, only resolve tests under the specified cwd.""" 378 # Pretend we're under '/services' and ask for 'common'. This should 379 # pick up all tests from '/services/common' 380 tests = list( 381 resolver.resolve_tests( 382 paths=["currant"], cwd=os.path.join(resolver.topsrcdir, "banana") 383 ) 384 ) 385 386 assert len(tests) == 2 387 388 # Tests should be rewritten to objdir. 389 for t in tests: 390 assert t["here"] == mozpath.join( 391 resolver.topobjdir, "_tests/xpcshell/banana/currant" 392 ) 393 394 395 def test_various_cwd(resolver): 396 """Test various cwd conditions are all equal.""" 397 expected = list(resolver.resolve_tests(paths=["banana"])) 398 actual = list(resolver.resolve_tests(paths=["banana"], cwd="/")) 399 assert actual == expected 400 401 actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topsrcdir)) 402 assert actual == expected 403 404 actual = list(resolver.resolve_tests(paths=["banana"], cwd=resolver.topobjdir)) 405 assert actual == expected 406 407 408 def test_subsuites(resolver): 409 """Test filtering by subsuite.""" 410 tests = list(resolver.resolve_tests(paths=["fig"])) 411 assert len(tests) == 2 412 413 tests = list(resolver.resolve_tests(paths=["fig"], subsuite="browser")) 414 assert len(tests) == 1 415 assert tests[0]["name"] == "src/TestInstrumentationB.java" 416 417 tests = list(resolver.resolve_tests(paths=["fig"], subsuite="background")) 418 assert len(tests) == 1 419 assert tests[0]["name"] == "src/TestInstrumentationA.java" 420 421 # Resolve tests *without* a subsuite. 422 tests = list(resolver.resolve_tests(flavor="browser-chrome", subsuite="undefined")) 423 assert len(tests) == 1 424 assert tests[0]["name"] == "browser_chrome.js" 425 426 427 def test_wildcard_patterns(resolver): 428 """Test matching paths by wildcard.""" 429 tests = list(resolver.resolve_tests(paths=["fig/**"])) 430 assert len(tests) == 2 431 for t in tests: 432 assert t["file_relpath"].startswith("fig") 433 434 tests = list(resolver.resolve_tests(paths=["**/**.js", "apple/**"])) 435 assert len(tests) == 9 436 for t in tests: 437 path = t["file_relpath"] 438 assert path.startswith("apple") or path.endswith(".js") 439 440 441 def test_resolve_metadata(resolver): 442 """Test finding metadata from outgoing files.""" 443 suites, tests = resolver.resolve_metadata(["bc"]) 444 assert suites == {"mochitest-browser-chrome"} 445 assert tests == [] 446 447 suites, tests = resolver.resolve_metadata([ 448 "mochitest-a11y", 449 "/browser", 450 "xpcshell", 451 ]) 452 assert suites == {"mochitest-a11y", "xpcshell"} 453 assert sorted(t["file_relpath"] for t in tests) == [ 454 "juniper/browser_chrome.js", 455 "kiwi/browser_devtools.js", 456 ] 457 458 459 def test_ancestor_manifest_defaults(resolver, topsrcdir, defaults): 460 """Test that defaults from ancestor manifests are found.""" 461 tests = list(resolver._resolve(paths=["carrot/test_included.js"])) 462 assert len(tests) == 2 463 464 if tests[0]["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.toml"): 465 [testOne, testTwo] = tests 466 else: 467 [testTwo, testOne] = tests 468 469 assert testOne["ancestor_manifest"] == os.path.join("carrot", "xpcshell-one.toml") 470 assert testOne["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.toml") 471 assert testOne["head"] == "head_one.js" 472 assert testOne["stick"] == "one" 473 474 assert testTwo["ancestor_manifest"] == os.path.join("carrot", "xpcshell-two.toml") 475 assert testTwo["manifest_relpath"] == os.path.join("carrot", "xpcshell-shared.toml") 476 assert testTwo["head"] == "head_two.js" 477 assert testTwo["stick"] == "two" 478 479 480 def test_task_regexes(): 481 """Test the task_regexes defined in TEST_SUITES.""" 482 task_labels = [ 483 "test-linux64/opt-marionette", 484 "test-linux64/opt-mochitest-plain", 485 "test-linux64/debug-mochitest-plain-e10s", 486 "test-linux64/opt-mochitest-a11y", 487 "test-linux64/opt-mochitest-browser", 488 "test-linux64/opt-mochitest-browser-chrome", 489 "test-linux64/opt-mochitest-browser-chrome-e10s", 490 "test-linux64/opt-mochitest-browser-chrome-e10s-11", 491 "test-linux64/opt-mochitest-chrome", 492 "test-linux64/opt-mochitest-devtools", 493 "test-linux64/opt-mochitest-devtools-chrome", 494 "test-linux64/opt-mochitest-gpu", 495 "test-linux64/opt-mochitest-gpu-e10s", 496 "test-linux64/opt-mochitest-media-e10s-1", 497 "test-linux64/opt-mochitest-media-e10s-11", 498 "test-linux64/opt-mochitest-browser-screenshots-1", 499 "test-linux64/opt-mochitest-browser-screenshots-e10s-1", 500 "test-linux64/opt-reftest", 501 "test-linux64/opt-geckoview-reftest", 502 "test-linux64/debug-reftest-e10s-1", 503 "test-linux64/debug-reftest-e10s-11", 504 "test-linux64/opt-robocop", 505 "test-linux64/opt-robocop-1", 506 "test-linux64/opt-robocop-e10s", 507 "test-linux64/opt-robocop-e10s-1", 508 "test-linux64/opt-robocop-e10s-11", 509 "test-linux64/opt-web-platform-tests-e10s-1", 510 "test-linux64/opt-web-platform-tests-reftest-e10s-1", 511 "test-linux64/opt-web-platform-tests-wdspec-e10s-1", 512 "test-linux64/opt-web-platform-tests-1", 513 "test-linux64/opt-web-platform-test-e10s-1", 514 "test-linux64/opt-xpcshell", 515 "test-linux64/opt-xpcshell-1", 516 "test-linux64/opt-xpcshell-2", 517 ] 518 519 test_cases = { 520 "mochitest-browser-chrome": [ 521 "test-linux64/opt-mochitest-browser-chrome", 522 "test-linux64/opt-mochitest-browser-chrome-e10s", 523 ], 524 "mochitest-chrome": [ 525 "test-linux64/opt-mochitest-chrome", 526 ], 527 "mochitest-devtools-chrome": [ 528 "test-linux64/opt-mochitest-devtools-chrome", 529 ], 530 "mochitest-media": [ 531 "test-linux64/opt-mochitest-media-e10s-1", 532 ], 533 "mochitest-plain": [ 534 "test-linux64/opt-mochitest-plain", 535 "test-linux64/debug-mochitest-plain-e10s", 536 ], 537 "mochitest-plain-gpu": [ 538 "test-linux64/opt-mochitest-gpu", 539 "test-linux64/opt-mochitest-gpu-e10s", 540 ], 541 "mochitest-browser-screenshots": [ 542 "test-linux64/opt-mochitest-browser-screenshots-1", 543 "test-linux64/opt-mochitest-browser-screenshots-e10s-1", 544 ], 545 "reftest": [ 546 "test-linux64/opt-reftest", 547 "test-linux64/opt-geckoview-reftest", 548 "test-linux64/debug-reftest-e10s-1", 549 ], 550 "robocop": [ 551 "test-linux64/opt-robocop", 552 "test-linux64/opt-robocop-1", 553 "test-linux64/opt-robocop-e10s", 554 "test-linux64/opt-robocop-e10s-1", 555 ], 556 "web-platform-tests": [ 557 "test-linux64/opt-web-platform-tests-e10s-1", 558 "test-linux64/opt-web-platform-tests-1", 559 ], 560 "web-platform-tests-reftest": [ 561 "test-linux64/opt-web-platform-tests-reftest-e10s-1", 562 ], 563 "web-platform-tests-wdspec": [ 564 "test-linux64/opt-web-platform-tests-wdspec-e10s-1", 565 ], 566 "xpcshell": [ 567 "test-linux64/opt-xpcshell", 568 "test-linux64/opt-xpcshell-1", 569 ], 570 } 571 572 regexes = [] 573 574 def match_task(task): 575 return any(re.search(pattern, task) for pattern in regexes) 576 577 for suite, expected in sorted(test_cases.items()): 578 print(suite) 579 regexes = TEST_SUITES[suite]["task_regex"] 580 assert set(filter(match_task, task_labels)) == set(expected) 581 582 583 if __name__ == "__main__": 584 mozunit.main()