test_session.py (15424B)
1 # mypy: allow-untyped-defs 2 from _pytest.config import ExitCode 3 from _pytest.monkeypatch import MonkeyPatch 4 from _pytest.pytester import Pytester 5 import pytest 6 7 8 class SessionTests: 9 def test_basic_testitem_events(self, pytester: Pytester) -> None: 10 tfile = pytester.makepyfile( 11 """ 12 def test_one(): 13 pass 14 def test_one_one(): 15 assert 0 16 def test_other(): 17 raise ValueError(23) 18 class TestClass(object): 19 def test_two(self, someargs): 20 pass 21 """ 22 ) 23 reprec = pytester.inline_run(tfile) 24 passed, skipped, failed = reprec.listoutcomes() 25 assert len(skipped) == 0 26 assert len(passed) == 1 27 assert len(failed) == 3 28 29 def end(x): 30 return x.nodeid.split("::")[-1] 31 32 assert end(failed[0]) == "test_one_one" 33 assert end(failed[1]) == "test_other" 34 itemstarted = reprec.getcalls("pytest_itemcollected") 35 assert len(itemstarted) == 4 36 # XXX check for failing funcarg setup 37 # colreports = reprec.getcalls("pytest_collectreport") 38 # assert len(colreports) == 4 39 # assert colreports[1].report.failed 40 41 def test_nested_import_error(self, pytester: Pytester) -> None: 42 tfile = pytester.makepyfile( 43 """ 44 import import_fails 45 def test_this(): 46 assert import_fails.a == 1 47 """, 48 import_fails=""" 49 import does_not_work 50 a = 1 51 """, 52 ) 53 reprec = pytester.inline_run(tfile) 54 values = reprec.getfailedcollections() 55 assert len(values) == 1 56 out = str(values[0].longrepr) 57 assert out.find("does_not_work") != -1 58 59 def test_raises_output(self, pytester: Pytester) -> None: 60 reprec = pytester.inline_runsource( 61 """ 62 import pytest 63 def test_raises_doesnt(): 64 pytest.raises(ValueError, int, "3") 65 """ 66 ) 67 passed, skipped, failed = reprec.listoutcomes() 68 assert len(failed) == 1 69 out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] 70 assert "DID NOT RAISE" in out 71 72 def test_syntax_error_module(self, pytester: Pytester) -> None: 73 reprec = pytester.inline_runsource("this is really not python") 74 values = reprec.getfailedcollections() 75 assert len(values) == 1 76 out = str(values[0].longrepr) 77 assert out.find("not python") != -1 78 79 def test_exit_first_problem(self, pytester: Pytester) -> None: 80 reprec = pytester.inline_runsource( 81 """ 82 def test_one(): assert 0 83 def test_two(): assert 0 84 """, 85 "--exitfirst", 86 ) 87 passed, skipped, failed = reprec.countoutcomes() 88 assert failed == 1 89 assert passed == skipped == 0 90 91 def test_maxfail(self, pytester: Pytester) -> None: 92 reprec = pytester.inline_runsource( 93 """ 94 def test_one(): assert 0 95 def test_two(): assert 0 96 def test_three(): assert 0 97 """, 98 "--maxfail=2", 99 ) 100 passed, skipped, failed = reprec.countoutcomes() 101 assert failed == 2 102 assert passed == skipped == 0 103 104 def test_broken_repr(self, pytester: Pytester) -> None: 105 p = pytester.makepyfile( 106 """ 107 import pytest 108 109 class reprexc(BaseException): 110 def __str__(self): 111 return "Ha Ha fooled you, I'm a broken repr()." 112 113 class BrokenRepr1(object): 114 foo=0 115 def __repr__(self): 116 raise reprexc 117 118 class TestBrokenClass(object): 119 def test_explicit_bad_repr(self): 120 t = BrokenRepr1() 121 with pytest.raises(BaseException, match="broken repr"): 122 repr(t) 123 124 def test_implicit_bad_repr1(self): 125 t = BrokenRepr1() 126 assert t.foo == 1 127 128 """ 129 ) 130 reprec = pytester.inline_run(p) 131 passed, skipped, failed = reprec.listoutcomes() 132 assert (len(passed), len(skipped), len(failed)) == (1, 0, 1) 133 out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] 134 assert out.find("<[reprexc() raised in repr()] BrokenRepr1") != -1 135 136 def test_broken_repr_with_showlocals_verbose(self, pytester: Pytester) -> None: 137 p = pytester.makepyfile( 138 """ 139 class ObjWithErrorInRepr: 140 def __repr__(self): 141 raise NotImplementedError 142 143 def test_repr_error(): 144 x = ObjWithErrorInRepr() 145 assert x == "value" 146 """ 147 ) 148 reprec = pytester.inline_run("--showlocals", "-vv", p) 149 passed, skipped, failed = reprec.listoutcomes() 150 assert (len(passed), len(skipped), len(failed)) == (0, 0, 1) 151 entries = failed[0].longrepr.reprtraceback.reprentries # type: ignore[union-attr] 152 assert len(entries) == 1 153 repr_locals = entries[0].reprlocals 154 assert repr_locals.lines 155 assert len(repr_locals.lines) == 1 156 assert repr_locals.lines[0].startswith( 157 "x = <[NotImplementedError() raised in repr()] ObjWithErrorInRepr" 158 ) 159 160 def test_skip_file_by_conftest(self, pytester: Pytester) -> None: 161 pytester.makepyfile( 162 conftest=""" 163 import pytest 164 def pytest_collect_file(): 165 pytest.skip("intentional") 166 """, 167 test_file=""" 168 def test_one(): pass 169 """, 170 ) 171 try: 172 reprec = pytester.inline_run(pytester.path) 173 except pytest.skip.Exception: # pragma: no cover 174 pytest.fail("wrong skipped caught") 175 reports = reprec.getreports("pytest_collectreport") 176 # Session, Dir 177 assert len(reports) == 2 178 assert reports[1].skipped 179 180 181 class TestNewSession(SessionTests): 182 def test_order_of_execution(self, pytester: Pytester) -> None: 183 reprec = pytester.inline_runsource( 184 """ 185 values = [] 186 def test_1(): 187 values.append(1) 188 def test_2(): 189 values.append(2) 190 def test_3(): 191 assert values == [1,2] 192 class Testmygroup(object): 193 reslist = values 194 def test_1(self): 195 self.reslist.append(1) 196 def test_2(self): 197 self.reslist.append(2) 198 def test_3(self): 199 self.reslist.append(3) 200 def test_4(self): 201 assert self.reslist == [1,2,1,2,3] 202 """ 203 ) 204 passed, skipped, failed = reprec.countoutcomes() 205 assert failed == skipped == 0 206 assert passed == 7 207 208 def test_collect_only_with_various_situations(self, pytester: Pytester) -> None: 209 p = pytester.makepyfile( 210 test_one=""" 211 def test_one(): 212 raise ValueError() 213 214 class TestX(object): 215 def test_method_one(self): 216 pass 217 218 class TestY(TestX): 219 pass 220 """, 221 test_three="xxxdsadsadsadsa", 222 __init__="", 223 ) 224 reprec = pytester.inline_run("--collect-only", p.parent) 225 226 itemstarted = reprec.getcalls("pytest_itemcollected") 227 assert len(itemstarted) == 3 228 assert not reprec.getreports("pytest_runtest_logreport") 229 started = reprec.getcalls("pytest_collectstart") 230 finished = reprec.getreports("pytest_collectreport") 231 assert len(started) == len(finished) 232 assert len(started) == 6 233 colfail = [x for x in finished if x.failed] 234 assert len(colfail) == 1 235 236 def test_minus_x_import_error(self, pytester: Pytester) -> None: 237 pytester.makepyfile(__init__="") 238 pytester.makepyfile(test_one="xxxx", test_two="yyyy") 239 reprec = pytester.inline_run("-x", pytester.path) 240 finished = reprec.getreports("pytest_collectreport") 241 colfail = [x for x in finished if x.failed] 242 assert len(colfail) == 1 243 244 def test_minus_x_overridden_by_maxfail(self, pytester: Pytester) -> None: 245 pytester.makepyfile(__init__="") 246 pytester.makepyfile(test_one="xxxx", test_two="yyyy", test_third="zzz") 247 reprec = pytester.inline_run("-x", "--maxfail=2", pytester.path) 248 finished = reprec.getreports("pytest_collectreport") 249 colfail = [x for x in finished if x.failed] 250 assert len(colfail) == 2 251 252 253 def test_plugin_specify(pytester: Pytester) -> None: 254 with pytest.raises(ImportError): 255 pytester.parseconfig("-p", "nqweotexistent") 256 # pytest.raises(ImportError, 257 # "config.do_configure(config)" 258 # ) 259 260 261 def test_plugin_already_exists(pytester: Pytester) -> None: 262 config = pytester.parseconfig("-p", "terminal") 263 assert config.option.plugins == ["terminal"] 264 config._do_configure() 265 config._ensure_unconfigure() 266 267 268 def test_exclude(pytester: Pytester) -> None: 269 hellodir = pytester.mkdir("hello") 270 hellodir.joinpath("test_hello.py").write_text("x y syntaxerror", encoding="utf-8") 271 hello2dir = pytester.mkdir("hello2") 272 hello2dir.joinpath("test_hello2.py").write_text("x y syntaxerror", encoding="utf-8") 273 pytester.makepyfile(test_ok="def test_pass(): pass") 274 result = pytester.runpytest("--ignore=hello", "--ignore=hello2") 275 assert result.ret == 0 276 result.stdout.fnmatch_lines(["*1 passed*"]) 277 278 279 def test_exclude_glob(pytester: Pytester) -> None: 280 hellodir = pytester.mkdir("hello") 281 hellodir.joinpath("test_hello.py").write_text("x y syntaxerror", encoding="utf-8") 282 hello2dir = pytester.mkdir("hello2") 283 hello2dir.joinpath("test_hello2.py").write_text("x y syntaxerror", encoding="utf-8") 284 hello3dir = pytester.mkdir("hallo3") 285 hello3dir.joinpath("test_hello3.py").write_text("x y syntaxerror", encoding="utf-8") 286 subdir = pytester.mkdir("sub") 287 subdir.joinpath("test_hello4.py").write_text("x y syntaxerror", encoding="utf-8") 288 pytester.makepyfile(test_ok="def test_pass(): pass") 289 result = pytester.runpytest("--ignore-glob=*h[ea]llo*") 290 assert result.ret == 0 291 result.stdout.fnmatch_lines(["*1 passed*"]) 292 293 294 def test_deselect(pytester: Pytester) -> None: 295 pytester.makepyfile( 296 test_a=""" 297 import pytest 298 299 def test_a1(): pass 300 301 @pytest.mark.parametrize('b', range(3)) 302 def test_a2(b): pass 303 304 class TestClass: 305 def test_c1(self): pass 306 307 def test_c2(self): pass 308 """ 309 ) 310 result = pytester.runpytest( 311 "-v", 312 "--deselect=test_a.py::test_a2[1]", 313 "--deselect=test_a.py::test_a2[2]", 314 "--deselect=test_a.py::TestClass::test_c1", 315 ) 316 assert result.ret == 0 317 result.stdout.fnmatch_lines(["*3 passed, 3 deselected*"]) 318 for line in result.stdout.lines: 319 assert not line.startswith(("test_a.py::test_a2[1]", "test_a.py::test_a2[2]")) 320 321 322 def test_sessionfinish_with_start(pytester: Pytester) -> None: 323 pytester.makeconftest( 324 """ 325 import os 326 values = [] 327 def pytest_sessionstart(): 328 values.append(os.getcwd()) 329 os.chdir("..") 330 331 def pytest_sessionfinish(): 332 assert values[0] == os.getcwd() 333 334 """ 335 ) 336 res = pytester.runpytest("--collect-only") 337 assert res.ret == ExitCode.NO_TESTS_COLLECTED 338 339 340 def test_collection_args_do_not_duplicate_modules(pytester: Pytester) -> None: 341 """Test that when multiple collection args are specified on the command line 342 for the same module, only a single Module collector is created. 343 344 Regression test for #723, #3358. 345 """ 346 pytester.makepyfile( 347 **{ 348 "d/test_it": """ 349 def test_1(): pass 350 def test_2(): pass 351 """ 352 } 353 ) 354 355 result = pytester.runpytest( 356 "--collect-only", 357 "d/test_it.py::test_1", 358 "d/test_it.py::test_2", 359 ) 360 result.stdout.fnmatch_lines( 361 [ 362 " <Dir d>", 363 " <Module test_it.py>", 364 " <Function test_1>", 365 " <Function test_2>", 366 ], 367 consecutive=True, 368 ) 369 370 # Different, but related case. 371 result = pytester.runpytest( 372 "--collect-only", 373 "--keep-duplicates", 374 "d", 375 "d", 376 ) 377 result.stdout.fnmatch_lines( 378 [ 379 " <Dir d>", 380 " <Module test_it.py>", 381 " <Function test_1>", 382 " <Function test_2>", 383 " <Function test_1>", 384 " <Function test_2>", 385 ], 386 consecutive=True, 387 ) 388 389 390 @pytest.mark.parametrize("path", ["root", "{relative}/root", "{environment}/root"]) 391 def test_rootdir_option_arg( 392 pytester: Pytester, monkeypatch: MonkeyPatch, path: str 393 ) -> None: 394 monkeypatch.setenv("PY_ROOTDIR_PATH", str(pytester.path)) 395 path = path.format(relative=str(pytester.path), environment="$PY_ROOTDIR_PATH") 396 397 rootdir = pytester.path / "root" / "tests" 398 rootdir.mkdir(parents=True) 399 pytester.makepyfile( 400 """ 401 import os 402 def test_one(): 403 assert 1 404 """ 405 ) 406 407 result = pytester.runpytest(f"--rootdir={path}") 408 result.stdout.fnmatch_lines( 409 [ 410 f"*rootdir: {pytester.path}/root", 411 "root/test_rootdir_option_arg.py *", 412 "*1 passed*", 413 ] 414 ) 415 416 417 def test_rootdir_wrong_option_arg(pytester: Pytester) -> None: 418 result = pytester.runpytest("--rootdir=wrong_dir") 419 result.stderr.fnmatch_lines( 420 ["*Directory *wrong_dir* not found. Check your '--rootdir' option.*"] 421 ) 422 423 424 def test_shouldfail_is_sticky(pytester: Pytester) -> None: 425 """Test that session.shouldfail cannot be reset to False after being set. 426 427 Issue #11706. 428 """ 429 pytester.makeconftest( 430 """ 431 def pytest_sessionfinish(session): 432 assert session.shouldfail 433 session.shouldfail = False 434 assert session.shouldfail 435 """ 436 ) 437 pytester.makepyfile( 438 """ 439 import pytest 440 441 def test_foo(): 442 pytest.fail("This is a failing test") 443 444 def test_bar(): pass 445 """ 446 ) 447 448 result = pytester.runpytest("--maxfail=1", "-Wall") 449 450 result.assert_outcomes(failed=1, warnings=1) 451 result.stdout.fnmatch_lines("*session.shouldfail cannot be unset*") 452 453 454 def test_shouldstop_is_sticky(pytester: Pytester) -> None: 455 """Test that session.shouldstop cannot be reset to False after being set. 456 457 Issue #11706. 458 """ 459 pytester.makeconftest( 460 """ 461 def pytest_sessionfinish(session): 462 assert session.shouldstop 463 session.shouldstop = False 464 assert session.shouldstop 465 """ 466 ) 467 pytester.makepyfile( 468 """ 469 import pytest 470 471 def test_foo(): 472 pytest.fail("This is a failing test") 473 474 def test_bar(): pass 475 """ 476 ) 477 478 result = pytester.runpytest("--stepwise", "-Wall") 479 480 result.assert_outcomes(failed=1, warnings=1) 481 result.stdout.fnmatch_lines("*session.shouldstop cannot be unset*")