commit b464b81a7d686b9d3ca510b89da1a6b3a79957f1
parent 5886fe1000529658864d2e1d34dcd46e1b9fbbdc
Author: Sam Sneddon <gsnedders@apple.com>
Date: Fri, 10 Oct 2025 07:49:32 +0000
Bug 1992845 [wpt PR 55145] - Give wptrunner a distinct exit code for CRITICAL errors, a=testonly
Automatic update from web-platform-tests
Give wptrunner a distinct exit code for CRITICAL errors
Previously, when running tests without --stability, wptrunner would
exit with 1 if there were unexpected results or if there was a
critical log message. With --stability, wptrunner would exit with
either 1 or 2 for stability failures, and 1 if the stability check
succeeded but there were critical log messages. In either case, wpt
run would then simplify this to always exiting with 0 or 1.
We now reserve exit codes above 64 for wptrunner's own usage, and exit
with 64 if there is a critical log message. This now takes priority
over the original exit code.
We also now allow wpt run to exit with the exit code that wptrunner
returns.
--
wpt-commits: 7902cfcbc75279eaff0e3874819ce954a213d761
wpt-pr: 55145
Diffstat:
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/testing/web-platform/tests/tools/wpt/run.py b/testing/web-platform/tests/tools/wpt/run.py
@@ -1012,7 +1012,7 @@ def run(venv, **kwargs):
setup_cls, wptrunner_kwargs = setup_wptrunner(venv, **kwargs)
try:
- rv = run_single(venv, **wptrunner_kwargs) > 0
+ rv = run_single(venv, **wptrunner_kwargs)
finally:
setup_cls.teardown()
diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -568,14 +568,14 @@ def check_stability(**kwargs):
**kwargs)
-def start(**kwargs):
+def start(**kwargs: Any) -> int:
assert logger is not None
logged_critical = wptlogging.LoggedAboveLevelHandler("CRITICAL")
handler = handlers.LogLevelFilter(logged_critical, "CRITICAL")
logger.add_handler(handler)
- rv = False
+ rv = 0
try:
if kwargs["list_test_groups"]:
list_test_groups(**kwargs)
@@ -586,12 +586,19 @@ def start(**kwargs):
elif kwargs["list_tests_json"]:
list_tests_json(**kwargs)
elif kwargs["verify"] or kwargs["stability"]:
- rv = check_stability(**kwargs) or logged_critical.has_log
+ rv = check_stability(**kwargs)
else:
- rv = not run_tests(**kwargs)[0] or logged_critical.has_log
+ rv = not run_tests(**kwargs)[0]
finally:
logger.shutdown()
logger.remove_handler(handler)
+
+ # Reserve everything above 64 for our global usage.
+ assert 0 <= rv < 64, "Exit codes above 64 are reserved"
+ if logged_critical.has_log:
+ print("Did log critical")
+ rv = 64
+
return rv
