tests(vcr): emit per-test verdicts via xdist controller's terminalreporter

Previous attempt wrote to sys.__stderr__ from the test fixture. Under
xdist, fixtures run inside worker subprocesses whose stderr is captured
by the controller and only released to the live log on test failure —
so passing tests' verdicts were silently swallowed.

Round-trip via report.user_properties: the worker-side fixture stashes
the verdict on user_properties, xdist serializes it onto the report,
and a controller-side pytest_runtest_logreport hook writes it via the
TerminalReporter (the same plugin that emits PASSED/FAILED markers).
TerminalReporter is resolved lazily on first hook call because it's
not yet registered when conftest's pytest_configure runs.

Verified locally in both serial and xdist modes.
This commit is contained in:
mateo-berri 2026-05-01 14:29:06 -07:00
parent 965185c106
commit 53f71fbf4d
3 changed files with 118 additions and 32 deletions

View file

@ -216,30 +216,6 @@ def vcr_verbose_enabled() -> bool:
return os.environ.get(VCR_VERBOSE_ENV) == "1"
def emit_vcr_verbose_line(line: str) -> None:
"""Write a one-line VCR verdict to the un-redirected stderr.
Pytest's stdout/stderr capture would otherwise hide this output (and
only surface it as 'Captured stdout teardown' on failing tests). Under
xdist each worker has its own stderr; CircleCI aggregates them. Writing
to ``sys.__stderr__`` bypasses pytest's capture entirely so the line
reaches the live CI log alongside the per-test PASSED/FAILED markers.
"""
import sys
try:
stream = sys.__stderr__
if stream is not None:
stream.write(line + "\n")
stream.flush()
return
except Exception: # pragma: no cover - last-ditch fallback
pass
# If __stderr__ isn't writable for some reason, fall back to the
# captured stderr — better than swallowing.
print(line, file=sys.stderr)
def format_vcr_verdict(cassette: Any) -> str:
"""Build a one-line hit/miss verdict for a vcrpy Cassette.

View file

@ -14,7 +14,6 @@ sys.path.insert(
import litellm # noqa: E402
from tests._vcr_redis_persister import ( # noqa: E402
emit_vcr_verbose_line,
filter_non_2xx_response,
format_vcr_verdict,
make_redis_persister,
@ -24,6 +23,12 @@ from tests._vcr_redis_persister import ( # noqa: E402
)
# Controller-side handles for writing per-test VCR verdicts to the live
# terminal. See the matching comment in tests/llm_translation/conftest.py.
_controller_pluginmanager = None
_controller_terminal_reporter = None
_FILTERED_REQUEST_HEADERS = (
"authorization",
"x-api-key",
@ -128,10 +133,12 @@ def _vcr_outcome_gate(request, vcr):
above, so we can mark the cassette key passed/failed before vcrpy's
Cassette.__exit__ triggers persister.save_cassette.
Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1.
Stashes a per-test hit/miss verdict on ``user_properties`` so the
controller-side ``pytest_runtest_logreport`` hook can surface it to the
live terminal under xdist.
"""
yield
cassette = vcr # name kept for the verbose-output line
cassette = vcr
rep_call = getattr(request.node, "rep_call", None)
test_passed = bool(rep_call and rep_call.passed)
cassette_path = getattr(cassette, "_path", None) if cassette is not None else None
@ -141,7 +148,47 @@ def _vcr_outcome_gate(request, vcr):
if not vcr_verbose_enabled():
return
verdict = format_vcr_verdict(cassette)
emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}")
request.node.user_properties.append(("vcr_verdict", verdict))
def pytest_configure(config):
"""Stash the pluginmanager so the logreport hook can find TerminalReporter."""
global _controller_pluginmanager
if os.environ.get("PYTEST_XDIST_WORKER"):
return
_controller_pluginmanager = config.pluginmanager
def _resolve_terminal_reporter():
global _controller_terminal_reporter
if _controller_terminal_reporter is not None:
return _controller_terminal_reporter
if _controller_pluginmanager is None:
return None
_controller_terminal_reporter = _controller_pluginmanager.getplugin(
"terminalreporter"
)
return _controller_terminal_reporter
def pytest_runtest_logreport(report):
"""Emit per-test VCR verdicts on the controller's live terminal."""
if report.when != "teardown":
return
if os.environ.get("PYTEST_XDIST_WORKER"):
return
if not vcr_verbose_enabled():
return
reporter = _resolve_terminal_reporter()
if reporter is None:
return
verdict = next(
(v for k, v in (report.user_properties or []) if k == "vcr_verdict"),
None,
)
if not verdict:
return
reporter.write_line(f"{verdict} :: {report.nodeid}")
@pytest.fixture(scope="session")

View file

@ -19,7 +19,6 @@ sys.path.insert(
import litellm # noqa: E402
from tests._vcr_redis_persister import ( # noqa: E402
emit_vcr_verbose_line,
filter_non_2xx_response,
format_vcr_verdict,
make_redis_persister,
@ -29,6 +28,16 @@ from tests._vcr_redis_persister import ( # noqa: E402
)
# Controller-side handles for writing per-test VCR verdicts to the live
# terminal. ``pytest_configure`` stashes the pluginmanager (workers don't get
# a TerminalReporter — their output is captured and aggregated by the
# controller), and ``pytest_runtest_logreport`` resolves the TerminalReporter
# lazily on first use because it isn't registered yet at conftest configure
# time.
_controller_pluginmanager = None
_controller_terminal_reporter = None
# vcrpy and respx both patch the httpx transport — applying both makes one
# silently win. Files in this set use respx and are skipped by the
# auto-marker below.
@ -182,10 +191,14 @@ def _vcr_outcome_gate(request, vcr):
above, so we can mark the cassette key passed/failed before vcrpy's
Cassette.__exit__ triggers persister.save_cassette.
Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1.
Stashes a per-test hit/miss verdict on ``user_properties`` so the
controller-side ``pytest_runtest_logreport`` hook can surface it to the
live terminal. xdist serializes ``user_properties`` on each phase's
report back to the controller, which is the only process that has a
TerminalReporter wired to CI's live log.
"""
yield
cassette = vcr # name kept for the verbose-output line
cassette = vcr
rep_call = getattr(request.node, "rep_call", None)
test_passed = bool(rep_call and rep_call.passed)
cassette_path = getattr(cassette, "_path", None) if cassette is not None else None
@ -195,7 +208,57 @@ def _vcr_outcome_gate(request, vcr):
if not vcr_verbose_enabled():
return
verdict = format_vcr_verdict(cassette)
emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}")
request.node.user_properties.append(("vcr_verdict", verdict))
def pytest_configure(config):
"""Stash the pluginmanager so the logreport hook can find TerminalReporter.
We can't grab TerminalReporter directly here — it's not registered until
pytest's own ``pytest_configure`` runs, and conftest hooks may run first.
Stashing the config is enough; the hook resolves on first use.
"""
global _controller_pluginmanager
if os.environ.get("PYTEST_XDIST_WORKER"):
return # workers don't have a live-log TerminalReporter
_controller_pluginmanager = config.pluginmanager
def _resolve_terminal_reporter():
"""Lazy-resolve the TerminalReporter once it's been registered."""
global _controller_terminal_reporter
if _controller_terminal_reporter is not None:
return _controller_terminal_reporter
if _controller_pluginmanager is None:
return None
_controller_terminal_reporter = _controller_pluginmanager.getplugin(
"terminalreporter"
)
return _controller_terminal_reporter
def pytest_runtest_logreport(report):
"""Print VCR verdicts on the controller, alongside PASSED/FAILED markers.
Runs once per phase per test. We pick teardown so the verdict (appended
in ``_vcr_outcome_gate`` teardown) is present in ``report.user_properties``.
"""
if report.when != "teardown":
return
if os.environ.get("PYTEST_XDIST_WORKER"):
return # only the controller has a live-log TerminalReporter
if not vcr_verbose_enabled():
return
reporter = _resolve_terminal_reporter()
if reporter is None:
return
verdict = next(
(v for k, v in (report.user_properties or []) if k == "vcr_verdict"),
None,
)
if not verdict:
return
reporter.write_line(f"{verdict} :: {report.nodeid}")
# ---------------------------------------------------------------------------