mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
tests(vcr): emit per-test verdicts via xdist controller's terminalreporter
Previous attempt wrote to sys.__stderr__ from the test fixture. Under xdist, fixtures run inside worker subprocesses whose stderr is captured by the controller and only released to the live log on test failure — so passing tests' verdicts were silently swallowed. Round-trip via report.user_properties: the worker-side fixture stashes the verdict on user_properties, xdist serializes it onto the report, and a controller-side pytest_runtest_logreport hook writes it via the TerminalReporter (the same plugin that emits PASSED/FAILED markers). TerminalReporter is resolved lazily on first hook call because it's not yet registered when conftest's pytest_configure runs. Verified locally in both serial and xdist modes.
This commit is contained in:
parent
965185c106
commit
53f71fbf4d
3 changed files with 118 additions and 32 deletions
|
|
@ -216,30 +216,6 @@ def vcr_verbose_enabled() -> bool:
|
|||
return os.environ.get(VCR_VERBOSE_ENV) == "1"
|
||||
|
||||
|
||||
def emit_vcr_verbose_line(line: str) -> None:
|
||||
"""Write a one-line VCR verdict to the un-redirected stderr.
|
||||
|
||||
Pytest's stdout/stderr capture would otherwise hide this output (and
|
||||
only surface it as 'Captured stdout teardown' on failing tests). Under
|
||||
xdist each worker has its own stderr; CircleCI aggregates them. Writing
|
||||
to ``sys.__stderr__`` bypasses pytest's capture entirely so the line
|
||||
reaches the live CI log alongside the per-test PASSED/FAILED markers.
|
||||
"""
|
||||
import sys
|
||||
|
||||
try:
|
||||
stream = sys.__stderr__
|
||||
if stream is not None:
|
||||
stream.write(line + "\n")
|
||||
stream.flush()
|
||||
return
|
||||
except Exception: # pragma: no cover - last-ditch fallback
|
||||
pass
|
||||
# If __stderr__ isn't writable for some reason, fall back to the
|
||||
# captured stderr — better than swallowing.
|
||||
print(line, file=sys.stderr)
|
||||
|
||||
|
||||
def format_vcr_verdict(cassette: Any) -> str:
|
||||
"""Build a one-line hit/miss verdict for a vcrpy Cassette.
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ sys.path.insert(
|
|||
import litellm # noqa: E402
|
||||
|
||||
from tests._vcr_redis_persister import ( # noqa: E402
|
||||
emit_vcr_verbose_line,
|
||||
filter_non_2xx_response,
|
||||
format_vcr_verdict,
|
||||
make_redis_persister,
|
||||
|
|
@ -24,6 +23,12 @@ from tests._vcr_redis_persister import ( # noqa: E402
|
|||
)
|
||||
|
||||
|
||||
# Controller-side handles for writing per-test VCR verdicts to the live
|
||||
# terminal. See the matching comment in tests/llm_translation/conftest.py.
|
||||
_controller_pluginmanager = None
|
||||
_controller_terminal_reporter = None
|
||||
|
||||
|
||||
_FILTERED_REQUEST_HEADERS = (
|
||||
"authorization",
|
||||
"x-api-key",
|
||||
|
|
@ -128,10 +133,12 @@ def _vcr_outcome_gate(request, vcr):
|
|||
above, so we can mark the cassette key passed/failed before vcrpy's
|
||||
Cassette.__exit__ triggers persister.save_cassette.
|
||||
|
||||
Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1.
|
||||
Stashes a per-test hit/miss verdict on ``user_properties`` so the
|
||||
controller-side ``pytest_runtest_logreport`` hook can surface it to the
|
||||
live terminal under xdist.
|
||||
"""
|
||||
yield
|
||||
cassette = vcr # name kept for the verbose-output line
|
||||
cassette = vcr
|
||||
rep_call = getattr(request.node, "rep_call", None)
|
||||
test_passed = bool(rep_call and rep_call.passed)
|
||||
cassette_path = getattr(cassette, "_path", None) if cassette is not None else None
|
||||
|
|
@ -141,7 +148,47 @@ def _vcr_outcome_gate(request, vcr):
|
|||
if not vcr_verbose_enabled():
|
||||
return
|
||||
verdict = format_vcr_verdict(cassette)
|
||||
emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}")
|
||||
request.node.user_properties.append(("vcr_verdict", verdict))
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Stash the pluginmanager so the logreport hook can find TerminalReporter."""
|
||||
global _controller_pluginmanager
|
||||
if os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
return
|
||||
_controller_pluginmanager = config.pluginmanager
|
||||
|
||||
|
||||
def _resolve_terminal_reporter():
|
||||
global _controller_terminal_reporter
|
||||
if _controller_terminal_reporter is not None:
|
||||
return _controller_terminal_reporter
|
||||
if _controller_pluginmanager is None:
|
||||
return None
|
||||
_controller_terminal_reporter = _controller_pluginmanager.getplugin(
|
||||
"terminalreporter"
|
||||
)
|
||||
return _controller_terminal_reporter
|
||||
|
||||
|
||||
def pytest_runtest_logreport(report):
|
||||
"""Emit per-test VCR verdicts on the controller's live terminal."""
|
||||
if report.when != "teardown":
|
||||
return
|
||||
if os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
return
|
||||
if not vcr_verbose_enabled():
|
||||
return
|
||||
reporter = _resolve_terminal_reporter()
|
||||
if reporter is None:
|
||||
return
|
||||
verdict = next(
|
||||
(v for k, v in (report.user_properties or []) if k == "vcr_verdict"),
|
||||
None,
|
||||
)
|
||||
if not verdict:
|
||||
return
|
||||
reporter.write_line(f"{verdict} :: {report.nodeid}")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ sys.path.insert(
|
|||
import litellm # noqa: E402
|
||||
|
||||
from tests._vcr_redis_persister import ( # noqa: E402
|
||||
emit_vcr_verbose_line,
|
||||
filter_non_2xx_response,
|
||||
format_vcr_verdict,
|
||||
make_redis_persister,
|
||||
|
|
@ -29,6 +28,16 @@ from tests._vcr_redis_persister import ( # noqa: E402
|
|||
)
|
||||
|
||||
|
||||
# Controller-side handles for writing per-test VCR verdicts to the live
|
||||
# terminal. ``pytest_configure`` stashes the pluginmanager (workers don't get
|
||||
# a TerminalReporter — their output is captured and aggregated by the
|
||||
# controller), and ``pytest_runtest_logreport`` resolves the TerminalReporter
|
||||
# lazily on first use because it isn't registered yet at conftest configure
|
||||
# time.
|
||||
_controller_pluginmanager = None
|
||||
_controller_terminal_reporter = None
|
||||
|
||||
|
||||
# vcrpy and respx both patch the httpx transport — applying both makes one
|
||||
# silently win. Files in this set use respx and are skipped by the
|
||||
# auto-marker below.
|
||||
|
|
@ -182,10 +191,14 @@ def _vcr_outcome_gate(request, vcr):
|
|||
above, so we can mark the cassette key passed/failed before vcrpy's
|
||||
Cassette.__exit__ triggers persister.save_cassette.
|
||||
|
||||
Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1.
|
||||
Stashes a per-test hit/miss verdict on ``user_properties`` so the
|
||||
controller-side ``pytest_runtest_logreport`` hook can surface it to the
|
||||
live terminal. xdist serializes ``user_properties`` on each phase's
|
||||
report back to the controller, which is the only process that has a
|
||||
TerminalReporter wired to CI's live log.
|
||||
"""
|
||||
yield
|
||||
cassette = vcr # name kept for the verbose-output line
|
||||
cassette = vcr
|
||||
rep_call = getattr(request.node, "rep_call", None)
|
||||
test_passed = bool(rep_call and rep_call.passed)
|
||||
cassette_path = getattr(cassette, "_path", None) if cassette is not None else None
|
||||
|
|
@ -195,7 +208,57 @@ def _vcr_outcome_gate(request, vcr):
|
|||
if not vcr_verbose_enabled():
|
||||
return
|
||||
verdict = format_vcr_verdict(cassette)
|
||||
emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}")
|
||||
request.node.user_properties.append(("vcr_verdict", verdict))
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Stash the pluginmanager so the logreport hook can find TerminalReporter.
|
||||
|
||||
We can't grab TerminalReporter directly here — it's not registered until
|
||||
pytest's own ``pytest_configure`` runs, and conftest hooks may run first.
|
||||
Stashing the config is enough; the hook resolves on first use.
|
||||
"""
|
||||
global _controller_pluginmanager
|
||||
if os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
return # workers don't have a live-log TerminalReporter
|
||||
_controller_pluginmanager = config.pluginmanager
|
||||
|
||||
|
||||
def _resolve_terminal_reporter():
|
||||
"""Lazy-resolve the TerminalReporter once it's been registered."""
|
||||
global _controller_terminal_reporter
|
||||
if _controller_terminal_reporter is not None:
|
||||
return _controller_terminal_reporter
|
||||
if _controller_pluginmanager is None:
|
||||
return None
|
||||
_controller_terminal_reporter = _controller_pluginmanager.getplugin(
|
||||
"terminalreporter"
|
||||
)
|
||||
return _controller_terminal_reporter
|
||||
|
||||
|
||||
def pytest_runtest_logreport(report):
|
||||
"""Print VCR verdicts on the controller, alongside PASSED/FAILED markers.
|
||||
|
||||
Runs once per phase per test. We pick teardown so the verdict (appended
|
||||
in ``_vcr_outcome_gate`` teardown) is present in ``report.user_properties``.
|
||||
"""
|
||||
if report.when != "teardown":
|
||||
return
|
||||
if os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
return # only the controller has a live-log TerminalReporter
|
||||
if not vcr_verbose_enabled():
|
||||
return
|
||||
reporter = _resolve_terminal_reporter()
|
||||
if reporter is None:
|
||||
return
|
||||
verdict = next(
|
||||
(v for k, v in (report.user_properties or []) if k == "vcr_verdict"),
|
||||
None,
|
||||
)
|
||||
if not verdict:
|
||||
return
|
||||
reporter.write_line(f"{verdict} :: {report.nodeid}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue