From cf4c9ede6102fdfe34e8a8894b0c6701359b5328 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 1 May 2026 13:00:20 -0700
Subject: [PATCH] tests(vcr): add LITELLM_VCR_VERBOSE per-test hit/miss
 reporting

Set LITELLM_VCR_VERBOSE=1 to print a one-line cassette verdict per
test (HIT / MISS / PARTIAL / NOOP) showing replay vs new-recording
counts. Useful for local QA to confirm which tests actually exercised
the cache and which fell through to the live provider.
---
 tests/_vcr_redis_persister.py               | 39 +++++++++++++++++++++
 tests/llm_responses_api_testing/conftest.py | 20 +++++++++++
 tests/llm_translation/conftest.py           | 20 +++++++++++
 3 files changed, 79 insertions(+)

diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py
index 537f314bfb7..fb1183f5575 100644
--- a/tests/_vcr_redis_persister.py
+++ b/tests/_vcr_redis_persister.py
@@ -146,3 +146,42 @@ def patch_vcrpy_aiohttp_record_path() -> None:
 
     _aiohttp_stubs.record_response = _record_response_preserving_body
     _PATCHED_AIOHTTP_RECORD = True
+
+
+VCR_VERBOSE_ENV = "LITELLM_VCR_VERBOSE"
+
+
+def vcr_verbose_enabled() -> bool:
+    return os.environ.get(VCR_VERBOSE_ENV) == "1"
+
+
+def format_vcr_verdict(cassette: Any) -> str:
+    """Build a one-line hit/miss verdict for a vcrpy Cassette.
+
+    HIT  — at least one request was served from cache and nothing new was
+           recorded. (Pure replay.)
+    MISS — nothing from cache; one or more requests went live and were
+           recorded. (Cold cache.)
+    PARTIAL — mix of replay and new recordings. Usually means the cassette
+              matches some but not all requests for this test (e.g. retries,
+              new branches, or vcrpy match_on too strict).
+    NOOP — test made no HTTP calls (or VCR not engaged for it).
+    """
+    if cassette is None:
+        return "[VCR NOOP]"
+    played = getattr(cassette, "play_count", 0) or 0
+    # cassette.data is the recorded request/response list; len(cassette) counts
+    # recorded episodes. New recordings during this test = len - prior_len, but
+    # we don't have prior_len here, so we use cassette.dirty (set when an append
+    # happened during this run) as the "new recording" signal.
+    dirty = getattr(cassette, "dirty", False)
+    total = len(cassette) if hasattr(cassette, "__len__") else 0
+    if played == 0 and not dirty:
+        return "[VCR NOOP] (no http traffic)"
+    if played > 0 and not dirty:
+        return f"[VCR HIT] {played} replayed, 0 new ({total} cassette entries)"
+    if played == 0 and dirty:
+        return f"[VCR MISS] 0 replayed, recorded new ({total} cassette entries)"
+    return (
+        f"[VCR PARTIAL] {played} replayed + new recordings ({total} cassette entries)"
+    )
diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py
index cbf59182d77..05e0597d60f 100644
--- a/tests/llm_responses_api_testing/conftest.py
+++ b/tests/llm_responses_api_testing/conftest.py
@@ -15,8 +15,10 @@ import litellm  # noqa: E402
 
 from tests._vcr_redis_persister import (  # noqa: E402
     filter_non_2xx_response,
+    format_vcr_verdict,
     make_redis_persister,
     patch_vcrpy_aiohttp_record_path,
+    vcr_verbose_enabled,
 )
 
 
@@ -102,6 +104,24 @@ def pytest_recording_configure(config, vcr):
     patch_vcrpy_aiohttp_record_path()
 
 
+@pytest.fixture(autouse=True)
+def _vcr_hit_miss_report(request, vcr):
+    """When LITELLM_VCR_VERBOSE=1, print a one-line cassette verdict per test.
+
+    Runs after the `vcr` fixture (which yields the active Cassette), so we can
+    inspect play_count / dirty / len in teardown."""
+    yield
+    if not vcr_verbose_enabled():
+        return
+    verdict = format_vcr_verdict(vcr)
+    reporter = request.config.pluginmanager.get_plugin("terminalreporter")
+    line = f"{verdict} :: {request.node.nodeid}"
+    if reporter is not None:
+        reporter.write_line(line)
+    else:  # pragma: no cover - reporter is always present in normal runs
+        print(line)
+
+
 @pytest.fixture(scope="session")
 def event_loop():
     try:
diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py
index 0691113a61f..24742438f1f 100644
--- a/tests/llm_translation/conftest.py
+++ b/tests/llm_translation/conftest.py
@@ -20,8 +20,10 @@ import litellm  # noqa: E402
 
 from tests._vcr_redis_persister import (  # noqa: E402
     filter_non_2xx_response,
+    format_vcr_verdict,
     make_redis_persister,
     patch_vcrpy_aiohttp_record_path,
+    vcr_verbose_enabled,
 )
 
 
@@ -126,6 +128,24 @@ def pytest_recording_configure(config, vcr):
     patch_vcrpy_aiohttp_record_path()
 
 
+@pytest.fixture(autouse=True)
+def _vcr_hit_miss_report(request, vcr):
+    """When LITELLM_VCR_VERBOSE=1, print a one-line cassette verdict per test.
+
+    Runs after the `vcr` fixture (which yields the active Cassette), so we can
+    inspect play_count / dirty / len in teardown."""
+    yield
+    if not vcr_verbose_enabled():
+        return
+    verdict = format_vcr_verdict(vcr)
+    reporter = request.config.pluginmanager.get_plugin("terminalreporter")
+    line = f"{verdict} :: {request.node.nodeid}"
+    if reporter is not None:
+        reporter.write_line(line)
+    else:  # pragma: no cover - reporter is always present in normal runs
+        print(line)
+
+
 # ---------------------------------------------------------------------------
 # Capture TRUE defaults at conftest import time (before test modules pollute).
 # ---------------------------------------------------------------------------