tests(vcr): assert response shape, not exact bytes, in replay tests

The Anthropic replay tests hardcoded specific token counts and content
strings ('Hello! How can I help you today?', prompt_tokens == 12). On a
fresh CI Redis those values must match a pre-recorded cassette that
doesn't exist, so the first run hits the live API and gets different
real bytes back.

Assert on shape instead: non-empty content, positive token counts,
finish_reason in the known set, and (for streaming) more than one chunk.
The tests still exercise the full transformation pipeline end-to-end and
catch shape regressions; drift in the exact text/token counts is
expected and now tolerated.
This commit is contained in:
mateo-berri 2026-04-30 18:35:01 -07:00
parent 722a1a9f8f
commit 95bce9a72e

View file

@ -1888,19 +1888,28 @@ def test_metadata_filter_applies_to_azure_anthropic():
def test_anthropic_basic_completion_replay():
"""Smoke-test the Anthropic completion pipeline end-to-end via VCR.
Asserts on response shape rather than specific bytes, so the test is
valid both on a fresh CI Redis (records on first run) and on a hot
cache (replays). Drift in the *shape* of Anthropic's response surfaces
here; drift in the exact text/token counts is expected and ignored.
"""
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Hello!"}],
)
assert response is not None
assert response.choices[0].message.content == ("Hello! How can I help you today?")
assert response.usage.prompt_tokens == 12
assert response.usage.completion_tokens == 11
assert response.choices[0].finish_reason == "stop"
content = response.choices[0].message.content
assert isinstance(content, str) and content.strip(), content
assert response.usage.prompt_tokens > 0
assert response.usage.completion_tokens > 0
assert response.choices[0].finish_reason in {"stop", "length"}
def test_anthropic_streaming_completion_replay():
"""Same as above for the streaming path; asserts on shape, not bytes."""
stream = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Hello!"}],
@ -1909,7 +1918,9 @@ def test_anthropic_streaming_completion_replay():
collected_text = ""
finish_reason = None
chunk_count = 0
for chunk in stream:
chunk_count += 1
if not chunk.choices:
continue
delta = chunk.choices[0].delta
@ -1918,5 +1929,6 @@ def test_anthropic_streaming_completion_replay():
if chunk.choices[0].finish_reason:
finish_reason = chunk.choices[0].finish_reason
assert collected_text == "Hello from LiteLLM!"
assert finish_reason == "stop"
assert chunk_count > 1, "expected multiple SSE chunks from streaming response"
assert collected_text.strip(), collected_text
assert finish_reason in {"stop", "length"}