mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
tests(vcr): assert response shape, not exact bytes, in replay tests
The Anthropic replay tests hardcoded specific token counts and content
strings ('Hello! How can I help you today?', prompt_tokens == 12). On a
fresh CI Redis those values must match a pre-recorded cassette that
doesn't exist, so the first run hits the live API and gets different
real bytes back.
Assert on shape instead: non-empty content, positive token counts,
finish_reason in the known set, and (for streaming) more than one chunk.
The tests still exercise the full transformation pipeline end-to-end and
catch shape regressions; drift in the exact text/token counts is
expected and now tolerated.
This commit is contained in:
parent
722a1a9f8f
commit
95bce9a72e
1 changed files with 18 additions and 6 deletions
|
|
@ -1888,19 +1888,28 @@ def test_metadata_filter_applies_to_azure_anthropic():
|
|||
|
||||
|
||||
def test_anthropic_basic_completion_replay():
|
||||
"""Smoke-test the Anthropic completion pipeline end-to-end via VCR.
|
||||
|
||||
Asserts on response shape rather than specific bytes, so the test is
|
||||
valid both on a fresh CI Redis (records on first run) and on a hot
|
||||
cache (replays). Drift in the *shape* of Anthropic's response surfaces
|
||||
here; drift in the exact text/token counts is expected and ignored.
|
||||
"""
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.choices[0].message.content == ("Hello! How can I help you today?")
|
||||
assert response.usage.prompt_tokens == 12
|
||||
assert response.usage.completion_tokens == 11
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
content = response.choices[0].message.content
|
||||
assert isinstance(content, str) and content.strip(), content
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.choices[0].finish_reason in {"stop", "length"}
|
||||
|
||||
|
||||
def test_anthropic_streaming_completion_replay():
|
||||
"""Same as above for the streaming path; asserts on shape, not bytes."""
|
||||
stream = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
|
|
@ -1909,7 +1918,9 @@ def test_anthropic_streaming_completion_replay():
|
|||
|
||||
collected_text = ""
|
||||
finish_reason = None
|
||||
chunk_count = 0
|
||||
for chunk in stream:
|
||||
chunk_count += 1
|
||||
if not chunk.choices:
|
||||
continue
|
||||
delta = chunk.choices[0].delta
|
||||
|
|
@ -1918,5 +1929,6 @@ def test_anthropic_streaming_completion_replay():
|
|||
if chunk.choices[0].finish_reason:
|
||||
finish_reason = chunk.choices[0].finish_reason
|
||||
|
||||
assert collected_text == "Hello from LiteLLM!"
|
||||
assert finish_reason == "stop"
|
||||
assert chunk_count > 1, "expected multiple SSE chunks from streaming response"
|
||||
assert collected_text.strip(), collected_text
|
||||
assert finish_reason in {"stop", "length"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue