mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
fix(tests): make Anthropic VCR fixture script reproducible
Delete existing cassettes before recording (record_mode='all' with vcrpy appends rather than overwriting), and strip non-deterministic response headers (Date, Server) so re-running the helper produces a byte-stable diff. Regenerate the committed cassettes with the fixed script so they match what contributors get when following the README.
This commit is contained in:
Cursor Agent
parent
94b319c577
commit
72c92920b2
3 changed files with 30 additions and 173 deletions
|
|
@ -26,6 +26,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
|
@ -161,12 +162,37 @@ def _serve(port: int, mode: str) -> ThreadingHTTPServer:
|
|||
return srv
|
||||
|
||||
|
||||
# Headers that vary every run (timestamps, server build) and must be stripped
|
||||
# so the cassette is byte-stable across regenerations. Replay does not depend
|
||||
# on them.
|
||||
_NON_DETERMINISTIC_HEADERS = ("Date", "Server")
|
||||
|
||||
|
||||
def _strip_nondeterministic_headers(path: Path) -> None:
|
||||
"""Remove headers whose values change every run from the cassette."""
|
||||
text = path.read_text()
|
||||
for header in _NON_DETERMINISTIC_HEADERS:
|
||||
# Matches a YAML block like::
|
||||
#
|
||||
# Date:
|
||||
# - Thu, 30 Apr 2026 00:43:16 GMT
|
||||
#
|
||||
# under the response ``headers:`` mapping. Indentation is fixed by vcrpy.
|
||||
pattern = re.compile(
|
||||
rf"^ {re.escape(header)}:\n - .*\n",
|
||||
re.MULTILINE,
|
||||
)
|
||||
text = pattern.sub("", text)
|
||||
path.write_text(text)
|
||||
|
||||
|
||||
def _rewrite_cassette_to_real_host(path: Path, mock_host_port: str) -> None:
|
||||
"""Replace mock host/port in the cassette with the real Anthropic host."""
|
||||
text = path.read_text()
|
||||
text = text.replace(f"http://{mock_host_port}", f"https://{REAL_ANTHROPIC_HOST}")
|
||||
text = text.replace(mock_host_port, REAL_ANTHROPIC_HOST)
|
||||
path.write_text(text)
|
||||
_strip_nondeterministic_headers(path)
|
||||
|
||||
|
||||
def _consume(iterable: Iterable[Any]) -> None:
|
||||
|
|
@ -176,6 +202,8 @@ def _consume(iterable: Iterable[Any]) -> None:
|
|||
|
||||
def record_non_streaming() -> None:
|
||||
cassette = CASSETTE_DIR / "anthropic_basic_completion.yaml"
|
||||
if cassette.exists():
|
||||
cassette.unlink()
|
||||
server = _serve(NON_STREAM_PORT, "json")
|
||||
try:
|
||||
my_vcr = vcr.VCR(
|
||||
|
|
@ -197,6 +225,8 @@ def record_non_streaming() -> None:
|
|||
|
||||
def record_streaming() -> None:
|
||||
cassette = CASSETTE_DIR / "anthropic_streaming_completion.yaml"
|
||||
if cassette.exists():
|
||||
cassette.unlink()
|
||||
server = _serve(STREAM_PORT, "stream")
|
||||
try:
|
||||
my_vcr = vcr.VCR(
|
||||
|
|
|
|||
|
|
@ -1,43 +1,4 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content":
|
||||
[{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}'
|
||||
headers:
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Host:
|
||||
- api.anthropic.com
|
||||
User-Agent:
|
||||
- litellm/1.84.0
|
||||
accept:
|
||||
- application/json
|
||||
content-type:
|
||||
- application/json
|
||||
method: POST
|
||||
uri: https://api.anthropic.com/v1/messages
|
||||
response:
|
||||
body:
|
||||
string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant",
|
||||
"model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text":
|
||||
"Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence":
|
||||
null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens":
|
||||
0, "output_tokens": 11}}'
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json
|
||||
anthropic-ratelimit-requests-limit:
|
||||
- '4000'
|
||||
anthropic-ratelimit-requests-remaining:
|
||||
- '3999'
|
||||
anthropic-ratelimit-tokens-limit:
|
||||
- '400000'
|
||||
anthropic-ratelimit-tokens-remaining:
|
||||
- '399988'
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content":
|
||||
[{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}'
|
||||
|
|
@ -70,53 +31,6 @@ interactions:
|
|||
- '358'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Thu, 30 Apr 2026 00:43:16 GMT
|
||||
Server:
|
||||
- BaseHTTP/0.6 Python/3.12.3
|
||||
anthropic-ratelimit-requests-limit:
|
||||
- '4000'
|
||||
anthropic-ratelimit-requests-remaining:
|
||||
- '3999'
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content":
|
||||
[{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}'
|
||||
headers:
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '141'
|
||||
Host:
|
||||
- api.anthropic.com
|
||||
User-Agent:
|
||||
- litellm/1.84.0
|
||||
accept:
|
||||
- application/json
|
||||
content-type:
|
||||
- application/json
|
||||
method: POST
|
||||
uri: https://api.anthropic.com/v1/messages
|
||||
response:
|
||||
body:
|
||||
string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant",
|
||||
"model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text":
|
||||
"Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence":
|
||||
null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens":
|
||||
0, "output_tokens": 11}}'
|
||||
headers:
|
||||
Content-Length:
|
||||
- '358'
|
||||
Content-Type:
|
||||
- application/json
|
||||
Date:
|
||||
- Thu, 30 Apr 2026 00:45:24 GMT
|
||||
Server:
|
||||
- BaseHTTP/0.6 Python/3.12.3
|
||||
anthropic-ratelimit-requests-limit:
|
||||
- '4000'
|
||||
anthropic-ratelimit-requests-remaining:
|
||||
|
|
|
|||
|
|
@ -75,93 +75,6 @@ interactions:
|
|||
- no-cache
|
||||
Content-Type:
|
||||
- text/event-stream
|
||||
Date:
|
||||
- Thu, 30 Apr 2026 00:43:17 GMT
|
||||
Server:
|
||||
- BaseHTTP/0.6 Python/3.12.3
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content":
|
||||
[{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000, "stream": true}'
|
||||
headers:
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '157'
|
||||
Host:
|
||||
- api.anthropic.com
|
||||
User-Agent:
|
||||
- litellm/1.84.0
|
||||
accept:
|
||||
- application/json
|
||||
content-type:
|
||||
- application/json
|
||||
method: POST
|
||||
uri: https://api.anthropic.com/v1/messages
|
||||
response:
|
||||
body:
|
||||
string: 'event: message_start
|
||||
|
||||
data: {"type": "message_start", "message": {"id": "msg_01STREAMABCDEFGH",
|
||||
"type": "message", "role": "assistant", "model": "claude-sonnet-4-5-20250929",
|
||||
"content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens":
|
||||
14, "output_tokens": 1}}}
|
||||
|
||||
|
||||
event: content_block_start
|
||||
|
||||
data: {"type": "content_block_start", "index": 0, "content_block": {"type":
|
||||
"text", "text": ""}}
|
||||
|
||||
|
||||
event: content_block_delta
|
||||
|
||||
data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta",
|
||||
"text": "Hello"}}
|
||||
|
||||
|
||||
event: content_block_delta
|
||||
|
||||
data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta",
|
||||
"text": " from"}}
|
||||
|
||||
|
||||
event: content_block_delta
|
||||
|
||||
data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta",
|
||||
"text": " LiteLLM!"}}
|
||||
|
||||
|
||||
event: content_block_stop
|
||||
|
||||
data: {"type": "content_block_stop", "index": 0}
|
||||
|
||||
|
||||
event: message_delta
|
||||
|
||||
data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence":
|
||||
null}, "usage": {"output_tokens": 5}}
|
||||
|
||||
|
||||
event: message_stop
|
||||
|
||||
data: {"type": "message_stop"}
|
||||
|
||||
|
||||
'
|
||||
headers:
|
||||
Cache-Control:
|
||||
- no-cache
|
||||
Content-Type:
|
||||
- text/event-stream
|
||||
Date:
|
||||
- Thu, 30 Apr 2026 00:45:25 GMT
|
||||
Server:
|
||||
- BaseHTTP/0.6 Python/3.12.3
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue