From 72c92920b25a2e575df54a4d661e06ffdb6c327c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 01:34:55 +0000 Subject: [PATCH] fix(tests): make Anthropic VCR fixture script reproducible Delete existing cassettes before recording (record_mode='all' with vcrpy appends rather than overwriting), and strip non-deterministic response headers (Date, Server) so re-running the helper produces a byte-stable diff. Regenerate the committed cassettes with the fixed script so they match what contributors get when following the README. --- .../cassettes/_record_anthropic_fixtures.py | 30 +++++++ .../cassettes/anthropic_basic_completion.yaml | 86 ------------------ .../anthropic_streaming_completion.yaml | 87 ------------------- 3 files changed, 30 insertions(+), 173 deletions(-) diff --git a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py index 0e75fd96505..c17c56169ec 100644 --- a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py +++ b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py @@ -26,6 +26,7 @@ from __future__ import annotations import json import os +import re import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -161,12 +162,37 @@ def _serve(port: int, mode: str) -> ThreadingHTTPServer: return srv +# Headers that vary every run (timestamps, server build) and must be stripped +# so the cassette is byte-stable across regenerations. Replay does not depend +# on them. +_NON_DETERMINISTIC_HEADERS = ("Date", "Server") + + +def _strip_nondeterministic_headers(path: Path) -> None: + """Remove headers whose values change every run from the cassette.""" + text = path.read_text() + for header in _NON_DETERMINISTIC_HEADERS: + # Matches a YAML block like:: + # + # Date: + # - Thu, 30 Apr 2026 00:43:16 GMT + # + # under the response ``headers:`` mapping. Indentation is fixed by vcrpy. + pattern = re.compile( + rf"^ {re.escape(header)}:\n - .*\n", + re.MULTILINE, + ) + text = pattern.sub("", text) + path.write_text(text) + + def _rewrite_cassette_to_real_host(path: Path, mock_host_port: str) -> None: """Replace mock host/port in the cassette with the real Anthropic host.""" text = path.read_text() text = text.replace(f"http://{mock_host_port}", f"https://{REAL_ANTHROPIC_HOST}") text = text.replace(mock_host_port, REAL_ANTHROPIC_HOST) path.write_text(text) + _strip_nondeterministic_headers(path) def _consume(iterable: Iterable[Any]) -> None: @@ -176,6 +202,8 @@ def _consume(iterable: Iterable[Any]) -> None: def record_non_streaming() -> None: cassette = CASSETTE_DIR / "anthropic_basic_completion.yaml" + if cassette.exists(): + cassette.unlink() server = _serve(NON_STREAM_PORT, "json") try: my_vcr = vcr.VCR( @@ -197,6 +225,8 @@ def record_non_streaming() -> None: def record_streaming() -> None: cassette = CASSETTE_DIR / "anthropic_streaming_completion.yaml" + if cassette.exists(): + cassette.unlink() server = _serve(STREAM_PORT, "stream") try: my_vcr = vcr.VCR( diff --git a/tests/llm_translation/cassettes/anthropic_basic_completion.yaml b/tests/llm_translation/cassettes/anthropic_basic_completion.yaml index 458d41f69ef..f593c5ab968 100644 --- a/tests/llm_translation/cassettes/anthropic_basic_completion.yaml +++ b/tests/llm_translation/cassettes/anthropic_basic_completion.yaml @@ -1,43 +1,4 @@ interactions: -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", - "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": - "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": - null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": - 0, "output_tokens": 11}}' - headers: - Content-Type: - - application/json - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - anthropic-ratelimit-tokens-limit: - - '400000' - anthropic-ratelimit-tokens-remaining: - - '399988' - status: - code: 200 - message: OK - request: body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' @@ -70,53 +31,6 @@ interactions: - '358' Content-Type: - application/json - Date: - - Thu, 30 Apr 2026 00:43:16 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - status: - code: 200 - message: OK -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '141' - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", - "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": - "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": - null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": - 0, "output_tokens": 11}}' - headers: - Content-Length: - - '358' - Content-Type: - - application/json - Date: - - Thu, 30 Apr 2026 00:45:24 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 anthropic-ratelimit-requests-limit: - '4000' anthropic-ratelimit-requests-remaining: diff --git a/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml b/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml index 3f80a85ec53..25c0c6c3d24 100644 --- a/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml +++ b/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml @@ -75,93 +75,6 @@ interactions: - no-cache Content-Type: - text/event-stream - Date: - - Thu, 30 Apr 2026 00:43:17 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 - status: - code: 200 - message: OK -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000, "stream": true}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '157' - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: 'event: message_start - - data: {"type": "message_start", "message": {"id": "msg_01STREAMABCDEFGH", - "type": "message", "role": "assistant", "model": "claude-sonnet-4-5-20250929", - "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": - 14, "output_tokens": 1}}} - - - event: content_block_start - - data: {"type": "content_block_start", "index": 0, "content_block": {"type": - "text", "text": ""}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": "Hello"}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": " from"}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": " LiteLLM!"}} - - - event: content_block_stop - - data: {"type": "content_block_stop", "index": 0} - - - event: message_delta - - data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": - null}, "usage": {"output_tokens": 5}} - - - event: message_stop - - data: {"type": "message_stop"} - - - ' - headers: - Cache-Control: - - no-cache - Content-Type: - - text/event-stream - Date: - - Thu, 30 Apr 2026 00:45:25 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 status: code: 200 message: OK