mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(integration): name Claude Code wire tests by behavior and move provider-agnostic ones to routing and streaming
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
0ea21ef9e0
commit
77e3b8552a
15 changed files with 25 additions and 23 deletions
|
|
@ -30,6 +30,8 @@ Streaming checks send real HTTP transfer chunks, including one-byte partitions,
|
|||
|
||||
The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: native-provider backends under `providers/` (`anthropic`, `bedrock`, `gemini`) and the translation bridges (`responses_bridge`, `chat_bridge`) at the top level. It runs in the providers shard; `run.py` selects test files recursively under each scheduled directory
|
||||
|
||||
A provider folder holds only what depends on that provider's wire format: request and response fidelity, multi-turn history, request parameters, content types, and provider-specific pricing. Behavior every provider shares, such as fallback or billing after a client disconnect, lives in the feature directory it exercises (`routing/`, `streaming/`, `spend/`). `_support/claude_code.py` holds a captured Claude Code request and stream builders that any directory can use as a realistic agent payload
|
||||
|
||||
The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy, database or Redis; a case that reaches the gateway belongs in one of the other shards
|
||||
|
||||
The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Shared Claude Code-shaped request builders and upstream stream fixtures for the /v1/messages contracts."""
|
||||
"""Shared Claude Code-shaped request builders and upstream stream fixtures for integration contracts."""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ def _diff(expected: dict[str, JsonValue], body: dict[str, JsonValue]) -> dict[st
|
|||
}
|
||||
|
||||
|
||||
def test_claude_code_adaptive_thinking_and_effort_reach_anthropic_intact(gateway: Gateway) -> None:
|
||||
def test_adaptive_thinking_and_effort_reach_anthropic_intact(gateway: Gateway) -> None:
|
||||
identity: Final = f"msg_fable_{uuid.uuid4().hex}"
|
||||
request_body: Final = cc.frontier_request(f"cache-bust-{uuid.uuid4().hex}", "high", 64000)
|
||||
cli_beta: Final = frozenset(cc.FRONTIER_CLI_BETA.split(","))
|
||||
|
|
@ -63,7 +63,7 @@ def test_claude_code_adaptive_thinking_and_effort_reach_anthropic_intact(gateway
|
|||
assert rows[0]["prompt_tokens"] == 12 and rows[0]["completion_tokens"] == 4
|
||||
|
||||
|
||||
def test_claude_code_xhigh_effort_reaches_anthropic_and_charges_by_usage(gateway: Gateway) -> None:
|
||||
def test_xhigh_effort_reaches_anthropic_and_charges_by_usage(gateway: Gateway) -> None:
|
||||
identity: Final = f"msg_opus_{uuid.uuid4().hex}"
|
||||
request_body: Final = cc.frontier_request(f"cache-bust-{uuid.uuid4().hex}", "xhigh", 128000)
|
||||
cli_beta: Final = frozenset(cc.FRONTIER_CLI_BETA.split(","))
|
||||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
_CONTEXT_MANAGEMENT: Final = {
|
||||
|
|
@ -4,7 +4,7 @@ from typing import Final
|
|||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
_PDF_BYTES: Final = (
|
||||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
_PNG_B64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAEElEQVR4nGP4z8AARwzEcQCukw/x0F8jngAAAABJRU5ErkJggg=="
|
||||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
_TURN1_TOOL: Final = ("toolu_a", "Read", {"file_path": "/tmp/cc_probe/a.txt"})
|
||||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
|
||||
_BETA_1M: Final = f"{cc.FRONTIER_CLI_BETA.replace(',effort-2025-11-24', ',context-1m-2025-08-07,effort-2025-11-24')}"
|
||||
|
||||
|
|
@ -4,11 +4,11 @@ from typing import Final
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
def test_claude_code_mid_loop_model_switch_replays_history_byte_identical(gateway: Gateway) -> None:
|
||||
def test_mid_loop_model_switch_replays_history_byte_identical(gateway: Gateway) -> None:
|
||||
identity1: Final = f"msg_sw1_{uuid.uuid4().hex}"
|
||||
identity2: Final = f"msg_sw2_{uuid.uuid4().hex}"
|
||||
turn1: Final = cc.frontier_request(
|
||||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
_USAGE: Final = {
|
||||
|
|
@ -16,7 +16,7 @@ _USAGE: Final = {
|
|||
}
|
||||
|
||||
|
||||
def test_claude_code_cached_turn_charges_cache_read_and_creation_rates(gateway: Gateway) -> None:
|
||||
def test_cached_turn_charges_cache_read_and_creation_rates(gateway: Gateway) -> None:
|
||||
identity: Final = f"msg_pc_{uuid.uuid4().hex}"
|
||||
turn1: Final = cc.frontier_request(
|
||||
f"cache-bust-{uuid.uuid4().hex}",
|
||||
|
|
@ -4,12 +4,12 @@ from typing import Final
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
|
||||
_MODEL: Final = cc.SONNET
|
||||
|
||||
|
||||
def test_claude_code_streaming_request_reaches_anthropic_intact_and_streams_back(gateway: Gateway) -> None:
|
||||
def test_streaming_request_reaches_anthropic_intact_and_streams_back(gateway: Gateway) -> None:
|
||||
identity: Final = f"msg_cc_{uuid.uuid4().hex}"
|
||||
request_body: Final = cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}")
|
||||
cli_beta: Final = frozenset(cc.CLI_BETA.split(","))
|
||||
|
|
@ -4,7 +4,7 @@ from typing import Final
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
from pydantic import JsonValue
|
||||
|
||||
_THINKING: Final = "need to read the file"
|
||||
|
|
@ -20,7 +20,7 @@ def _diff(expected: dict[str, JsonValue], body: dict[str, JsonValue]) -> dict[st
|
|||
}
|
||||
|
||||
|
||||
def test_claude_code_tool_loop_round_trips_thinking_tool_use_and_tool_result(gateway: Gateway) -> None:
|
||||
def test_tool_loop_round_trips_thinking_tool_use_and_tool_result(gateway: Gateway) -> None:
|
||||
identity1: Final = f"msg_tl1_{uuid.uuid4().hex}"
|
||||
identity2: Final = f"msg_tl2_{uuid.uuid4().hex}"
|
||||
turn1: Final = cc.frontier_request(
|
||||
|
|
@ -114,7 +114,7 @@ def test_claude_code_tool_loop_round_trips_thinking_tool_use_and_tool_result(gat
|
|||
assert rows[0]["prompt_tokens"] == 30
|
||||
|
||||
|
||||
def test_claude_code_parallel_tool_results_reach_anthropic_in_client_order(gateway: Gateway) -> None:
|
||||
def test_parallel_tool_results_reach_anthropic_in_client_order(gateway: Gateway) -> None:
|
||||
turn1: Final = cc.frontier_request(
|
||||
f"cache-bust-{uuid.uuid4().hex}",
|
||||
"high",
|
||||
|
|
@ -4,7 +4,7 @@ from typing import Final
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
|
||||
WEB_SEARCH_TOOL: Final = {"type": "web_search_20250305", "name": "web_search", "max_uses": 8}
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ def _web_search_stream(identity: str) -> tuple[bytes, ...]:
|
|||
)
|
||||
|
||||
|
||||
def test_claude_code_web_search_tool_passthrough_and_cited_response(gateway: Gateway) -> None:
|
||||
def test_web_search_tool_passthrough_and_cited_response(gateway: Gateway) -> None:
|
||||
identity: Final = f"msg_ws_{uuid.uuid4().hex}"
|
||||
base: Final = cc.frontier_request(
|
||||
f"cache-bust-{uuid.uuid4().hex}",
|
||||
|
|
@ -9,7 +9,7 @@ from integration._support.client import Gateway, eventually
|
|||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
|
||||
|
||||
def _error_529() -> Reply:
|
||||
|
|
@ -19,7 +19,7 @@ def _error_529() -> Reply:
|
|||
)
|
||||
|
||||
|
||||
def test_anthropic_overloaded_primary_falls_back_to_second_deployment(gateway: Gateway, tmp_path: Path) -> None:
|
||||
def test_overloaded_primary_falls_back_to_second_deployment(gateway: Gateway, tmp_path: Path) -> None:
|
||||
identity: Final = f"msg_fb_{uuid.uuid4().hex}"
|
||||
request_body: Final = cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}")
|
||||
|
||||
|
|
@ -4,7 +4,7 @@ from typing import Final
|
|||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.messages_endpoint import _claude_code as cc
|
||||
from integration._support import claude_code as cc
|
||||
|
||||
|
||||
def test_client_disconnect_mid_stream_still_bills_the_message(gateway: Gateway) -> None:
|
||||
Loading…
Add table
Reference in a new issue