diff --git a/tests/integration/README.md b/tests/integration/README.md index 1865366123a..cf4e3be3f02 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -30,6 +30,8 @@ Streaming checks send real HTTP transfer chunks, including one-byte partitions, The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: native-provider backends under `providers/` (`anthropic`, `bedrock`, `gemini`) and the translation bridges (`responses_bridge`, `chat_bridge`) at the top level. It runs in the providers shard; `run.py` selects test files recursively under each scheduled directory +A provider folder holds only what depends on that provider's wire format: request and response fidelity, multi-turn history, request parameters, content types, and provider-specific pricing. Behavior every provider shares, such as fallback or billing after a client disconnect, lives in the feature directory it exercises (`routing/`, `streaming/`, `spend/`). `_support/claude_code.py` holds a captured Claude Code request and stream builders that any directory can use as a realistic agent payload + The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy, database or Redis; a case that reaches the gateway belongs in one of the other shards The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions diff --git a/tests/integration/messages_endpoint/_claude_code.py b/tests/integration/_support/claude_code.py similarity index 99% rename from tests/integration/messages_endpoint/_claude_code.py rename to tests/integration/_support/claude_code.py index 53c439fbc82..8f2502952be 100644 --- a/tests/integration/messages_endpoint/_claude_code.py +++ b/tests/integration/_support/claude_code.py @@ -1,4 +1,4 @@ -"""Shared Claude Code-shaped request builders and upstream stream fixtures for the /v1/messages contracts.""" +"""Shared Claude Code-shaped request builders and upstream stream fixtures for integration contracts.""" import json from collections.abc import Mapping diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_frontier_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_adaptive_thinking_effort_wire.py similarity index 94% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_frontier_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_adaptive_thinking_effort_wire.py index a9a41b96751..d327181a491 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_frontier_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_adaptive_thinking_effort_wire.py @@ -5,7 +5,7 @@ import pytest from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue @@ -17,7 +17,7 @@ def _diff(expected: dict[str, JsonValue], body: dict[str, JsonValue]) -> dict[st } -def test_claude_code_adaptive_thinking_and_effort_reach_anthropic_intact(gateway: Gateway) -> None: +def test_adaptive_thinking_and_effort_reach_anthropic_intact(gateway: Gateway) -> None: identity: Final = f"msg_fable_{uuid.uuid4().hex}" request_body: Final = cc.frontier_request(f"cache-bust-{uuid.uuid4().hex}", "high", 64000) cli_beta: Final = frozenset(cc.FRONTIER_CLI_BETA.split(",")) @@ -63,7 +63,7 @@ def test_claude_code_adaptive_thinking_and_effort_reach_anthropic_intact(gateway assert rows[0]["prompt_tokens"] == 12 and rows[0]["completion_tokens"] == 4 -def test_claude_code_xhigh_effort_reaches_anthropic_and_charges_by_usage(gateway: Gateway) -> None: +def test_xhigh_effort_reaches_anthropic_and_charges_by_usage(gateway: Gateway) -> None: identity: Final = f"msg_opus_{uuid.uuid4().hex}" request_body: Final = cc.frontier_request(f"cache-bust-{uuid.uuid4().hex}", "xhigh", 128000) cli_beta: Final = frozenset(cc.FRONTIER_CLI_BETA.split(",")) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_compaction_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_compaction_wire.py similarity index 98% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_compaction_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_compaction_wire.py index 4be3a2b9977..ce87a76a5f2 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_compaction_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_compaction_wire.py @@ -3,7 +3,7 @@ from typing import Final from integration._support.client import Gateway from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue _CONTEXT_MANAGEMENT: Final = { diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_document_input_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_document_input_wire.py similarity index 98% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_document_input_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_document_input_wire.py index d4a827d7ece..29ab1a70449 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_document_input_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_document_input_wire.py @@ -4,7 +4,7 @@ from typing import Final from integration._support.client import Gateway from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue _PDF_BYTES: Final = ( diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_image_input_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_image_input_wire.py similarity index 97% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_image_input_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_image_input_wire.py index b77f328403b..271f064becc 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_image_input_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_image_input_wire.py @@ -3,7 +3,7 @@ from typing import Final from integration._support.client import Gateway from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue _PNG_B64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAEElEQVR4nGP4z8AARwzEcQCukw/x0F8jngAAAABJRU5ErkJggg==" diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_interleaved_thinking_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_interleaved_thinking_history_wire.py similarity index 99% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_interleaved_thinking_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_interleaved_thinking_history_wire.py index ff8ccf0df4f..a1d1a9bcb03 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_interleaved_thinking_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_interleaved_thinking_history_wire.py @@ -3,7 +3,7 @@ from typing import Final from integration._support.client import Gateway from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue _TURN1_TOOL: Final = ("toolu_a", "Read", {"file_path": "/tmp/cc_probe/a.txt"}) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_long_context_beta_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_long_context_beta_wire.py similarity index 97% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_long_context_beta_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_long_context_beta_wire.py index a198ec8500a..4c4b075fcc1 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_long_context_beta_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_long_context_beta_wire.py @@ -5,7 +5,7 @@ import pytest from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc _BETA_1M: Final = f"{cc.FRONTIER_CLI_BETA.replace(',effort-2025-11-24', ',context-1m-2025-08-07,effort-2025-11-24')}" diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_model_switch_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_model_switch_history_wire.py similarity index 95% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_model_switch_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_model_switch_history_wire.py index 05b936c93f3..d70f689ce07 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_model_switch_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_model_switch_history_wire.py @@ -4,11 +4,11 @@ from typing import Final from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue -def test_claude_code_mid_loop_model_switch_replays_history_byte_identical(gateway: Gateway) -> None: +def test_mid_loop_model_switch_replays_history_byte_identical(gateway: Gateway) -> None: identity1: Final = f"msg_sw1_{uuid.uuid4().hex}" identity2: Final = f"msg_sw2_{uuid.uuid4().hex}" turn1: Final = cc.frontier_request( diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_prompt_cache_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_prompt_cache_billing_wire.py similarity index 95% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_prompt_cache_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_prompt_cache_billing_wire.py index 54bbe2a0704..8d4338f4910 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_prompt_cache_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_prompt_cache_billing_wire.py @@ -5,7 +5,7 @@ import pytest from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue _USAGE: Final = { @@ -16,7 +16,7 @@ _USAGE: Final = { } -def test_claude_code_cached_turn_charges_cache_read_and_creation_rates(gateway: Gateway) -> None: +def test_cached_turn_charges_cache_read_and_creation_rates(gateway: Gateway) -> None: identity: Final = f"msg_pc_{uuid.uuid4().hex}" turn1: Final = cc.frontier_request( f"cache-bust-{uuid.uuid4().hex}", diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_native_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_request_fidelity_wire.py similarity index 94% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_native_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_request_fidelity_wire.py index 5eaf96cd696..288787f5765 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_native_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_request_fidelity_wire.py @@ -4,12 +4,12 @@ from typing import Final from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc _MODEL: Final = cc.SONNET -def test_claude_code_streaming_request_reaches_anthropic_intact_and_streams_back(gateway: Gateway) -> None: +def test_streaming_request_reaches_anthropic_intact_and_streams_back(gateway: Gateway) -> None: identity: Final = f"msg_cc_{uuid.uuid4().hex}" request_body: Final = cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}") cli_beta: Final = frozenset(cc.CLI_BETA.split(",")) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_tool_loop_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_tool_loop_wire.py similarity index 96% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_tool_loop_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_tool_loop_wire.py index 39bda48c2c4..7d421345ed6 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_tool_loop_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_tool_loop_wire.py @@ -4,7 +4,7 @@ from typing import Final from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc from pydantic import JsonValue _THINKING: Final = "need to read the file" @@ -20,7 +20,7 @@ def _diff(expected: dict[str, JsonValue], body: dict[str, JsonValue]) -> dict[st } -def test_claude_code_tool_loop_round_trips_thinking_tool_use_and_tool_result(gateway: Gateway) -> None: +def test_tool_loop_round_trips_thinking_tool_use_and_tool_result(gateway: Gateway) -> None: identity1: Final = f"msg_tl1_{uuid.uuid4().hex}" identity2: Final = f"msg_tl2_{uuid.uuid4().hex}" turn1: Final = cc.frontier_request( @@ -114,7 +114,7 @@ def test_claude_code_tool_loop_round_trips_thinking_tool_use_and_tool_result(gat assert rows[0]["prompt_tokens"] == 30 -def test_claude_code_parallel_tool_results_reach_anthropic_in_client_order(gateway: Gateway) -> None: +def test_parallel_tool_results_reach_anthropic_in_client_order(gateway: Gateway) -> None: turn1: Final = cc.frontier_request( f"cache-bust-{uuid.uuid4().hex}", "high", diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_web_search_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_web_search_citations_wire.py similarity index 97% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_web_search_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_web_search_citations_wire.py index aed0a5abd40..f63126116be 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_web_search_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_web_search_citations_wire.py @@ -4,7 +4,7 @@ from typing import Final from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc WEB_SEARCH_TOOL: Final = {"type": "web_search_20250305", "name": "web_search", "max_uses": 8} @@ -103,7 +103,7 @@ def _web_search_stream(identity: str) -> tuple[bytes, ...]: ) -def test_claude_code_web_search_tool_passthrough_and_cited_response(gateway: Gateway) -> None: +def test_web_search_tool_passthrough_and_cited_response(gateway: Gateway) -> None: identity: Final = f"msg_ws_{uuid.uuid4().hex}" base: Final = cc.frontier_request( f"cache-bust-{uuid.uuid4().hex}", diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_fallback_wire.py b/tests/integration/routing/test_overloaded_deployment_fallback.py similarity index 95% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_fallback_wire.py rename to tests/integration/routing/test_overloaded_deployment_fallback.py index 830f84c858c..d5ca213d3d9 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_fallback_wire.py +++ b/tests/integration/routing/test_overloaded_deployment_fallback.py @@ -9,7 +9,7 @@ from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc def _error_529() -> Reply: @@ -19,7 +19,7 @@ def _error_529() -> Reply: ) -def test_anthropic_overloaded_primary_falls_back_to_second_deployment(gateway: Gateway, tmp_path: Path) -> None: +def test_overloaded_primary_falls_back_to_second_deployment(gateway: Gateway, tmp_path: Path) -> None: identity: Final = f"msg_fb_{uuid.uuid4().hex}" request_body: Final = cc.claude_code_request(f"cache-bust-{uuid.uuid4().hex}") diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_client_disconnect_wire.py b/tests/integration/streaming/test_client_disconnect_still_bills.py similarity index 98% rename from tests/integration/messages_endpoint/providers/anthropic/test_claude_code_client_disconnect_wire.py rename to tests/integration/streaming/test_client_disconnect_still_bills.py index 0e7f00943c2..5a80e13cf44 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_claude_code_client_disconnect_wire.py +++ b/tests/integration/streaming/test_client_disconnect_still_bills.py @@ -4,7 +4,7 @@ from typing import Final from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server -from integration.messages_endpoint import _claude_code as cc +from integration._support import claude_code as cc def test_client_disconnect_mid_stream_still_bills_the_message(gateway: Gateway) -> None: