diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 01a01b1034b..a483dcec9d7 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -516,7 +516,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens str(path.relative_to(repo_root)) for folders in groups.values() for folder in folders - for path in (integration_root / folder).glob("test_*.py") + for path in (integration_root / folder).rglob("test_*.py") ) browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json" browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else () diff --git a/tests/integration/README.md b/tests/integration/README.md index ac9b01786b9..1865366123a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -28,6 +28,8 @@ Provider contracts exercise actual TCP requests with synthetic credentials and l Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. CircleCI reuses its existing Redis image for that extra process; it never pulls an image during tests +The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: native-provider backends under `providers/` (`anthropic`, `bedrock`, `gemini`) and the translation bridges (`responses_bridge`, `chat_bridge`) at the top level. It runs in the providers shard; `run.py` selects test files recursively under each scheduled directory + The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy, database or Redis; a case that reaches the gateway belongs in one of the other shards The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index aa0b27eceda..376c2a515b7 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -10,6 +10,7 @@ OWNED_DIRECTORIES: Final = frozenset( "routing", "providers", "streaming", + "messages_endpoint", "configuration", "mcp", "observability", diff --git a/tests/integration/providers/test_anthropic_messages_fireworks_stop_wire.py b/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_fireworks_stop_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_messages_fireworks_stop_wire.py rename to tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_fireworks_stop_wire.py diff --git a/tests/integration/providers/test_anthropic_advisor_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_advisor_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_advisor_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_advisor_wire.py diff --git a/tests/integration/providers/test_anthropic_legacy_thinking_budget_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_legacy_thinking_budget_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_legacy_thinking_budget_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_legacy_thinking_budget_wire.py diff --git a/tests/integration/providers/test_anthropic_messages_timeout_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_timeout_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_messages_timeout_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_timeout_wire.py diff --git a/tests/integration/providers/test_anthropic_thinking_signature_retry_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_thinking_signature_retry_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_thinking_signature_retry_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_thinking_signature_retry_wire.py diff --git a/tests/integration/providers/test_anthropic_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py diff --git a/tests/integration/providers/test_websearch_interception_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py similarity index 100% rename from tests/integration/providers/test_websearch_interception_wire.py rename to tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py diff --git a/tests/integration/providers/test_bedrock_invoke_tool_search_wire.py b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_invoke_tool_search_wire.py similarity index 100% rename from tests/integration/providers/test_bedrock_invoke_tool_search_wire.py rename to tests/integration/messages_endpoint/providers/bedrock/test_bedrock_invoke_tool_search_wire.py diff --git a/tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_web_search_replay_wire.py similarity index 100% rename from tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py rename to tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_web_search_replay_wire.py diff --git a/tests/integration/providers/test_gemini_messages_cache_control_wire.py b/tests/integration/messages_endpoint/providers/gemini/test_gemini_messages_cache_control_wire.py similarity index 100% rename from tests/integration/providers/test_gemini_messages_cache_control_wire.py rename to tests/integration/messages_endpoint/providers/gemini/test_gemini_messages_cache_control_wire.py diff --git a/tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py b/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_claude_code_cache_key_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py rename to tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_claude_code_cache_key_wire.py diff --git a/tests/integration/providers/test_anthropic_messages_openai_bridge_wire.py b/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_messages_openai_bridge_wire.py rename to tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_bridge_wire.py diff --git a/tests/integration/providers/test_anthropic_messages_openai_tools_wire.py b/tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_tools_wire.py similarity index 100% rename from tests/integration/providers/test_anthropic_messages_openai_tools_wire.py rename to tests/integration/messages_endpoint/responses_bridge/test_anthropic_messages_openai_tools_wire.py diff --git a/tests/integration/providers/test_responses_bridge_stream_options.py b/tests/integration/messages_endpoint/responses_bridge/test_responses_bridge_stream_options.py similarity index 100% rename from tests/integration/providers/test_responses_bridge_stream_options.py rename to tests/integration/messages_endpoint/responses_bridge/test_responses_bridge_stream_options.py diff --git a/tests/integration/run.py b/tests/integration/run.py index 9ef585def3d..b416e4869dd 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -14,7 +14,7 @@ GROUPS: Final = MappingProxyType( "management": ("management", "authorization", "configuration"), "accounting": ("pricing", "spend"), "database": ("database",), - "providers": ("providers", "routing", "streaming"), + "providers": ("providers", "routing", "streaming", "messages_endpoint"), "extensions": ("observability", "compatibility"), "mcp": ("mcp",), "sdk": ("sdk",), @@ -35,7 +35,7 @@ def main() -> int: selected: Final = tuple( str(path.relative_to(root)) for folder in GROUPS[options.group] - for path in sorted((root / "tests/integration" / folder).glob("test_*.py")) + for path in sorted((root / "tests/integration" / folder).rglob("test_*.py")) ) if not selected: parser.error(f"No integration test files selected for {options.group}") diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index 4ac7cecb97a..982e14660b7 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -133,46 +133,6 @@ def test_get_llm_provider_azure_o1(): assert model == "o1-mini" -def test_default_api_base(): - from litellm.litellm_core_utils.get_llm_provider_logic import ( - _get_openai_compatible_provider_info, - ) - from litellm.types.utils import LlmProviders - - # Patch environment variable to remove API base if it's set - with patch.dict(os.environ, {}, clear=True): - for provider in litellm.openai_compatible_providers: - # Get the API base for the given provider - if provider == "github_copilot": - continue - # Skip chatgpt as it requires OAuth authentication - if provider == "chatgpt": - continue - # Skip ragflow as it requires specific model format: ragflow/chat/{id}/{model} or ragflow/agent/{id}/{model} - if provider == "ragflow": - continue - _, _, _, api_base = _get_openai_compatible_provider_info( - model=f"{provider}/*", api_base=None, api_key=None, dynamic_api_key=None - ) - if api_base is None: - continue - - for other_provider in LlmProviders: - if other_provider.value != provider and provider != "{}_chat".format( - other_provider.value - ): - if provider == "codestral" and other_provider.value == "mistral": - continue - elif provider == "github" and other_provider.value == "azure": - continue - elif ( - provider in ("qwencloud", "qwen_ai_platform") - and other_provider.value == "dashscope" - ): - continue - assert other_provider.value not in api_base.replace("/openai", "") - - def test_hosted_vllm_default_api_key(): from litellm.litellm_core_utils.get_llm_provider_logic import ( _get_openai_compatible_provider_info,