diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 703ccf13c27..6726b21d7b2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -19,6 +19,7 @@ from typing import ( Union, cast, ) +from urllib.parse import urlparse import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -31,6 +32,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -51,15 +53,36 @@ from .utils import AnthropicMessagesRequestUtils, mock_response _RESPONSES_API_PROVIDERS = frozenset({"openai"}) -def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: +def _is_openai_native_api_base(api_base: str | None) -> bool: + """Whether the effective OpenAI base URL points at the genuine OpenAI API. + + OpenAI-compatible third parties (a custom ``api_base``, e.g. Zhipu / GLM) + generally expose only chat/completions, not the Responses API, so routing + /v1/messages there would hit a non-existent ``/responses`` path and 404. + Genuine OpenAI (no override, or an ``api.openai.com`` host) supports it. + """ + effective = api_base or get_secret_str("OPENAI_API_BASE") or litellm.api_base + if not effective: + return True + hostname = urlparse(effective).hostname or "" + return hostname == "api.openai.com" or hostname.endswith(".api.openai.com") + + +def _should_route_to_responses_api(custom_llm_provider: str | None, api_base: str | None) -> bool: """Return True when the provider should use the Responses API path. Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to opt out and route OpenAI/Azure requests through chat/completions instead. + + Only genuine OpenAI endpoints route to the Responses API. OpenAI-compatible + providers configured with a custom ``api_base`` fall back to chat/completions + since they typically do not implement ``/responses``. """ if litellm.use_chat_completions_url_for_anthropic_messages: return False - return custom_llm_provider in _RESPONSES_API_PROVIDERS + if custom_llm_provider not in _RESPONSES_API_PROVIDERS: + return False + return _is_openai_native_api_base(api_base) def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: @@ -523,7 +546,7 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, **kwargs, ) - if _should_route_to_responses_api(custom_llm_provider): + if _should_route_to_responses_api(custom_llm_provider, api_base or dynamic_api_base): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs) # The in-gateway context_management polyfill runs inside diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 3327fc39f73..cf4e4524f1a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -9,6 +9,7 @@ sys.path.insert(0, os.path.abspath("../../../../..")) from unittest.mock import AsyncMock, MagicMock, patch +import litellm from litellm.anthropic_interface import messages from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import Delta, ModelResponse, StreamingChoices @@ -821,3 +822,96 @@ def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypat assert result == "translated" assert translation_calls["count"] == 1 assert "config" not in captured + + +def _responses_vs_completions_stubs(monkeypatch): + """Patch the two Anthropic->OpenAI translation handlers with distinct + return values so the caller can assert which endpoint a request routed to. + """ + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + monkeypatch.setattr( + handler.LiteLLMMessagesToResponsesAPIHandler, + "anthropic_messages_handler", + staticmethod(lambda **kwargs: "responses"), + ) + monkeypatch.setattr( + handler.LiteLLMMessagesToCompletionTransformationHandler, + "anthropic_messages_handler", + staticmethod(lambda **kwargs: "chat-completions"), + ) + + +@pytest.mark.parametrize( + "api_base", + [ + "https://open.bigmodel.cn/api/paas/v4", + "https://api.groq.com/openai/v1", + "http://0.0.0.0:4000/v1", + ], +) +def test_openai_compatible_custom_api_base_routes_to_chat_completions(monkeypatch, api_base): + """Regression for #33824: an OpenAI-compatible deployment (provider=openai) + with a non-openai api_base must route /v1/messages through chat/completions, + not the Responses API which those endpoints do not implement (404 /responses). + """ + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + _responses_vs_completions_stubs(monkeypatch) + + result = anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="openai/glm-4.6", + api_key="sk-test", + api_base=api_base, + ) + + assert result == "chat-completions" + + +@pytest.mark.parametrize("api_base", [None, "https://api.openai.com/v1"]) +def test_genuine_openai_routes_to_responses_api(monkeypatch, api_base): + """Genuine OpenAI endpoints (no override, or an api.openai.com host) still + route to the Responses API.""" + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + _responses_vs_completions_stubs(monkeypatch) + + result = anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="openai/gpt-5.1", + api_key="sk-test", + api_base=api_base, + ) + + assert result == "responses" + + +def test_should_route_to_responses_api_honors_api_base(): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _should_route_to_responses_api, + ) + + assert _should_route_to_responses_api("openai", None) is True + assert _should_route_to_responses_api("openai", "https://api.openai.com/v1") is True + assert _should_route_to_responses_api("openai", "https://us.api.openai.com/v1") is True + assert _should_route_to_responses_api("openai", "https://open.bigmodel.cn/api/paas/v4") is False + assert _should_route_to_responses_api("anthropic", None) is False + + +def test_should_route_to_responses_api_respects_global_optout(monkeypatch): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _should_route_to_responses_api, + ) + + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + assert _should_route_to_responses_api("openai", None) is False