From 85cac3b98f04bbf4f568e9591b3cb16aaf8f9687 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 23:42:40 +0000 Subject: [PATCH] fix(anthropic): honor supported_endpoints when routing /v1/messages /v1/messages picked the upstream Responses API for any deployment whose provider resolved to `openai`, based on the provider string alone. OpenAI compatible gateways behind a custom api_base commonly serve chat/completions only, so they rejected the bridged request outright. The chat/completions path already guards against this in `responses_api_bridge_check`; the messages path never got an equivalent. Rather than add a new flag, honor the per-deployment declaration that already exists: a deployment whose `model_info.supported_endpoints` omits `/v1/responses` is now bridged through chat/completions. Deployments that declare nothing keep today's behavior byte for byte, and the global `use_chat_completions_url_for_anthropic_messages` opt-out still wins. --- .../messages/handler.py | 52 +++++++--- ...erimental_pass_through_messages_handler.py | 96 +++++++++++++++++++ 2 files changed, 134 insertions(+), 14 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 3ef298aa336..4a087b20148 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -39,30 +39,52 @@ from .utils import AnthropicMessagesRequestUtils, mock_response # Providers that are routed directly to the OpenAI Responses API instead of # going through chat/completions. _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) +_MESSAGES_ENDPOINT: Final = "/v1/messages" +_RESPONSES_ENDPOINT: Final = "/v1/responses" -def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool: +def _declared_supported_endpoints(model_info: object) -> frozenset[str] | None: + """Endpoints the deployment declares it serves, or ``None`` when it declares nothing. + + ``model_info.supported_endpoints`` is the operator's per-deployment declaration in + config.yaml, plumbed here as ``kwargs["model_info"]`` by the router. It is distinct from + the same-named cost-map field, which is model metadata rather than operator intent. + """ + if not isinstance(model_info, dict): + return None + declared: Final = model_info.get("supported_endpoints") + if not isinstance(declared, (list, tuple)): + return None + return frozenset(endpoint for endpoint in declared if isinstance(endpoint, str)) + + +def _should_route_to_responses_api(custom_llm_provider: str | None, model_info: object) -> bool: """Return True when the provider should use the Responses API path. - Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to - opt out and route OpenAI/Azure requests through chat/completions instead. + Only the providers in ``_RESPONSES_API_PROVIDERS`` are candidates, and a deployment that + declares its ``supported_endpoints`` without ``/v1/responses`` opts out: an + OpenAI-compatible backend behind a custom ``api_base`` commonly serves chat/completions + only and rejects a Responses request outright. A deployment that declares nothing keeps + the historical behavior. + + Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to opt every + deployment out at once. """ if litellm.use_chat_completions_url_for_anthropic_messages: return False - return custom_llm_provider in _RESPONSES_API_PROVIDERS + if custom_llm_provider not in _RESPONSES_API_PROVIDERS: + return False + declared: Final = _declared_supported_endpoints(model_info) + return declared is None or _RESPONSES_ENDPOINT in declared def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: """Whether the deployment opted into forwarding /v1/messages untranslated. - The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``, - declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]`` - by the router. + The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``. """ - if not isinstance(model_info, dict): - return False - supported_endpoints: Final = model_info.get("supported_endpoints") - return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints + declared: Final = _declared_supported_endpoints(model_info) + return declared is not None and _MESSAGES_ENDPOINT in declared ####### ENVIRONMENT VARIABLES ################### @@ -511,6 +533,7 @@ def anthropic_messages_handler( ) anthropic_messages_provider_config: BaseAnthropicMessagesConfig | None = None + deployment_model_info: Final = kwargs.get("model_info") if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: anthropic_messages_provider_config = ProviderConfigManager.get_provider_anthropic_messages_config( @@ -518,7 +541,7 @@ def anthropic_messages_handler( provider=litellm.LlmProviders(custom_llm_provider), ) if anthropic_messages_provider_config is None and _deployment_passes_through_anthropic_messages( - kwargs.get("model_info") + deployment_model_info ): from litellm.llms.openai_like.messages.transformation import ( OpenAILikeAnthropicMessagesConfig, @@ -526,7 +549,8 @@ def anthropic_messages_handler( anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig() if anthropic_messages_provider_config is None: - # Route to Responses API for OpenAI / Azure, chat/completions for everything else. + # Route to the Responses API for OpenAI deployments that can serve it, + # chat/completions for everything else. _shared_kwargs: Final = dict( max_tokens=max_tokens, messages=messages, @@ -548,7 +572,7 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, **kwargs, ) - if _should_route_to_responses_api(custom_llm_provider): + if _should_route_to_responses_api(custom_llm_provider, deployment_model_info): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs) # The in-gateway context_management polyfill runs inside diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index df3db3d2c57..476406cd45b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -905,3 +905,99 @@ def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypat assert result == "translated" assert translation_calls["count"] == 1 assert "config" not in captured + + +def _translation_target_stubs(monkeypatch): + """Patch the two Anthropic->OpenAI translation handlers with distinguishable fakes. + + Returns a list that records ``"responses"`` or ``"chat_completions"`` for whichever + bridge ``anthropic_messages_handler`` dispatched to. + """ + from litellm.llms.anthropic.experimental_pass_through.messages import handler + + dispatched: list = [] + + def fake_responses(**kwargs): + dispatched.append("responses") + return "responses-bridge" + + def fake_chat_completions(**kwargs): + dispatched.append("chat_completions") + return "chat-completions-bridge" + + monkeypatch.setattr( + handler.LiteLLMMessagesToResponsesAPIHandler, + "anthropic_messages_handler", + staticmethod(fake_responses), + ) + monkeypatch.setattr( + handler.LiteLLMMessagesToCompletionTransformationHandler, + "anthropic_messages_handler", + staticmethod(fake_chat_completions), + ) + return dispatched + + +def _call_messages_handler(model_info=None): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + kwargs = {"model_info": model_info} if model_info is not None else {} + return anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="openai/some-gateway-model", + api_key="sk-test", + api_base="https://custom-gateway.example.com/v1", + **kwargs, + ) + + +def test_messages_uses_chat_completions_when_supported_endpoints_omit_responses(monkeypatch): + """Regression: an OpenAI-compatible backend that declares its endpoints without + /v1/responses must not have /v1/messages bridged to the Responses API. Guessing + /v1/responses from the provider string alone made such gateways 400.""" + dispatched = _translation_target_stubs(monkeypatch) + + result = _call_messages_handler(model_info={"supported_endpoints": ["/v1/chat/completions"]}) + + assert result == "chat-completions-bridge" + assert dispatched == ["chat_completions"] + + +def test_messages_uses_responses_when_supported_endpoints_declare_responses(monkeypatch): + """A deployment that explicitly declares /v1/responses still gets the Responses bridge.""" + dispatched = _translation_target_stubs(monkeypatch) + + result = _call_messages_handler(model_info={"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]}) + + assert result == "responses-bridge" + assert dispatched == ["responses"] + + +@pytest.mark.parametrize( + "model_info", + [None, {}, {"supported_endpoints": None}, {"supported_endpoints": "/v1/chat/completions"}], + ids=["absent", "empty", "null-endpoints", "non-list-endpoints"], +) +def test_messages_uses_responses_when_supported_endpoints_undeclared(monkeypatch, model_info): + """Backward compatibility: without a usable per-deployment declaration, openai + deployments keep going to the Responses API exactly as before.""" + dispatched = _translation_target_stubs(monkeypatch) + + result = _call_messages_handler(model_info=model_info) + + assert result == "responses-bridge" + assert dispatched == ["responses"] + + +def test_global_chat_completions_flag_still_overrides_declared_responses_support(monkeypatch): + """The global opt-out wins even when the deployment declares /v1/responses.""" + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + dispatched = _translation_target_stubs(monkeypatch) + + result = _call_messages_handler(model_info={"supported_endpoints": ["/v1/responses"]}) + + assert result == "chat-completions-bridge" + assert dispatched == ["chat_completions"]