From 060e40021def9354de8732bdfad0ffb578c634c4 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 21 Aug 2026 19:47:47 -0700 Subject: [PATCH] fix(anthropic): resolve the provider exactly once on /v1/messages (#37757) get_llm_provider ran in the messages handler and again inside completion, so a provider/vendor/model id lost its vendor segment and reached upstream bare. Pass the caller's unresolved model down the bridge instead, and move the responses marker into the canonical provider/responses/model slot. Reporting stays provider-local on both bridges: message_start names the id the provider itself knows, through a shared local_model_name helper. Fixes #37716 --- .../adapters/handler.py | 11 +- .../messages/handler.py | 41 ++++++- .../responses_adapters/handler.py | 9 +- .../experimental_pass_through/utils.py | 5 + .../adapters/test_handler_prompt_cache_key.py | 2 +- ...erimental_pass_through_messages_handler.py | 112 +++++++++++++++++- .../test_responses_adapters_handler.py | 39 ++++++ 7 files changed, 204 insertions(+), 15 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 89066e33cbc..9d61701d26d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import ) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, + local_model_name, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -358,9 +359,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: except Exception: pass - if isinstance(model, str) and model and not model.startswith("responses/"): - # Prefix model with "responses/" to route to OpenAI Responses API - completion_kwargs["model"] = f"responses/{model}" + if isinstance(model, str) and model and "responses/" not in model: + local_model: Final = model.removeprefix(f"{custom_llm_provider}/") + completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}" auto_summary: Final = is_reasoning_auto_summary_enabled() @@ -616,7 +617,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( completion_response, - model=model, + model=local_model_name(model, kwargs.get("custom_llm_provider")), tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=True, @@ -750,7 +751,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( completion_response, - model=model, + model=local_model_name(model, kwargs.get("custom_llm_provider")), tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=False, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 26aef666172..f4d24bb933c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -42,15 +42,46 @@ from .utils import AnthropicMessagesRequestUtils, mock_response _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) -def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool: - """Return True when the provider should use the Responses API path. +def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool: + from litellm.main import responses_api_bridge_check + + model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider) + return model_info.get("mode") == "responses" + + +def _responses_mode_is_lost_by_prefix_strip( + requested_model: str, resolved_model: str, custom_llm_provider: str +) -> bool: + """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped. + + ``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a + deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the + chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions. + """ + if requested_model == resolved_model: + return False + return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api( + resolved_model, custom_llm_provider + ) + + +def _should_route_to_responses_api( + custom_llm_provider: str | None, + requested_model: str | None = None, + resolved_model: str | None = None, +) -> bool: + """Return True when the request should use the Responses API path. Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to opt out and route OpenAI/Azure requests through chat/completions instead. """ if litellm.use_chat_completions_url_for_anthropic_messages: return False - return custom_llm_provider in _RESPONSES_API_PROVIDERS + if custom_llm_provider in _RESPONSES_API_PROVIDERS: + return True + if custom_llm_provider is None or requested_model is None or resolved_model is None: + return False + return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider) def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: @@ -533,7 +564,7 @@ def anthropic_messages_handler( _shared_kwargs: Final = dict( max_tokens=max_tokens, messages=messages, - model=model, + model=original_model, metadata=metadata, stop_sequences=stop_sequences, stream=stream, @@ -551,7 +582,7 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, **kwargs, ) - if _should_route_to_responses_api(custom_llm_provider): + if _should_route_to_responses_api(custom_llm_provider, original_model, model): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs) # The in-gateway context_management polyfill runs inside diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 843cda249c5..c1ea39fd72c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import ResponsesAPIResponse +from ..utils import local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -179,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler: result: Final = await litellm.aresponses(**responses_kwargs) if stream: - wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + wrapper: Final = AnthropicResponsesStreamWrapper( + responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")) + ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): @@ -257,7 +260,9 @@ class LiteLLMMessagesToResponsesAPIHandler: result: Final = litellm.responses(**responses_kwargs) if stream: - wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + wrapper: Final = AnthropicResponsesStreamWrapper( + responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")) + ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index c5abcf8c04c..29661572b73 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -13,6 +13,11 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None: return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None +def local_model_name(model: str, custom_llm_provider: object) -> str: + """The id the provider itself knows, for reporting back to the caller in ``message_start``.""" + return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model + + def is_reasoning_auto_summary_enabled() -> bool: """Check whether the default 'summary: detailed' injection is enabled (opt-in).""" return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py index 5b7f2a60f68..f48d51dbe1e 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py @@ -66,5 +66,5 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero {"custom_llm_provider": "openai"}, thinking={"type": "enabled", "budget_tokens": 1024}, ) - assert completion_kwargs["model"] == "responses/openai/gpt-5.6-luna" + assert completion_kwargs["model"] == "openai/responses/gpt-5.6-luna" assert completion_kwargs["prompt_cache_key"] == "session-abc" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 91f5023496a..15ac73ed352 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -217,7 +217,10 @@ async def _async_return(value): def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): """ - Test that litellm.completion is called when a custom LLM provider is given + Test that litellm.completion is called when a custom LLM provider is given. + + Provider resolution now happens exactly once, inside litellm.completion itself + (BerriAI/litellm#37716), so the handler passes the original unresolved model through. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, @@ -241,7 +244,7 @@ def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provide # Verify that the custom provider was passed through call_kwargs = mock_completion.call_args.kwargs assert call_kwargs["custom_llm_provider"] == "my-custom-llm" - assert call_kwargs["model"] == "my-custom-llm/my-custom-model" + assert call_kwargs["model"] == "my-custom-model" assert call_kwargs["api_key"] == "test-api-key" @@ -997,3 +1000,108 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys and info.get("supports_mid_conversation_system") is not True ] assert missing == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, expected_wire_model, expected_url", + [ + ( + "perplexity/perplexity/kimi-k3", + "perplexity/kimi-k3", + "https://api.perplexity.ai/v1/responses", + ), + ( + "perplexity/perplexity/sonar", + "perplexity/sonar", + "https://api.perplexity.ai/v1/responses", + ), + ("perplexity/sonar", "sonar", "https://api.perplexity.ai/chat/completions"), + ], +) +async def test_messages_strips_provider_prefix_exactly_once( + requested_model, expected_wire_model, expected_url +): + """ + BerriAI/litellm#37716: only the leading provider segment may be stripped on the way upstream. + + A multi-segment id such as perplexity/perplexity/kimi-k3 must reach the provider as + perplexity/kimi-k3, matching what /v1/chat/completions and /v1/responses already send. + + The endpoint is asserted alongside the body because perplexity/perplexity/sonar is a + Responses-only deployment whose bare id perplexity/sonar is an ordinary chat model, so + stripping the prefix must not also move the request onto chat/completions. + + The subject is the outbound request, so the transport is cut at the wire rather than + stubbed with a response body: these ids take different bridges (chat completions + versus the Responses API) and would otherwise need different response shapes. + """ + captured = {} + + async def fake_send(self, request, **kwargs): + captured["body"] = json.loads(request.content) + captured["url"] = str(request.url) + raise httpx.ConnectError("cut at the wire", request=request) + + with ( + patch.object(httpx.AsyncClient, "send", fake_send), + pytest.raises(litellm.exceptions.InternalServerError), + ): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model=requested_model, + api_key="test-api-key", + ) + + assert captured["body"]["model"] == expected_wire_model + assert captured["url"] == expected_url + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, expected_reported_model", + [ + ("perplexity/perplexity/kimi-k3", "perplexity/kimi-k3"), + ("perplexity/sonar", "sonar"), + ], +) +async def test_messages_streaming_reports_provider_local_model(requested_model, expected_reported_model): + """ + BerriAI/litellm#37716: the wire keeps every segment, so ``message_start`` must still + report the id the provider itself knows rather than the caller's prefixed deployment id. + """ + + class _EmptyStream: + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + with patch("litellm.acompletion", new=AsyncMock(return_value=_EmptyStream())): + stream = await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model=requested_model, + api_key="test-api-key", + stream=True, + ) + first_event = await stream.__anext__() + + assert json.loads(first_event.decode().split("data: ", 1)[1])["message"]["model"] == expected_reported_model + + +def test_messages_sync_streaming_reports_provider_local_model(): + """Same guarantee as the async bridge, at the sync call site.""" + with patch("litellm.completion", new=MagicMock(return_value=iter(()))): + stream = litellm.anthropic.messages.create( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="perplexity/perplexity/kimi-k3", + api_key="test-api-key", + stream=True, + ) + first_event = next(iter(stream)) + + assert json.loads(first_event.decode().split("data: ", 1)[1])["message"]["model"] == "perplexity/kimi-k3" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 7ef3077f9d7..589dc64f9b9 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -1,9 +1,15 @@ +import json import os import sys +from unittest.mock import AsyncMock, patch + +import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +import litellm from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import ( + LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, ) @@ -43,3 +49,36 @@ def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): ) assert "user" not in responses_kwargs assert "prompt_cache_key" not in responses_kwargs + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, expected_reported_model", + [ + ("openai/gpt-5.6-luna", "gpt-5.6-luna"), + ("perplexity/perplexity/kimi-k3", "perplexity/kimi-k3"), + ], +) +async def test_streaming_message_start_reports_the_provider_local_model(requested_model, expected_reported_model): + """ + BerriAI/litellm#37716 sends the caller's unresolved id down this bridge so the provider + resolves it once. ``message_start`` is a reporting field rather than a wire value, so it + keeps naming the model as the provider knows it, with only the leading provider segment gone. + """ + + async def empty_stream(): + return + yield + + with patch.object(litellm, "aresponses", AsyncMock(return_value=empty_stream())): + sse = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model=requested_model, + stream=True, + custom_llm_provider=requested_model.split("/")[0], + ) + events = [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in sse] + + message_start = next(e for e in events if e["type"] == "message_start") + assert message_start["message"]["model"] == expected_reported_model