From 2213a141218fd4e08ba476852d2dfbe5c189c13b Mon Sep 17 00:00:00 2001 From: Devon Krisman Date: Sat, 29 Aug 2026 19:54:56 -0400 Subject: [PATCH] fix(anthropic adapter): report client-requested model in streaming message_start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-streaming /v1/messages responses already report the model the client requested: the proxy restamps the response object after the call (_override_openai_response_model). Streaming responses bypass that restamp entirely — the adapter serializes message_start to SSE bytes before it runs, and AnthropicStreamWrapper is constructed with the post-routing deployment model. For aliased/wildcard model_list deployments this makes streaming and non-streaming responses disagree, and leaks internal deployment identifiers (e.g. hosted_vllm served-model names) to Anthropic-protocol clients that read message_start.model. Fix: prefer the client-requested name that the router already stamps into litellm_metadata["model_group"] when constructing the streaming translation, in both the async and sync handlers. SDK callers that bypass the proxy carry no such metadata and keep the existing behavior. --- .../anthropic/messages/adapters/handler.py | 35 +++- .../test_handler_stream_model_mirror.py | 149 ++++++++++++++++++ 2 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 tests/unit/llms/anthropic/messages/adapters/test_handler_stream_model_mirror.py diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 71ccfbaf915..099a646e3b6 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -141,6 +141,29 @@ def request_safeguards_evaluator( ) +def _client_facing_model( + model: str, + proxy_litellm_metadata: "Mapping[str, object] | None", +) -> str: + """Return the model name to expose in translated Anthropic responses. + + When the call arrives through the router/proxy, ``model`` is the + post-routing deployment identifier (e.g. ``hosted_vllm/my-internal-name``) + and the name the client actually requested travels in + ``litellm_metadata["model_group"]`` (stamped by the router). Prefer it so + the streaming ``message_start`` event reports the same model the + non-streaming path returns after the proxy's response-model restamp + (``_override_openai_response_model`` — which cannot reach the streaming + path because SSE bytes are serialized before it runs). SDK callers that + bypass the proxy carry no such metadata and keep the current behavior. + """ + if isinstance(proxy_litellm_metadata, Mapping): + model_group: Final = proxy_litellm_metadata.get("model_group") + if isinstance(model_group, str) and model_group: + return model_group + return model + + async def prepare_context_managed_request( *, model: str, @@ -802,7 +825,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( completion_response, - model=local_model_name(model, kwargs.get("custom_llm_provider")), + model=_client_facing_model( + local_model_name(model, kwargs.get("custom_llm_provider")), + proxy_litellm_metadata, + ), tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=True, @@ -898,10 +924,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: # ``context_management`` spec and no client-sent ``compaction`` block, # and bridging through a worker-thread event loop just to discover # there is no work is pure overhead. + proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) if context_management is None and not _messages_have_compaction_block(messages): polyfill_result: PolyfillResult | None = None else: - proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) polyfill_result = run_async_function( prepare_context_managed_request, @@ -949,7 +975,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( completion_response, - model=local_model_name(model, kwargs.get("custom_llm_provider")), + model=_client_facing_model( + local_model_name(model, kwargs.get("custom_llm_provider")), + proxy_litellm_metadata, + ), tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=False, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_stream_model_mirror.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_stream_model_mirror.py new file mode 100644 index 00000000000..a355b147edb --- /dev/null +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_stream_model_mirror.py @@ -0,0 +1,149 @@ +""" +Regression tests: streaming ``/v1/messages`` must report the client-requested +model, not the internal post-routing deployment identifier. + +Background — what was broken: +* Non-streaming ``/v1/messages`` responses already report the model name the + client asked for: the proxy restamps the response object after the call + (``_override_openai_response_model`` in + ``litellm/proxy/common_request_processing.py``). +* Streaming responses bypass that restamp entirely: the adapter serializes the + ``message_start`` event to SSE bytes before the proxy-level restamp runs (a + bytes async-generator has no ``model`` attribute to restamp), and + ``AnthropicStreamWrapper`` is constructed with the POST-routing model the + handler received from the router (e.g. ``hosted_vllm/``). +* Net effect: for aliased/wildcard ``model_list`` deployments, streaming and + non-streaming responses to the same endpoint disagree, and the streaming + path leaks internal deployment identifiers to Anthropic-protocol clients + (agent harnesses read ``message_start.model``). + +The fix threads the client-requested name — which the router already stamps +into ``litellm_metadata["model_group"]`` — into the streaming translation via +``_client_facing_model``. SDK callers that bypass the proxy carry no such +metadata and keep the existing behavior. +""" + +import json +from unittest.mock import patch + +import pytest + +from litellm.llms.anthropic.messages.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + _client_facing_model, +) + +MESSAGES = [{"role": "user", "content": "hello"}] + + +# --------------------------------------------------------------------------- +# Helper-level coverage +# --------------------------------------------------------------------------- + + +def test_client_facing_model_prefers_model_group(): + assert ( + _client_facing_model( + "hosted_vllm/internal-served-name", + {"model_group": "claude-opus-5"}, + ) + == "claude-opus-5" + ) + + +def test_client_facing_model_without_metadata_keeps_model(): + assert _client_facing_model("hosted_vllm/internal-served-name", None) == ( + "hosted_vllm/internal-served-name" + ) + + +@pytest.mark.parametrize("bad_group", [None, "", 42, {"nested": "dict"}]) +def test_client_facing_model_ignores_unusable_model_group(bad_group): + assert ( + _client_facing_model("deployment-model", {"model_group": bad_group}) + == "deployment-model" + ) + + +# --------------------------------------------------------------------------- +# End-to-end through the async handler: message_start carries the alias +# --------------------------------------------------------------------------- + + +class _EmptyAsyncStream: + """Minimal async completion stream: ends immediately. + + ``AnthropicStreamWrapper`` emits ``message_start`` before consuming the + underlying stream, which is all these tests need. + """ + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + +async def _first_message_start(transformed_stream) -> dict: + async for raw in transformed_stream: + text = raw.decode("utf-8") if isinstance(raw, bytes) else str(raw) + for line in text.splitlines(): + line = line.strip() + if line.startswith("data: "): + event = json.loads(line[len("data: ") :]) + if event.get("type") == "message_start": + return event + raise AssertionError("no message_start event emitted") + + +@pytest.mark.asyncio +async def test_streaming_message_start_mirrors_requested_alias(): + """Proxy-routed call: litellm_metadata.model_group wins over the + post-routing deployment model.""" + with patch("litellm.acompletion", return_value=_EmptyAsyncStream()): # test-quality-ok: the handler calls litellm.acompletion directly, no injection seam + transformed_stream = await ( + LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=16, + messages=MESSAGES, + model="hosted_vllm/internal-served-name", + stream=True, + litellm_metadata={"model_group": "claude-opus-5"}, + ) + ) + event = await _first_message_start(transformed_stream) + assert event["message"]["model"] == "claude-opus-5" + + +@pytest.mark.asyncio +async def test_streaming_message_start_without_proxy_metadata_unchanged(): + """SDK-style call (no proxy metadata, no provider): the model passed to + the handler is reported as-is.""" + with patch("litellm.acompletion", return_value=_EmptyAsyncStream()): # test-quality-ok: the handler calls litellm.acompletion directly, no injection seam + transformed_stream = await ( + LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=16, + messages=MESSAGES, + model="hosted_vllm/internal-served-name", + stream=True, + ) + ) + event = await _first_message_start(transformed_stream) + assert event["message"]["model"] == "hosted_vllm/internal-served-name" + + +@pytest.mark.asyncio +async def test_streaming_message_start_without_proxy_metadata_strips_provider_prefix(): + """SDK-style call with a resolved provider: falls back to the + provider-local name (``local_model_name``), not the alias path.""" + with patch("litellm.acompletion", return_value=_EmptyAsyncStream()): # test-quality-ok: the handler calls litellm.acompletion directly, no injection seam + transformed_stream = await ( + LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=16, + messages=MESSAGES, + model="hosted_vllm/internal-served-name", + stream=True, + custom_llm_provider="hosted_vllm", + ) + ) + event = await _first_message_start(transformed_stream) + assert event["message"]["model"] == "internal-served-name"