scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-09 08:18:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 67

fix(anthropic adapter): report client-requested model in streaming message_start

Browse source 
Non-streaming /v1/messages responses already report the model the client
requested: the proxy restamps the response object after the call
(_override_openai_response_model). Streaming responses bypass that restamp
entirely — the adapter serializes message_start to SSE bytes before it runs,
and AnthropicStreamWrapper is constructed with the post-routing deployment
model. For aliased/wildcard model_list deployments this makes streaming and
non-streaming responses disagree, and leaks internal deployment identifiers
(e.g. hosted_vllm served-model names) to Anthropic-protocol clients that
read message_start.model.

Fix: prefer the client-requested name that the router already stamps into
litellm_metadata["model_group"] when constructing the streaming
translation, in both the async and sync handlers. SDK callers that bypass
the proxy carry no such metadata and keep the existing behavior.
This commit is contained in:
Devon Krisman 2026-08-29 19:54:56 -04:00 • committed by dev
parent cbf4e40d38
commit 2213a14121
2 changed files with 181 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -141,6 +141,29 @@ def request_safeguards_evaluator(
)
def _client_facing_model(
model: str,
proxy_litellm_metadata: "Mapping[str, object] | None",
) -> str:
"""Return the model name to expose in translated Anthropic responses.
When the call arrives through the router/proxy, ``model`` is the
post-routing deployment identifier (e.g. ``hosted_vllm/my-internal-name``)
and the name the client actually requested travels in
``litellm_metadata["model_group"]`` (stamped by the router). Prefer it so
the streaming ``message_start`` event reports the same model the
non-streaming path returns after the proxy's response-model restamp
(``_override_openai_response_model`` — which cannot reach the streaming
path because SSE bytes are serialized before it runs). SDK callers that
bypass the proxy carry no such metadata and keep the current behavior.
"""
if isinstance(proxy_litellm_metadata, Mapping):
model_group: Final = proxy_litellm_metadata.get("model_group")
if isinstance(model_group, str) and model_group:
return model_group
return model
async def prepare_context_managed_request(
*,
model: str,
@ -802,7 +825,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
model=_client_facing_model(
local_model_name(model, kwargs.get("custom_llm_provider")),
proxy_litellm_metadata,
),
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=True,
@ -898,10 +924,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# ``context_management`` spec and no client-sent ``compaction`` block,
# and bridging through a worker-thread event loop just to discover
# there is no work is pure overhead.
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
if context_management is None and not _messages_have_compaction_block(messages):
polyfill_result: PolyfillResult | None = None
else:
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
polyfill_result = run_async_function(
prepare_context_managed_request,
@ -949,7 +975,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
model=_client_facing_model(
local_model_name(model, kwargs.get("custom_llm_provider")),
proxy_litellm_metadata,
),
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=False,

149
tests/unit/llms/anthropic/messages/adapters/test_handler_stream_model_mirror.py Normal file
View file

@ -0,0 +1,149 @@
"""
Regression tests: streaming ``/v1/messages`` must report the client-requested
model, not the internal post-routing deployment identifier.
Background — what was broken:
* Non-streaming ``/v1/messages`` responses already report the model name the
client asked for: the proxy restamps the response object after the call
(``_override_openai_response_model`` in
``litellm/proxy/common_request_processing.py``).
* Streaming responses bypass that restamp entirely: the adapter serializes the
``message_start`` event to SSE bytes before the proxy-level restamp runs (a
bytes async-generator has no ``model`` attribute to restamp), and
``AnthropicStreamWrapper`` is constructed with the POST-routing model the
handler received from the router (e.g. ``hosted_vllm/<internal-name>``).
* Net effect: for aliased/wildcard ``model_list`` deployments, streaming and
non-streaming responses to the same endpoint disagree, and the streaming
path leaks internal deployment identifiers to Anthropic-protocol clients
(agent harnesses read ``message_start.model``).
The fix threads the client-requested name — which the router already stamps
into ``litellm_metadata["model_group"]`` — into the streaming translation via
``_client_facing_model``. SDK callers that bypass the proxy carry no such
metadata and keep the existing behavior.
"""
import json
from unittest.mock import patch
import pytest
from litellm.llms.anthropic.messages.adapters.handler import (
LiteLLMMessagesToCompletionTransformationHandler,
_client_facing_model,
)
MESSAGES = [{"role": "user", "content": "hello"}]
# ---------------------------------------------------------------------------
# Helper-level coverage
# ---------------------------------------------------------------------------
def test_client_facing_model_prefers_model_group():
assert (
_client_facing_model(
"hosted_vllm/internal-served-name",
{"model_group": "claude-opus-5"},
)
== "claude-opus-5"
)
def test_client_facing_model_without_metadata_keeps_model():
assert _client_facing_model("hosted_vllm/internal-served-name", None) == (
"hosted_vllm/internal-served-name"
)
@pytest.mark.parametrize("bad_group", [None, "", 42, {"nested": "dict"}])
def test_client_facing_model_ignores_unusable_model_group(bad_group):
assert (
_client_facing_model("deployment-model", {"model_group": bad_group})
== "deployment-model"
)
# ---------------------------------------------------------------------------
# End-to-end through the async handler: message_start carries the alias
# ---------------------------------------------------------------------------
class _EmptyAsyncStream:
"""Minimal async completion stream: ends immediately.
``AnthropicStreamWrapper`` emits ``message_start`` before consuming the
underlying stream, which is all these tests need.
"""
def __aiter__(self):
return self
async def __anext__(self):
raise StopAsyncIteration
async def _first_message_start(transformed_stream) -> dict:
async for raw in transformed_stream:
text = raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)
for line in text.splitlines():
line = line.strip()
if line.startswith("data: "):
event = json.loads(line[len("data: ") :])
if event.get("type") == "message_start":
return event
raise AssertionError("no message_start event emitted")
@pytest.mark.asyncio
async def test_streaming_message_start_mirrors_requested_alias():
"""Proxy-routed call: litellm_metadata.model_group wins over the
post-routing deployment model."""
with patch("litellm.acompletion", return_value=_EmptyAsyncStream()): # test-quality-ok: the handler calls litellm.acompletion directly, no injection seam
transformed_stream = await (
LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
max_tokens=16,
messages=MESSAGES,
model="hosted_vllm/internal-served-name",
stream=True,
litellm_metadata={"model_group": "claude-opus-5"},
)
)
event = await _first_message_start(transformed_stream)
assert event["message"]["model"] == "claude-opus-5"
@pytest.mark.asyncio
async def test_streaming_message_start_without_proxy_metadata_unchanged():
"""SDK-style call (no proxy metadata, no provider): the model passed to
the handler is reported as-is."""
with patch("litellm.acompletion", return_value=_EmptyAsyncStream()): # test-quality-ok: the handler calls litellm.acompletion directly, no injection seam
transformed_stream = await (
LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
max_tokens=16,
messages=MESSAGES,
model="hosted_vllm/internal-served-name",
stream=True,
)
)
event = await _first_message_start(transformed_stream)
assert event["message"]["model"] == "hosted_vllm/internal-served-name"
@pytest.mark.asyncio
async def test_streaming_message_start_without_proxy_metadata_strips_provider_prefix():
"""SDK-style call with a resolved provider: falls back to the
provider-local name (``local_model_name``), not the alias path."""
with patch("litellm.acompletion", return_value=_EmptyAsyncStream()): # test-quality-ok: the handler calls litellm.acompletion directly, no injection seam
transformed_stream = await (
LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
max_tokens=16,
messages=MESSAGES,
model="hosted_vllm/internal-served-name",
stream=True,
custom_llm_provider="hosted_vllm",
)
)
event = await _first_message_start(transformed_stream)
assert event["message"]["model"] == "internal-served-name"