From 847172d311f625eba25c97b7c70cf317a735175c Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 17:31:04 +0000 Subject: [PATCH 1/3] fix(anthropic): carry the served model from message_start onto stream chunks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/chat/handler.py | 4 + .../chat/test_anthropic_chat_handler.py | 78 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 4dd0deeb62b..e3912138d8c 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -632,6 +632,7 @@ class ModelResponseIterator: self.tool_name_reverse_map: dict[str, str] = tool_name_reverse_map or {} # Generate response ID once per stream to match OpenAI-compatible behavior self.response_id = _generate_id() + self.served_model: str | None = None # Track if we're currently streaming a response_format tool self.is_response_format_tool: bool = False @@ -1067,6 +1068,8 @@ class ModelResponseIterator: } """ message_start_block: Final = MessageStartBlock(**chunk) + start_message: Final = message_start_block["message"] + self.served_model = start_message["model"] if "model" in start_message else None if "usage" in message_start_block["message"]: usage = self._handle_usage(anthropic_usage_chunk=message_start_block["message"]["usage"]) elif type_chunk == "error": @@ -1098,6 +1101,7 @@ class ModelResponseIterator: ], usage=usage, id=self.response_id, + model=self.served_model, ) return returned_chunk diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index c3400dc40c3..1e6639ef537 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2719,3 +2719,81 @@ class TestRustChatCompletionsHook: "model": "m", "messages": [], } + + +def _served_model_stream_chunks(model: str | None) -> list[dict]: + message: Final = { + "id": "msg_served", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + if model is not None: + message["model"] = model + return [ + {"type": "message_start", "message": message}, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Hello"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 2}, + }, + {"type": "message_stop"}, + ] + + +def test_message_start_model_is_carried_on_stream_chunks(): + iterator: Final = ModelResponseIterator(None, sync_stream=True) + + parsed: Final = [ + iterator.chunk_parser(chunk) for chunk in _served_model_stream_chunks("claude-served-1") + ] + + assert all(chunk.model == "claude-served-1" for chunk in parsed) + + +def test_message_start_without_model_leaves_chunk_model_unset(): + iterator: Final = ModelResponseIterator(None, sync_stream=True) + + parsed: Final = [ + iterator.chunk_parser(chunk) for chunk in _served_model_stream_chunks(None) + ] + + assert all(chunk.model is None for chunk in parsed) + + +def test_served_model_reaches_assembled_stream_through_custom_stream_wrapper(): + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + served_model: Final = "claude-served-1" + sse_lines: Final = [ + f"data: {json.dumps(chunk)}\n".encode() for chunk in _served_model_stream_chunks(served_model) + ] + iterator: Final = ModelResponseIterator(iter(sse_lines), sync_stream=True) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(iterator), + model="anthropic/claude-requested", + custom_llm_provider="anthropic", + logging_obj=MagicMock(), + ) + + chunks: Final = list(wrapper) + + assert len(chunks) > 1 + for chunk in chunks[1:]: + assert chunk._hidden_params["provider_response_model"] == served_model + assembled: Final = litellm.stream_chunk_builder( + chunks=list(chunks), messages=[{"role": "user", "content": "hi"}] + ) + assert assembled._hidden_params["provider_response_model"] == served_model From 07d49364285b083190393893600101e186158a59 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 17:44:59 +0000 Subject: [PATCH 2/3] fix(anthropic): satisfy strict lint and update tests pinned to the dropped served model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/chat/handler.py | 3 ++- .../chat/test_anthropic_chat_handler.py | 16 ++++------------ .../guardrail_hooks/test_tool_permission.py | 7 ++++--- ...test_anthropic_passthrough_logging_handler.py | 6 +++--- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index e3912138d8c..58ed9b38ccb 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -1069,7 +1069,8 @@ class ModelResponseIterator: """ message_start_block: Final = MessageStartBlock(**chunk) start_message: Final = message_start_block["message"] - self.served_model = start_message["model"] if "model" in start_message else None + if "model" in start_message: + self.served_model = start_message["model"] if "usage" in message_start_block["message"]: usage = self._handle_usage(anthropic_usage_chunk=message_start_block["message"]["usage"]) elif type_chunk == "error": diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 1e6639ef537..7db44b4709e 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2756,9 +2756,7 @@ def _served_model_stream_chunks(model: str | None) -> list[dict]: def test_message_start_model_is_carried_on_stream_chunks(): iterator: Final = ModelResponseIterator(None, sync_stream=True) - parsed: Final = [ - iterator.chunk_parser(chunk) for chunk in _served_model_stream_chunks("claude-served-1") - ] + parsed: Final = [iterator.chunk_parser(chunk) for chunk in _served_model_stream_chunks("claude-served-1")] assert all(chunk.model == "claude-served-1" for chunk in parsed) @@ -2766,9 +2764,7 @@ def test_message_start_model_is_carried_on_stream_chunks(): def test_message_start_without_model_leaves_chunk_model_unset(): iterator: Final = ModelResponseIterator(None, sync_stream=True) - parsed: Final = [ - iterator.chunk_parser(chunk) for chunk in _served_model_stream_chunks(None) - ] + parsed: Final = [iterator.chunk_parser(chunk) for chunk in _served_model_stream_chunks(None)] assert all(chunk.model is None for chunk in parsed) @@ -2777,9 +2773,7 @@ def test_served_model_reaches_assembled_stream_through_custom_stream_wrapper(): from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper served_model: Final = "claude-served-1" - sse_lines: Final = [ - f"data: {json.dumps(chunk)}\n".encode() for chunk in _served_model_stream_chunks(served_model) - ] + sse_lines: Final = [f"data: {json.dumps(chunk)}\n".encode() for chunk in _served_model_stream_chunks(served_model)] iterator: Final = ModelResponseIterator(iter(sse_lines), sync_stream=True) wrapper: Final = CustomStreamWrapper( completion_stream=iter(iterator), @@ -2793,7 +2787,5 @@ def test_served_model_reaches_assembled_stream_through_custom_stream_wrapper(): assert len(chunks) > 1 for chunk in chunks[1:]: assert chunk._hidden_params["provider_response_model"] == served_model - assembled: Final = litellm.stream_chunk_builder( - chunks=list(chunks), messages=[{"role": "user", "content": "hi"}] - ) + assembled: Final = litellm.stream_chunk_builder(chunks=list(chunks), messages=[{"role": "user", "content": "hi"}]) assert assembled._hidden_params["provider_response_model"] == served_model diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 87a1b84acc5..427fa43ffd5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -1291,8 +1291,9 @@ class TestToolPermissionGuardrailAnthropicMessages: async def test_rewrite_mode_keeps_the_stream_identity_it_had_before_the_shared_helper(self): """Well-formed SSE must round-trip exactly as it did before the helpers were shared. - The shared module can stamp the upstream message id and model onto the assembled response - for callers that ask for it; this path never did, and a client reads those bytes. + The shared module can stamp the upstream message id onto the assembled response for + callers that ask for it; this path never did, and a client reads those bytes. The model, + though, is now the upstream's, matching what the untouched passthrough shows clients. """ with patch.object(self.rewriting, "should_run_guardrail", return_value=True): out = await self._drain(self.rewriting, self._sse_chunks("Read")) @@ -1304,7 +1305,7 @@ class TestToolPermissionGuardrailAnthropicMessages: if line.startswith("data: ") and json.loads(line[6:]).get("type") == "message_start" )["message"] assert message_start["id"].startswith("chatcmpl-"), "the rewritten stream must not adopt the upstream message id" - assert message_start["model"] == "unknown-model", "the rewritten stream must not adopt the upstream model" + assert message_start["model"] == "claude-sonnet-4-5", "the rewritten stream reports the model the upstream served" @pytest.mark.asyncio async def test_message_start_without_a_dict_message_fails_closed(self): diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 2d7397594aa..ba8b5fa3ac4 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2541,7 +2541,7 @@ class TestRecordPartialUsageForFailure: function_id="test-partial-usage-failure", ) - def _interrupted_chunks(self): + def _interrupted_chunks(self, *, model: str = "claude-sonnet-5"): return [ self._sse( "message_start", @@ -2551,7 +2551,7 @@ class TestRecordPartialUsageForFailure: "id": "msg_abc", "type": "message", "role": "assistant", - "model": "claude-sonnet-5", + "model": model, "content": [], "stop_reason": None, "stop_sequence": None, @@ -2588,7 +2588,7 @@ class TestRecordPartialUsageForFailure: AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure( litellm_logging_obj=logging_obj, request_body={"model": "claude-unpriced-test-model", "stream": True}, - all_chunks=self._interrupted_chunks(), + all_chunks=self._interrupted_chunks(model="claude-unpriced-test-model"), ) usage = logging_obj.model_call_details["combined_usage_object"] From 686556fffc475a0ae083286440b50bba771da26c Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 17:49:15 +0000 Subject: [PATCH 3/3] test(anthropic): build served-model stream chunks immutably Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/test_anthropic_chat_handler.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 7db44b4709e..a241fc03d57 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2721,18 +2721,19 @@ class TestRustChatCompletionsHook: } -def _served_model_stream_chunks(model: str | None) -> list[dict]: - message: Final = { - "id": "msg_served", - "type": "message", - "role": "assistant", - "content": [], - "usage": {"input_tokens": 10, "output_tokens": 1}, - } - if model is not None: - message["model"] = model +def _served_model_stream_chunks(model: str | None) -> list[dict[str, object]]: return [ - {"type": "message_start", "message": message}, + { + "type": "message_start", + "message": { + "id": "msg_served", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + **({"model": model} if model is not None else {}), + }, + }, { "type": "content_block_start", "index": 0,