From 04416f76f14452b3a0056a3dc9d150fa650c70df Mon Sep 17 00:00:00 2001 From: Senthurapandi <127392902+senthurapandi2000@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:36:09 -0700 Subject: [PATCH 1/4] fix(proxy): preserve routed model for streamed spend logging --- .../litellm_core_utils/streaming_handler.py | 35 ++++++ .../test_streaming_handler.py | 102 ++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f97a274708f..dbbdcba6a88 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2115,6 +2115,41 @@ class CustomStreamWrapper: except Exception: complete_streaming_response = None + if complete_streaming_response is not None and isinstance(self.model, str) and self.model: + litellm_params = self.logging_obj.model_call_details.get( + "litellm_params", {} + ) + proxy_request = litellm_params.get("proxy_server_request", {}) + request_body = ( + proxy_request.get("body", {}) + if isinstance(proxy_request, dict) + else {} + ) + client_requested_model = ( + request_body.get("model") + if isinstance(request_body, dict) + else None + ) + + assembled_model = complete_streaming_response.model + builder_recovered_routed_model = ( + isinstance(assembled_model, str) + and bool(assembled_model) + and any( + ( + chunk.get("model") + if isinstance(chunk, dict) + else getattr(chunk, "model", None) + ) + == assembled_model + for chunk in self.chunks[1:] + ) + and assembled_model != client_requested_model + ) + + if not builder_recovered_routed_model: + complete_streaming_response.model = self.model + response: Final = self.model_response_creator() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 3af79c709cc..01c133bc0f9 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -4360,6 +4360,108 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin session_id_var.set("") +@pytest.mark.asyncio +async def test_finalize_completed_stream_uses_routed_model_for_logging(logging_obj): + routed_model = "anthropic/claude-haiku-4-5" + client_alias = "myalias/haiku" + + async def _empty_aiter(): + if False: + yield None + + logging_obj.dispatch_success_handlers = AsyncMock() + + wrapper = CustomStreamWrapper( + completion_stream=_empty_aiter(), + model=routed_model, + logging_obj=logging_obj, + custom_llm_provider="anthropic", + ) + + wrapper.sent_last_chunk = True + wrapper.chunks = [MagicMock()] + wrapper.async_success_handler = AsyncMock() + + assembled_response = ModelResponse( + model=client_alias, + choices=[], + usage=Usage( + prompt_tokens=27, + completion_tokens=1, + total_tokens=28, + ), + ) + + with patch( + "litellm.stream_chunk_builder", + return_value=assembled_response, + ): + with pytest.raises(StopAsyncIteration): + await wrapper._finalize_completed_stream(cache_hit=False) + + await asyncio.sleep(0) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + + logged_response = logging_obj.dispatch_success_handlers.await_args.args[0] + + assert logged_response.model == routed_model + + +@pytest.mark.asyncio +async def test_finalize_completed_stream_preserves_model_recovered_from_later_chunk( + logging_obj, +): + wrapper_model = "anthropic/claude-haiku-4-5" + client_alias = "myalias/haiku" + recovered_routed_model = "anthropic/claude-sonnet-4-5" + + async def _empty_aiter(): + if False: + yield None + + logging_obj.dispatch_success_handlers = AsyncMock() + + wrapper = CustomStreamWrapper( + completion_stream=_empty_aiter(), + model=wrapper_model, + logging_obj=logging_obj, + custom_llm_provider="anthropic", + ) + + wrapper.sent_last_chunk = True + wrapper.chunks = [ + {"model": client_alias}, + {"model": recovered_routed_model}, + ] + wrapper.async_success_handler = AsyncMock() + + assembled_response = ModelResponse( + model=recovered_routed_model, + choices=[], + usage=Usage( + prompt_tokens=27, + completion_tokens=1, + total_tokens=28, + ), + ) + + with patch( + "litellm.stream_chunk_builder", + return_value=assembled_response, + ): + with pytest.raises(StopAsyncIteration): + await wrapper._finalize_completed_stream(cache_hit=False) + + await asyncio.sleep(0) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + + logged_response = logging_obj.dispatch_success_handlers.await_args.args[0] + + assert logged_response.model == recovered_routed_model + + @pytest.mark.asyncio async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch): """_check_max_streaming_duration() raises litellm.Timeout when a client keeps From 034f3938f36ab2ffbd9efa302901104dc981396f Mon Sep 17 00:00:00 2001 From: Senthurapandi <127392902+senthurapandi2000@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:12:49 -0700 Subject: [PATCH 2/4] style: format streaming handler --- .../litellm_core_utils/streaming_handler.py | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index dbbdcba6a88..660e85f676b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2116,31 +2116,17 @@ class CustomStreamWrapper: complete_streaming_response = None if complete_streaming_response is not None and isinstance(self.model, str) and self.model: - litellm_params = self.logging_obj.model_call_details.get( - "litellm_params", {} - ) + litellm_params = self.logging_obj.model_call_details.get("litellm_params", {}) proxy_request = litellm_params.get("proxy_server_request", {}) - request_body = ( - proxy_request.get("body", {}) - if isinstance(proxy_request, dict) - else {} - ) - client_requested_model = ( - request_body.get("model") - if isinstance(request_body, dict) - else None - ) + request_body = proxy_request.get("body", {}) if isinstance(proxy_request, dict) else {} + client_requested_model = request_body.get("model") if isinstance(request_body, dict) else None assembled_model = complete_streaming_response.model builder_recovered_routed_model = ( isinstance(assembled_model, str) and bool(assembled_model) and any( - ( - chunk.get("model") - if isinstance(chunk, dict) - else getattr(chunk, "model", None) - ) + (chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None)) == assembled_model for chunk in self.chunks[1:] ) From 30cba9b5fdfa10689b20c2d4ccb633d891596b8a Mon Sep 17 00:00:00 2001 From: Senthurapandi <127392902+senthurapandi2000@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:37:36 -0700 Subject: [PATCH 3/4] fix(proxy): avoid mutable defaults in stream model handling --- litellm/litellm_core_utils/streaming_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 660e85f676b..c0cd7a536de 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2116,9 +2116,10 @@ class CustomStreamWrapper: complete_streaming_response = None if complete_streaming_response is not None and isinstance(self.model, str) and self.model: - litellm_params = self.logging_obj.model_call_details.get("litellm_params", {}) - proxy_request = litellm_params.get("proxy_server_request", {}) - request_body = proxy_request.get("body", {}) if isinstance(proxy_request, dict) else {} + litellm_params = self.logging_obj.model_call_details.get("litellm_params") + proxy_request = litellm_params.get("proxy_server_request") if isinstance(litellm_params, dict) else None + request_body = proxy_request.get("body") if isinstance(proxy_request, dict) else None + client_requested_model = request_body.get("model") if isinstance(request_body, dict) else None assembled_model = complete_streaming_response.model From 261ad0074bfa079a3a47864f97f3c8c9b0030016 Mon Sep 17 00:00:00 2001 From: Senthurapandi <127392902+senthurapandi2000@users.noreply.github.com> Date: Mon, 21 Sep 2026 03:02:35 -0700 Subject: [PATCH 4/4] fix(proxy): preserve routed model for streaming alias spend --- .../streaming_chunk_builder_utils.py | 43 +++++++++++ .../litellm_core_utils/streaming_handler.py | 38 ++++++---- litellm/proxy/common_request_processing.py | 57 ++------------ litellm/utils.py | 1 + .../test_streaming_handler.py | 74 ++++++++++++++++++- 5 files changed, 144 insertions(+), 69 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index fcd55c844c6..72654b247ca 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -49,6 +49,49 @@ if TYPE_CHECKING: ) +def _assembled_model_came_from_a_later_chunk( + chunks: Sequence[object], + assembled_model: object, +) -> bool: + """Report whether stream_chunk_builder picked a model the first chunk did not carry. + + Azure Model Router puts the routed model on the chunks after the first one, and the + proxy deliberately leaves those chunks unrestamped so the builder can recover it. + + A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so + an alias-restamped stream reaches the builder with the same shape: a first chunk that + disagrees with the rest. Those two are only told apart by what the client asked for. + """ + first_chunk: Final = chunks[0] + first_chunk_model: Final = ( + first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None) + ) + return ( + isinstance(first_chunk_model, str) + and isinstance(assembled_model, str) + and bool(assembled_model) + and assembled_model != first_chunk_model + ) + + +def _assembled_model_is_the_name_the_client_asked_for( + request_data: Mapping[str, object], + assembled_model: object, +) -> bool: + """Report whether the assembled model is the public name the proxy stamps onto chunks. + + That stamp is what leaves an unpriced alias on the partial response, so the deployment's + own model has to go back on before the row is costed. Pre-call processing rewrites + `request_data["model"]` for aliasing and routing, so the client's own name wins when it + is there, in the same order the proxy picks the name it stamps. + """ + client_requested_model: Final = request_data.get("_litellm_client_requested_model") + stamped_model: Final = ( + client_requested_model if isinstance(client_requested_model, str) else request_data.get("model") + ) + return isinstance(stamped_model, str) and assembled_model == stamped_model + + class _ThinkingBlockFragment(TypedDict, total=False): type: str | None data: str | None diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index c0cd7a536de..76c7f339748 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -24,6 +24,10 @@ from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, ) from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject +from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( + _assembled_model_came_from_a_later_chunk, + _assembled_model_is_the_name_the_client_asked_for, +) from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.types.llms.openai import OpenAIChatCompletionChunk from litellm.types.router import GenericLiteLLMParams @@ -1864,6 +1868,18 @@ class CustomStreamWrapper: except Exception: complete_streaming_response = None + if complete_streaming_response is not None and isinstance(self.model, str) and self.model: + builder_recovered_routed_model = _assembled_model_came_from_a_later_chunk( + self.chunks, + complete_streaming_response.model, + ) and not _assembled_model_is_the_name_the_client_asked_for( + self.logging_obj.model_call_details, + complete_streaming_response.model, + ) + + if not builder_recovered_routed_model: + complete_streaming_response.model = self.model + response = self.model_response_creator() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) @@ -2116,22 +2132,12 @@ class CustomStreamWrapper: complete_streaming_response = None if complete_streaming_response is not None and isinstance(self.model, str) and self.model: - litellm_params = self.logging_obj.model_call_details.get("litellm_params") - proxy_request = litellm_params.get("proxy_server_request") if isinstance(litellm_params, dict) else None - request_body = proxy_request.get("body") if isinstance(proxy_request, dict) else None - - client_requested_model = request_body.get("model") if isinstance(request_body, dict) else None - - assembled_model = complete_streaming_response.model - builder_recovered_routed_model = ( - isinstance(assembled_model, str) - and bool(assembled_model) - and any( - (chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None)) - == assembled_model - for chunk in self.chunks[1:] - ) - and assembled_model != client_requested_model + builder_recovered_routed_model = _assembled_model_came_from_a_later_chunk( + self.chunks, + complete_streaming_response.model, + ) and not _assembled_model_is_the_name_the_client_asked_for( + self.logging_obj.model_call_details, + complete_streaming_response.model, ) if not builder_recovered_routed_model: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9484fd7c723..538de4d93b2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3,7 +3,7 @@ import contextlib import json import logging import math -from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence +from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping from datetime import datetime from functools import lru_cache from types import MappingProxyType @@ -61,6 +61,10 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( + _assembled_model_came_from_a_later_chunk, + _assembled_model_is_the_name_the_client_asked_for, +) from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) @@ -326,46 +330,6 @@ def _deferred_stream_logging_is_armed(request_data: dict) -> bool: ) -def _assembled_model_came_from_a_later_chunk(chunks: Sequence[object], assembled_model: object) -> bool: - """Report whether stream_chunk_builder picked a model the first chunk did not carry. - - Azure Model Router puts the routed model on the chunks after the first one, and the - proxy deliberately leaves those chunks unrestamped so the builder can recover it. - - A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so - an alias-restamped stream reaches the builder with the same shape: a first chunk that - disagrees with the rest. Those two are only told apart by what the client asked for. - """ - first_chunk: Final = chunks[0] - first_chunk_model: Final = ( - first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None) - ) - return ( - isinstance(first_chunk_model, str) - and isinstance(assembled_model, str) - and bool(assembled_model) - and assembled_model != first_chunk_model - ) - - -def _assembled_model_is_the_name_the_client_asked_for( - request_data: Mapping[str, object], - assembled_model: object, -) -> bool: - """Report whether the assembled model is the public name the proxy stamps onto chunks. - - That stamp is what leaves an unpriced alias on the partial response, so the deployment's - own model has to go back on before the row is costed. Pre-call processing rewrites - `request_data["model"]` for aliasing and routing, so the client's own name wins when it - is there, in the same order the proxy picks the name it stamps. - """ - client_requested_model: Final = request_data.get("_litellm_client_requested_model") - stamped_model: Final = ( - client_requested_model if isinstance(client_requested_model, str) else request_data.get("model") - ) - return isinstance(stamped_model, str) and assembled_model == stamped_model - - async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, response: object) -> bool: """ A client disconnect throws GeneratorExit/CancelledError into the streaming @@ -2460,6 +2424,8 @@ class ProxyBaseLLMRequestProcessing: """ client_model: Final = get_client_requested_model(request) or self.data.get("model") requested_model_from_client: Final[str | None] = client_model if isinstance(client_model, str) else None + if requested_model_from_client: + self.data["_litellm_client_requested_model"] = requested_model_from_client self._debug_log_request_payload() if skip_pre_call_logic: @@ -2605,13 +2571,6 @@ class ProxyBaseLLMRequestProcessing: callback_headers=stream_callback_headers or MappingProxyType({}), ) - # Preserve the original client-requested model (pre-alias mapping) for downstream - # streaming generators. Pre-call processing can rewrite `self.data["model"]` for - # aliasing/routing, but the OpenAI-compatible response `model` field should reflect - # what the client sent. - if requested_model_from_client: - self.data["_litellm_client_requested_model"] = requested_model_from_client - if _post_call_guardrails_active: self._arm_deferred_stream_dispatch( response=response, @@ -3501,7 +3460,7 @@ class ProxyBaseLLMRequestProcessing: getattr(cb, "guardrail_name", type(cb).__name__), e, ) - if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "model_call_details"): + if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "`model_call_details`"): captured_logging_obj.model_call_details.setdefault("metadata", {})["guardrail_blocked"] = True except Exception as e: verbose_proxy_logger.exception( diff --git a/litellm/utils.py b/litellm/utils.py index 252bc691301..9c60058f834 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1216,6 +1216,7 @@ def function_setup( optional_params={}, litellm_params=litellm_params, stream_options=kwargs.get("stream_options", None), + _litellm_client_requested_model=kwargs.get("_litellm_client_requested_model"), ) return logging_obj, kwargs except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 01c133bc0f9..462030abab6 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -4361,15 +4361,20 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin @pytest.mark.asyncio -async def test_finalize_completed_stream_uses_routed_model_for_logging(logging_obj): - routed_model = "anthropic/claude-haiku-4-5" - client_alias = "myalias/haiku" +async def test_finalize_completed_stream_restores_routed_model_for_key_alias( + logging_obj, +): + routed_model = "anthropic/claude-opus-4-20250514" + client_alias = "claude" + alias_target = "some-deployment" async def _empty_aiter(): if False: yield None logging_obj.dispatch_success_handlers = AsyncMock() + logging_obj.model_call_details["_litellm_client_requested_model"] = client_alias + logging_obj.model_call_details["model"] = alias_target wrapper = CustomStreamWrapper( completion_stream=_empty_aiter(), @@ -4379,7 +4384,12 @@ async def test_finalize_completed_stream_uses_routed_model_for_logging(logging_o ) wrapper.sent_last_chunk = True - wrapper.chunks = [MagicMock()] + wrapper.chunks = [ + {"model": routed_model}, + {"model": client_alias}, + {"model": client_alias}, + {"model": routed_model}, + ] wrapper.async_success_handler = AsyncMock() assembled_response = ModelResponse( @@ -4461,6 +4471,62 @@ async def test_finalize_completed_stream_preserves_model_recovered_from_later_ch assert logged_response.model == recovered_routed_model +def test_sync_completed_stream_restores_routed_model_for_key_alias(logging_obj): + routed_model = "anthropic/claude-opus-4-20250514" + client_alias = "claude" + alias_target = "some-deployment" + + logging_obj.success_handler = MagicMock() + logging_obj.model_call_details["_litellm_client_requested_model"] = client_alias + logging_obj.model_call_details["model"] = alias_target + + wrapper = CustomStreamWrapper( + completion_stream=iter(()), + model=routed_model, + logging_obj=logging_obj, + custom_llm_provider="anthropic", + ) + + wrapper.sent_last_chunk = True + wrapper.send_stream_usage = False + wrapper.chunks = [ + {"model": routed_model}, + {"model": client_alias}, + {"model": client_alias}, + {"model": routed_model}, + ] + + assembled_response = ModelResponse( + model=client_alias, + choices=[], + usage=Usage( + prompt_tokens=27, + completion_tokens=1, + total_tokens=28, + ), + ) + + with ( + patch( + "litellm.stream_chunk_builder", + return_value=assembled_response, + ), + patch.object(wrapper, "cache_streaming_response") as cache_mock, + patch( + "litellm.litellm_core_utils.streaming_handler.executor.submit" + ) as submit_mock, + ): + with pytest.raises(StopIteration): + next(wrapper) + + submit_mock.assert_called_once() + + logged_response = submit_mock.call_args.args[1] + assert logged_response.model == routed_model + + cache_mock.assert_called_once() + cached_response = cache_mock.call_args.kwargs["processed_chunk"] + assert cached_response.model == routed_model @pytest.mark.asyncio async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch):