From 04416f76f14452b3a0056a3dc9d150fa650c70df Mon Sep 17 00:00:00 2001 From: Senthurapandi <127392902+senthurapandi2000@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:36:09 -0700 Subject: [PATCH] fix(proxy): preserve routed model for streamed spend logging --- .../litellm_core_utils/streaming_handler.py | 35 ++++++ .../test_streaming_handler.py | 102 ++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f97a274708f..dbbdcba6a88 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2115,6 +2115,41 @@ class CustomStreamWrapper: except Exception: complete_streaming_response = None + if complete_streaming_response is not None and isinstance(self.model, str) and self.model: + litellm_params = self.logging_obj.model_call_details.get( + "litellm_params", {} + ) + proxy_request = litellm_params.get("proxy_server_request", {}) + request_body = ( + proxy_request.get("body", {}) + if isinstance(proxy_request, dict) + else {} + ) + client_requested_model = ( + request_body.get("model") + if isinstance(request_body, dict) + else None + ) + + assembled_model = complete_streaming_response.model + builder_recovered_routed_model = ( + isinstance(assembled_model, str) + and bool(assembled_model) + and any( + ( + chunk.get("model") + if isinstance(chunk, dict) + else getattr(chunk, "model", None) + ) + == assembled_model + for chunk in self.chunks[1:] + ) + and assembled_model != client_requested_model + ) + + if not builder_recovered_routed_model: + complete_streaming_response.model = self.model + response: Final = self.model_response_creator() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 3af79c709cc..01c133bc0f9 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -4360,6 +4360,108 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin session_id_var.set("") +@pytest.mark.asyncio +async def test_finalize_completed_stream_uses_routed_model_for_logging(logging_obj): + routed_model = "anthropic/claude-haiku-4-5" + client_alias = "myalias/haiku" + + async def _empty_aiter(): + if False: + yield None + + logging_obj.dispatch_success_handlers = AsyncMock() + + wrapper = CustomStreamWrapper( + completion_stream=_empty_aiter(), + model=routed_model, + logging_obj=logging_obj, + custom_llm_provider="anthropic", + ) + + wrapper.sent_last_chunk = True + wrapper.chunks = [MagicMock()] + wrapper.async_success_handler = AsyncMock() + + assembled_response = ModelResponse( + model=client_alias, + choices=[], + usage=Usage( + prompt_tokens=27, + completion_tokens=1, + total_tokens=28, + ), + ) + + with patch( + "litellm.stream_chunk_builder", + return_value=assembled_response, + ): + with pytest.raises(StopAsyncIteration): + await wrapper._finalize_completed_stream(cache_hit=False) + + await asyncio.sleep(0) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + + logged_response = logging_obj.dispatch_success_handlers.await_args.args[0] + + assert logged_response.model == routed_model + + +@pytest.mark.asyncio +async def test_finalize_completed_stream_preserves_model_recovered_from_later_chunk( + logging_obj, +): + wrapper_model = "anthropic/claude-haiku-4-5" + client_alias = "myalias/haiku" + recovered_routed_model = "anthropic/claude-sonnet-4-5" + + async def _empty_aiter(): + if False: + yield None + + logging_obj.dispatch_success_handlers = AsyncMock() + + wrapper = CustomStreamWrapper( + completion_stream=_empty_aiter(), + model=wrapper_model, + logging_obj=logging_obj, + custom_llm_provider="anthropic", + ) + + wrapper.sent_last_chunk = True + wrapper.chunks = [ + {"model": client_alias}, + {"model": recovered_routed_model}, + ] + wrapper.async_success_handler = AsyncMock() + + assembled_response = ModelResponse( + model=recovered_routed_model, + choices=[], + usage=Usage( + prompt_tokens=27, + completion_tokens=1, + total_tokens=28, + ), + ) + + with patch( + "litellm.stream_chunk_builder", + return_value=assembled_response, + ): + with pytest.raises(StopAsyncIteration): + await wrapper._finalize_completed_stream(cache_hit=False) + + await asyncio.sleep(0) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + + logged_response = logging_obj.dispatch_success_handlers.await_args.args[0] + + assert logged_response.model == recovered_routed_model + + @pytest.mark.asyncio async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch): """_check_max_streaming_duration() raises litellm.Timeout when a client keeps