fix(proxy): preserve routed model for streamed spend logging

This commit is contained in:
Senthurapandi 2026-09-20 16:36:09 -07:00
parent 0e61ed51c0
commit 04416f76f1
2 changed files with 137 additions and 0 deletions

View file

@ -2115,6 +2115,41 @@ class CustomStreamWrapper:
except Exception:
complete_streaming_response = None
if complete_streaming_response is not None and isinstance(self.model, str) and self.model:
litellm_params = self.logging_obj.model_call_details.get(
"litellm_params", {}
)
proxy_request = litellm_params.get("proxy_server_request", {})
request_body = (
proxy_request.get("body", {})
if isinstance(proxy_request, dict)
else {}
)
client_requested_model = (
request_body.get("model")
if isinstance(request_body, dict)
else None
)
assembled_model = complete_streaming_response.model
builder_recovered_routed_model = (
isinstance(assembled_model, str)
and bool(assembled_model)
and any(
(
chunk.get("model")
if isinstance(chunk, dict)
else getattr(chunk, "model", None)
)
== assembled_model
for chunk in self.chunks[1:]
)
and assembled_model != client_requested_model
)
if not builder_recovered_routed_model:
complete_streaming_response.model = self.model
response: Final = self.model_response_creator()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)

View file

@ -4360,6 +4360,108 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin
session_id_var.set("")
@pytest.mark.asyncio
async def test_finalize_completed_stream_uses_routed_model_for_logging(logging_obj):
routed_model = "anthropic/claude-haiku-4-5"
client_alias = "myalias/haiku"
async def _empty_aiter():
if False:
yield None
logging_obj.dispatch_success_handlers = AsyncMock()
wrapper = CustomStreamWrapper(
completion_stream=_empty_aiter(),
model=routed_model,
logging_obj=logging_obj,
custom_llm_provider="anthropic",
)
wrapper.sent_last_chunk = True
wrapper.chunks = [MagicMock()]
wrapper.async_success_handler = AsyncMock()
assembled_response = ModelResponse(
model=client_alias,
choices=[],
usage=Usage(
prompt_tokens=27,
completion_tokens=1,
total_tokens=28,
),
)
with patch(
"litellm.stream_chunk_builder",
return_value=assembled_response,
):
with pytest.raises(StopAsyncIteration):
await wrapper._finalize_completed_stream(cache_hit=False)
await asyncio.sleep(0)
logging_obj.dispatch_success_handlers.assert_awaited_once()
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
assert logged_response.model == routed_model
@pytest.mark.asyncio
async def test_finalize_completed_stream_preserves_model_recovered_from_later_chunk(
logging_obj,
):
wrapper_model = "anthropic/claude-haiku-4-5"
client_alias = "myalias/haiku"
recovered_routed_model = "anthropic/claude-sonnet-4-5"
async def _empty_aiter():
if False:
yield None
logging_obj.dispatch_success_handlers = AsyncMock()
wrapper = CustomStreamWrapper(
completion_stream=_empty_aiter(),
model=wrapper_model,
logging_obj=logging_obj,
custom_llm_provider="anthropic",
)
wrapper.sent_last_chunk = True
wrapper.chunks = [
{"model": client_alias},
{"model": recovered_routed_model},
]
wrapper.async_success_handler = AsyncMock()
assembled_response = ModelResponse(
model=recovered_routed_model,
choices=[],
usage=Usage(
prompt_tokens=27,
completion_tokens=1,
total_tokens=28,
),
)
with patch(
"litellm.stream_chunk_builder",
return_value=assembled_response,
):
with pytest.raises(StopAsyncIteration):
await wrapper._finalize_completed_stream(cache_hit=False)
await asyncio.sleep(0)
logging_obj.dispatch_success_handlers.assert_awaited_once()
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
assert logged_response.model == recovered_routed_model
@pytest.mark.asyncio
async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch):
"""_check_max_streaming_duration() raises litellm.Timeout when a client keeps