mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): preserve routed model for streamed spend logging
This commit is contained in:
parent
0e61ed51c0
commit
04416f76f1
2 changed files with 137 additions and 0 deletions
|
|
@ -2115,6 +2115,41 @@ class CustomStreamWrapper:
|
|||
except Exception:
|
||||
complete_streaming_response = None
|
||||
|
||||
if complete_streaming_response is not None and isinstance(self.model, str) and self.model:
|
||||
litellm_params = self.logging_obj.model_call_details.get(
|
||||
"litellm_params", {}
|
||||
)
|
||||
proxy_request = litellm_params.get("proxy_server_request", {})
|
||||
request_body = (
|
||||
proxy_request.get("body", {})
|
||||
if isinstance(proxy_request, dict)
|
||||
else {}
|
||||
)
|
||||
client_requested_model = (
|
||||
request_body.get("model")
|
||||
if isinstance(request_body, dict)
|
||||
else None
|
||||
)
|
||||
|
||||
assembled_model = complete_streaming_response.model
|
||||
builder_recovered_routed_model = (
|
||||
isinstance(assembled_model, str)
|
||||
and bool(assembled_model)
|
||||
and any(
|
||||
(
|
||||
chunk.get("model")
|
||||
if isinstance(chunk, dict)
|
||||
else getattr(chunk, "model", None)
|
||||
)
|
||||
== assembled_model
|
||||
for chunk in self.chunks[1:]
|
||||
)
|
||||
and assembled_model != client_requested_model
|
||||
)
|
||||
|
||||
if not builder_recovered_routed_model:
|
||||
complete_streaming_response.model = self.model
|
||||
|
||||
response: Final = self.model_response_creator()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
|
|
|||
|
|
@ -4360,6 +4360,108 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin
|
|||
session_id_var.set("")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_completed_stream_uses_routed_model_for_logging(logging_obj):
|
||||
routed_model = "anthropic/claude-haiku-4-5"
|
||||
client_alias = "myalias/haiku"
|
||||
|
||||
async def _empty_aiter():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
logging_obj.dispatch_success_handlers = AsyncMock()
|
||||
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=_empty_aiter(),
|
||||
model=routed_model,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
wrapper.sent_last_chunk = True
|
||||
wrapper.chunks = [MagicMock()]
|
||||
wrapper.async_success_handler = AsyncMock()
|
||||
|
||||
assembled_response = ModelResponse(
|
||||
model=client_alias,
|
||||
choices=[],
|
||||
usage=Usage(
|
||||
prompt_tokens=27,
|
||||
completion_tokens=1,
|
||||
total_tokens=28,
|
||||
),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.stream_chunk_builder",
|
||||
return_value=assembled_response,
|
||||
):
|
||||
with pytest.raises(StopAsyncIteration):
|
||||
await wrapper._finalize_completed_stream(cache_hit=False)
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
logging_obj.dispatch_success_handlers.assert_awaited_once()
|
||||
|
||||
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
|
||||
|
||||
assert logged_response.model == routed_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_completed_stream_preserves_model_recovered_from_later_chunk(
|
||||
logging_obj,
|
||||
):
|
||||
wrapper_model = "anthropic/claude-haiku-4-5"
|
||||
client_alias = "myalias/haiku"
|
||||
recovered_routed_model = "anthropic/claude-sonnet-4-5"
|
||||
|
||||
async def _empty_aiter():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
logging_obj.dispatch_success_handlers = AsyncMock()
|
||||
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=_empty_aiter(),
|
||||
model=wrapper_model,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
wrapper.sent_last_chunk = True
|
||||
wrapper.chunks = [
|
||||
{"model": client_alias},
|
||||
{"model": recovered_routed_model},
|
||||
]
|
||||
wrapper.async_success_handler = AsyncMock()
|
||||
|
||||
assembled_response = ModelResponse(
|
||||
model=recovered_routed_model,
|
||||
choices=[],
|
||||
usage=Usage(
|
||||
prompt_tokens=27,
|
||||
completion_tokens=1,
|
||||
total_tokens=28,
|
||||
),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.stream_chunk_builder",
|
||||
return_value=assembled_response,
|
||||
):
|
||||
with pytest.raises(StopAsyncIteration):
|
||||
await wrapper._finalize_completed_stream(cache_hit=False)
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
logging_obj.dispatch_success_handlers.assert_awaited_once()
|
||||
|
||||
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
|
||||
|
||||
assert logged_response.model == recovered_routed_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch):
|
||||
"""_check_max_streaming_duration() raises litellm.Timeout when a client keeps
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue