mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 261ad0074b into 1a4e2b1698
This commit is contained in:
commit
db03cf8f24
5 changed files with 248 additions and 49 deletions
|
|
@ -49,6 +49,49 @@ if TYPE_CHECKING:
|
|||
)
|
||||
|
||||
|
||||
def _assembled_model_came_from_a_later_chunk(
|
||||
chunks: Sequence[object],
|
||||
assembled_model: object,
|
||||
) -> bool:
|
||||
"""Report whether stream_chunk_builder picked a model the first chunk did not carry.
|
||||
|
||||
Azure Model Router puts the routed model on the chunks after the first one, and the
|
||||
proxy deliberately leaves those chunks unrestamped so the builder can recover it.
|
||||
|
||||
A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so
|
||||
an alias-restamped stream reaches the builder with the same shape: a first chunk that
|
||||
disagrees with the rest. Those two are only told apart by what the client asked for.
|
||||
"""
|
||||
first_chunk: Final = chunks[0]
|
||||
first_chunk_model: Final = (
|
||||
first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None)
|
||||
)
|
||||
return (
|
||||
isinstance(first_chunk_model, str)
|
||||
and isinstance(assembled_model, str)
|
||||
and bool(assembled_model)
|
||||
and assembled_model != first_chunk_model
|
||||
)
|
||||
|
||||
|
||||
def _assembled_model_is_the_name_the_client_asked_for(
|
||||
request_data: Mapping[str, object],
|
||||
assembled_model: object,
|
||||
) -> bool:
|
||||
"""Report whether the assembled model is the public name the proxy stamps onto chunks.
|
||||
|
||||
That stamp is what leaves an unpriced alias on the partial response, so the deployment's
|
||||
own model has to go back on before the row is costed. Pre-call processing rewrites
|
||||
`request_data["model"]` for aliasing and routing, so the client's own name wins when it
|
||||
is there, in the same order the proxy picks the name it stamps.
|
||||
"""
|
||||
client_requested_model: Final = request_data.get("_litellm_client_requested_model")
|
||||
stamped_model: Final = (
|
||||
client_requested_model if isinstance(client_requested_model, str) else request_data.get("model")
|
||||
)
|
||||
return isinstance(stamped_model, str) and assembled_model == stamped_model
|
||||
|
||||
|
||||
class _ThinkingBlockFragment(TypedDict, total=False):
|
||||
type: str | None
|
||||
data: str | None
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ from litellm.litellm_core_utils.model_response_utils import (
|
|||
is_model_response_stream_empty,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
|
||||
_assembled_model_came_from_a_later_chunk,
|
||||
_assembled_model_is_the_name_the_client_asked_for,
|
||||
)
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.types.llms.openai import OpenAIChatCompletionChunk
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -1864,6 +1868,18 @@ class CustomStreamWrapper:
|
|||
except Exception:
|
||||
complete_streaming_response = None
|
||||
|
||||
if complete_streaming_response is not None and isinstance(self.model, str) and self.model:
|
||||
builder_recovered_routed_model = _assembled_model_came_from_a_later_chunk(
|
||||
self.chunks,
|
||||
complete_streaming_response.model,
|
||||
) and not _assembled_model_is_the_name_the_client_asked_for(
|
||||
self.logging_obj.model_call_details,
|
||||
complete_streaming_response.model,
|
||||
)
|
||||
|
||||
if not builder_recovered_routed_model:
|
||||
complete_streaming_response.model = self.model
|
||||
|
||||
response = self.model_response_creator()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
|
@ -2115,6 +2131,18 @@ class CustomStreamWrapper:
|
|||
except Exception:
|
||||
complete_streaming_response = None
|
||||
|
||||
if complete_streaming_response is not None and isinstance(self.model, str) and self.model:
|
||||
builder_recovered_routed_model = _assembled_model_came_from_a_later_chunk(
|
||||
self.chunks,
|
||||
complete_streaming_response.model,
|
||||
) and not _assembled_model_is_the_name_the_client_asked_for(
|
||||
self.logging_obj.model_call_details,
|
||||
complete_streaming_response.model,
|
||||
)
|
||||
|
||||
if not builder_recovered_routed_model:
|
||||
complete_streaming_response.model = self.model
|
||||
|
||||
response: Final = self.model_response_creator()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import contextlib
|
|||
import json
|
||||
import logging
|
||||
import math
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
|
|
@ -61,6 +61,10 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|||
get_response_headers,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
|
||||
_assembled_model_came_from_a_later_chunk,
|
||||
_assembled_model_is_the_name_the_client_asked_for,
|
||||
)
|
||||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
backfill_missing_cache_usage_fields,
|
||||
)
|
||||
|
|
@ -326,46 +330,6 @@ def _deferred_stream_logging_is_armed(request_data: dict) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _assembled_model_came_from_a_later_chunk(chunks: Sequence[object], assembled_model: object) -> bool:
|
||||
"""Report whether stream_chunk_builder picked a model the first chunk did not carry.
|
||||
|
||||
Azure Model Router puts the routed model on the chunks after the first one, and the
|
||||
proxy deliberately leaves those chunks unrestamped so the builder can recover it.
|
||||
|
||||
A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so
|
||||
an alias-restamped stream reaches the builder with the same shape: a first chunk that
|
||||
disagrees with the rest. Those two are only told apart by what the client asked for.
|
||||
"""
|
||||
first_chunk: Final = chunks[0]
|
||||
first_chunk_model: Final = (
|
||||
first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None)
|
||||
)
|
||||
return (
|
||||
isinstance(first_chunk_model, str)
|
||||
and isinstance(assembled_model, str)
|
||||
and bool(assembled_model)
|
||||
and assembled_model != first_chunk_model
|
||||
)
|
||||
|
||||
|
||||
def _assembled_model_is_the_name_the_client_asked_for(
|
||||
request_data: Mapping[str, object],
|
||||
assembled_model: object,
|
||||
) -> bool:
|
||||
"""Report whether the assembled model is the public name the proxy stamps onto chunks.
|
||||
|
||||
That stamp is what leaves an unpriced alias on the partial response, so the deployment's
|
||||
own model has to go back on before the row is costed. Pre-call processing rewrites
|
||||
`request_data["model"]` for aliasing and routing, so the client's own name wins when it
|
||||
is there, in the same order the proxy picks the name it stamps.
|
||||
"""
|
||||
client_requested_model: Final = request_data.get("_litellm_client_requested_model")
|
||||
stamped_model: Final = (
|
||||
client_requested_model if isinstance(client_requested_model, str) else request_data.get("model")
|
||||
)
|
||||
return isinstance(stamped_model, str) and assembled_model == stamped_model
|
||||
|
||||
|
||||
async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, response: object) -> bool:
|
||||
"""
|
||||
A client disconnect throws GeneratorExit/CancelledError into the streaming
|
||||
|
|
@ -2460,6 +2424,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"""
|
||||
client_model: Final = get_client_requested_model(request) or self.data.get("model")
|
||||
requested_model_from_client: Final[str | None] = client_model if isinstance(client_model, str) else None
|
||||
if requested_model_from_client:
|
||||
self.data["_litellm_client_requested_model"] = requested_model_from_client
|
||||
self._debug_log_request_payload()
|
||||
|
||||
if skip_pre_call_logic:
|
||||
|
|
@ -2605,13 +2571,6 @@ class ProxyBaseLLMRequestProcessing:
|
|||
callback_headers=stream_callback_headers or MappingProxyType({}),
|
||||
)
|
||||
|
||||
# Preserve the original client-requested model (pre-alias mapping) for downstream
|
||||
# streaming generators. Pre-call processing can rewrite `self.data["model"]` for
|
||||
# aliasing/routing, but the OpenAI-compatible response `model` field should reflect
|
||||
# what the client sent.
|
||||
if requested_model_from_client:
|
||||
self.data["_litellm_client_requested_model"] = requested_model_from_client
|
||||
|
||||
if _post_call_guardrails_active:
|
||||
self._arm_deferred_stream_dispatch(
|
||||
response=response,
|
||||
|
|
@ -3501,7 +3460,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
getattr(cb, "guardrail_name", type(cb).__name__),
|
||||
e,
|
||||
)
|
||||
if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "model_call_details"):
|
||||
if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "`model_call_details`"):
|
||||
captured_logging_obj.model_call_details.setdefault("metadata", {})["guardrail_blocked"] = True
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -1216,6 +1216,7 @@ def function_setup(
|
|||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
stream_options=kwargs.get("stream_options", None),
|
||||
_litellm_client_requested_model=kwargs.get("_litellm_client_requested_model"),
|
||||
)
|
||||
return logging_obj, kwargs
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -4360,6 +4360,174 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin
|
|||
session_id_var.set("")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_completed_stream_restores_routed_model_for_key_alias(
|
||||
logging_obj,
|
||||
):
|
||||
routed_model = "anthropic/claude-opus-4-20250514"
|
||||
client_alias = "claude"
|
||||
alias_target = "some-deployment"
|
||||
|
||||
async def _empty_aiter():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
logging_obj.dispatch_success_handlers = AsyncMock()
|
||||
logging_obj.model_call_details["_litellm_client_requested_model"] = client_alias
|
||||
logging_obj.model_call_details["model"] = alias_target
|
||||
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=_empty_aiter(),
|
||||
model=routed_model,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
wrapper.sent_last_chunk = True
|
||||
wrapper.chunks = [
|
||||
{"model": routed_model},
|
||||
{"model": client_alias},
|
||||
{"model": client_alias},
|
||||
{"model": routed_model},
|
||||
]
|
||||
wrapper.async_success_handler = AsyncMock()
|
||||
|
||||
assembled_response = ModelResponse(
|
||||
model=client_alias,
|
||||
choices=[],
|
||||
usage=Usage(
|
||||
prompt_tokens=27,
|
||||
completion_tokens=1,
|
||||
total_tokens=28,
|
||||
),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.stream_chunk_builder",
|
||||
return_value=assembled_response,
|
||||
):
|
||||
with pytest.raises(StopAsyncIteration):
|
||||
await wrapper._finalize_completed_stream(cache_hit=False)
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
logging_obj.dispatch_success_handlers.assert_awaited_once()
|
||||
|
||||
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
|
||||
|
||||
assert logged_response.model == routed_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_completed_stream_preserves_model_recovered_from_later_chunk(
|
||||
logging_obj,
|
||||
):
|
||||
wrapper_model = "anthropic/claude-haiku-4-5"
|
||||
client_alias = "myalias/haiku"
|
||||
recovered_routed_model = "anthropic/claude-sonnet-4-5"
|
||||
|
||||
async def _empty_aiter():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
logging_obj.dispatch_success_handlers = AsyncMock()
|
||||
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=_empty_aiter(),
|
||||
model=wrapper_model,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
wrapper.sent_last_chunk = True
|
||||
wrapper.chunks = [
|
||||
{"model": client_alias},
|
||||
{"model": recovered_routed_model},
|
||||
]
|
||||
wrapper.async_success_handler = AsyncMock()
|
||||
|
||||
assembled_response = ModelResponse(
|
||||
model=recovered_routed_model,
|
||||
choices=[],
|
||||
usage=Usage(
|
||||
prompt_tokens=27,
|
||||
completion_tokens=1,
|
||||
total_tokens=28,
|
||||
),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.stream_chunk_builder",
|
||||
return_value=assembled_response,
|
||||
):
|
||||
with pytest.raises(StopAsyncIteration):
|
||||
await wrapper._finalize_completed_stream(cache_hit=False)
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
logging_obj.dispatch_success_handlers.assert_awaited_once()
|
||||
|
||||
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
|
||||
|
||||
assert logged_response.model == recovered_routed_model
|
||||
|
||||
def test_sync_completed_stream_restores_routed_model_for_key_alias(logging_obj):
|
||||
routed_model = "anthropic/claude-opus-4-20250514"
|
||||
client_alias = "claude"
|
||||
alias_target = "some-deployment"
|
||||
|
||||
logging_obj.success_handler = MagicMock()
|
||||
logging_obj.model_call_details["_litellm_client_requested_model"] = client_alias
|
||||
logging_obj.model_call_details["model"] = alias_target
|
||||
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=iter(()),
|
||||
model=routed_model,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
wrapper.sent_last_chunk = True
|
||||
wrapper.send_stream_usage = False
|
||||
wrapper.chunks = [
|
||||
{"model": routed_model},
|
||||
{"model": client_alias},
|
||||
{"model": client_alias},
|
||||
{"model": routed_model},
|
||||
]
|
||||
|
||||
assembled_response = ModelResponse(
|
||||
model=client_alias,
|
||||
choices=[],
|
||||
usage=Usage(
|
||||
prompt_tokens=27,
|
||||
completion_tokens=1,
|
||||
total_tokens=28,
|
||||
),
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.stream_chunk_builder",
|
||||
return_value=assembled_response,
|
||||
),
|
||||
patch.object(wrapper, "cache_streaming_response") as cache_mock,
|
||||
patch(
|
||||
"litellm.litellm_core_utils.streaming_handler.executor.submit"
|
||||
) as submit_mock,
|
||||
):
|
||||
with pytest.raises(StopIteration):
|
||||
next(wrapper)
|
||||
|
||||
submit_mock.assert_called_once()
|
||||
|
||||
logged_response = submit_mock.call_args.args[1]
|
||||
assert logged_response.model == routed_model
|
||||
|
||||
cache_mock.assert_called_once()
|
||||
cached_response = cache_mock.call_args.kwargs["processed_chunk"]
|
||||
assert cached_response.model == routed_model
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch):
|
||||
"""_check_max_streaming_duration() raises litellm.Timeout when a client keeps
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue