This commit is contained in:
Senthurapandi Rajendran 2026-09-22 06:50:34 +08:00 • committed by GitHub
commit db03cf8f24
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 248 additions and 49 deletions

View file

@ -49,6 +49,49 @@ if TYPE_CHECKING:
)
def _assembled_model_came_from_a_later_chunk(
chunks: Sequence[object],
assembled_model: object,
) -> bool:
"""Report whether stream_chunk_builder picked a model the first chunk did not carry.
Azure Model Router puts the routed model on the chunks after the first one, and the
proxy deliberately leaves those chunks unrestamped so the builder can recover it.
A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so
an alias-restamped stream reaches the builder with the same shape: a first chunk that
disagrees with the rest. Those two are only told apart by what the client asked for.
"""
first_chunk: Final = chunks[0]
first_chunk_model: Final = (
first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None)
)
return (
isinstance(first_chunk_model, str)
and isinstance(assembled_model, str)
and bool(assembled_model)
and assembled_model != first_chunk_model
)
def _assembled_model_is_the_name_the_client_asked_for(
request_data: Mapping[str, object],
assembled_model: object,
) -> bool:
"""Report whether the assembled model is the public name the proxy stamps onto chunks.
That stamp is what leaves an unpriced alias on the partial response, so the deployment's
own model has to go back on before the row is costed. Pre-call processing rewrites
`request_data["model"]` for aliasing and routing, so the client's own name wins when it
is there, in the same order the proxy picks the name it stamps.
"""
client_requested_model: Final = request_data.get("_litellm_client_requested_model")
stamped_model: Final = (
client_requested_model if isinstance(client_requested_model, str) else request_data.get("model")
)
return isinstance(stamped_model, str) and assembled_model == stamped_model
class _ThinkingBlockFragment(TypedDict, total=False):
type: str | None
data: str | None

View file

@ -24,6 +24,10 @@ from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
_assembled_model_came_from_a_later_chunk,
_assembled_model_is_the_name_the_client_asked_for,
)
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.types.llms.openai import OpenAIChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
@ -1864,6 +1868,18 @@ class CustomStreamWrapper:
except Exception:
complete_streaming_response = None
if complete_streaming_response is not None and isinstance(self.model, str) and self.model:
builder_recovered_routed_model = _assembled_model_came_from_a_later_chunk(
self.chunks,
complete_streaming_response.model,
) and not _assembled_model_is_the_name_the_client_asked_for(
self.logging_obj.model_call_details,
complete_streaming_response.model,
)
if not builder_recovered_routed_model:
complete_streaming_response.model = self.model
response = self.model_response_creator()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
@ -2115,6 +2131,18 @@ class CustomStreamWrapper:
except Exception:
complete_streaming_response = None
if complete_streaming_response is not None and isinstance(self.model, str) and self.model:
builder_recovered_routed_model = _assembled_model_came_from_a_later_chunk(
self.chunks,
complete_streaming_response.model,
) and not _assembled_model_is_the_name_the_client_asked_for(
self.logging_obj.model_call_details,
complete_streaming_response.model,
)
if not builder_recovered_routed_model:
complete_streaming_response.model = self.model
response: Final = self.model_response_creator()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)

View file

@ -3,7 +3,7 @@ import contextlib
import json
import logging
import math
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
@ -61,6 +61,10 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
get_response_headers,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
_assembled_model_came_from_a_later_chunk,
_assembled_model_is_the_name_the_client_asked_for,
)
from litellm.litellm_core_utils.streaming_handler import (
backfill_missing_cache_usage_fields,
)
@ -326,46 +330,6 @@ def _deferred_stream_logging_is_armed(request_data: dict) -> bool:
)
def _assembled_model_came_from_a_later_chunk(chunks: Sequence[object], assembled_model: object) -> bool:
"""Report whether stream_chunk_builder picked a model the first chunk did not carry.
Azure Model Router puts the routed model on the chunks after the first one, and the
proxy deliberately leaves those chunks unrestamped so the builder can recover it.
A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so
an alias-restamped stream reaches the builder with the same shape: a first chunk that
disagrees with the rest. Those two are only told apart by what the client asked for.
"""
first_chunk: Final = chunks[0]
first_chunk_model: Final = (
first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None)
)
return (
isinstance(first_chunk_model, str)
and isinstance(assembled_model, str)
and bool(assembled_model)
and assembled_model != first_chunk_model
)
def _assembled_model_is_the_name_the_client_asked_for(
request_data: Mapping[str, object],
assembled_model: object,
) -> bool:
"""Report whether the assembled model is the public name the proxy stamps onto chunks.
That stamp is what leaves an unpriced alias on the partial response, so the deployment's
own model has to go back on before the row is costed. Pre-call processing rewrites
`request_data["model"]` for aliasing and routing, so the client's own name wins when it
is there, in the same order the proxy picks the name it stamps.
"""
client_requested_model: Final = request_data.get("_litellm_client_requested_model")
stamped_model: Final = (
client_requested_model if isinstance(client_requested_model, str) else request_data.get("model")
)
return isinstance(stamped_model, str) and assembled_model == stamped_model
async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, response: object) -> bool:
"""
A client disconnect throws GeneratorExit/CancelledError into the streaming
@ -2460,6 +2424,8 @@ class ProxyBaseLLMRequestProcessing:
"""
client_model: Final = get_client_requested_model(request) or self.data.get("model")
requested_model_from_client: Final[str | None] = client_model if isinstance(client_model, str) else None
if requested_model_from_client:
self.data["_litellm_client_requested_model"] = requested_model_from_client
self._debug_log_request_payload()
if skip_pre_call_logic:
@ -2605,13 +2571,6 @@ class ProxyBaseLLMRequestProcessing:
callback_headers=stream_callback_headers or MappingProxyType({}),
)
# Preserve the original client-requested model (pre-alias mapping) for downstream
# streaming generators. Pre-call processing can rewrite `self.data["model"]` for
# aliasing/routing, but the OpenAI-compatible response `model` field should reflect
# what the client sent.
if requested_model_from_client:
self.data["_litellm_client_requested_model"] = requested_model_from_client
if _post_call_guardrails_active:
self._arm_deferred_stream_dispatch(
response=response,
@ -3501,7 +3460,7 @@ class ProxyBaseLLMRequestProcessing:
getattr(cb, "guardrail_name", type(cb).__name__),
e,
)
if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "model_call_details"):
if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "`model_call_details`"):
captured_logging_obj.model_call_details.setdefault("metadata", {})["guardrail_blocked"] = True
except Exception as e:
verbose_proxy_logger.exception(

View file

@ -1216,6 +1216,7 @@ def function_setup(
optional_params={},
litellm_params=litellm_params,
stream_options=kwargs.get("stream_options", None),
_litellm_client_requested_model=kwargs.get("_litellm_client_requested_model"),
)
return logging_obj, kwargs
except Exception as e:

View file

@ -4360,6 +4360,174 @@ async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_fin
session_id_var.set("")
@pytest.mark.asyncio
async def test_finalize_completed_stream_restores_routed_model_for_key_alias(
logging_obj,
):
routed_model = "anthropic/claude-opus-4-20250514"
client_alias = "claude"
alias_target = "some-deployment"
async def _empty_aiter():
if False:
yield None
logging_obj.dispatch_success_handlers = AsyncMock()
logging_obj.model_call_details["_litellm_client_requested_model"] = client_alias
logging_obj.model_call_details["model"] = alias_target
wrapper = CustomStreamWrapper(
completion_stream=_empty_aiter(),
model=routed_model,
logging_obj=logging_obj,
custom_llm_provider="anthropic",
)
wrapper.sent_last_chunk = True
wrapper.chunks = [
{"model": routed_model},
{"model": client_alias},
{"model": client_alias},
{"model": routed_model},
]
wrapper.async_success_handler = AsyncMock()
assembled_response = ModelResponse(
model=client_alias,
choices=[],
usage=Usage(
prompt_tokens=27,
completion_tokens=1,
total_tokens=28,
),
)
with patch(
"litellm.stream_chunk_builder",
return_value=assembled_response,
):
with pytest.raises(StopAsyncIteration):
await wrapper._finalize_completed_stream(cache_hit=False)
await asyncio.sleep(0)
logging_obj.dispatch_success_handlers.assert_awaited_once()
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
assert logged_response.model == routed_model
@pytest.mark.asyncio
async def test_finalize_completed_stream_preserves_model_recovered_from_later_chunk(
logging_obj,
):
wrapper_model = "anthropic/claude-haiku-4-5"
client_alias = "myalias/haiku"
recovered_routed_model = "anthropic/claude-sonnet-4-5"
async def _empty_aiter():
if False:
yield None
logging_obj.dispatch_success_handlers = AsyncMock()
wrapper = CustomStreamWrapper(
completion_stream=_empty_aiter(),
model=wrapper_model,
logging_obj=logging_obj,
custom_llm_provider="anthropic",
)
wrapper.sent_last_chunk = True
wrapper.chunks = [
{"model": client_alias},
{"model": recovered_routed_model},
]
wrapper.async_success_handler = AsyncMock()
assembled_response = ModelResponse(
model=recovered_routed_model,
choices=[],
usage=Usage(
prompt_tokens=27,
completion_tokens=1,
total_tokens=28,
),
)
with patch(
"litellm.stream_chunk_builder",
return_value=assembled_response,
):
with pytest.raises(StopAsyncIteration):
await wrapper._finalize_completed_stream(cache_hit=False)
await asyncio.sleep(0)
logging_obj.dispatch_success_handlers.assert_awaited_once()
logged_response = logging_obj.dispatch_success_handlers.await_args.args[0]
assert logged_response.model == recovered_routed_model
def test_sync_completed_stream_restores_routed_model_for_key_alias(logging_obj):
routed_model = "anthropic/claude-opus-4-20250514"
client_alias = "claude"
alias_target = "some-deployment"
logging_obj.success_handler = MagicMock()
logging_obj.model_call_details["_litellm_client_requested_model"] = client_alias
logging_obj.model_call_details["model"] = alias_target
wrapper = CustomStreamWrapper(
completion_stream=iter(()),
model=routed_model,
logging_obj=logging_obj,
custom_llm_provider="anthropic",
)
wrapper.sent_last_chunk = True
wrapper.send_stream_usage = False
wrapper.chunks = [
{"model": routed_model},
{"model": client_alias},
{"model": client_alias},
{"model": routed_model},
]
assembled_response = ModelResponse(
model=client_alias,
choices=[],
usage=Usage(
prompt_tokens=27,
completion_tokens=1,
total_tokens=28,
),
)
with (
patch(
"litellm.stream_chunk_builder",
return_value=assembled_response,
),
patch.object(wrapper, "cache_streaming_response") as cache_mock,
patch(
"litellm.litellm_core_utils.streaming_handler.executor.submit"
) as submit_mock,
):
with pytest.raises(StopIteration):
next(wrapper)
submit_mock.assert_called_once()
logged_response = submit_mock.call_args.args[1]
assert logged_response.model == routed_model
cache_mock.assert_called_once()
cached_response = cache_mock.call_args.kwargs["processed_chunk"]
assert cached_response.model == routed_model
@pytest.mark.asyncio
async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_correlation_context(monkeypatch):
"""_check_max_streaming_duration() raises litellm.Timeout when a client keeps