diff --git a/litellm/router.py b/litellm/router.py index 9f770b23608..64db136a791 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -422,6 +422,9 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +# The only response_format that leaves a stream continuation-eligible: the rest +# ask for structured output that cannot resume from an arbitrary cut point. +_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: @@ -3060,7 +3063,7 @@ class Router: # Structured output cannot be resumed from an arbitrary cut point; # `{"type": "text"}` is the unconstrained default and stays eligible. response_format: Final = request_kwargs.get("response_format") - if response_format is not None and response_format != {"type": "text"}: + if response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT: return False tool_choice: Final = request_kwargs.get("tool_choice") if tool_choice == "required" or isinstance(tool_choice, Mapping): @@ -3071,17 +3074,17 @@ class Router: @staticmethod def _build_completion_continuation_input( - messages: list[dict[str, str]], + messages: Sequence[Mapping[str, str]], generated_content: str, ) -> Sequence[Mapping[str, object]]: """Append the partial output as an assistant prefill for a prefill-capable fallback to continue. A nested break folds the new partial into an existing trailing prefill rather than appending a second assistant turn.""" - if messages and messages[-1].get("role") == "assistant" and messages[-1].get("prefix"): - last: Final = messages[-1] - merged: dict[str, object] = {**last, "content": str(last.get("content") or "") + generated_content} + last: Final = messages[-1] if messages else None + if last is not None and last.get("role") == "assistant" and last.get("prefix"): + merged: Final = {**last, "content": str(last.get("content") or "") + generated_content} return [*messages[:-1], merged] - prefill: dict[str, object] = {"role": "assistant", "content": generated_content, "prefix": True} + prefill: Final = {"role": "assistant", "content": generated_content, "prefix": True} return [*messages, prefill] @staticmethod diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 2a527c8b249..688470b2b77 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -6,6 +6,7 @@ assistant prefill are dropped, so the partial text is continued rather than regenerated or rejected. Requests without the marker pass through untouched. """ +from collections.abc import Mapping, Sequence from typing import Final from pydantic import TypeAdapter, ValidationError @@ -32,11 +33,12 @@ class ContinuationPrefillDeploymentCheck(CustomLogger): async def async_filter_deployments( self, model: str, - healthy_deployments: list[dict[str, object]], - messages: list[AllMessageValues] | None, - request_kwargs: dict[str, object] | None = None, + healthy_deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: Sequence[AllMessageValues] | None, + request_kwargs: Mapping[str, object] | None = None, parent_otel_span: Span | None = None, - ) -> list[dict[str, object]]: + ) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list if not (request_kwargs or {}).get(MID_STREAM_CONTINUATION_KWARG): return healthy_deployments - return [deployment for deployment in healthy_deployments if _deployment_supports_prefill(deployment)] + eligible: Final = (deployment for deployment in healthy_deployments if _deployment_supports_prefill(deployment)) + return list(eligible) # mutable-ok: downstream deployment selection consumes a mutable list diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 7aa244a8433..f5cc8fb6b03 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -4966,7 +4966,6 @@ class TestContinuationDisqualifiers: wrapper._accumulate_streamed_delta({"content": "Hi"}) wrapper._accumulate_streamed_delta({"tool_calls": [{"index": 0}]}) - # a later plain-text delta must not clear the latch wrapper._accumulate_streamed_delta({"content": "there"}) assert wrapper.response_uptil_now == "Hithere"