This commit is contained in:
mosifu 2026-11-07 00:53:34 +00:00 • committed by GitHub
commit 0c09eb7fb7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 175 additions and 5 deletions

View file

@ -237,6 +237,28 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
return status_code is not None and 400 <= status_code < 500 and status_code != 429
def _mid_stream_error_is_terminal(
mapped_exception: Exception,
mapped_status_code: int | None,
original_status_code: int | None,
*,
router_fallback_available: bool = False,
) -> bool:
"""Whether a mid-stream failure is surfaced instead of retried on another deployment.
A 4xx is terminal, with two exceptions: 429 is transient, and a context-window overflow
is satisfiable by a deployment with a larger window (#48887). That second exemption is
the Router's, not the exception's: a direct SDK call has no other deployment to try, so
it keeps receiving the 400 it got (#15785).
"""
if (
isinstance(mapped_exception, litellm.ContextWindowExceededError)
and router_fallback_available
):
return False
return _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
def is_async_iterable(obj: object) -> bool:
"""
Check if an object is an async iterable (can be used with 'async for').
@ -473,6 +495,9 @@ class CustomStreamWrapper:
self.response_uptil_now = ""
self._emitted_disqualifying_content = False
_model_info: Final[dict] = litellm_params.model_info or {}
# Set by the Router when it resolved a deployment. A direct SDK call has none, which is
# what separates "try another deployment" from "this 400 is the answer".
self._has_router_model_info: bool = bool(_model_info)
_api_base: Final = get_api_base(
model=model or "",
@ -2930,8 +2955,11 @@ class CustomStreamWrapper:
# Raise non-retriable client errors directly (skip fallback).
# Exception: 429 (rate-limit) IS retriable/transient — allow it
# through so the Router can switch to a different model group.
if not _is_transport_timeout(e) and (
_skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
if not _is_transport_timeout(e) and _mid_stream_error_is_terminal(
mapped_exception,
mapped_status_code,
original_status_code,
router_fallback_available=self._has_router_model_info,
):
raise mapped_exception

View file

@ -373,7 +373,26 @@ def _error_event_error_object(event: object) -> object:
return {field: value for field in ("message", "code") if _is_error_field_value(value := _event_field(event, field))}
def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
def _has_router_deployment(litellm_metadata: Mapping[str, object] | None) -> bool:
"""Whether the Router is the caller, and so has another deployment to try.
`model_info` is the deployment the Router resolved before the call. A direct SDK
call has no deployment, and therefore nothing to fall back to.
"""
model_info: Final = litellm_metadata.get("model_info") if litellm_metadata else None
return _is_json_object(model_info) and bool(model_info)
def _mid_stream_fallback_eligible(
mapped_exception: Exception,
litellm_metadata: Mapping[str, object] | None = None,
) -> bool:
# A context-window overflow arrives as a 400, but another deployment with a larger
# window can still satisfy it, so it has to reach the Router's fallback chain (#48887).
# Only when the Router is actually the caller: a direct SDK call has no other deployment
# to consult, and keeps the 400 it got (#15785).
if isinstance(mapped_exception, litellm.ContextWindowExceededError):
return _has_router_deployment(litellm_metadata)
if isinstance(mapped_exception, litellm.ContentPolicyViolationError):
return True
status_code: Final = getattr(mapped_exception, "status_code", None)
@ -803,7 +822,7 @@ class BaseResponsesAPIStreamingIterator:
)
mapped_exception: Final = self._map_error_event_exception(error_obj)
if not _mid_stream_fallback_eligible(mapped_exception):
if not _mid_stream_fallback_eligible(mapped_exception, self.litellm_metadata):
raise mapped_exception
mapped_status: Final = getattr(mapped_exception, "status_code", None)
raise MidStreamFallbackError(

View file

@ -0,0 +1,91 @@
"""Regression tests for #48887.
A context-window overflow reported *inside* a stream arrives after HTTP 200, so it is not
covered by any status-code-based fallback gate. It is still satisfiable by a deployment
with a larger window, so both the Responses iterator and the chat stream wrapper must let
it through to the Router instead of raising it straight to the caller.
"""
from __future__ import annotations
import pytest
from litellm.exceptions import BadRequestError, ContextWindowExceededError
from litellm.litellm_core_utils.streaming_handler import _mid_stream_error_is_terminal
from litellm.responses.streaming_iterator import _mid_stream_fallback_eligible
def _context_window_error() -> ContextWindowExceededError:
return ContextWindowExceededError(
message="Your input exceeds the context window of this model.",
model="gpt-6-luna",
llm_provider="openai",
)
def _bad_request(status_code: int | None = 400) -> BadRequestError:
error = BadRequestError(
message="Invalid value for 'temperature'.",
model="gpt-6-luna",
llm_provider="openai",
)
error.status_code = status_code # type: ignore[misc]
return error
# --- Responses API streams (litellm/responses/streaming_iterator.py) -------------------
ROUTER_METADATA = {"model_info": {"id": "deployment-1"}}
def test_context_window_error_is_fallback_eligible_on_responses_stream() -> None:
assert _mid_stream_fallback_eligible(_context_window_error(), ROUTER_METADATA) is True
def test_context_window_error_is_not_eligible_for_a_direct_sdk_call() -> None:
"""No deployment means no larger window to try; the caller keeps the 400 (#15785)."""
assert _mid_stream_fallback_eligible(_context_window_error()) is False
assert _mid_stream_fallback_eligible(_context_window_error(), {"model_info": {}}) is False
def test_plain_bad_request_still_skips_responses_fallback() -> None:
assert _mid_stream_fallback_eligible(_bad_request()) is False
@pytest.mark.parametrize("status_code", [None, 429, 500, 503])
def test_transient_and_server_errors_stay_fallback_eligible(status_code: int | None) -> None:
assert _mid_stream_fallback_eligible(_bad_request(status_code)) is True
# --- Chat completion streams (litellm/litellm_core_utils/streaming_handler.py) ---------
def test_context_window_error_is_not_terminal_on_chat_stream() -> None:
error = _context_window_error()
assert _mid_stream_error_is_terminal(
error, error.status_code, error.status_code, router_fallback_available=True
) is False
def test_context_window_error_is_terminal_for_a_direct_sdk_call() -> None:
error = _context_window_error()
assert _mid_stream_error_is_terminal(error, error.status_code, error.status_code) is True
def test_context_window_error_is_not_terminal_even_when_envelope_reports_400() -> None:
"""The original exception can be the only one carrying the context-window type."""
assert _mid_stream_error_is_terminal(_bad_request(400), 400, 400) is True
assert _mid_stream_error_is_terminal(
_context_window_error(), 400, 400, router_fallback_available=True
) is False
def test_plain_bad_request_is_terminal_on_chat_stream() -> None:
assert _mid_stream_error_is_terminal(_bad_request(), 400, 400) is True
@pytest.mark.parametrize("status_code", [429, 500, 503])
def test_retriable_status_codes_are_not_terminal_on_chat_stream(status_code: int) -> None:
error = _bad_request(status_code)
assert _mid_stream_error_is_terminal(error, status_code, status_code) is False

View file

@ -56,7 +56,7 @@ from litellm.types.llms.openai import (
)
def _make_iterator() -> BaseResponsesAPIStreamingIterator:
def _make_iterator(litellm_metadata: Mapping[str, object] | None = None) -> BaseResponsesAPIStreamingIterator:
mock_logging_obj = Mock(spec=LiteLLMLoggingObj)
mock_logging_obj.model_call_details = {"litellm_params": {}}
mock_config = Mock(spec=BaseResponsesAPIConfig)
@ -68,6 +68,7 @@ def _make_iterator() -> BaseResponsesAPIStreamingIterator:
responses_api_provider_config=mock_config,
logging_obj=mock_logging_obj,
custom_llm_provider="openai",
litellm_metadata=litellm_metadata,
)
@ -959,3 +960,34 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se
assert failure is not None
_assert_names_the_azure_deployment(failure, reported_api_base)
def test_context_window_error_reaches_fallback_only_under_the_router() -> None:
"""The exemption is the Router's, not the exception's.
The case above pins the direct-SDK contract (#15785): no deployment means the 400 is
the answer. When the Router resolved a deployment, the same event must be wrapped so
`context_window_fallbacks` can route it to a larger window (#48887).
"""
iterator: Final = _make_iterator(litellm_metadata={"model_info": {"id": "deployment-1"}})
chunk: Final = _make_error_chunk(
"invalid_request_error",
"context_length_exceeded",
"Your input exceeds the context window of this model.",
)
with pytest.raises(MidStreamFallbackError) as exc_info:
iterator._maybe_raise_for_error_event(chunk)
assert isinstance(exc_info.value.original_exception, litellm.ContextWindowExceededError)
def test_empty_model_info_does_not_count_as_a_router_deployment() -> None:
"""`model_info` present but empty is what a caller can pass by hand; it selects nothing."""
iterator: Final = _make_iterator(litellm_metadata={"model_info": {}})
chunk: Final = _make_error_chunk(
"invalid_request_error",
"context_length_exceeded",
"Your input exceeds the context window of this model.",
)
with pytest.raises(litellm.ContextWindowExceededError) as exc_info:
iterator._maybe_raise_for_error_event(chunk)
assert not isinstance(exc_info.value, MidStreamFallbackError)