diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index aa6ce6ea99e..d3a375f2356 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -228,6 +228,28 @@ def _skips_stream_fallback(status_code: int | None) -> bool: return status_code is not None and 400 <= status_code < 500 and status_code != 429 +def _mid_stream_error_is_terminal( + mapped_exception: Exception, + mapped_status_code: int | None, + original_status_code: int | None, + *, + router_fallback_available: bool = False, +) -> bool: + """Whether a mid-stream failure is surfaced instead of retried on another deployment. + + A 4xx is terminal, with two exceptions: 429 is transient, and a context-window overflow + is satisfiable by a deployment with a larger window (#48887). That second exemption is + the Router's, not the exception's: a direct SDK call has no other deployment to try, so + it keeps receiving the 400 it got (#15785). + """ + if ( + isinstance(mapped_exception, litellm.ContextWindowExceededError) + and router_fallback_available + ): + return False + return _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code) + + def is_async_iterable(obj: object) -> bool: """ Check if an object is an async iterable (can be used with 'async for'). @@ -462,6 +484,9 @@ class CustomStreamWrapper: self.response_uptil_now = "" self._emitted_disqualifying_content = False _model_info: Final[dict] = litellm_params.model_info or {} + # Set by the Router when it resolved a deployment. A direct SDK call has none, which is + # what separates "try another deployment" from "this 400 is the answer". + self._has_router_model_info: bool = bool(_model_info) _api_base: Final = get_api_base( model=model or "", @@ -2916,8 +2941,11 @@ class CustomStreamWrapper: # Raise non-retriable client errors directly (skip fallback). # Exception: 429 (rate-limit) IS retriable/transient — allow it # through so the Router can switch to a different model group. - if not _is_transport_timeout(e) and ( - _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code) + if not _is_transport_timeout(e) and _mid_stream_error_is_terminal( + mapped_exception, + mapped_status_code, + original_status_code, + router_fallback_available=self._has_router_model_info, ): raise mapped_exception diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 288e4a7119d..9ffa3f7fdd4 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -371,7 +371,26 @@ def _error_event_error_object(event: object) -> object: return {field: value for field in ("message", "code") if _is_error_field_value(value := _event_field(event, field))} -def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool: +def _has_router_deployment(litellm_metadata: Mapping[str, object] | None) -> bool: + """Whether the Router is the caller, and so has another deployment to try. + + `model_info` is the deployment the Router resolved before the call. A direct SDK + call has no deployment, and therefore nothing to fall back to. + """ + model_info: Final = litellm_metadata.get("model_info") if litellm_metadata else None + return _is_json_object(model_info) and bool(model_info) + + +def _mid_stream_fallback_eligible( + mapped_exception: Exception, + litellm_metadata: Mapping[str, object] | None = None, +) -> bool: + # A context-window overflow arrives as a 400, but another deployment with a larger + # window can still satisfy it, so it has to reach the Router's fallback chain (#48887). + # Only when the Router is actually the caller: a direct SDK call has no other deployment + # to consult, and keeps the 400 it got (#15785). + if isinstance(mapped_exception, litellm.ContextWindowExceededError): + return _has_router_deployment(litellm_metadata) if isinstance(mapped_exception, litellm.ContentPolicyViolationError): return True status_code: Final = getattr(mapped_exception, "status_code", None) @@ -801,7 +820,7 @@ class BaseResponsesAPIStreamingIterator: ) mapped_exception: Final = self._map_error_event_exception(error_obj) - if not _mid_stream_fallback_eligible(mapped_exception): + if not _mid_stream_fallback_eligible(mapped_exception, self.litellm_metadata): raise mapped_exception mapped_status: Final = getattr(mapped_exception, "status_code", None) raise MidStreamFallbackError( diff --git a/tests/unit/responses/test_mid_stream_context_window_fallback.py b/tests/unit/responses/test_mid_stream_context_window_fallback.py new file mode 100644 index 00000000000..d8ecebbc85d --- /dev/null +++ b/tests/unit/responses/test_mid_stream_context_window_fallback.py @@ -0,0 +1,91 @@ +"""Regression tests for #48887. + +A context-window overflow reported *inside* a stream arrives after HTTP 200, so it is not +covered by any status-code-based fallback gate. It is still satisfiable by a deployment +with a larger window, so both the Responses iterator and the chat stream wrapper must let +it through to the Router instead of raising it straight to the caller. +""" + +from __future__ import annotations + +import pytest + +from litellm.exceptions import BadRequestError, ContextWindowExceededError +from litellm.litellm_core_utils.streaming_handler import _mid_stream_error_is_terminal +from litellm.responses.streaming_iterator import _mid_stream_fallback_eligible + + +def _context_window_error() -> ContextWindowExceededError: + return ContextWindowExceededError( + message="Your input exceeds the context window of this model.", + model="gpt-6-luna", + llm_provider="openai", + ) + + +def _bad_request(status_code: int | None = 400) -> BadRequestError: + error = BadRequestError( + message="Invalid value for 'temperature'.", + model="gpt-6-luna", + llm_provider="openai", + ) + error.status_code = status_code # type: ignore[misc] + return error + + +# --- Responses API streams (litellm/responses/streaming_iterator.py) ------------------- + + +ROUTER_METADATA = {"model_info": {"id": "deployment-1"}} + + +def test_context_window_error_is_fallback_eligible_on_responses_stream() -> None: + assert _mid_stream_fallback_eligible(_context_window_error(), ROUTER_METADATA) is True + + +def test_context_window_error_is_not_eligible_for_a_direct_sdk_call() -> None: + """No deployment means no larger window to try; the caller keeps the 400 (#15785).""" + assert _mid_stream_fallback_eligible(_context_window_error()) is False + assert _mid_stream_fallback_eligible(_context_window_error(), {"model_info": {}}) is False + + +def test_plain_bad_request_still_skips_responses_fallback() -> None: + assert _mid_stream_fallback_eligible(_bad_request()) is False + + +@pytest.mark.parametrize("status_code", [None, 429, 500, 503]) +def test_transient_and_server_errors_stay_fallback_eligible(status_code: int | None) -> None: + assert _mid_stream_fallback_eligible(_bad_request(status_code)) is True + + +# --- Chat completion streams (litellm/litellm_core_utils/streaming_handler.py) --------- + + +def test_context_window_error_is_not_terminal_on_chat_stream() -> None: + error = _context_window_error() + assert _mid_stream_error_is_terminal( + error, error.status_code, error.status_code, router_fallback_available=True + ) is False + + +def test_context_window_error_is_terminal_for_a_direct_sdk_call() -> None: + error = _context_window_error() + assert _mid_stream_error_is_terminal(error, error.status_code, error.status_code) is True + + +def test_context_window_error_is_not_terminal_even_when_envelope_reports_400() -> None: + """The original exception can be the only one carrying the context-window type.""" + assert _mid_stream_error_is_terminal(_bad_request(400), 400, 400) is True + assert _mid_stream_error_is_terminal( + _context_window_error(), 400, 400, router_fallback_available=True + ) is False + + +def test_plain_bad_request_is_terminal_on_chat_stream() -> None: + assert _mid_stream_error_is_terminal(_bad_request(), 400, 400) is True + + +@pytest.mark.parametrize("status_code", [429, 500, 503]) +def test_retriable_status_codes_are_not_terminal_on_chat_stream(status_code: int) -> None: + error = _bad_request(status_code) + assert _mid_stream_error_is_terminal(error, status_code, status_code) is False diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 14ecb185f06..ee38f964632 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -55,7 +55,7 @@ from litellm.types.llms.openai import ( ) -def _make_iterator() -> BaseResponsesAPIStreamingIterator: +def _make_iterator(litellm_metadata: Mapping[str, object] | None = None) -> BaseResponsesAPIStreamingIterator: mock_logging_obj = Mock(spec=LiteLLMLoggingObj) mock_logging_obj.model_call_details = {"litellm_params": {}} mock_config = Mock(spec=BaseResponsesAPIConfig) @@ -67,6 +67,7 @@ def _make_iterator() -> BaseResponsesAPIStreamingIterator: responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, custom_llm_provider="openai", + litellm_metadata=litellm_metadata, ) @@ -881,3 +882,34 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se assert failure is not None _assert_names_the_azure_deployment(failure, reported_api_base) + + +def test_context_window_error_reaches_fallback_only_under_the_router() -> None: + """The exemption is the Router's, not the exception's. + + The case above pins the direct-SDK contract (#15785): no deployment means the 400 is + the answer. When the Router resolved a deployment, the same event must be wrapped so + `context_window_fallbacks` can route it to a larger window (#48887). + """ + iterator: Final = _make_iterator(litellm_metadata={"model_info": {"id": "deployment-1"}}) + chunk: Final = _make_error_chunk( + "invalid_request_error", + "context_length_exceeded", + "Your input exceeds the context window of this model.", + ) + with pytest.raises(MidStreamFallbackError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + assert isinstance(exc_info.value.original_exception, litellm.ContextWindowExceededError) + + +def test_empty_model_info_does_not_count_as_a_router_deployment() -> None: + """`model_info` present but empty is what a caller can pass by hand; it selects nothing.""" + iterator: Final = _make_iterator(litellm_metadata={"model_info": {}}) + chunk: Final = _make_error_chunk( + "invalid_request_error", + "context_length_exceeded", + "Your input exceeds the context window of this model.", + ) + with pytest.raises(litellm.ContextWindowExceededError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + assert not isinstance(exc_info.value, MidStreamFallbackError)