fix(streaming): reach context_window_fallbacks without breaking the SDK 400

A context-window overflow reported inside a stream arrives after HTTP 200, so it
never reaches the status-code gates that decide whether the Router may try another
deployment. Both stream paths treated it as terminal and raised it to the caller.

Exempt it the way ContentPolicyViolationError already is (#28599), but gate the
exemption on the Router actually being the caller: `litellm_metadata.model_info`
on the Responses path, `litellm_params.model_info` on the chat path. A direct SDK
call has no second deployment to try and keeps the 400 it got (#15785), which is
what test_..._raises_context_window_exceeded_directly pins.

No router.py change: the Rust core already reads
MidStreamFallbackError.original_exception when it classifies the error
(router_rust/invoke/streaming.py::_is_context_window), so the envelope no longer
hides the trigger from the fallback dispatch.

Fixes #48887

litellm/litellm_core_utils/streaming_handler.py

@@ -228,6 +228,28 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
     return status_code is not None and 400 <= status_code < 500 and status_code != 429
 
 
+def _mid_stream_error_is_terminal(
+    mapped_exception: Exception,
+    mapped_status_code: int | None,
+    original_status_code: int | None,
+    *,
+    router_fallback_available: bool = False,
+) -> bool:
+    """Whether a mid-stream failure is surfaced instead of retried on another deployment.
+
+    A 4xx is terminal, with two exceptions: 429 is transient, and a context-window overflow
+    is satisfiable by a deployment with a larger window (#48887). That second exemption is
+    the Router's, not the exception's: a direct SDK call has no other deployment to try, so
+    it keeps receiving the 400 it got (#15785).
+    """
+    if (
+        isinstance(mapped_exception, litellm.ContextWindowExceededError)
+        and router_fallback_available
+    ):
+        return False
+    return _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
+
+
 def is_async_iterable(obj: object) -> bool:
     """
     Check if an object is an async iterable (can be used with 'async for').
@@ -462,6 +484,9 @@ class CustomStreamWrapper:
         self.response_uptil_now = ""
         self._emitted_disqualifying_content = False
         _model_info: Final[dict] = litellm_params.model_info or {}
+        # Set by the Router when it resolved a deployment. A direct SDK call has none, which is
+        # what separates "try another deployment" from "this 400 is the answer".
+        self._has_router_model_info: bool = bool(_model_info)
 
         _api_base: Final = get_api_base(
             model=model or "",
@@ -2916,8 +2941,11 @@ class CustomStreamWrapper:
         # Raise non-retriable client errors directly (skip fallback).
         # Exception: 429 (rate-limit) IS retriable/transient — allow it
         # through so the Router can switch to a different model group.
-        if not _is_transport_timeout(e) and (
-            _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
+        if not _is_transport_timeout(e) and _mid_stream_error_is_terminal(
+            mapped_exception,
+            mapped_status_code,
+            original_status_code,
+            router_fallback_available=self._has_router_model_info,
         ):
             raise mapped_exception
 

litellm/responses/streaming_iterator.py

@@ -371,7 +371,26 @@ def _error_event_error_object(event: object) -> object:
     return {field: value for field in ("message", "code") if _is_error_field_value(value := _event_field(event, field))}
 
 
-def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
+def _has_router_deployment(litellm_metadata: Mapping[str, object] | None) -> bool:
+    """Whether the Router is the caller, and so has another deployment to try.
+
+    `model_info` is the deployment the Router resolved before the call. A direct SDK
+    call has no deployment, and therefore nothing to fall back to.
+    """
+    model_info: Final = litellm_metadata.get("model_info") if litellm_metadata else None
+    return _is_json_object(model_info) and bool(model_info)
+
+
+def _mid_stream_fallback_eligible(
+    mapped_exception: Exception,
+    litellm_metadata: Mapping[str, object] | None = None,
+) -> bool:
+    # A context-window overflow arrives as a 400, but another deployment with a larger
+    # window can still satisfy it, so it has to reach the Router's fallback chain (#48887).
+    # Only when the Router is actually the caller: a direct SDK call has no other deployment
+    # to consult, and keeps the 400 it got (#15785).
+    if isinstance(mapped_exception, litellm.ContextWindowExceededError):
+        return _has_router_deployment(litellm_metadata)
     if isinstance(mapped_exception, litellm.ContentPolicyViolationError):
         return True
     status_code: Final = getattr(mapped_exception, "status_code", None)
@@ -801,7 +820,7 @@ class BaseResponsesAPIStreamingIterator:
         )
 
         mapped_exception: Final = self._map_error_event_exception(error_obj)
-        if not _mid_stream_fallback_eligible(mapped_exception):
+        if not _mid_stream_fallback_eligible(mapped_exception, self.litellm_metadata):
             raise mapped_exception
         mapped_status: Final = getattr(mapped_exception, "status_code", None)
         raise MidStreamFallbackError(

tests/unit/responses/test_mid_stream_context_window_fallback.py

@@ -0,0 +1,91 @@
+"""Regression tests for #48887.
+
+A context-window overflow reported *inside* a stream arrives after HTTP 200, so it is not
+covered by any status-code-based fallback gate. It is still satisfiable by a deployment
+with a larger window, so both the Responses iterator and the chat stream wrapper must let
+it through to the Router instead of raising it straight to the caller.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from litellm.exceptions import BadRequestError, ContextWindowExceededError
+from litellm.litellm_core_utils.streaming_handler import _mid_stream_error_is_terminal
+from litellm.responses.streaming_iterator import _mid_stream_fallback_eligible
+
+
+def _context_window_error() -> ContextWindowExceededError:
+    return ContextWindowExceededError(
+        message="Your input exceeds the context window of this model.",
+        model="gpt-6-luna",
+        llm_provider="openai",
+    )
+
+
+def _bad_request(status_code: int | None = 400) -> BadRequestError:
+    error = BadRequestError(
+        message="Invalid value for 'temperature'.",
+        model="gpt-6-luna",
+        llm_provider="openai",
+    )
+    error.status_code = status_code  # type: ignore[misc]
+    return error
+
+
+# --- Responses API streams (litellm/responses/streaming_iterator.py) -------------------
+
+
+ROUTER_METADATA = {"model_info": {"id": "deployment-1"}}
+
+
+def test_context_window_error_is_fallback_eligible_on_responses_stream() -> None:
+    assert _mid_stream_fallback_eligible(_context_window_error(), ROUTER_METADATA) is True
+
+
+def test_context_window_error_is_not_eligible_for_a_direct_sdk_call() -> None:
+    """No deployment means no larger window to try; the caller keeps the 400 (#15785)."""
+    assert _mid_stream_fallback_eligible(_context_window_error()) is False
+    assert _mid_stream_fallback_eligible(_context_window_error(), {"model_info": {}}) is False
+
+
+def test_plain_bad_request_still_skips_responses_fallback() -> None:
+    assert _mid_stream_fallback_eligible(_bad_request()) is False
+
+
+@pytest.mark.parametrize("status_code", [None, 429, 500, 503])
+def test_transient_and_server_errors_stay_fallback_eligible(status_code: int | None) -> None:
+    assert _mid_stream_fallback_eligible(_bad_request(status_code)) is True
+
+
+# --- Chat completion streams (litellm/litellm_core_utils/streaming_handler.py) ---------
+
+
+def test_context_window_error_is_not_terminal_on_chat_stream() -> None:
+    error = _context_window_error()
+    assert _mid_stream_error_is_terminal(
+        error, error.status_code, error.status_code, router_fallback_available=True
+    ) is False
+
+
+def test_context_window_error_is_terminal_for_a_direct_sdk_call() -> None:
+    error = _context_window_error()
+    assert _mid_stream_error_is_terminal(error, error.status_code, error.status_code) is True
+
+
+def test_context_window_error_is_not_terminal_even_when_envelope_reports_400() -> None:
+    """The original exception can be the only one carrying the context-window type."""
+    assert _mid_stream_error_is_terminal(_bad_request(400), 400, 400) is True
+    assert _mid_stream_error_is_terminal(
+        _context_window_error(), 400, 400, router_fallback_available=True
+    ) is False
+
+
+def test_plain_bad_request_is_terminal_on_chat_stream() -> None:
+    assert _mid_stream_error_is_terminal(_bad_request(), 400, 400) is True
+
+
+@pytest.mark.parametrize("status_code", [429, 500, 503])
+def test_retriable_status_codes_are_not_terminal_on_chat_stream(status_code: int) -> None:
+    error = _bad_request(status_code)
+    assert _mid_stream_error_is_terminal(error, status_code, status_code) is False

tests/unit/responses/test_streaming_iterator_error_events.py

@@ -55,7 +55,7 @@ from litellm.types.llms.openai import (
 )
 
 
-def _make_iterator() -> BaseResponsesAPIStreamingIterator:
+def _make_iterator(litellm_metadata: Mapping[str, object] | None = None) -> BaseResponsesAPIStreamingIterator:
     mock_logging_obj = Mock(spec=LiteLLMLoggingObj)
     mock_logging_obj.model_call_details = {"litellm_params": {}}
     mock_config = Mock(spec=BaseResponsesAPIConfig)
@@ -67,6 +67,7 @@ def _make_iterator() -> BaseResponsesAPIStreamingIterator:
         responses_api_provider_config=mock_config,
         logging_obj=mock_logging_obj,
         custom_llm_provider="openai",
+        litellm_metadata=litellm_metadata,
     )
 
 
@@ -881,3 +882,34 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se
 
     assert failure is not None
     _assert_names_the_azure_deployment(failure, reported_api_base)
+
+
+def test_context_window_error_reaches_fallback_only_under_the_router() -> None:
+    """The exemption is the Router's, not the exception's.
+
+    The case above pins the direct-SDK contract (#15785): no deployment means the 400 is
+    the answer. When the Router resolved a deployment, the same event must be wrapped so
+    `context_window_fallbacks` can route it to a larger window (#48887).
+    """
+    iterator: Final = _make_iterator(litellm_metadata={"model_info": {"id": "deployment-1"}})
+    chunk: Final = _make_error_chunk(
+        "invalid_request_error",
+        "context_length_exceeded",
+        "Your input exceeds the context window of this model.",
+    )
+    with pytest.raises(MidStreamFallbackError) as exc_info:
+        iterator._maybe_raise_for_error_event(chunk)
+    assert isinstance(exc_info.value.original_exception, litellm.ContextWindowExceededError)
+
+
+def test_empty_model_info_does_not_count_as_a_router_deployment() -> None:
+    """`model_info` present but empty is what a caller can pass by hand; it selects nothing."""
+    iterator: Final = _make_iterator(litellm_metadata={"model_info": {}})
+    chunk: Final = _make_error_chunk(
+        "invalid_request_error",
+        "context_length_exceeded",
+        "Your input exceeds the context window of this model.",
+    )
+    with pytest.raises(litellm.ContextWindowExceededError) as exc_info:
+        iterator._maybe_raise_for_error_event(chunk)
+    assert not isinstance(exc_info.value, MidStreamFallbackError)