Fix: trigger fallbacks on mid-stream httpx.TimeoutException

The async CustomStreamWrapper.__anext__ caught httpx.TimeoutException
and re-raised it raw, bypassing _handle_stream_fallback_error. The
Router's FallbackStreamWrapper only reacts to MidStreamFallbackError,
so stream_timeout firing mid-stream never triggered fallbacks the way
connection-phase timeout does.

Route timeouts through _handle_stream_fallback_error like every other
exception so they wrap into MidStreamFallbackError and the Router can
switch to a fallback model.
This commit is contained in:
Ryan Crabbe 2026-05-01 11:46:17 -07:00
parent ebbe2f49ff
commit 35133e3550
No known key found for this signature in database
2 changed files with 34 additions and 1 deletions

View file

@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception)
)
raise e
self._handle_stream_fallback_error(e)
except Exception as e:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:

View file

@ -878,6 +878,39 @@ def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging):
assert "invalid maxOutputTokens" in str(excinfo.value)
@pytest.mark.asyncio
async def test_async_streaming_read_timeout_triggers_midstream_fallback(
logging_obj: Logging,
):
"""A mid-stream httpx.ReadTimeout must wrap into MidStreamFallbackError so
the Router's FallbackStreamWrapper can switch to a fallback model.
Previously __anext__ caught httpx.TimeoutException and re-raised it raw,
which bypassed _handle_stream_fallback_error and prevented stream_timeout
from triggering fallbacks the way connection-phase timeout does.
"""
import httpx
from litellm.exceptions import MidStreamFallbackError
async def _raise_read_timeout(**kwargs):
raise httpx.ReadTimeout("Timeout on reading data from socket")
response = CustomStreamWrapper(
completion_stream=None,
model="gpt-4",
logging_obj=logging_obj,
custom_llm_provider="openai",
make_call=_raise_read_timeout,
)
with pytest.raises(MidStreamFallbackError) as excinfo:
await response.__anext__()
assert excinfo.value.is_pre_first_chunk is True
assert isinstance(excinfo.value.original_exception, Exception)
def test_streaming_handler_with_created_time_propagation(
initialized_custom_stream_wrapper: CustomStreamWrapper, logging_obj: Logging
):