mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
Fix: trigger fallbacks on mid-stream httpx.TimeoutException
The async CustomStreamWrapper.__anext__ caught httpx.TimeoutException and re-raised it raw, bypassing _handle_stream_fallback_error. The Router's FallbackStreamWrapper only reacts to MidStreamFallbackError, so stream_timeout firing mid-stream never triggered fallbacks the way connection-phase timeout does. Route timeouts through _handle_stream_fallback_error like every other exception so they wrap into MidStreamFallbackError and the Router can switch to a fallback model.
This commit is contained in:
parent
ebbe2f49ff
commit
35133e3550
2 changed files with 34 additions and 1 deletions
|
|
@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
|
|||
asyncio.create_task(
|
||||
self.logging_obj.async_failure_handler(e, traceback_exception)
|
||||
)
|
||||
raise e
|
||||
self._handle_stream_fallback_error(e)
|
||||
except Exception as e:
|
||||
traceback_exception = traceback.format_exc()
|
||||
if self.logging_obj is not None:
|
||||
|
|
|
|||
|
|
@ -878,6 +878,39 @@ def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging):
|
|||
assert "invalid maxOutputTokens" in str(excinfo.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_streaming_read_timeout_triggers_midstream_fallback(
|
||||
logging_obj: Logging,
|
||||
):
|
||||
"""A mid-stream httpx.ReadTimeout must wrap into MidStreamFallbackError so
|
||||
the Router's FallbackStreamWrapper can switch to a fallback model.
|
||||
|
||||
Previously __anext__ caught httpx.TimeoutException and re-raised it raw,
|
||||
which bypassed _handle_stream_fallback_error and prevented stream_timeout
|
||||
from triggering fallbacks the way connection-phase timeout does.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
async def _raise_read_timeout(**kwargs):
|
||||
raise httpx.ReadTimeout("Timeout on reading data from socket")
|
||||
|
||||
response = CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="gpt-4",
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="openai",
|
||||
make_call=_raise_read_timeout,
|
||||
)
|
||||
|
||||
with pytest.raises(MidStreamFallbackError) as excinfo:
|
||||
await response.__anext__()
|
||||
|
||||
assert excinfo.value.is_pre_first_chunk is True
|
||||
assert isinstance(excinfo.value.original_exception, Exception)
|
||||
|
||||
|
||||
def test_streaming_handler_with_created_time_propagation(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper, logging_obj: Logging
|
||||
):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue