scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(responses): keep context-window events out of mid-stream fallback and fix stale exception assertions

Browse source
This commit is contained in:
mateo-berri 2026-09-13 03:18:59 -07:00
parent c246372859
commit fff7a2cecf
5 changed files with 20 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/responses/streaming_iterator.py
View file

@ -222,7 +222,7 @@ def _status_code_for_error_fields(error_type: str | None, error_code: str | None
def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
if isinstance(mapped_exception, (litellm.ContentPolicyViolationError, litellm.ContextWindowExceededError)):
if isinstance(mapped_exception, litellm.ContentPolicyViolationError):
return True
status_code: Final = getattr(mapped_exception, "status_code", None)
return not isinstance(status_code, int) or status_code >= 500 or status_code == 429

7
litellm/router.py
View file

@ -3271,12 +3271,11 @@ class Router:
kwargs=initial_kwargs,
metadata_variable_name="litellm_metadata",
)
# The content-policy dispatch branch matches on the trigger's own type, so a refusal's
# MidStreamFallbackError envelope is unwrapped here or the wrong fallback list is consulted.
fallback_trigger: Final[Exception] = (
e.original_exception
if isinstance(
e.original_exception,
(litellm.ContentPolicyViolationError, litellm.ContextWindowExceededError),
)
if isinstance(e.original_exception, litellm.ContentPolicyViolationError)
else e
)
fallback_response = await self.async_function_with_fallbacks_common_utils(

9
tests/llm_responses_api_testing/test_openai_responses_api.py
View file

@ -1627,9 +1627,10 @@ async def test_openai_responses_api_token_limit_error():
Parsing the in-stream ErrorEvent must not raise
"pydantic_core._pydantic_core.ValidationError: 3 validation errors for ErrorEvent".
The iterator now surfaces the event as litellm.APIError with status 400
(invalid_request_error is a non-retriable client error, so no
MidStreamFallbackError wrapping) carrying the provider's message.
The iterator routes the event through litellm.exception_type, so it surfaces as
the typed 400 client error the non-streaming path raises (litellm.BadRequestError)
carrying the provider's message. invalid_request_error is a non-retriable client
error, so there is no MidStreamFallbackError wrapping.
"""
litellm._turn_on_debug()
@ -1644,7 +1645,7 @@ async def test_openai_responses_api_token_limit_error():
async for event in response:
print(event)
with pytest.raises(litellm.APIError) as exc_info:
with pytest.raises(litellm.BadRequestError) as exc_info:
await _drain()
assert exc_info.value.status_code == 400

2
tests/router_unit_tests/test_router_aresponses_streaming_fallback.py
View file

@ -372,7 +372,7 @@ async def test_aresponses_fallback_on_in_stream_error_event():
raised = mock_fallback.await_args.kwargs["e"]
assert isinstance(raised, MidStreamFallbackError)
assert raised.status_code == 429
assert isinstance(raised.original_exception, litellm.APIError)
assert isinstance(raised.original_exception, litellm.RateLimitError)
assert raised.original_exception.status_code == 429
assert mock_fallback.await_args.kwargs["kwargs"]["input"] == "original question"

18
tests/test_litellm/responses/test_streaming_iterator_error_events.py
View file

@ -4,11 +4,11 @@ raise instead of being returned as benign chunks, mirroring chat streaming
semantics (_handle_stream_fallback_error). The event's code, type and status go
through litellm.exception_type, so each event raises the same typed exception
the non-streaming path raises for that provider error: non-retriable 4xx
(except 429) raise that typed exception directly, while 429, 5xx,
ContentPolicyViolationError and ContextWindowExceededError are wrapped in
(except 429) raise that typed exception directly, so a context-length event
surfaces as ContextWindowExceededError(400) with no MidStreamFallbackError
wrapping, while 429, 5xx and ContentPolicyViolationError are wrapped in
MidStreamFallbackError so the Router's mid-stream fallback machinery fires and
its content_policy_fallbacks / context_window_fallbacks dispatch sees the
trigger it matches on.
its content_policy_fallbacks dispatch sees the trigger it matches on.
Status mapping must consider both the OpenAI error `type` (e.g.
"invalid_request_error") and `code` (e.g. "invalid_prompt",
@ -110,19 +110,21 @@ def test_maybe_raise_for_error_event_maps_context_length_code_to_400():
assert not isinstance(exc_info.value, MidStreamFallbackError)
def test_maybe_raise_for_error_event_wraps_context_window_exceeded_for_context_window_fallbacks():
def test_maybe_raise_for_error_event_raises_context_window_exceeded_directly():
"""A context-length error event maps to ContextWindowExceededError exactly like the non-streaming
path and is wrapped so the Router's context_window_fallbacks dispatch fires mid-stream."""
path and, being a non-retriable client error, is raised directly rather than wrapped for mid-stream
fallback, preserving the direct-SDK 400 contract from issue #15785."""
iterator = _make_iterator()
chunk = _make_error_chunk(
"invalid_request_error",
"context_length_exceeded",
"This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.",
)
with pytest.raises(MidStreamFallbackError) as exc_info:
with pytest.raises(litellm.ContextWindowExceededError) as exc_info:
iterator._maybe_raise_for_error_event(chunk)
assert isinstance(exc_info.value.original_exception, litellm.ContextWindowExceededError)
assert exc_info.value.status_code == 400
assert not isinstance(exc_info.value, MidStreamFallbackError)
assert "maximum context length" in str(exc_info.value)
CONTENT_POLICY_MESSAGE = "This content was flagged for possible cybersecurity risk. The response was halted mid-stream."