scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix(router): hold streamed /v1/messages frames for an overflow only on context-window-only groups

Browse source 
A group with only context_window_fallbacks now lets every non-overflow error before content reach the
client after message_start, as it did before the hold. qwen's Mantle overflow keeps the prompt is too
long prefix only when the requested output leaves room for a prompt, and matches its repr form.
This commit is contained in:
mateo-berri 2026-10-30 23:34:46 -07:00
parent 4545a0c785
commit 9de1f570f8
4 changed files with 121 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -936,6 +936,9 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = (
re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"),
re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"),
)
_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile(
r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens"
)
_BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile(
r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']"""
)
@ -968,13 +971,19 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded
code, or in a wording the other providers share, either in a 400 body or in a streamed error
event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow
(litellm/litellm#36546).
(litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested
output, so only the output budget says whether a shorter prompt could fit at all; when it cannot,
the overflow keeps Mantle's own message so the client is not told to shorten the prompt.
"""
matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS)
match: Final = next((found for found in matches if found is not None), None)
if match is not None:
prompt_tokens, max_tokens = match.groups()
return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str)
if output_budget is not None:
context_window, requested_output_tokens = (int(group) for group in output_budget.groups())
return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str
if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(
error_str
) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str):

38
litellm/router.py
View file

@ -861,6 +861,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error
return has_generated_content or not error.is_pre_first_chunk
def _anthropic_stream_error_skips_recovery(
error: Exception, has_generated_content: bool, holds_for_overflow_only: bool
) -> bool:
"""
A held stream whose only possible recovery is a context window fallback lets every other failure through
with its held frames, as it would have reached the client with no hold.
"""
from litellm.exceptions import MidStreamFallbackError
if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
has_generated_content, error
):
return True
if not holds_for_overflow_only:
return False
underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error
return not isinstance(underlying, litellm.ContextWindowExceededError)
def _stream_raised_error_status(error: Exception) -> int | None:
raw_status: Final = getattr(error, "status_code", None)
if isinstance(raw_status, int):
@ -6936,9 +6955,14 @@ class Router(DeprecatedRouterMembers):
# to take over there is nothing to buffer for, so every frame,
# including pings and provider error frames, is forwarded live.
model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry(
initial_kwargs
) or self._stream_can_fall_back(model, initial_kwargs)
holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow(
model, initial_kwargs
)
has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over
self._anthropic_messages_stream_can_retry(initial_kwargs)
or self._stream_can_fall_back_on_overflow(model, initial_kwargs)
can_retry_or_fall_back or holds_for_overflow_only
)
buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline
try:
@ -7000,6 +7024,7 @@ class Router(DeprecatedRouterMembers):
async for item in self._aanthropic_messages_recover_stream_error(
stream_error,
has_generated_content,
holds_for_overflow_only,
buffered_lifecycle_chunks,
model,
initial_kwargs,
@ -7020,6 +7045,7 @@ class Router(DeprecatedRouterMembers):
self,
stream_error: Exception,
has_generated_content: bool,
holds_for_overflow_only: bool,
buffered_lifecycle_chunks: tuple[bytes, ...],
model: str,
initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it
@ -7029,12 +7055,10 @@ class Router(DeprecatedRouterMembers):
"""Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
from litellm.exceptions import MidStreamFallbackError
if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
has_generated_content, stream_error
):
if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only):
for buffered_chunk in buffered_lifecycle_chunks:
yield buffered_chunk
if stream_error.original_exception is not None:
if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None:
raise stream_error.original_exception from stream_error
raise stream_error
fallback_error: Final = (
@ -7180,7 +7204,7 @@ class Router(DeprecatedRouterMembers):
context_window_fallbacks=_request_fallback_list(
kwargs,
"context_window_fallbacks",
cast( # cast-ok: untyped router attribute
cast( # cast-ok: list is invariant and the helper takes list[object]
"list[object] | None", self._core_context_window_fallbacks
),
),

51
tests/unit/litellm_core_utils/test_exception_mapping_utils.py
View file

@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
import httpx
import openai
import pytest
from _pytest.mark.structures import ParameterSet
from fastapi import HTTPException
from openai import AsyncOpenAI, OpenAI
@ -1824,17 +1825,19 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex
assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message
_QWEN_OVERFLOW_EVENT: Final = (
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum '
"context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least "
"32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the "
'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }'
)
def _qwen_overflow_event(requested_output_tokens: int) -> str:
counted_input_tokens: Final = 32768 - requested_output_tokens + 1
return (
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum '
f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your "
f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please "
"reduce the length of the input prompt or the number of requested output tokens. "
f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}'
)
@pytest.mark.parametrize(
"original_exception",
[
def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]:
return (
pytest.param(
BaseLLMException(
status_code=400,
@ -1842,7 +1845,7 @@ _QWEN_OVERFLOW_EVENT: Final = (
{
"error": {
"code": "validation_error",
"message": _QWEN_OVERFLOW_EVENT,
"message": event,
"param": None,
"type": "invalid_request_error",
}
@ -1851,9 +1854,15 @@ _QWEN_OVERFLOW_EVENT: Final = (
),
id="400-body",
),
pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"),
],
)
pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"),
pytest.param(
BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"),
id="python-repr",
),
)
@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16)))
def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded(
original_exception: BaseLLMException,
) -> None:
@ -1869,6 +1878,22 @@ def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_win
assert "prompt is too long" in excinfo.value.message
@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768)))
def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message(
original_exception: BaseLLMException,
) -> None:
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
exception_type(
model="qwen.qwen3-32b",
original_exception=original_exception,
custom_llm_provider="bedrock_mantle",
)
assert excinfo.value.status_code == 400
assert "prompt is too long" not in excinfo.value.message
assert "you requested 32768 output tokens" in excinfo.value.message
def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded():
from litellm.llms.base_llm.chat.transformation import BaseLLMException

44
tests/unit/test_router/test_router.py
View file

@ -19313,6 +19313,39 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_
assert len(provider.calls) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("raised", "expected_error"),
[
pytest.param(
litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"),
litellm.ServiceUnavailableError,
id="raised-503",
),
pytest.param(
litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"),
litellm.BadRequestError,
id="raised-400",
),
pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"),
],
)
async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start(
raised: Exception, expected_error: type[Exception]
) -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
provider: Final = _AnthropicMessagesScriptedProvider(
lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised)
)
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
assert isinstance(outcome[-1], expected_error)
assert len(provider.calls) == 1
@pytest.mark.parametrize(
("context_window_fallbacks", "request_kwargs", "expected"),
[
@ -20210,7 +20243,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_
with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt:
recovery = router._aanthropic_messages_recover_stream_error(
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
declined,
True,
False,
buffered,
"primary",
{"model": "primary"},
_anthropic_messages_make_wrapper(),
AsyncMock(),
)
with pytest.raises(BedrockError) as exc_info:
await drain(recovery)
@ -20231,7 +20271,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to
with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt):
recovery = router._aanthropic_messages_recover_stream_error(
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
)
collected = [chunk async for chunk in recovery]
assert collected == [b"fallback"]