From 9de1f570f8d73ed4c88dcc89b79f12ee6348e74f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:34:46 -0700 Subject: [PATCH] fix(router): hold streamed /v1/messages frames for an overflow only on context-window-only groups A group with only context_window_fallbacks now lets every non-overflow error before content reach the client after message_start, as it did before the hold. qwen's Mantle overflow keeps the prompt is too long prefix only when the requested output leaves room for a prompt, and matches its repr form. --- .../exception_mapping_utils.py | 11 +++- litellm/router.py | 38 +++++++++++--- .../test_exception_mapping_utils.py | 51 ++++++++++++++----- tests/unit/test_router/test_router.py | 44 +++++++++++++++- 4 files changed, 121 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 4a062ebd132..5e16563ae26 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -936,6 +936,9 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), ) +_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile( + r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens" +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -968,13 +971,19 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded code, or in a wording the other providers share, either in a 400 body or in a streamed error event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow - (litellm/litellm#36546). + (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested + output, so only the output budget says whether a shorter prompt could fit at all; when it cannot, + the overflow keeps Mantle's own message so the client is not told to shorten the prompt. """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" + output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str) + if output_budget is not None: + context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) + return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( error_str ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): diff --git a/litellm/router.py b/litellm/router.py index 0c3975ebf61..67b7400f411 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -861,6 +861,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error return has_generated_content or not error.is_pre_first_chunk +def _anthropic_stream_error_skips_recovery( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool +) -> bool: + """ + A held stream whose only possible recovery is a context window fallback lets every other failure through + with its held frames, as it would have reached the client with no hold. + """ + from litellm.exceptions import MidStreamFallbackError + + if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, error + ): + return True + if not holds_for_overflow_only: + return False + underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error + return not isinstance(underlying, litellm.ContextWindowExceededError) + + def _stream_raised_error_status(error: Exception) -> int | None: raw_status: Final = getattr(error, "status_code", None) if isinstance(raw_status, int): @@ -6936,9 +6955,14 @@ class Router(DeprecatedRouterMembers): # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group + can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry( + initial_kwargs + ) or self._stream_can_fall_back(model, initial_kwargs) + holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow( + model, initial_kwargs + ) has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over - self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back_on_overflow(model, initial_kwargs) + can_retry_or_fall_back or holds_for_overflow_only ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -7000,6 +7024,7 @@ class Router(DeprecatedRouterMembers): async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, + holds_for_overflow_only, buffered_lifecycle_chunks, model, initial_kwargs, @@ -7020,6 +7045,7 @@ class Router(DeprecatedRouterMembers): self, stream_error: Exception, has_generated_content: bool, + holds_for_overflow_only: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it @@ -7029,12 +7055,10 @@ class Router(DeprecatedRouterMembers): """Turns a source-iterator failure into a fallback attempt or the error reaching the caller.""" from litellm.exceptions import MidStreamFallbackError - if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( - has_generated_content, stream_error - ): + if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk - if stream_error.original_exception is not None: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: raise stream_error.original_exception from stream_error raise stream_error fallback_error: Final = ( @@ -7180,7 +7204,7 @@ class Router(DeprecatedRouterMembers): context_window_fallbacks=_request_fallback_list( kwargs, "context_window_fallbacks", - cast( # cast-ok: untyped router attribute + cast( # cast-ok: list is invariant and the helper takes list[object] "list[object] | None", self._core_context_window_fallbacks ), ), diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index f0a062308fc..be4927cf26f 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import openai import pytest +from _pytest.mark.structures import ParameterSet from fastapi import HTTPException from openai import AsyncOpenAI, OpenAI @@ -1824,17 +1825,19 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message -_QWEN_OVERFLOW_EVENT: Final = ( - 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' - "context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least " - "32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the " - 'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }' -) +def _qwen_overflow_event(requested_output_tokens: int) -> str: + counted_input_tokens: Final = 32768 - requested_output_tokens + 1 + return ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your " + f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please " + "reduce the length of the input prompt or the number of requested output tokens. " + f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}' + ) -@pytest.mark.parametrize( - "original_exception", - [ +def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]: + return ( pytest.param( BaseLLMException( status_code=400, @@ -1842,7 +1845,7 @@ _QWEN_OVERFLOW_EVENT: Final = ( { "error": { "code": "validation_error", - "message": _QWEN_OVERFLOW_EVENT, + "message": event, "param": None, "type": "invalid_request_error", } @@ -1851,9 +1854,15 @@ _QWEN_OVERFLOW_EVENT: Final = ( ), id="400-body", ), - pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"), - ], -) + pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"), + pytest.param( + BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"), + id="python-repr", + ), + ) + + +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16))) def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( original_exception: BaseLLMException, ) -> None: @@ -1869,6 +1878,22 @@ def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_win assert "prompt is too long" in excinfo.value.message +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768))) +def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "you requested 32768 output tokens" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index bbf706d079d..147880ee6b8 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19313,6 +19313,39 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_ assert len(provider.calls) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raised", "expected_error"), + [ + pytest.param( + litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"), + litellm.ServiceUnavailableError, + id="raised-503", + ), + pytest.param( + litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"), + litellm.BadRequestError, + id="raised-400", + ), + pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"), + ], +) +async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start( + raised: Exception, expected_error: type[Exception] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised) + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], expected_error) + assert len(provider.calls) == 1 + + @pytest.mark.parametrize( ("context_window_fallbacks", "request_kwargs", "expected"), [ @@ -20210,7 +20243,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_ with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt: recovery = router._aanthropic_messages_recover_stream_error( - declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + declined, + True, + False, + buffered, + "primary", + {"model": "primary"}, + _anthropic_messages_make_wrapper(), + AsyncMock(), ) with pytest.raises(BedrockError) as exc_info: await drain(recovery) @@ -20231,7 +20271,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt): recovery = router._aanthropic_messages_recover_stream_error( - raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) collected = [chunk async for chunk in recovery] assert collected == [b"fallback"]