From 8cfb20122e162830625c0dc2354e21cb60af7da8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:04:48 -0700 Subject: [PATCH 1/5] fix(bedrock_mantle): map gpt-oss context overflow to ContextWindowExceededError Bedrock Mantle's gpt-oss models report a too-long prompt as "Input length (N) exceeds model's maximum context length (M)", which the Mantle mapping missed, so callers got a BadRequestError (or a 503 when streaming), context window fallbacks never fired, and Claude Code never compacted. Match that wording too and return the existing "prompt is too long: N tokens > M maximum" message. --- .../exception_mapping_utils.py | 16 ++++--- .../test_exception_mapping_utils.py | 47 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47ec8331c83..c37db350c6d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -892,7 +892,10 @@ def _map_openai_like_exception( ) -_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)") +_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( + re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"), + re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -912,12 +915,13 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ - Mantle reports context overflow as a validation_error carrying the token counts, or as - OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event - with no error type. Clients such as Claude Code only treat "prompt is too long" as - overflow (litellm/litellm#36546). + Mantle reports context overflow as a validation_error carrying the token counts (worded + differently for gpt-oss), or as OpenAI's context_length_exceeded code, either in a 400 body + or in a streamed error event with no error type. Clients such as Claude Code only treat + "prompt is too long" as overflow (litellm/litellm#36546). """ - match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str) + matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) + match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 55e7c47e2c9..1a96a090f3b 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1491,6 +1491,53 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message +_GPT_OSS_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' + "message: \"Input length (360068) exceeds model's maximum context length (131072).\", param: None } }" +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _GPT_OSS_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), + pytest.param( + BaseLLMException( + status_code=400, message=f"Error code: 400 - {repr({'error': {'message': _GPT_OSS_OVERFLOW_EVENT}})}" + ), + id="python-repr", + ), + ], +) +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(original_exception): + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="openai.gpt-oss-20b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From 66a11a6287f6c63bca90f9de82e2905824deeda2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:49:59 -0700 Subject: [PATCH 2/5] fix(bedrock_mantle): map the gpt-oss Responses API overflow wording too Mantle's /v1/responses words gpt-oss overflow as "The engine prompt length N exceeds the max_model_len M", so a streamed /v1/responses overflow still ended in a response.failed carrying a plain BadRequestError. Match that wording as well. --- .../exception_mapping_utils.py | 7 ++++--- .../test_exception_mapping_utils.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index c37db350c6d..6aac1918993 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -895,6 +895,7 @@ def _map_openai_like_exception( _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"), re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), + re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), ) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" @@ -916,9 +917,9 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ Mantle reports context overflow as a validation_error carrying the token counts (worded - differently for gpt-oss), or as OpenAI's context_length_exceeded code, either in a 400 body - or in a streamed error event with no error type. Clients such as Claude Code only treat - "prompt is too long" as overflow (litellm/litellm#36546). + differently for gpt-oss, and again on its Responses API), or as OpenAI's context_length_exceeded + code, either in a 400 body or in a streamed error event with no error type. Clients such as + Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 1a96a090f3b..4ac7c644cda 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1538,6 +1538,27 @@ def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exc assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded(): + from litellm.responses.streaming_iterator import map_stream_error_to_exception + + mapped_exception = map_stream_error_to_exception( + { + "code": "invalid_prompt", + "message": ( + 'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: ' + '"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", ' + 'param: Some("input") } }' + ), + }, + model="openai.gpt-oss-20b", + custom_llm_provider="bedrock_mantle", + ) + + assert isinstance(mapped_exception, litellm.ContextWindowExceededError) + assert mapped_exception.status_code == 400 + assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From fdb163e51c9c8c12e3b09ca96b59253da90fd901 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:17:03 -0700 Subject: [PATCH 3/5] fix(router): send a streamed /v1/messages context overflow before content to context_window_fallbacks A streamed /v1/messages request whose overflow arrives before any content now holds its lifecycle frames and takes context_window_fallbacks or the plain fallbacks, the same way the non-streamed route and the streamed chat and Responses routes do. Bedrock Mantle's qwen overflow wording ("This model's maximum context length is ...") also maps to ContextWindowExceededError with the generic prompt is too long message --- .../exception_mapping_utils.py | 11 ++-- litellm/router.py | 30 ++++++++- ..._mantle_context_overflow_fallbacks_wire.py | 18 ++--- .../test_exception_mapping_utils.py | 57 ++++++++++++++-- tests/unit/test_router/test_router.py | 65 +++++++++++++++++++ 5 files changed, 156 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 6aac1918993..ec4e58fda6b 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -917,16 +917,19 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ Mantle reports context overflow as a validation_error carrying the token counts (worded - differently for gpt-oss, and again on its Responses API), or as OpenAI's context_length_exceeded - code, either in a 400 body or in a streamed error event with no error type. Clients such as - Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). + differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded + code, or in a wording the other providers share, either in a 400 body or in a streamed error + event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow + (litellm/litellm#36546). """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" - if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: + if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( + error_str + ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE return None diff --git a/litellm/router.py b/litellm/router.py index fc888495e99..7e1cc569504 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -821,9 +821,15 @@ def _stream_raised_error_status(error: Exception) -> int | None: def _stream_fallback_error_for_raised( error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" ) -> "MidStreamFallbackError | None": - """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + """ + The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain + sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates + unchanged. + """ if has_generated_content: return None + if isinstance(error, litellm.ContextWindowExceededError): + return _stream_pre_content_error(error, model, llm_provider) status_code: Final = _stream_raised_error_status(error) if status_code is None: return _stream_pre_content_error(error, model, llm_provider) @@ -1628,7 +1634,9 @@ class Router: else: self.fallbacks = [{"*": _fallbacks}] - self.context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks + self.context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolver contract + context_window_fallbacks or litellm.context_window_fallbacks + ) _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) @@ -6611,7 +6619,7 @@ class Router: model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back(model, initial_kwargs) + or self._stream_can_fall_back_on_overflow(model, initial_kwargs) ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -10025,6 +10033,22 @@ class Router: ) ) + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks + can serve as well as every route _stream_can_fall_back counts.""" + if fallbacks_disabled_for_request(kwargs): + return False + if self._stream_can_fall_back(model_group, kwargs): + return True + context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) + if not context_window_fallbacks: + return False + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=context_window_fallbacks, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return has_unattempted_fallback_target(resolved, kwargs) + def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: """ Whether async_function_with_fallbacks_common_utils could still route a diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 37aa96c2467..e069dfc510e 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet _assert_fell_back(cwf_rig, "overflow", prompt, text) -def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None: +def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None: prompt: Final = _prompt("cwf messages stream") response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True)) assert response.status_code == 200, text - error: Final = _only_error(text) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(cwf_rig, "overflow", prompt) + _assert_fell_back(cwf_rig, "overflow", prompt, text) def test_context_window_fallback_does_not_fire_on_streamed_responses(cwf_rig: Rig) -> None: @@ -455,16 +452,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r _assert_fell_back(fb_rig, "overflow", prompt, text) -def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back( - fb_rig: Rig, -) -> None: +def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None: prompt: Final = _prompt("fb messages stream") response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True)) - assert response.status_code == 400, text - error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"]) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(fb_rig, "overflow", prompt) + assert response.status_code == 200, text + _assert_fell_back(fb_rig, "overflow", prompt, text) def test_plain_fallback_does_not_fire_on_streamed_responses(fb_rig: Rig) -> None: diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 4ac7c644cda..2388713acf8 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1493,7 +1493,7 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): _GPT_OSS_OVERFLOW_EVENT: Final = ( 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' - "message: \"Input length (360068) exceeds model's maximum context length (131072).\", param: None } }" + 'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }' ) @@ -1519,13 +1519,15 @@ _GPT_OSS_OVERFLOW_EVENT: Final = ( pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), pytest.param( BaseLLMException( - status_code=400, message=f"Error code: 400 - {repr({'error': {'message': _GPT_OSS_OVERFLOW_EVENT}})}" + status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}" ), id="python-repr", ), ], ) -def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(original_exception): +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: with pytest.raises(litellm.ContextWindowExceededError) as excinfo: exception_type( model="openai.gpt-oss-20b", @@ -1538,10 +1540,10 @@ def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exc assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message -def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded(): +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None: from litellm.responses.streaming_iterator import map_stream_error_to_exception - mapped_exception = map_stream_error_to_exception( + mapped_exception: Final = map_stream_error_to_exception( { "code": "invalid_prompt", "message": ( @@ -1559,6 +1561,51 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message +_QWEN_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + "context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least " + "32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the " + 'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }' +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _QWEN_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"), + ], +) +def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 300c9cff6ff..086253039ec 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18970,6 +18970,71 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" +def _anthropic_messages_overflow_before_content(): + return _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" + ), + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback(): + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs"), + [ + pytest.param(None, {}, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"), + ], +) +async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], litellm.ContextWindowExceededError) + assert len(provider.calls) == 1 + + +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs", "expected"), + [ + pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"), + pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), + pytest.param(None, {}, False, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + ], +) +def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + + assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected + + def test_anthropic_messages_retry_budget_precedence_direct_call(): """A retry policy naming the error class outranks the request's num_retries, which outranks the routed deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too.""" From 4545a0c785be0f7b30d64cd1bd6c4d7f4250df85 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:29:48 -0700 Subject: [PATCH 4/5] fix(router): validate the context_window_fallbacks chain in the streamed overflow hold check --- litellm/router.py | 16 +++++++++++----- tests/unit/test_router/test_router.py | 5 +++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index fcd7d89a2b3..0c3975ebf61 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -680,6 +680,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams ) @@ -10383,18 +10384,23 @@ class Router(DeprecatedRouterMembers): ) ) - def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool: """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks - can serve as well as every route _stream_can_fall_back counts.""" + can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists + errs toward True, as _stream_can_fall_back does for non-standard fallbacks.""" if fallbacks_disabled_for_request(kwargs): return False if self._stream_can_fall_back(model_group, kwargs): return True - context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) - if not context_window_fallbacks: + configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) + if not configured: return False + try: + chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured) + except ValidationError: + return True resolved, _ = get_fallback_model_group_for_lookup_groups( - fallbacks=context_window_fallbacks, + fallbacks=chain, lookup_groups=fallback_lookup_groups(kwargs, model_group), ) return has_unattempted_fallback_target(resolved, kwargs) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 2d91f2a02c0..bbf706d079d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19265,7 +19265,7 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" -def _anthropic_messages_overflow_before_content(): +def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream: return _AnthropicMessagesRaisingByteStream( [_anthropic_messages_message_start_chunk()], litellm.ContextWindowExceededError( @@ -19275,7 +19275,7 @@ def _anthropic_messages_overflow_before_content(): @pytest.mark.asyncio -async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback(): +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None: router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) provider: Final = _AnthropicMessagesScriptedProvider( _anthropic_messages_overflow_before_content, @@ -19320,6 +19320,7 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_ pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), pytest.param(None, {}, False, id="no-fallback-configured"), pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"), ], ) def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( From 9de1f570f8d73ed4c88dcc89b79f12ee6348e74f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:34:46 -0700 Subject: [PATCH 5/5] fix(router): hold streamed /v1/messages frames for an overflow only on context-window-only groups A group with only context_window_fallbacks now lets every non-overflow error before content reach the client after message_start, as it did before the hold. qwen's Mantle overflow keeps the prompt is too long prefix only when the requested output leaves room for a prompt, and matches its repr form. --- .../exception_mapping_utils.py | 11 +++- litellm/router.py | 38 +++++++++++--- .../test_exception_mapping_utils.py | 51 ++++++++++++++----- tests/unit/test_router/test_router.py | 44 +++++++++++++++- 4 files changed, 121 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 4a062ebd132..5e16563ae26 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -936,6 +936,9 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), ) +_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile( + r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens" +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -968,13 +971,19 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded code, or in a wording the other providers share, either in a 400 body or in a streamed error event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow - (litellm/litellm#36546). + (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested + output, so only the output budget says whether a shorter prompt could fit at all; when it cannot, + the overflow keeps Mantle's own message so the client is not told to shorten the prompt. """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" + output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str) + if output_budget is not None: + context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) + return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( error_str ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): diff --git a/litellm/router.py b/litellm/router.py index 0c3975ebf61..67b7400f411 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -861,6 +861,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error return has_generated_content or not error.is_pre_first_chunk +def _anthropic_stream_error_skips_recovery( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool +) -> bool: + """ + A held stream whose only possible recovery is a context window fallback lets every other failure through + with its held frames, as it would have reached the client with no hold. + """ + from litellm.exceptions import MidStreamFallbackError + + if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, error + ): + return True + if not holds_for_overflow_only: + return False + underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error + return not isinstance(underlying, litellm.ContextWindowExceededError) + + def _stream_raised_error_status(error: Exception) -> int | None: raw_status: Final = getattr(error, "status_code", None) if isinstance(raw_status, int): @@ -6936,9 +6955,14 @@ class Router(DeprecatedRouterMembers): # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group + can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry( + initial_kwargs + ) or self._stream_can_fall_back(model, initial_kwargs) + holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow( + model, initial_kwargs + ) has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over - self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back_on_overflow(model, initial_kwargs) + can_retry_or_fall_back or holds_for_overflow_only ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -7000,6 +7024,7 @@ class Router(DeprecatedRouterMembers): async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, + holds_for_overflow_only, buffered_lifecycle_chunks, model, initial_kwargs, @@ -7020,6 +7045,7 @@ class Router(DeprecatedRouterMembers): self, stream_error: Exception, has_generated_content: bool, + holds_for_overflow_only: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it @@ -7029,12 +7055,10 @@ class Router(DeprecatedRouterMembers): """Turns a source-iterator failure into a fallback attempt or the error reaching the caller.""" from litellm.exceptions import MidStreamFallbackError - if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( - has_generated_content, stream_error - ): + if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk - if stream_error.original_exception is not None: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: raise stream_error.original_exception from stream_error raise stream_error fallback_error: Final = ( @@ -7180,7 +7204,7 @@ class Router(DeprecatedRouterMembers): context_window_fallbacks=_request_fallback_list( kwargs, "context_window_fallbacks", - cast( # cast-ok: untyped router attribute + cast( # cast-ok: list is invariant and the helper takes list[object] "list[object] | None", self._core_context_window_fallbacks ), ), diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index f0a062308fc..be4927cf26f 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import openai import pytest +from _pytest.mark.structures import ParameterSet from fastapi import HTTPException from openai import AsyncOpenAI, OpenAI @@ -1824,17 +1825,19 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message -_QWEN_OVERFLOW_EVENT: Final = ( - 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' - "context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least " - "32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the " - 'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }' -) +def _qwen_overflow_event(requested_output_tokens: int) -> str: + counted_input_tokens: Final = 32768 - requested_output_tokens + 1 + return ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your " + f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please " + "reduce the length of the input prompt or the number of requested output tokens. " + f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}' + ) -@pytest.mark.parametrize( - "original_exception", - [ +def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]: + return ( pytest.param( BaseLLMException( status_code=400, @@ -1842,7 +1845,7 @@ _QWEN_OVERFLOW_EVENT: Final = ( { "error": { "code": "validation_error", - "message": _QWEN_OVERFLOW_EVENT, + "message": event, "param": None, "type": "invalid_request_error", } @@ -1851,9 +1854,15 @@ _QWEN_OVERFLOW_EVENT: Final = ( ), id="400-body", ), - pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"), - ], -) + pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"), + pytest.param( + BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"), + id="python-repr", + ), + ) + + +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16))) def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( original_exception: BaseLLMException, ) -> None: @@ -1869,6 +1878,22 @@ def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_win assert "prompt is too long" in excinfo.value.message +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768))) +def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "you requested 32768 output tokens" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index bbf706d079d..147880ee6b8 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19313,6 +19313,39 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_ assert len(provider.calls) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raised", "expected_error"), + [ + pytest.param( + litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"), + litellm.ServiceUnavailableError, + id="raised-503", + ), + pytest.param( + litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"), + litellm.BadRequestError, + id="raised-400", + ), + pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"), + ], +) +async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start( + raised: Exception, expected_error: type[Exception] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised) + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], expected_error) + assert len(provider.calls) == 1 + + @pytest.mark.parametrize( ("context_window_fallbacks", "request_kwargs", "expected"), [ @@ -20210,7 +20243,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_ with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt: recovery = router._aanthropic_messages_recover_stream_error( - declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + declined, + True, + False, + buffered, + "primary", + {"model": "primary"}, + _anthropic_messages_make_wrapper(), + AsyncMock(), ) with pytest.raises(BedrockError) as exc_info: await drain(recovery) @@ -20231,7 +20271,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt): recovery = router._aanthropic_messages_recover_stream_error( - raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) collected = [chunk async for chunk in recovery] assert collected == [b"fallback"]