From 8cfb20122e162830625c0dc2354e21cb60af7da8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:04:48 -0700 Subject: [PATCH 01/11] fix(bedrock_mantle): map gpt-oss context overflow to ContextWindowExceededError Bedrock Mantle's gpt-oss models report a too-long prompt as "Input length (N) exceeds model's maximum context length (M)", which the Mantle mapping missed, so callers got a BadRequestError (or a 503 when streaming), context window fallbacks never fired, and Claude Code never compacted. Match that wording too and return the existing "prompt is too long: N tokens > M maximum" message. --- .../exception_mapping_utils.py | 16 ++++--- .../test_exception_mapping_utils.py | 47 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 47ec8331c83..c37db350c6d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -892,7 +892,10 @@ def _map_openai_like_exception( ) -_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)") +_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( + re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"), + re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -912,12 +915,13 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ - Mantle reports context overflow as a validation_error carrying the token counts, or as - OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event - with no error type. Clients such as Claude Code only treat "prompt is too long" as - overflow (litellm/litellm#36546). + Mantle reports context overflow as a validation_error carrying the token counts (worded + differently for gpt-oss), or as OpenAI's context_length_exceeded code, either in a 400 body + or in a streamed error event with no error type. Clients such as Claude Code only treat + "prompt is too long" as overflow (litellm/litellm#36546). """ - match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str) + matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) + match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 55e7c47e2c9..1a96a090f3b 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1491,6 +1491,53 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message +_GPT_OSS_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' + "message: \"Input length (360068) exceeds model's maximum context length (131072).\", param: None } }" +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _GPT_OSS_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), + pytest.param( + BaseLLMException( + status_code=400, message=f"Error code: 400 - {repr({'error': {'message': _GPT_OSS_OVERFLOW_EVENT}})}" + ), + id="python-repr", + ), + ], +) +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(original_exception): + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="openai.gpt-oss-20b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From 66a11a6287f6c63bca90f9de82e2905824deeda2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:49:59 -0700 Subject: [PATCH 02/11] fix(bedrock_mantle): map the gpt-oss Responses API overflow wording too Mantle's /v1/responses words gpt-oss overflow as "The engine prompt length N exceeds the max_model_len M", so a streamed /v1/responses overflow still ended in a response.failed carrying a plain BadRequestError. Match that wording as well. --- .../exception_mapping_utils.py | 7 ++++--- .../test_exception_mapping_utils.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index c37db350c6d..6aac1918993 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -895,6 +895,7 @@ def _map_openai_like_exception( _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"), re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), + re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), ) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" @@ -916,9 +917,9 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ Mantle reports context overflow as a validation_error carrying the token counts (worded - differently for gpt-oss), or as OpenAI's context_length_exceeded code, either in a 400 body - or in a streamed error event with no error type. Clients such as Claude Code only treat - "prompt is too long" as overflow (litellm/litellm#36546). + differently for gpt-oss, and again on its Responses API), or as OpenAI's context_length_exceeded + code, either in a 400 body or in a streamed error event with no error type. Clients such as + Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 1a96a090f3b..4ac7c644cda 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1538,6 +1538,27 @@ def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exc assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded(): + from litellm.responses.streaming_iterator import map_stream_error_to_exception + + mapped_exception = map_stream_error_to_exception( + { + "code": "invalid_prompt", + "message": ( + 'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: ' + '"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", ' + 'param: Some("input") } }' + ), + }, + model="openai.gpt-oss-20b", + custom_llm_provider="bedrock_mantle", + ) + + assert isinstance(mapped_exception, litellm.ContextWindowExceededError) + assert mapped_exception.status_code == 400 + assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From fdb163e51c9c8c12e3b09ca96b59253da90fd901 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:17:03 -0700 Subject: [PATCH 03/11] fix(router): send a streamed /v1/messages context overflow before content to context_window_fallbacks A streamed /v1/messages request whose overflow arrives before any content now holds its lifecycle frames and takes context_window_fallbacks or the plain fallbacks, the same way the non-streamed route and the streamed chat and Responses routes do. Bedrock Mantle's qwen overflow wording ("This model's maximum context length is ...") also maps to ContextWindowExceededError with the generic prompt is too long message --- .../exception_mapping_utils.py | 11 ++-- litellm/router.py | 30 ++++++++- ..._mantle_context_overflow_fallbacks_wire.py | 18 ++--- .../test_exception_mapping_utils.py | 57 ++++++++++++++-- tests/unit/test_router/test_router.py | 65 +++++++++++++++++++ 5 files changed, 156 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 6aac1918993..ec4e58fda6b 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -917,16 +917,19 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ Mantle reports context overflow as a validation_error carrying the token counts (worded - differently for gpt-oss, and again on its Responses API), or as OpenAI's context_length_exceeded - code, either in a 400 body or in a streamed error event with no error type. Clients such as - Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). + differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded + code, or in a wording the other providers share, either in a 400 body or in a streamed error + event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow + (litellm/litellm#36546). """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" - if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: + if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( + error_str + ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE return None diff --git a/litellm/router.py b/litellm/router.py index fc888495e99..7e1cc569504 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -821,9 +821,15 @@ def _stream_raised_error_status(error: Exception) -> int | None: def _stream_fallback_error_for_raised( error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" ) -> "MidStreamFallbackError | None": - """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + """ + The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain + sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates + unchanged. + """ if has_generated_content: return None + if isinstance(error, litellm.ContextWindowExceededError): + return _stream_pre_content_error(error, model, llm_provider) status_code: Final = _stream_raised_error_status(error) if status_code is None: return _stream_pre_content_error(error, model, llm_provider) @@ -1628,7 +1634,9 @@ class Router: else: self.fallbacks = [{"*": _fallbacks}] - self.context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks + self.context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolver contract + context_window_fallbacks or litellm.context_window_fallbacks + ) _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) @@ -6611,7 +6619,7 @@ class Router: model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back(model, initial_kwargs) + or self._stream_can_fall_back_on_overflow(model, initial_kwargs) ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -10025,6 +10033,22 @@ class Router: ) ) + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks + can serve as well as every route _stream_can_fall_back counts.""" + if fallbacks_disabled_for_request(kwargs): + return False + if self._stream_can_fall_back(model_group, kwargs): + return True + context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) + if not context_window_fallbacks: + return False + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=context_window_fallbacks, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return has_unattempted_fallback_target(resolved, kwargs) + def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: """ Whether async_function_with_fallbacks_common_utils could still route a diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 37aa96c2467..e069dfc510e 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet _assert_fell_back(cwf_rig, "overflow", prompt, text) -def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None: +def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None: prompt: Final = _prompt("cwf messages stream") response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True)) assert response.status_code == 200, text - error: Final = _only_error(text) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(cwf_rig, "overflow", prompt) + _assert_fell_back(cwf_rig, "overflow", prompt, text) def test_context_window_fallback_does_not_fire_on_streamed_responses(cwf_rig: Rig) -> None: @@ -455,16 +452,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r _assert_fell_back(fb_rig, "overflow", prompt, text) -def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back( - fb_rig: Rig, -) -> None: +def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None: prompt: Final = _prompt("fb messages stream") response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True)) - assert response.status_code == 400, text - error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"]) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(fb_rig, "overflow", prompt) + assert response.status_code == 200, text + _assert_fell_back(fb_rig, "overflow", prompt, text) def test_plain_fallback_does_not_fire_on_streamed_responses(fb_rig: Rig) -> None: diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 4ac7c644cda..2388713acf8 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1493,7 +1493,7 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): _GPT_OSS_OVERFLOW_EVENT: Final = ( 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' - "message: \"Input length (360068) exceeds model's maximum context length (131072).\", param: None } }" + 'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }' ) @@ -1519,13 +1519,15 @@ _GPT_OSS_OVERFLOW_EVENT: Final = ( pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), pytest.param( BaseLLMException( - status_code=400, message=f"Error code: 400 - {repr({'error': {'message': _GPT_OSS_OVERFLOW_EVENT}})}" + status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}" ), id="python-repr", ), ], ) -def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(original_exception): +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: with pytest.raises(litellm.ContextWindowExceededError) as excinfo: exception_type( model="openai.gpt-oss-20b", @@ -1538,10 +1540,10 @@ def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exc assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message -def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded(): +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None: from litellm.responses.streaming_iterator import map_stream_error_to_exception - mapped_exception = map_stream_error_to_exception( + mapped_exception: Final = map_stream_error_to_exception( { "code": "invalid_prompt", "message": ( @@ -1559,6 +1561,51 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message +_QWEN_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + "context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least " + "32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the " + 'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }' +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _QWEN_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"), + ], +) +def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 300c9cff6ff..086253039ec 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18970,6 +18970,71 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" +def _anthropic_messages_overflow_before_content(): + return _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" + ), + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback(): + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs"), + [ + pytest.param(None, {}, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"), + ], +) +async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], litellm.ContextWindowExceededError) + assert len(provider.calls) == 1 + + +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs", "expected"), + [ + pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"), + pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), + pytest.param(None, {}, False, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + ], +) +def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + + assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected + + def test_anthropic_messages_retry_budget_precedence_direct_call(): """A retry policy naming the error class outranks the request's num_retries, which outranks the routed deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too.""" From 4545a0c785be0f7b30d64cd1bd6c4d7f4250df85 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:29:48 -0700 Subject: [PATCH 04/11] fix(router): validate the context_window_fallbacks chain in the streamed overflow hold check --- litellm/router.py | 16 +++++++++++----- tests/unit/test_router/test_router.py | 5 +++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index fcd7d89a2b3..0c3975ebf61 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -680,6 +680,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams ) @@ -10383,18 +10384,23 @@ class Router(DeprecatedRouterMembers): ) ) - def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool: """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks - can serve as well as every route _stream_can_fall_back counts.""" + can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists + errs toward True, as _stream_can_fall_back does for non-standard fallbacks.""" if fallbacks_disabled_for_request(kwargs): return False if self._stream_can_fall_back(model_group, kwargs): return True - context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) - if not context_window_fallbacks: + configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) + if not configured: return False + try: + chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured) + except ValidationError: + return True resolved, _ = get_fallback_model_group_for_lookup_groups( - fallbacks=context_window_fallbacks, + fallbacks=chain, lookup_groups=fallback_lookup_groups(kwargs, model_group), ) return has_unattempted_fallback_target(resolved, kwargs) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 2d91f2a02c0..bbf706d079d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19265,7 +19265,7 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" -def _anthropic_messages_overflow_before_content(): +def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream: return _AnthropicMessagesRaisingByteStream( [_anthropic_messages_message_start_chunk()], litellm.ContextWindowExceededError( @@ -19275,7 +19275,7 @@ def _anthropic_messages_overflow_before_content(): @pytest.mark.asyncio -async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback(): +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None: router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) provider: Final = _AnthropicMessagesScriptedProvider( _anthropic_messages_overflow_before_content, @@ -19320,6 +19320,7 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_ pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), pytest.param(None, {}, False, id="no-fallback-configured"), pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"), ], ) def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( From 9de1f570f8d73ed4c88dcc89b79f12ee6348e74f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:34:46 -0700 Subject: [PATCH 05/11] fix(router): hold streamed /v1/messages frames for an overflow only on context-window-only groups A group with only context_window_fallbacks now lets every non-overflow error before content reach the client after message_start, as it did before the hold. qwen's Mantle overflow keeps the prompt is too long prefix only when the requested output leaves room for a prompt, and matches its repr form. --- .../exception_mapping_utils.py | 11 +++- litellm/router.py | 38 +++++++++++--- .../test_exception_mapping_utils.py | 51 ++++++++++++++----- tests/unit/test_router/test_router.py | 44 +++++++++++++++- 4 files changed, 121 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 4a062ebd132..5e16563ae26 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -936,6 +936,9 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), ) +_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile( + r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens" +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -968,13 +971,19 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded code, or in a wording the other providers share, either in a 400 body or in a streamed error event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow - (litellm/litellm#36546). + (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested + output, so only the output budget says whether a shorter prompt could fit at all; when it cannot, + the overflow keeps Mantle's own message so the client is not told to shorten the prompt. """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" + output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str) + if output_budget is not None: + context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) + return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( error_str ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): diff --git a/litellm/router.py b/litellm/router.py index 0c3975ebf61..67b7400f411 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -861,6 +861,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error return has_generated_content or not error.is_pre_first_chunk +def _anthropic_stream_error_skips_recovery( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool +) -> bool: + """ + A held stream whose only possible recovery is a context window fallback lets every other failure through + with its held frames, as it would have reached the client with no hold. + """ + from litellm.exceptions import MidStreamFallbackError + + if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, error + ): + return True + if not holds_for_overflow_only: + return False + underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error + return not isinstance(underlying, litellm.ContextWindowExceededError) + + def _stream_raised_error_status(error: Exception) -> int | None: raw_status: Final = getattr(error, "status_code", None) if isinstance(raw_status, int): @@ -6936,9 +6955,14 @@ class Router(DeprecatedRouterMembers): # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group + can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry( + initial_kwargs + ) or self._stream_can_fall_back(model, initial_kwargs) + holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow( + model, initial_kwargs + ) has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over - self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back_on_overflow(model, initial_kwargs) + can_retry_or_fall_back or holds_for_overflow_only ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -7000,6 +7024,7 @@ class Router(DeprecatedRouterMembers): async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, + holds_for_overflow_only, buffered_lifecycle_chunks, model, initial_kwargs, @@ -7020,6 +7045,7 @@ class Router(DeprecatedRouterMembers): self, stream_error: Exception, has_generated_content: bool, + holds_for_overflow_only: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it @@ -7029,12 +7055,10 @@ class Router(DeprecatedRouterMembers): """Turns a source-iterator failure into a fallback attempt or the error reaching the caller.""" from litellm.exceptions import MidStreamFallbackError - if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( - has_generated_content, stream_error - ): + if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk - if stream_error.original_exception is not None: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: raise stream_error.original_exception from stream_error raise stream_error fallback_error: Final = ( @@ -7180,7 +7204,7 @@ class Router(DeprecatedRouterMembers): context_window_fallbacks=_request_fallback_list( kwargs, "context_window_fallbacks", - cast( # cast-ok: untyped router attribute + cast( # cast-ok: list is invariant and the helper takes list[object] "list[object] | None", self._core_context_window_fallbacks ), ), diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index f0a062308fc..be4927cf26f 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import openai import pytest +from _pytest.mark.structures import ParameterSet from fastapi import HTTPException from openai import AsyncOpenAI, OpenAI @@ -1824,17 +1825,19 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message -_QWEN_OVERFLOW_EVENT: Final = ( - 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' - "context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least " - "32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the " - 'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }' -) +def _qwen_overflow_event(requested_output_tokens: int) -> str: + counted_input_tokens: Final = 32768 - requested_output_tokens + 1 + return ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your " + f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please " + "reduce the length of the input prompt or the number of requested output tokens. " + f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}' + ) -@pytest.mark.parametrize( - "original_exception", - [ +def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]: + return ( pytest.param( BaseLLMException( status_code=400, @@ -1842,7 +1845,7 @@ _QWEN_OVERFLOW_EVENT: Final = ( { "error": { "code": "validation_error", - "message": _QWEN_OVERFLOW_EVENT, + "message": event, "param": None, "type": "invalid_request_error", } @@ -1851,9 +1854,15 @@ _QWEN_OVERFLOW_EVENT: Final = ( ), id="400-body", ), - pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"), - ], -) + pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"), + pytest.param( + BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"), + id="python-repr", + ), + ) + + +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16))) def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( original_exception: BaseLLMException, ) -> None: @@ -1869,6 +1878,22 @@ def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_win assert "prompt is too long" in excinfo.value.message +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768))) +def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "you requested 32768 output tokens" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index bbf706d079d..147880ee6b8 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19313,6 +19313,39 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_ assert len(provider.calls) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raised", "expected_error"), + [ + pytest.param( + litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"), + litellm.ServiceUnavailableError, + id="raised-503", + ), + pytest.param( + litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"), + litellm.BadRequestError, + id="raised-400", + ), + pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"), + ], +) +async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start( + raised: Exception, expected_error: type[Exception] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised) + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], expected_error) + assert len(provider.calls) == 1 + + @pytest.mark.parametrize( ("context_window_fallbacks", "request_kwargs", "expected"), [ @@ -20210,7 +20243,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_ with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt: recovery = router._aanthropic_messages_recover_stream_error( - declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + declined, + True, + False, + buffered, + "primary", + {"model": "primary"}, + _anthropic_messages_make_wrapper(), + AsyncMock(), ) with pytest.raises(BedrockError) as exc_info: await drain(recovery) @@ -20231,7 +20271,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt): recovery = router._aanthropic_messages_recover_stream_error( - raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) collected = [chunk async for chunk in recovery] assert collected == [b"fallback"] From 88d504bd548f2dd5d9653bc1579d856ffc591cc6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:50:11 -0700 Subject: [PATCH 06/11] fix(bedrock_mantle): keep Mantle's text when an overflow blames max_tokens Anthropic's and vLLM's max_tokens overflow wordings ask the client to lower max_tokens, so they map to ContextWindowExceededError with Mantle's own message instead of the generic prompt is too long text --- .../exception_mapping_utils.py | 9 +++++- litellm/router.py | 2 +- .../test_exception_mapping_utils.py | 31 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 5e16563ae26..f5bce908f10 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -939,6 +939,10 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile( r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens" ) +_BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS: Final = ( + re.compile(r"input length and `max_tokens` exceed context limit"), + re.compile(r"\\?'max_tokens\\?' or \\?'max_completion_tokens\\?' is too large"), +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -973,7 +977,8 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested output, so only the output budget says whether a shorter prompt could fit at all; when it cannot, - the overflow keeps Mantle's own message so the client is not told to shorten the prompt. + or when the wording blames `max_tokens` (Anthropic's and vLLM's), the overflow keeps Mantle's own + message so the client lowers `max_tokens` instead of shortening the prompt. """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) @@ -984,6 +989,8 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: if output_budget is not None: context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str + if any(pattern.search(error_str) is not None for pattern in _BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS): + return error_str if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( error_str ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): diff --git a/litellm/router.py b/litellm/router.py index 67b7400f411..93ca82271bc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1874,7 +1874,7 @@ class Router(DeprecatedRouterMembers): else: self._core_fallbacks = [{"*": _fallbacks}] - self._core_context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolver contract + self._core_context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolvers take lists context_window_fallbacks or litellm.context_window_fallbacks ) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index be4927cf26f..0e6c00ab311 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1894,6 +1894,37 @@ def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_mess assert "you requested 32768 output tokens" in excinfo.value.message +_ANTHROPIC_MAX_TOKENS_OVERFLOW: Final = ( + "input length and `max_tokens` exceed context limit: 197202 + 21333 > 200000, decrease input length or " + "`max_tokens` and try again" +) +_VLLM_MAX_TOKENS_OVERFLOW: Final = ( + "'max_tokens' or 'max_completion_tokens' is too large: 200000. This model's maximum context length is 131072 " + "tokens and your request has 83 input tokens (200000 > 131072 - 83)." +) + + +@pytest.mark.parametrize( + "original_exception", + ( + *_mantle_overflow_serializations(_ANTHROPIC_MAX_TOKENS_OVERFLOW), + *_mantle_overflow_serializations(_VLLM_MAX_TOKENS_OVERFLOW), + ), +) +def test_bedrock_mantle_max_tokens_overflow_keeps_mantles_message(original_exception: BaseLLMException) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="anthropic.claude-haiku-4-5", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "max_tokens" in excinfo.value.message + assert "200000" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From f140baef0f427975270bbd7651b868eb284c229e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:27:44 -0700 Subject: [PATCH 07/11] test(router): cover the stream recovery decision for overflow-only holds --- tests/unit/test_router/test_router.py | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 147880ee6b8..d317bb12b28 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -53,6 +53,7 @@ from litellm.router import ( FallbackAwareAnthropicMessagesStream, _stream_commits_now, _anthropic_stream_error_is_gateway_verdict, + _anthropic_stream_error_skips_recovery, _stream_fallback_error_for_raised, _anthropic_stream_forwards_ping_live, _stream_raised_error_status, @@ -18571,6 +18572,53 @@ def test_anthropic_stream_should_decline_fallback_direct_call(): assert _anthropic_stream_should_decline_fallback(False, post_first_chunk_error) is True +def _pre_first_chunk_stream_error(original_exception: Exception) -> MidStreamFallbackError: + return MidStreamFallbackError( + message="stream failed", + model="glm", + llm_provider="bedrock_mantle", + original_exception=original_exception, + is_pre_first_chunk=True, + ) + + +_STREAM_OVERFLOW: Final = litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" +) +_STREAM_UNAVAILABLE: Final = litellm.ServiceUnavailableError( + message="unavailable", llm_provider="bedrock_mantle", model="glm" +) + + +@pytest.mark.parametrize( + ("error", "has_generated_content", "holds_for_overflow_only", "skips"), + [ + pytest.param(_STREAM_UNAVAILABLE, False, False, False, id="full-hold-recovers-any-error"), + pytest.param(_STREAM_OVERFLOW, False, True, False, id="overflow-only-hold-recovers-an-overflow"), + pytest.param(_STREAM_UNAVAILABLE, False, True, True, id="overflow-only-hold-passes-a-503-through"), + pytest.param( + _pre_first_chunk_stream_error(_STREAM_OVERFLOW), + False, + True, + False, + id="overflow-only-hold-recovers-a-wrapped-overflow", + ), + pytest.param( + _pre_first_chunk_stream_error(_STREAM_UNAVAILABLE), + False, + True, + True, + id="overflow-only-hold-passes-a-wrapped-503-through", + ), + pytest.param(_pre_first_chunk_stream_error(_STREAM_OVERFLOW), True, False, True, id="content-already-sent"), + ], +) +def test_anthropic_stream_error_skips_recovery_direct_call( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool, skips: bool +) -> None: + assert _anthropic_stream_error_skips_recovery(error, has_generated_content, holds_for_overflow_only) is skips + + def test_anthropic_stream_commits_now_direct_call(): content = _anthropic_messages_content_chunk("hi") lifecycle_chunk = _anthropic_messages_message_start_chunk() From cc774225725437fd0d13f872b743b4752fd97c02 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:23:27 -0700 Subject: [PATCH 08/11] fix(bedrock_mantle): stop overflow retries, keep refusals and Mantle's text where compacting cannot help --- .../exception_mapping_utils.py | 20 ++++++--- litellm/router.py | 7 +-- .../test_exception_mapping_utils.py | 42 +++++++++++++++--- tests/unit/test_router/test_router.py | 44 +++++++++++++++++++ 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index f5bce908f10..6978fe76c6d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -976,9 +976,10 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: code, or in a wording the other providers share, either in a 400 body or in a streamed error event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested - output, so only the output budget says whether a shorter prompt could fit at all; when it cannot, - or when the wording blames `max_tokens` (Anthropic's and vLLM's), the overflow keeps Mantle's own - message so the client lowers `max_tokens` instead of shortening the prompt. + output, so only the output budget says whether shortening the prompt can help; once the output + takes more than half the window, or when the wording blames `max_tokens` (Anthropic's and vLLM's), + the overflow keeps Mantle's own message so the client lowers `max_tokens` instead of shortening the + prompt. A wording only the shared overflow list knows keeps Mantle's message too, as OpenAI's does. """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) @@ -988,13 +989,18 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str) if output_budget is not None: context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) - return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str + if requested_output_tokens * 2 > context_window: + return error_str + return ( + f"prompt is too long: the prompt and {requested_output_tokens} requested output tokens exceed the " + f"{context_window} token context window" + ) if any(pattern.search(error_str) is not None for pattern in _BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS): return error_str - if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( - error_str - ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): + if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE + if ExceptionCheckers.is_error_str_context_window_exceeded(error_str): + return error_str return None diff --git a/litellm/router.py b/litellm/router.py index 93ca82271bc..d9996c8c2f6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6986,7 +6986,7 @@ class Router(DeprecatedRouterMembers): ) refusal_stop_details = ( parse_anthropic_refusal_stop_details(parse_window) - if not has_generated_content and error_event is None + if not has_generated_content and error_event is None and not holds_for_overflow_only else None ) if ( @@ -7250,8 +7250,9 @@ class Router(DeprecatedRouterMembers): last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see for attempt in range(attempted_retries_for_request(initial_kwargs), budget): trigger = _mid_stream_retry_trigger(last_error) - if not policy_applies and not self._stream_should_retry( - trigger, healthy_deployments, all_deployments, initial_kwargs + if isinstance(trigger, litellm.ContextWindowExceededError) or ( + not policy_applies + and not self._stream_should_retry(trigger, healthy_deployments, all_deployments, initial_kwargs) ): return _StreamRetriesExhausted(last_error) if get_metadata_variable_name_from_kwargs(retry_kwargs) in retry_kwargs: diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 0e6c00ab311..56f4cc338ba 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1862,7 +1862,13 @@ def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]: ) -@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16))) +@pytest.mark.parametrize( + "original_exception", + [ + *_mantle_overflow_serializations(_qwen_overflow_event(16)), + *_mantle_overflow_serializations(_qwen_overflow_event(16384)), + ], +) def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( original_exception: BaseLLMException, ) -> None: @@ -1875,11 +1881,19 @@ def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_win assert excinfo.value.status_code == 400 assert excinfo.value.response.status_code == 400 - assert "prompt is too long" in excinfo.value.message + assert "prompt is too long: the prompt and" in excinfo.value.message + assert "requested output tokens exceed the 32768 token context window" in excinfo.value.message -@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768))) -def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message( +@pytest.mark.parametrize( + "original_exception", + [ + *_mantle_overflow_serializations(_qwen_overflow_event(16385)), + *_mantle_overflow_serializations(_qwen_overflow_event(32000)), + *_mantle_overflow_serializations(_qwen_overflow_event(32768)), + ], +) +def test_bedrock_mantle_qwen_output_budget_over_half_the_window_keeps_mantles_message( original_exception: BaseLLMException, ) -> None: with pytest.raises(litellm.ContextWindowExceededError) as excinfo: @@ -1891,7 +1905,7 @@ def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_mess assert excinfo.value.status_code == 400 assert "prompt is too long" not in excinfo.value.message - assert "you requested 32768 output tokens" in excinfo.value.message + assert "output tokens and your prompt contains at least" in excinfo.value.message _ANTHROPIC_MAX_TOKENS_OVERFLOW: Final = ( @@ -1925,6 +1939,24 @@ def test_bedrock_mantle_max_tokens_overflow_keeps_mantles_message(original_excep assert "200000" in excinfo.value.message +def test_bedrock_mantle_overflow_only_the_shared_list_knows_keeps_mantles_message() -> None: + vllm_total_overflow: Final = ( + "This model's maximum context length is 32768 tokens. However, you requested 40000 tokens (100 in the " + "messages, 39900 in the completion). Please reduce the length of the messages or completion." + ) + + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=BaseLLMException(status_code=400, message=vllm_total_overflow), + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "39900 in the completion" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d317bb12b28..b669b9905af 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19394,6 +19394,50 @@ async def test_anthropic_messages_stream_context_window_only_group_forwards_othe assert len(provider.calls) == 1 +def _anthropic_messages_refusal_chunk() -> bytes: + return ( + b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "refusal", ' + b'"stop_details": {"type": "refusal", "category": "cyber"}}, "usage": {"output_tokens": 0}}\n\n' + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_context_window_only_group_passes_a_refusal_through() -> None: + router: Final = _anthropic_messages_retry_router( + num_retries=0, fallbacks=[{"*": ["fb"]}], context_window_fallbacks=[{"glm": ["cw"]}] + ) + refusal_stream: Final = (_anthropic_messages_message_start_chunk(), _anthropic_messages_refusal_chunk()) + provider: Final = _AnthropicMessagesScriptedProvider(lambda: _AnthropicMessagesFakeByteStream(list(refusal_stream))) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, fallbacks=[]) + body: Final = [chunk async for chunk in stream] + + assert body == list(refusal_stream) + assert len(provider.calls) == 1 + + +@pytest.mark.asyncio +async def test_anthropic_messages_policy_retry_stops_at_a_context_window_error() -> None: + router: Final = _anthropic_messages_retry_router( + num_retries=0, retry_policy=RetryPolicy(InternalServerErrorRetries=3) + ) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.InternalServerError(message="upstream reset", llm_provider="bedrock_mantle", model="glm"), + ), + _anthropic_messages_raise_context_window_error, + _anthropic_messages_raise_context_window_error, + _anthropic_messages_raise_context_window_error, + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + with pytest.raises(litellm.ContextWindowExceededError): + [chunk async for chunk in stream] + + assert len(provider.calls) == 2 + + @pytest.mark.parametrize( ("context_window_fallbacks", "request_kwargs", "expected"), [ From 8438fa916a195ce7d618e260ea3f94c87636658b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:45:07 -0700 Subject: [PATCH 09/11] fix(router): retry a streamed overflow only when the retry policy names its error class --- litellm/router.py | 12 +++++++++++- tests/unit/test_router/test_router.py | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d9996c8c2f6..e4b8dbcd97c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7225,6 +7225,16 @@ class Router(DeprecatedRouterMembers): return False return True + def _stream_overflow_retry_declined(self, trigger: Exception, kwargs: Mapping[str, object]) -> bool: + """ + An overflow resends the same prompt to the same windows, so only a retry policy that names its error + class retries it, never a grant made for the error that opened the retries. + """ + return ( + isinstance(trigger, litellm.ContextWindowExceededError) + and self._anthropic_messages_policy_retries(trigger, kwargs) is None + ) + async def _astream_retry_same_group( self, e: "MidStreamFallbackError", @@ -7250,7 +7260,7 @@ class Router(DeprecatedRouterMembers): last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see for attempt in range(attempted_retries_for_request(initial_kwargs), budget): trigger = _mid_stream_retry_trigger(last_error) - if isinstance(trigger, litellm.ContextWindowExceededError) or ( + if self._stream_overflow_retry_declined(trigger, initial_kwargs) or ( not policy_applies and not self._stream_should_retry(trigger, healthy_deployments, all_deployments, initial_kwargs) ): diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b669b9905af..5259495c3aa 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19416,11 +19416,18 @@ async def test_anthropic_messages_stream_context_window_only_group_passes_a_refu assert len(provider.calls) == 1 +@pytest.mark.parametrize( + ("retry_policy", "expected_calls"), + [ + (RetryPolicy(InternalServerErrorRetries=3), 2), + (RetryPolicy(InternalServerErrorRetries=2, BadRequestErrorRetries=2), 3), + ], +) @pytest.mark.asyncio -async def test_anthropic_messages_policy_retry_stops_at_a_context_window_error() -> None: - router: Final = _anthropic_messages_retry_router( - num_retries=0, retry_policy=RetryPolicy(InternalServerErrorRetries=3) - ) +async def test_anthropic_messages_policy_retry_retries_an_overflow_only_when_the_policy_names_it( + retry_policy: RetryPolicy, expected_calls: int +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, retry_policy=retry_policy) provider: Final = _AnthropicMessagesScriptedProvider( lambda: _AnthropicMessagesRaisingByteStream( [_anthropic_messages_message_start_chunk()], @@ -19435,7 +19442,7 @@ async def test_anthropic_messages_policy_retry_stops_at_a_context_window_error() with pytest.raises(litellm.ContextWindowExceededError): [chunk async for chunk in stream] - assert len(provider.calls) == 2 + assert len(provider.calls) == expected_calls @pytest.mark.parametrize( From fc4ad87a6cdb53eab97d76b228cb167dffbe8d75 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:52:01 -0700 Subject: [PATCH 10/11] fix(router): read the context window fallbacks through the typed fallback list helper --- litellm/router_python_core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 96954ba5f7c..a3f4d0ec67d 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -10524,7 +10524,13 @@ class PythonCore(DeprecatedRouterMembers): return False if self._stream_can_fall_back(model_group, kwargs): return True - configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) + configured: Final = _request_fallback_list( + kwargs, + "context_window_fallbacks", + cast( # cast-ok: list is invariant and the helper takes list[object] + "list[object] | None", self._core_context_window_fallbacks + ), + ) if not configured: return False try: From 8b9c9632f09d4852325f71a4ec784e5a7c6021c5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:27:34 -0700 Subject: [PATCH 11/11] fix(router): fall back through plain fallbacks when context_window_fallbacks does not name the group An overflow on a group missing from context_window_fallbacks raised the original error and skipped the group's plain fallbacks, so a Mantle group that fell back before this mapping returned a 400. An empty {} entry in a fallback list no longer raises StopIteration. --- litellm/router_python_core.py | 26 ++++++++-------- .../router_utils/fallback_event_handlers.py | 2 +- tests/unit/test_router/core/test_fallbacks.py | 30 +++++++++++-------- tests/unit/test_router/test_router.py | 24 +++++++++++++++ 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index a3f4d0ec67d..60c691c30e5 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -9265,21 +9265,19 @@ class PythonCore(DeprecatedRouterMembers): lookup_groups=lookup_groups, ) ) - if context_window_fallback_model_group is None: - raise original_exception + if context_window_fallback_model_group is not None: + input_kwargs.update( + { + "fallback_model_group": context_window_fallback_model_group, + "original_model_group": original_model_group, + } + ) - input_kwargs.update( - { - "fallback_model_group": context_window_fallback_model_group, - "original_model_group": original_model_group, - } - ) - - response = await run_async_fallback( - *args, - **input_kwargs, - ) - return response + response = await run_async_fallback( + *args, + **input_kwargs, + ) + return response else: error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..7fc6c628822 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -622,7 +622,7 @@ def get_fallback_model_group( prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys) ## check for specific model group-specific fallbacks for idx, item in enumerate(fallbacks): - if isinstance(item, dict): + if isinstance(item, dict) and item: fallback_key = next(iter(item)) if fallback_key == model_group: # check exact match fallback_model_group = item[model_group] diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index e4e5c3120c8..8cda5e0c275 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -355,26 +355,30 @@ async def test_an_error_specific_chain_wins_over_the_general_chain( @pytest.mark.asyncio -@pytest.mark.parametrize( - ("error", "field", "raised"), - [ - (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError), - (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError), - ], - ids=["context-window", "content-policy"], -) -async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain( - make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception] +async def test_a_content_policy_chain_without_the_group_raises_without_trying_the_general_chain( + make_core: MakeCore, host: ScriptedHost ) -> None: - core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]}) - host.answer("openai/a", error()) + core = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", content_policy()) - with pytest.raises(raised): + with pytest.raises(litellm.ContentPolicyViolationError): await complete(core) assert host.deployments_called("acompletion") == ["a"] +@pytest.mark.asyncio +async def test_a_context_window_chain_without_the_group_falls_back_through_the_general_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], context_window_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", context_window()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + @pytest.mark.asyncio @pytest.mark.parametrize("error", [context_window, content_policy], ids=["context-window", "content-policy"]) async def test_without_an_error_specific_chain_the_general_chain_applies( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a2537dd3e38..13e165f2dc7 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19339,6 +19339,28 @@ async def test_anthropic_messages_stream_overflow_before_content_takes_the_conte assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_on_a_group_missing_from_the_context_window_chain_takes_its_fallbacks() -> ( + None +): + router: Final = _anthropic_messages_retry_router( + num_retries=0, fallbacks=[{"glm": ["fb"]}], context_window_fallbacks=[{"other": ["cw"]}] + ) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")] + assert [model in _ANTHROPIC_MESSAGES_RETRY_GROUP for model, _, _ in provider.calls] == [True, False] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/fb-model" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("context_window_fallbacks", "request_kwargs"), @@ -19451,8 +19473,10 @@ async def test_anthropic_messages_policy_retry_retries_an_overflow_only_when_the pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"), pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), pytest.param(None, {}, False, id="no-fallback-configured"), + pytest.param(None, {"fallbacks": [{"glm": ["fb"]}]}, True, id="request-plain-fallback"), pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"), + pytest.param(None, {"context_window_fallbacks": [{}]}, False, id="request-chain-of-an-empty-entry"), ], ) def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks(