From fdb163e51c9c8c12e3b09ca96b59253da90fd901 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:17:03 -0700 Subject: [PATCH] fix(router): send a streamed /v1/messages context overflow before content to context_window_fallbacks A streamed /v1/messages request whose overflow arrives before any content now holds its lifecycle frames and takes context_window_fallbacks or the plain fallbacks, the same way the non-streamed route and the streamed chat and Responses routes do. Bedrock Mantle's qwen overflow wording ("This model's maximum context length is ...") also maps to ContextWindowExceededError with the generic prompt is too long message --- .../exception_mapping_utils.py | 11 ++-- litellm/router.py | 30 ++++++++- ..._mantle_context_overflow_fallbacks_wire.py | 18 ++--- .../test_exception_mapping_utils.py | 57 ++++++++++++++-- tests/unit/test_router/test_router.py | 65 +++++++++++++++++++ 5 files changed, 156 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 6aac1918993..ec4e58fda6b 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -917,16 +917,19 @@ def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPExc def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ Mantle reports context overflow as a validation_error carrying the token counts (worded - differently for gpt-oss, and again on its Responses API), or as OpenAI's context_length_exceeded - code, either in a 400 body or in a streamed error event with no error type. Clients such as - Claude Code only treat "prompt is too long" as overflow (litellm/litellm#36546). + differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded + code, or in a wording the other providers share, either in a 400 body or in a streamed error + event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow + (litellm/litellm#36546). """ matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" - if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: + if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( + error_str + ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE return None diff --git a/litellm/router.py b/litellm/router.py index fc888495e99..7e1cc569504 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -821,9 +821,15 @@ def _stream_raised_error_status(error: Exception) -> int | None: def _stream_fallback_error_for_raised( error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" ) -> "MidStreamFallbackError | None": - """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + """ + The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain + sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates + unchanged. + """ if has_generated_content: return None + if isinstance(error, litellm.ContextWindowExceededError): + return _stream_pre_content_error(error, model, llm_provider) status_code: Final = _stream_raised_error_status(error) if status_code is None: return _stream_pre_content_error(error, model, llm_provider) @@ -1628,7 +1634,9 @@ class Router: else: self.fallbacks = [{"*": _fallbacks}] - self.context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks + self.context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolver contract + context_window_fallbacks or litellm.context_window_fallbacks + ) _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) @@ -6611,7 +6619,7 @@ class Router: model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back(model, initial_kwargs) + or self._stream_can_fall_back_on_overflow(model, initial_kwargs) ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -10025,6 +10033,22 @@ class Router: ) ) + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks + can serve as well as every route _stream_can_fall_back counts.""" + if fallbacks_disabled_for_request(kwargs): + return False + if self._stream_can_fall_back(model_group, kwargs): + return True + context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) + if not context_window_fallbacks: + return False + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=context_window_fallbacks, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return has_unattempted_fallback_target(resolved, kwargs) + def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: """ Whether async_function_with_fallbacks_common_utils could still route a diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 37aa96c2467..e069dfc510e 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet _assert_fell_back(cwf_rig, "overflow", prompt, text) -def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None: +def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None: prompt: Final = _prompt("cwf messages stream") response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True)) assert response.status_code == 200, text - error: Final = _only_error(text) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(cwf_rig, "overflow", prompt) + _assert_fell_back(cwf_rig, "overflow", prompt, text) def test_context_window_fallback_does_not_fire_on_streamed_responses(cwf_rig: Rig) -> None: @@ -455,16 +452,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r _assert_fell_back(fb_rig, "overflow", prompt, text) -def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back( - fb_rig: Rig, -) -> None: +def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None: prompt: Final = _prompt("fb messages stream") response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True)) - assert response.status_code == 400, text - error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"]) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(fb_rig, "overflow", prompt) + assert response.status_code == 200, text + _assert_fell_back(fb_rig, "overflow", prompt, text) def test_plain_fallback_does_not_fire_on_streamed_responses(fb_rig: Rig) -> None: diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 4ac7c644cda..2388713acf8 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1493,7 +1493,7 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): _GPT_OSS_OVERFLOW_EVENT: Final = ( 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' - "message: \"Input length (360068) exceeds model's maximum context length (131072).\", param: None } }" + 'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }' ) @@ -1519,13 +1519,15 @@ _GPT_OSS_OVERFLOW_EVENT: Final = ( pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), pytest.param( BaseLLMException( - status_code=400, message=f"Error code: 400 - {repr({'error': {'message': _GPT_OSS_OVERFLOW_EVENT}})}" + status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}" ), id="python-repr", ), ], ) -def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(original_exception): +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: with pytest.raises(litellm.ContextWindowExceededError) as excinfo: exception_type( model="openai.gpt-oss-20b", @@ -1538,10 +1540,10 @@ def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exc assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message -def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded(): +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None: from litellm.responses.streaming_iterator import map_stream_error_to_exception - mapped_exception = map_stream_error_to_exception( + mapped_exception: Final = map_stream_error_to_exception( { "code": "invalid_prompt", "message": ( @@ -1559,6 +1561,51 @@ def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_contex assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message +_QWEN_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + "context length is 32768 tokens. However, you requested 16 output tokens and your prompt contains at least " + "32753 input tokens, for a total of at least 32769 tokens. Please reduce the length of the input prompt or the " + 'number of requested output tokens. (parameter=input_tokens, value=32753)", param: None } }' +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _QWEN_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_QWEN_OVERFLOW_EVENT), id="streamed-event"), + ], +) +def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 300c9cff6ff..086253039ec 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18970,6 +18970,71 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" +def _anthropic_messages_overflow_before_content(): + return _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" + ), + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback(): + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs"), + [ + pytest.param(None, {}, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"), + ], +) +async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], litellm.ContextWindowExceededError) + assert len(provider.calls) == 1 + + +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs", "expected"), + [ + pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"), + pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), + pytest.param(None, {}, False, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + ], +) +def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + + assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected + + def test_anthropic_messages_retry_budget_precedence_direct_call(): """A retry policy naming the error class outranks the request's num_retries, which outranks the routed deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too."""