diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 85c232ad3f4..6978fe76c6d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -931,7 +931,18 @@ def _map_openai_like_exception( ) -_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)") +_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( + re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"), + re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), + re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), +) +_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile( + r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens" +) +_BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS: Final = ( + re.compile(r"input length and `max_tokens` exceed context limit"), + re.compile(r"\\?'max_tokens\\?' or \\?'max_completion_tokens\\?' is too large"), +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -960,17 +971,36 @@ def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> b def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ - Mantle reports context overflow as a validation_error carrying the token counts, or as - OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event - with no error type. Clients such as Claude Code only treat "prompt is too long" as - overflow (litellm/litellm#36546). + Mantle reports context overflow as a validation_error carrying the token counts (worded + differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded + code, or in a wording the other providers share, either in a 400 body or in a streamed error + event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow + (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested + output, so only the output budget says whether shortening the prompt can help; once the output + takes more than half the window, or when the wording blames `max_tokens` (Anthropic's and vLLM's), + the overflow keeps Mantle's own message so the client lowers `max_tokens` instead of shortening the + prompt. A wording only the shared overflow list knows keeps Mantle's message too, as OpenAI's does. """ - match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str) + matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) + match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" + output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str) + if output_budget is not None: + context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) + if requested_output_tokens * 2 > context_window: + return error_str + return ( + f"prompt is too long: the prompt and {requested_output_tokens} requested output tokens exceed the " + f"{context_window} token context window" + ) + if any(pattern.search(error_str) is not None for pattern in _BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS): + return error_str if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE + if ExceptionCheckers.is_error_str_context_window_exceeded(error_str): + return error_str return None diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 30b05456987..60c691c30e5 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -674,6 +674,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams ) @@ -883,6 +884,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error return has_generated_content or not error.is_pre_first_chunk +def _anthropic_stream_error_skips_recovery( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool +) -> bool: + """ + A held stream whose only possible recovery is a context window fallback lets every other failure through + with its held frames, as it would have reached the client with no hold. + """ + from litellm.exceptions import MidStreamFallbackError + + if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, error + ): + return True + if not holds_for_overflow_only: + return False + underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error + return not isinstance(underlying, litellm_exceptions.ContextWindowExceededError) + + def _stream_raised_error_status(error: Exception) -> int | None: raw_status: Final = getattr(error, "status_code", None) if isinstance(raw_status, int): @@ -896,9 +916,15 @@ def _stream_raised_error_status(error: Exception) -> int | None: def _stream_fallback_error_for_raised( error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" ) -> "MidStreamFallbackError | None": - """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + """ + The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain + sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates + unchanged. + """ if has_generated_content: return None + if isinstance(error, litellm_exceptions.ContextWindowExceededError): + return _stream_pre_content_error(error, model, llm_provider) status_code: Final = _stream_raised_error_status(error) if status_code is None: return _stream_pre_content_error(error, model, llm_provider) @@ -7023,9 +7049,14 @@ class PythonCore(DeprecatedRouterMembers): # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group + can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry( + initial_kwargs + ) or self._stream_can_fall_back(model, initial_kwargs) + holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow( + model, initial_kwargs + ) has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over - self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back(model, initial_kwargs) + can_retry_or_fall_back or holds_for_overflow_only ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -7049,7 +7080,7 @@ class PythonCore(DeprecatedRouterMembers): ) refusal_stop_details = ( parse_anthropic_refusal_stop_details(parse_window) - if not has_generated_content and error_event is None + if not has_generated_content and error_event is None and not holds_for_overflow_only else None ) if ( @@ -7087,6 +7118,7 @@ class PythonCore(DeprecatedRouterMembers): async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, + holds_for_overflow_only, buffered_lifecycle_chunks, model, initial_kwargs, @@ -7107,6 +7139,7 @@ class PythonCore(DeprecatedRouterMembers): self, stream_error: Exception, has_generated_content: bool, + holds_for_overflow_only: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it @@ -7116,12 +7149,10 @@ class PythonCore(DeprecatedRouterMembers): """Turns a source-iterator failure into a fallback attempt or the error reaching the caller.""" from litellm.exceptions import MidStreamFallbackError - if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( - has_generated_content, stream_error - ): + if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk - if stream_error.original_exception is not None: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: raise stream_error.original_exception from stream_error raise stream_error fallback_error: Final = ( @@ -7267,7 +7298,7 @@ class PythonCore(DeprecatedRouterMembers): context_window_fallbacks=_request_fallback_list( kwargs, "context_window_fallbacks", - cast( # cast-ok: untyped router attribute + cast( # cast-ok: list is invariant and the helper takes list[object] "list[object] | None", self._core_context_window_fallbacks ), ), @@ -7288,6 +7319,16 @@ class PythonCore(DeprecatedRouterMembers): return False return True + def _stream_overflow_retry_declined(self, trigger: Exception, kwargs: Mapping[str, object]) -> bool: + """ + An overflow resends the same prompt to the same windows, so only a retry policy that names its error + class retries it, never a grant made for the error that opened the retries. + """ + return ( + isinstance(trigger, litellm_exceptions.ContextWindowExceededError) + and self._anthropic_messages_policy_retries(trigger, kwargs) is None + ) + async def _astream_retry_same_group( self, e: "MidStreamFallbackError", @@ -7313,8 +7354,9 @@ class PythonCore(DeprecatedRouterMembers): last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see for attempt in range(attempted_retries_for_request(initial_kwargs), budget): trigger = _mid_stream_retry_trigger(last_error) - if not policy_applies and not self._stream_should_retry( - trigger, healthy_deployments, all_deployments, initial_kwargs + if self._stream_overflow_retry_declined(trigger, initial_kwargs) or ( + not policy_applies + and not self._stream_should_retry(trigger, healthy_deployments, all_deployments, initial_kwargs) ): return _StreamRetriesExhausted(last_error) if get_metadata_variable_name_from_kwargs(retry_kwargs) in retry_kwargs: @@ -9223,21 +9265,19 @@ class PythonCore(DeprecatedRouterMembers): lookup_groups=lookup_groups, ) ) - if context_window_fallback_model_group is None: - raise original_exception + if context_window_fallback_model_group is not None: + input_kwargs.update( + { + "fallback_model_group": context_window_fallback_model_group, + "original_model_group": original_model_group, + } + ) - input_kwargs.update( - { - "fallback_model_group": context_window_fallback_model_group, - "original_model_group": original_model_group, - } - ) - - response = await run_async_fallback( - *args, - **input_kwargs, - ) - return response + response = await run_async_fallback( + *args, + **input_kwargs, + ) + return response else: error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" @@ -10474,6 +10514,33 @@ class PythonCore(DeprecatedRouterMembers): ) ) + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool: + """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks + can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists + errs toward True, as _stream_can_fall_back does for non-standard fallbacks.""" + if fallbacks_disabled_for_request(kwargs): + return False + if self._stream_can_fall_back(model_group, kwargs): + return True + configured: Final = _request_fallback_list( + kwargs, + "context_window_fallbacks", + cast( # cast-ok: list is invariant and the helper takes list[object] + "list[object] | None", self._core_context_window_fallbacks + ), + ) + if not configured: + return False + try: + chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured) + except ValidationError: + return True + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=chain, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return has_unattempted_fallback_target(resolved, kwargs) + def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: """ Whether async_function_with_fallbacks_common_utils could still route a diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..7fc6c628822 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -622,7 +622,7 @@ def get_fallback_model_group( prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys) ## check for specific model group-specific fallbacks for idx, item in enumerate(fallbacks): - if isinstance(item, dict): + if isinstance(item, dict) and item: fallback_key = next(iter(item)) if fallback_key == model_group: # check exact match fallback_model_group = item[model_group] diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 23b75a5a8cf..3053c52f740 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet _assert_fell_back(cwf_rig, "overflow", prompt, text) -def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None: +def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None: prompt: Final = _prompt("cwf messages stream") response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True)) assert response.status_code == 200, text - error: Final = _only_error(text) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(cwf_rig, "overflow", prompt) + _assert_fell_back(cwf_rig, "overflow", prompt, text) def test_context_window_fallback_fires_on_streamed_responses(cwf_rig: Rig) -> None: @@ -454,16 +451,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r _assert_fell_back(fb_rig, "overflow", prompt, text) -def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back( - fb_rig: Rig, -) -> None: +def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None: prompt: Final = _prompt("fb messages stream") response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True)) - assert response.status_code == 400, text - error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"]) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(fb_rig, "overflow", prompt) + assert response.status_code == 200, text + _assert_fell_back(fb_rig, "overflow", prompt, text) def test_plain_fallback_fires_on_streamed_responses(fb_rig: Rig) -> None: diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 2b1e4a5eed3..56f4cc338ba 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import openai import pytest +from _pytest.mark.structures import ParameterSet from fastapi import HTTPException from openai import AsyncOpenAI, OpenAI @@ -1754,6 +1755,208 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message +_GPT_OSS_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' + 'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }' +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _GPT_OSS_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), + pytest.param( + BaseLLMException( + status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}" + ), + id="python-repr", + ), + ], +) +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="openai.gpt-oss-20b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message + + +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None: + from litellm.responses.streaming_iterator import map_stream_error_to_exception + + mapped_exception: Final = map_stream_error_to_exception( + { + "code": "invalid_prompt", + "message": ( + 'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: ' + '"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", ' + 'param: Some("input") } }' + ), + }, + model="openai.gpt-oss-20b", + custom_llm_provider="bedrock_mantle", + ) + + assert isinstance(mapped_exception, litellm.ContextWindowExceededError) + assert mapped_exception.status_code == 400 + assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message + + +def _qwen_overflow_event(requested_output_tokens: int) -> str: + counted_input_tokens: Final = 32768 - requested_output_tokens + 1 + return ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your " + f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please " + "reduce the length of the input prompt or the number of requested output tokens. " + f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}' + ) + + +def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]: + return ( + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": event, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"), + pytest.param( + BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"), + id="python-repr", + ), + ) + + +@pytest.mark.parametrize( + "original_exception", + [ + *_mantle_overflow_serializations(_qwen_overflow_event(16)), + *_mantle_overflow_serializations(_qwen_overflow_event(16384)), + ], +) +def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long: the prompt and" in excinfo.value.message + assert "requested output tokens exceed the 32768 token context window" in excinfo.value.message + + +@pytest.mark.parametrize( + "original_exception", + [ + *_mantle_overflow_serializations(_qwen_overflow_event(16385)), + *_mantle_overflow_serializations(_qwen_overflow_event(32000)), + *_mantle_overflow_serializations(_qwen_overflow_event(32768)), + ], +) +def test_bedrock_mantle_qwen_output_budget_over_half_the_window_keeps_mantles_message( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "output tokens and your prompt contains at least" in excinfo.value.message + + +_ANTHROPIC_MAX_TOKENS_OVERFLOW: Final = ( + "input length and `max_tokens` exceed context limit: 197202 + 21333 > 200000, decrease input length or " + "`max_tokens` and try again" +) +_VLLM_MAX_TOKENS_OVERFLOW: Final = ( + "'max_tokens' or 'max_completion_tokens' is too large: 200000. This model's maximum context length is 131072 " + "tokens and your request has 83 input tokens (200000 > 131072 - 83)." +) + + +@pytest.mark.parametrize( + "original_exception", + ( + *_mantle_overflow_serializations(_ANTHROPIC_MAX_TOKENS_OVERFLOW), + *_mantle_overflow_serializations(_VLLM_MAX_TOKENS_OVERFLOW), + ), +) +def test_bedrock_mantle_max_tokens_overflow_keeps_mantles_message(original_exception: BaseLLMException) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="anthropic.claude-haiku-4-5", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "max_tokens" in excinfo.value.message + assert "200000" in excinfo.value.message + + +def test_bedrock_mantle_overflow_only_the_shared_list_knows_keeps_mantles_message() -> None: + vllm_total_overflow: Final = ( + "This model's maximum context length is 32768 tokens. However, you requested 40000 tokens (100 in the " + "messages, 39900 in the completion). Please reduce the length of the messages or completion." + ) + + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=BaseLLMException(status_code=400, message=vllm_total_overflow), + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "39900 in the completion" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index e4e5c3120c8..8cda5e0c275 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -355,26 +355,30 @@ async def test_an_error_specific_chain_wins_over_the_general_chain( @pytest.mark.asyncio -@pytest.mark.parametrize( - ("error", "field", "raised"), - [ - (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError), - (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError), - ], - ids=["context-window", "content-policy"], -) -async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain( - make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception] +async def test_a_content_policy_chain_without_the_group_raises_without_trying_the_general_chain( + make_core: MakeCore, host: ScriptedHost ) -> None: - core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]}) - host.answer("openai/a", error()) + core = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", content_policy()) - with pytest.raises(raised): + with pytest.raises(litellm.ContentPolicyViolationError): await complete(core) assert host.deployments_called("acompletion") == ["a"] +@pytest.mark.asyncio +async def test_a_context_window_chain_without_the_group_falls_back_through_the_general_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], context_window_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", context_window()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + @pytest.mark.asyncio @pytest.mark.parametrize("error", [context_window, content_policy], ids=["context-window", "content-policy"]) async def test_without_an_error_specific_chain_the_general_chain_applies( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9996ffcb098..13e165f2dc7 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -53,6 +53,7 @@ from litellm.router import ( FallbackAwareAnthropicMessagesStream, _stream_commits_now, _anthropic_stream_error_is_gateway_verdict, + _anthropic_stream_error_skips_recovery, _stream_fallback_error_for_raised, _anthropic_stream_forwards_ping_live, _stream_raised_error_status, @@ -18571,6 +18572,53 @@ def test_anthropic_stream_should_decline_fallback_direct_call(): assert _anthropic_stream_should_decline_fallback(False, post_first_chunk_error) is True +def _pre_first_chunk_stream_error(original_exception: Exception) -> MidStreamFallbackError: + return MidStreamFallbackError( + message="stream failed", + model="glm", + llm_provider="bedrock_mantle", + original_exception=original_exception, + is_pre_first_chunk=True, + ) + + +_STREAM_OVERFLOW: Final = litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" +) +_STREAM_UNAVAILABLE: Final = litellm.ServiceUnavailableError( + message="unavailable", llm_provider="bedrock_mantle", model="glm" +) + + +@pytest.mark.parametrize( + ("error", "has_generated_content", "holds_for_overflow_only", "skips"), + [ + pytest.param(_STREAM_UNAVAILABLE, False, False, False, id="full-hold-recovers-any-error"), + pytest.param(_STREAM_OVERFLOW, False, True, False, id="overflow-only-hold-recovers-an-overflow"), + pytest.param(_STREAM_UNAVAILABLE, False, True, True, id="overflow-only-hold-passes-a-503-through"), + pytest.param( + _pre_first_chunk_stream_error(_STREAM_OVERFLOW), + False, + True, + False, + id="overflow-only-hold-recovers-a-wrapped-overflow", + ), + pytest.param( + _pre_first_chunk_stream_error(_STREAM_UNAVAILABLE), + False, + True, + True, + id="overflow-only-hold-passes-a-wrapped-503-through", + ), + pytest.param(_pre_first_chunk_stream_error(_STREAM_OVERFLOW), True, False, True, id="content-already-sent"), + ], +) +def test_anthropic_stream_error_skips_recovery_direct_call( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool, skips: bool +) -> None: + assert _anthropic_stream_error_skips_recovery(error, has_generated_content, holds_for_overflow_only) is skips + + def test_anthropic_stream_commits_now_direct_call(): content = _anthropic_messages_content_chunk("hi") lifecycle_chunk = _anthropic_messages_message_start_chunk() @@ -19265,6 +19313,180 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" +def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream: + return _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" + ), + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_on_a_group_missing_from_the_context_window_chain_takes_its_fallbacks() -> ( + None +): + router: Final = _anthropic_messages_retry_router( + num_retries=0, fallbacks=[{"glm": ["fb"]}], context_window_fallbacks=[{"other": ["cw"]}] + ) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")] + assert [model in _ANTHROPIC_MESSAGES_RETRY_GROUP for model, _, _ in provider.calls] == [True, False] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/fb-model" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs"), + [ + pytest.param(None, {}, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"), + ], +) +async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], litellm.ContextWindowExceededError) + assert len(provider.calls) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raised", "expected_error"), + [ + pytest.param( + litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"), + litellm.ServiceUnavailableError, + id="raised-503", + ), + pytest.param( + litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"), + litellm.BadRequestError, + id="raised-400", + ), + pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"), + ], +) +async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start( + raised: Exception, expected_error: type[Exception] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised) + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], expected_error) + assert len(provider.calls) == 1 + + +def _anthropic_messages_refusal_chunk() -> bytes: + return ( + b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "refusal", ' + b'"stop_details": {"type": "refusal", "category": "cyber"}}, "usage": {"output_tokens": 0}}\n\n' + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_context_window_only_group_passes_a_refusal_through() -> None: + router: Final = _anthropic_messages_retry_router( + num_retries=0, fallbacks=[{"*": ["fb"]}], context_window_fallbacks=[{"glm": ["cw"]}] + ) + refusal_stream: Final = (_anthropic_messages_message_start_chunk(), _anthropic_messages_refusal_chunk()) + provider: Final = _AnthropicMessagesScriptedProvider(lambda: _AnthropicMessagesFakeByteStream(list(refusal_stream))) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, fallbacks=[]) + body: Final = [chunk async for chunk in stream] + + assert body == list(refusal_stream) + assert len(provider.calls) == 1 + + +@pytest.mark.parametrize( + ("retry_policy", "expected_calls"), + [ + (RetryPolicy(InternalServerErrorRetries=3), 2), + (RetryPolicy(InternalServerErrorRetries=2, BadRequestErrorRetries=2), 3), + ], +) +@pytest.mark.asyncio +async def test_anthropic_messages_policy_retry_retries_an_overflow_only_when_the_policy_names_it( + retry_policy: RetryPolicy, expected_calls: int +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, retry_policy=retry_policy) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.InternalServerError(message="upstream reset", llm_provider="bedrock_mantle", model="glm"), + ), + _anthropic_messages_raise_context_window_error, + _anthropic_messages_raise_context_window_error, + _anthropic_messages_raise_context_window_error, + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + with pytest.raises(litellm.ContextWindowExceededError): + [chunk async for chunk in stream] + + assert len(provider.calls) == expected_calls + + +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs", "expected"), + [ + pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"), + pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), + pytest.param(None, {}, False, id="no-fallback-configured"), + pytest.param(None, {"fallbacks": [{"glm": ["fb"]}]}, True, id="request-plain-fallback"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"), + pytest.param(None, {"context_window_fallbacks": [{}]}, False, id="request-chain-of-an-empty-entry"), + ], +) +def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + + assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected + + def test_anthropic_messages_retry_budget_precedence_direct_call(): """A retry policy naming the error class outranks the request's num_retries, which outranks the routed deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too.""" @@ -20144,7 +20366,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_ with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt: recovery = router._aanthropic_messages_recover_stream_error( - declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + declined, + True, + False, + buffered, + "primary", + {"model": "primary"}, + _anthropic_messages_make_wrapper(), + AsyncMock(), ) with pytest.raises(BedrockError) as exc_info: await drain(recovery) @@ -20165,7 +20394,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt): recovery = router._aanthropic_messages_recover_stream_error( - raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) collected = [chunk async for chunk in recovery] assert collected == [b"fallback"]