This commit is contained in:
simonp9 2026-11-01 22:54:23 -08:00 • committed by GitHub
commit b49fee1bf6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 585 additions and 60 deletions

View file

@ -931,7 +931,18 @@ def _map_openai_like_exception(
)
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = (
re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"),
re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"),
re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"),
)
_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile(
r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens"
)
_BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS: Final = (
re.compile(r"input length and `max_tokens` exceed context limit"),
re.compile(r"\\?'max_tokens\\?' or \\?'max_completion_tokens\\?' is too large"),
)
_BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile(
r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']"""
)
@ -960,17 +971,36 @@ def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> b
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
"""
Mantle reports context overflow as a validation_error carrying the token counts, or as
OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event
with no error type. Clients such as Claude Code only treat "prompt is too long" as
overflow (litellm/litellm#36546).
Mantle reports context overflow as a validation_error carrying the token counts (worded
differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded
code, or in a wording the other providers share, either in a 400 body or in a streamed error
event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow
(litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested
output, so only the output budget says whether shortening the prompt can help; once the output
takes more than half the window, or when the wording blames `max_tokens` (Anthropic's and vLLM's),
the overflow keeps Mantle's own message so the client lowers `max_tokens` instead of shortening the
prompt. A wording only the shared overflow list knows keeps Mantle's message too, as OpenAI's does.
"""
match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS)
match: Final = next((found for found in matches if found is not None), None)
if match is not None:
prompt_tokens, max_tokens = match.groups()
return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str)
if output_budget is not None:
context_window, requested_output_tokens = (int(group) for group in output_budget.groups())
if requested_output_tokens * 2 > context_window:
return error_str
return (
f"prompt is too long: the prompt and {requested_output_tokens} requested output tokens exceed the "
f"{context_window} token context window"
)
if any(pattern.search(error_str) is not None for pattern in _BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS):
return error_str
if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None:
return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE
if ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
return error_str
return None

View file

@ -674,6 +674,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...])
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
Mapping[str, object] | GenericLiteLLMParams
)
@ -883,6 +884,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error
return has_generated_content or not error.is_pre_first_chunk
def _anthropic_stream_error_skips_recovery(
error: Exception, has_generated_content: bool, holds_for_overflow_only: bool
) -> bool:
"""
A held stream whose only possible recovery is a context window fallback lets every other failure through
with its held frames, as it would have reached the client with no hold.
"""
from litellm.exceptions import MidStreamFallbackError
if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
has_generated_content, error
):
return True
if not holds_for_overflow_only:
return False
underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error
return not isinstance(underlying, litellm_exceptions.ContextWindowExceededError)
def _stream_raised_error_status(error: Exception) -> int | None:
raw_status: Final = getattr(error, "status_code", None)
if isinstance(raw_status, int):
@ -896,9 +916,15 @@ def _stream_raised_error_status(error: Exception) -> int | None:
def _stream_fallback_error_for_raised(
error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic"
) -> "MidStreamFallbackError | None":
"""The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged."""
"""
The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain
sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates
unchanged.
"""
if has_generated_content:
return None
if isinstance(error, litellm_exceptions.ContextWindowExceededError):
return _stream_pre_content_error(error, model, llm_provider)
status_code: Final = _stream_raised_error_status(error)
if status_code is None:
return _stream_pre_content_error(error, model, llm_provider)
@ -7023,9 +7049,14 @@ class PythonCore(DeprecatedRouterMembers):
# to take over there is nothing to buffer for, so every frame,
# including pings and provider error frames, is forwarded live.
model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry(
initial_kwargs
) or self._stream_can_fall_back(model, initial_kwargs)
holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow(
model, initial_kwargs
)
has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over
self._anthropic_messages_stream_can_retry(initial_kwargs)
or self._stream_can_fall_back(model, initial_kwargs)
can_retry_or_fall_back or holds_for_overflow_only
)
buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline
try:
@ -7049,7 +7080,7 @@ class PythonCore(DeprecatedRouterMembers):
)
refusal_stop_details = (
parse_anthropic_refusal_stop_details(parse_window)
if not has_generated_content and error_event is None
if not has_generated_content and error_event is None and not holds_for_overflow_only
else None
)
if (
@ -7087,6 +7118,7 @@ class PythonCore(DeprecatedRouterMembers):
async for item in self._aanthropic_messages_recover_stream_error(
stream_error,
has_generated_content,
holds_for_overflow_only,
buffered_lifecycle_chunks,
model,
initial_kwargs,
@ -7107,6 +7139,7 @@ class PythonCore(DeprecatedRouterMembers):
self,
stream_error: Exception,
has_generated_content: bool,
holds_for_overflow_only: bool,
buffered_lifecycle_chunks: tuple[bytes, ...],
model: str,
initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it
@ -7116,12 +7149,10 @@ class PythonCore(DeprecatedRouterMembers):
"""Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
from litellm.exceptions import MidStreamFallbackError
if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
has_generated_content, stream_error
):
if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only):
for buffered_chunk in buffered_lifecycle_chunks:
yield buffered_chunk
if stream_error.original_exception is not None:
if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None:
raise stream_error.original_exception from stream_error
raise stream_error
fallback_error: Final = (
@ -7267,7 +7298,7 @@ class PythonCore(DeprecatedRouterMembers):
context_window_fallbacks=_request_fallback_list(
kwargs,
"context_window_fallbacks",
cast( # cast-ok: untyped router attribute
cast( # cast-ok: list is invariant and the helper takes list[object]
"list[object] | None", self._core_context_window_fallbacks
),
),
@ -7288,6 +7319,16 @@ class PythonCore(DeprecatedRouterMembers):
return False
return True
def _stream_overflow_retry_declined(self, trigger: Exception, kwargs: Mapping[str, object]) -> bool:
"""
An overflow resends the same prompt to the same windows, so only a retry policy that names its error
class retries it, never a grant made for the error that opened the retries.
"""
return (
isinstance(trigger, litellm_exceptions.ContextWindowExceededError)
and self._anthropic_messages_policy_retries(trigger, kwargs) is None
)
async def _astream_retry_same_group(
self,
e: "MidStreamFallbackError",
@ -7313,8 +7354,9 @@ class PythonCore(DeprecatedRouterMembers):
last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see
for attempt in range(attempted_retries_for_request(initial_kwargs), budget):
trigger = _mid_stream_retry_trigger(last_error)
if not policy_applies and not self._stream_should_retry(
trigger, healthy_deployments, all_deployments, initial_kwargs
if self._stream_overflow_retry_declined(trigger, initial_kwargs) or (
not policy_applies
and not self._stream_should_retry(trigger, healthy_deployments, all_deployments, initial_kwargs)
):
return _StreamRetriesExhausted(last_error)
if get_metadata_variable_name_from_kwargs(retry_kwargs) in retry_kwargs:
@ -9223,21 +9265,19 @@ class PythonCore(DeprecatedRouterMembers):
lookup_groups=lookup_groups,
)
)
if context_window_fallback_model_group is None:
raise original_exception
if context_window_fallback_model_group is not None:
input_kwargs.update(
{
"fallback_model_group": context_window_fallback_model_group,
"original_model_group": original_model_group,
}
)
input_kwargs.update(
{
"fallback_model_group": context_window_fallback_model_group,
"original_model_group": original_model_group,
}
)
response = await run_async_fallback(
*args,
**input_kwargs,
)
return response
response = await run_async_fallback(
*args,
**input_kwargs,
)
return response
else:
error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks"
@ -10474,6 +10514,33 @@ class PythonCore(DeprecatedRouterMembers):
)
)
def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool:
"""Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks
can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists
errs toward True, as _stream_can_fall_back does for non-standard fallbacks."""
if fallbacks_disabled_for_request(kwargs):
return False
if self._stream_can_fall_back(model_group, kwargs):
return True
configured: Final = _request_fallback_list(
kwargs,
"context_window_fallbacks",
cast( # cast-ok: list is invariant and the helper takes list[object]
"list[object] | None", self._core_context_window_fallbacks
),
)
if not configured:
return False
try:
chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured)
except ValidationError:
return True
resolved, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=chain,
lookup_groups=fallback_lookup_groups(kwargs, model_group),
)
return has_unattempted_fallback_target(resolved, kwargs)
def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
"""
Whether async_function_with_fallbacks_common_utils could still route a

View file

@ -622,7 +622,7 @@ def get_fallback_model_group(
prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys)
## check for specific model group-specific fallbacks
for idx, item in enumerate(fallbacks):
if isinstance(item, dict):
if isinstance(item, dict) and item:
fallback_key = next(iter(item))
if fallback_key == model_group: # check exact match
fallback_model_group = item[model_group]

View file

@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet
_assert_fell_back(cwf_rig, "overflow", prompt, text)
def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None:
def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None:
prompt: Final = _prompt("cwf messages stream")
response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True))
assert response.status_code == 200, text
error: Final = _only_error(text)
assert error["type"] == "invalid_request_error", text
assert _GENERIC in str(error["message"]), text
_assert_no_fallback(cwf_rig, "overflow", prompt)
_assert_fell_back(cwf_rig, "overflow", prompt, text)
def test_context_window_fallback_fires_on_streamed_responses(cwf_rig: Rig) -> None:
@ -454,16 +451,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r
_assert_fell_back(fb_rig, "overflow", prompt, text)
def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back(
fb_rig: Rig,
) -> None:
def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None:
prompt: Final = _prompt("fb messages stream")
response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True))
assert response.status_code == 400, text
error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"])
assert error["type"] == "invalid_request_error", text
assert _GENERIC in str(error["message"]), text
_assert_no_fallback(fb_rig, "overflow", prompt)
assert response.status_code == 200, text
_assert_fell_back(fb_rig, "overflow", prompt, text)
def test_plain_fallback_fires_on_streamed_responses(fb_rig: Rig) -> None:

View file

@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
import httpx
import openai
import pytest
from _pytest.mark.structures import ParameterSet
from fastapi import HTTPException
from openai import AsyncOpenAI, OpenAI
@ -1754,6 +1755,208 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message
_GPT_OSS_OVERFLOW_EVENT: Final = (
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), '
'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }'
)
@pytest.mark.parametrize(
"original_exception",
[
pytest.param(
BaseLLMException(
status_code=400,
message=json.dumps(
{
"error": {
"code": "validation_error",
"message": _GPT_OSS_OVERFLOW_EVENT,
"param": None,
"type": "invalid_request_error",
}
}
),
),
id="400-body",
),
pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"),
pytest.param(
BaseLLMException(
status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}"
),
id="python-repr",
),
],
)
def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(
original_exception: BaseLLMException,
) -> None:
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
exception_type(
model="openai.gpt-oss-20b",
original_exception=original_exception,
custom_llm_provider="bedrock_mantle",
)
assert excinfo.value.status_code == 400
assert excinfo.value.response.status_code == 400
assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message
def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None:
from litellm.responses.streaming_iterator import map_stream_error_to_exception
mapped_exception: Final = map_stream_error_to_exception(
{
"code": "invalid_prompt",
"message": (
'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: '
'"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", '
'param: Some("input") } }'
),
},
model="openai.gpt-oss-20b",
custom_llm_provider="bedrock_mantle",
)
assert isinstance(mapped_exception, litellm.ContextWindowExceededError)
assert mapped_exception.status_code == 400
assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message
def _qwen_overflow_event(requested_output_tokens: int) -> str:
counted_input_tokens: Final = 32768 - requested_output_tokens + 1
return (
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum '
f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your "
f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please "
"reduce the length of the input prompt or the number of requested output tokens. "
f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}'
)
def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]:
return (
pytest.param(
BaseLLMException(
status_code=400,
message=json.dumps(
{
"error": {
"code": "validation_error",
"message": event,
"param": None,
"type": "invalid_request_error",
}
}
),
),
id="400-body",
),
pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"),
pytest.param(
BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"),
id="python-repr",
),
)
@pytest.mark.parametrize(
"original_exception",
[
*_mantle_overflow_serializations(_qwen_overflow_event(16)),
*_mantle_overflow_serializations(_qwen_overflow_event(16384)),
],
)
def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded(
original_exception: BaseLLMException,
) -> None:
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
exception_type(
model="qwen.qwen3-32b",
original_exception=original_exception,
custom_llm_provider="bedrock_mantle",
)
assert excinfo.value.status_code == 400
assert excinfo.value.response.status_code == 400
assert "prompt is too long: the prompt and" in excinfo.value.message
assert "requested output tokens exceed the 32768 token context window" in excinfo.value.message
@pytest.mark.parametrize(
"original_exception",
[
*_mantle_overflow_serializations(_qwen_overflow_event(16385)),
*_mantle_overflow_serializations(_qwen_overflow_event(32000)),
*_mantle_overflow_serializations(_qwen_overflow_event(32768)),
],
)
def test_bedrock_mantle_qwen_output_budget_over_half_the_window_keeps_mantles_message(
original_exception: BaseLLMException,
) -> None:
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
exception_type(
model="qwen.qwen3-32b",
original_exception=original_exception,
custom_llm_provider="bedrock_mantle",
)
assert excinfo.value.status_code == 400
assert "prompt is too long" not in excinfo.value.message
assert "output tokens and your prompt contains at least" in excinfo.value.message
_ANTHROPIC_MAX_TOKENS_OVERFLOW: Final = (
"input length and `max_tokens` exceed context limit: 197202 + 21333 > 200000, decrease input length or "
"`max_tokens` and try again"
)
_VLLM_MAX_TOKENS_OVERFLOW: Final = (
"'max_tokens' or 'max_completion_tokens' is too large: 200000. This model's maximum context length is 131072 "
"tokens and your request has 83 input tokens (200000 > 131072 - 83)."
)
@pytest.mark.parametrize(
"original_exception",
(
*_mantle_overflow_serializations(_ANTHROPIC_MAX_TOKENS_OVERFLOW),
*_mantle_overflow_serializations(_VLLM_MAX_TOKENS_OVERFLOW),
),
)
def test_bedrock_mantle_max_tokens_overflow_keeps_mantles_message(original_exception: BaseLLMException) -> None:
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
exception_type(
model="anthropic.claude-haiku-4-5",
original_exception=original_exception,
custom_llm_provider="bedrock_mantle",
)
assert excinfo.value.status_code == 400
assert "prompt is too long" not in excinfo.value.message
assert "max_tokens" in excinfo.value.message
assert "200000" in excinfo.value.message
def test_bedrock_mantle_overflow_only_the_shared_list_knows_keeps_mantles_message() -> None:
vllm_total_overflow: Final = (
"This model's maximum context length is 32768 tokens. However, you requested 40000 tokens (100 in the "
"messages, 39900 in the completion). Please reduce the length of the messages or completion."
)
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
exception_type(
model="qwen.qwen3-32b",
original_exception=BaseLLMException(status_code=400, message=vllm_total_overflow),
custom_llm_provider="bedrock_mantle",
)
assert excinfo.value.status_code == 400
assert "prompt is too long" not in excinfo.value.message
assert "39900 in the completion" in excinfo.value.message
def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded():
from litellm.llms.base_llm.chat.transformation import BaseLLMException

View file

@ -355,26 +355,30 @@ async def test_an_error_specific_chain_wins_over_the_general_chain(
@pytest.mark.asyncio
@pytest.mark.parametrize(
("error", "field", "raised"),
[
(context_window, "context_window_fallbacks", litellm.ContextWindowExceededError),
(content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError),
],
ids=["context-window", "content-policy"],
)
async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain(
make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception]
async def test_a_content_policy_chain_without_the_group_raises_without_trying_the_general_chain(
make_core: MakeCore, host: ScriptedHost
) -> None:
core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]})
host.answer("openai/a", error())
core = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}])
host.answer("openai/a", content_policy())
with pytest.raises(raised):
with pytest.raises(litellm.ContentPolicyViolationError):
await complete(core)
assert host.deployments_called("acompletion") == ["a"]
@pytest.mark.asyncio
async def test_a_context_window_chain_without_the_group_falls_back_through_the_general_chain(
make_core: MakeCore, host: ScriptedHost
) -> None:
core = build(make_core, fallbacks=[{"a": ["b"]}], context_window_fallbacks=[{"other": ["c"]}])
host.answer("openai/a", context_window())
await complete(core)
assert host.deployments_called("acompletion") == ["a", "b"]
@pytest.mark.asyncio
@pytest.mark.parametrize("error", [context_window, content_policy], ids=["context-window", "content-policy"])
async def test_without_an_error_specific_chain_the_general_chain_applies(

View file

@ -53,6 +53,7 @@ from litellm.router import (
FallbackAwareAnthropicMessagesStream,
_stream_commits_now,
_anthropic_stream_error_is_gateway_verdict,
_anthropic_stream_error_skips_recovery,
_stream_fallback_error_for_raised,
_anthropic_stream_forwards_ping_live,
_stream_raised_error_status,
@ -18571,6 +18572,53 @@ def test_anthropic_stream_should_decline_fallback_direct_call():
assert _anthropic_stream_should_decline_fallback(False, post_first_chunk_error) is True
def _pre_first_chunk_stream_error(original_exception: Exception) -> MidStreamFallbackError:
return MidStreamFallbackError(
message="stream failed",
model="glm",
llm_provider="bedrock_mantle",
original_exception=original_exception,
is_pre_first_chunk=True,
)
_STREAM_OVERFLOW: Final = litellm.ContextWindowExceededError(
message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm"
)
_STREAM_UNAVAILABLE: Final = litellm.ServiceUnavailableError(
message="unavailable", llm_provider="bedrock_mantle", model="glm"
)
@pytest.mark.parametrize(
("error", "has_generated_content", "holds_for_overflow_only", "skips"),
[
pytest.param(_STREAM_UNAVAILABLE, False, False, False, id="full-hold-recovers-any-error"),
pytest.param(_STREAM_OVERFLOW, False, True, False, id="overflow-only-hold-recovers-an-overflow"),
pytest.param(_STREAM_UNAVAILABLE, False, True, True, id="overflow-only-hold-passes-a-503-through"),
pytest.param(
_pre_first_chunk_stream_error(_STREAM_OVERFLOW),
False,
True,
False,
id="overflow-only-hold-recovers-a-wrapped-overflow",
),
pytest.param(
_pre_first_chunk_stream_error(_STREAM_UNAVAILABLE),
False,
True,
True,
id="overflow-only-hold-passes-a-wrapped-503-through",
),
pytest.param(_pre_first_chunk_stream_error(_STREAM_OVERFLOW), True, False, True, id="content-already-sent"),
],
)
def test_anthropic_stream_error_skips_recovery_direct_call(
error: Exception, has_generated_content: bool, holds_for_overflow_only: bool, skips: bool
) -> None:
assert _anthropic_stream_error_skips_recovery(error, has_generated_content, holds_for_overflow_only) is skips
def test_anthropic_stream_commits_now_direct_call():
content = _anthropic_messages_content_chunk("hi")
lifecycle_chunk = _anthropic_messages_message_start_chunk()
@ -19265,6 +19313,180 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream:
return _AnthropicMessagesRaisingByteStream(
[_anthropic_messages_message_start_chunk()],
litellm.ContextWindowExceededError(
message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm"
),
)
@pytest.mark.asyncio
async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
provider: Final = _AnthropicMessagesScriptedProvider(
_anthropic_messages_overflow_before_content,
lambda: _AnthropicMessagesFakeByteStream(
[_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")]
),
)
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
body: Final = [chunk async for chunk in stream]
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")]
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
@pytest.mark.asyncio
async def test_anthropic_messages_stream_overflow_on_a_group_missing_from_the_context_window_chain_takes_its_fallbacks() -> (
None
):
router: Final = _anthropic_messages_retry_router(
num_retries=0, fallbacks=[{"glm": ["fb"]}], context_window_fallbacks=[{"other": ["cw"]}]
)
provider: Final = _AnthropicMessagesScriptedProvider(
_anthropic_messages_overflow_before_content,
lambda: _AnthropicMessagesFakeByteStream(
[_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")]
),
)
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
body: Final = [chunk async for chunk in stream]
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")]
assert [model in _ANTHROPIC_MESSAGES_RETRY_GROUP for model, _, _ in provider.calls] == [True, False]
assert [model for model, _, _ in provider.calls][-1] == "anthropic/fb-model"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("context_window_fallbacks", "request_kwargs"),
[
pytest.param(None, {}, id="no-fallback-configured"),
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"),
],
)
async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over(
context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object]
) -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks)
provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content)
stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs)
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
assert isinstance(outcome[-1], litellm.ContextWindowExceededError)
assert len(provider.calls) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("raised", "expected_error"),
[
pytest.param(
litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"),
litellm.ServiceUnavailableError,
id="raised-503",
),
pytest.param(
litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"),
litellm.BadRequestError,
id="raised-400",
),
pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"),
],
)
async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start(
raised: Exception, expected_error: type[Exception]
) -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
provider: Final = _AnthropicMessagesScriptedProvider(
lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised)
)
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
assert isinstance(outcome[-1], expected_error)
assert len(provider.calls) == 1
def _anthropic_messages_refusal_chunk() -> bytes:
return (
b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "refusal", '
b'"stop_details": {"type": "refusal", "category": "cyber"}}, "usage": {"output_tokens": 0}}\n\n'
)
@pytest.mark.asyncio
async def test_anthropic_messages_stream_context_window_only_group_passes_a_refusal_through() -> None:
router: Final = _anthropic_messages_retry_router(
num_retries=0, fallbacks=[{"*": ["fb"]}], context_window_fallbacks=[{"glm": ["cw"]}]
)
refusal_stream: Final = (_anthropic_messages_message_start_chunk(), _anthropic_messages_refusal_chunk())
provider: Final = _AnthropicMessagesScriptedProvider(lambda: _AnthropicMessagesFakeByteStream(list(refusal_stream)))
stream: Final = await _anthropic_messages_stream_through_router(router, provider, fallbacks=[])
body: Final = [chunk async for chunk in stream]
assert body == list(refusal_stream)
assert len(provider.calls) == 1
@pytest.mark.parametrize(
("retry_policy", "expected_calls"),
[
(RetryPolicy(InternalServerErrorRetries=3), 2),
(RetryPolicy(InternalServerErrorRetries=2, BadRequestErrorRetries=2), 3),
],
)
@pytest.mark.asyncio
async def test_anthropic_messages_policy_retry_retries_an_overflow_only_when_the_policy_names_it(
retry_policy: RetryPolicy, expected_calls: int
) -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, retry_policy=retry_policy)
provider: Final = _AnthropicMessagesScriptedProvider(
lambda: _AnthropicMessagesRaisingByteStream(
[_anthropic_messages_message_start_chunk()],
litellm.InternalServerError(message="upstream reset", llm_provider="bedrock_mantle", model="glm"),
),
_anthropic_messages_raise_context_window_error,
_anthropic_messages_raise_context_window_error,
_anthropic_messages_raise_context_window_error,
)
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
with pytest.raises(litellm.ContextWindowExceededError):
[chunk async for chunk in stream]
assert len(provider.calls) == expected_calls
@pytest.mark.parametrize(
("context_window_fallbacks", "request_kwargs", "expected"),
[
pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"),
pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"),
pytest.param(None, {}, False, id="no-fallback-configured"),
pytest.param(None, {"fallbacks": [{"glm": ["fb"]}]}, True, id="request-plain-fallback"),
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"),
pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"),
pytest.param(None, {"context_window_fallbacks": [{}]}, False, id="request-chain-of-an-empty-entry"),
],
)
def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks(
context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool
) -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks)
assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected
def test_anthropic_messages_retry_budget_precedence_direct_call():
"""A retry policy naming the error class outranks the request's num_retries, which outranks the routed
deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too."""
@ -20144,7 +20366,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_
with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt:
recovery = router._aanthropic_messages_recover_stream_error(
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
declined,
True,
False,
buffered,
"primary",
{"model": "primary"},
_anthropic_messages_make_wrapper(),
AsyncMock(),
)
with pytest.raises(BedrockError) as exc_info:
await drain(recovery)
@ -20165,7 +20394,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to
with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt):
recovery = router._aanthropic_messages_recover_stream_error(
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
)
collected = [chunk async for chunk in recovery]
assert collected == [b"fallback"]