mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 8b9c9632f0 into 87e32e7c6d
This commit is contained in:
commit
8cebd58e4d
7 changed files with 585 additions and 60 deletions
|
|
@ -931,7 +931,18 @@ def _map_openai_like_exception(
|
|||
)
|
||||
|
||||
|
||||
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
|
||||
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = (
|
||||
re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"),
|
||||
re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"),
|
||||
re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"),
|
||||
)
|
||||
_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile(
|
||||
r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens"
|
||||
)
|
||||
_BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS: Final = (
|
||||
re.compile(r"input length and `max_tokens` exceed context limit"),
|
||||
re.compile(r"\\?'max_tokens\\?' or \\?'max_completion_tokens\\?' is too large"),
|
||||
)
|
||||
_BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile(
|
||||
r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']"""
|
||||
)
|
||||
|
|
@ -960,17 +971,36 @@ def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> b
|
|||
|
||||
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
|
||||
"""
|
||||
Mantle reports context overflow as a validation_error carrying the token counts, or as
|
||||
OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event
|
||||
with no error type. Clients such as Claude Code only treat "prompt is too long" as
|
||||
overflow (litellm/litellm#36546).
|
||||
Mantle reports context overflow as a validation_error carrying the token counts (worded
|
||||
differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded
|
||||
code, or in a wording the other providers share, either in a 400 body or in a streamed error
|
||||
event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow
|
||||
(litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested
|
||||
output, so only the output budget says whether shortening the prompt can help; once the output
|
||||
takes more than half the window, or when the wording blames `max_tokens` (Anthropic's and vLLM's),
|
||||
the overflow keeps Mantle's own message so the client lowers `max_tokens` instead of shortening the
|
||||
prompt. A wording only the shared overflow list knows keeps Mantle's message too, as OpenAI's does.
|
||||
"""
|
||||
match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
|
||||
matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS)
|
||||
match: Final = next((found for found in matches if found is not None), None)
|
||||
if match is not None:
|
||||
prompt_tokens, max_tokens = match.groups()
|
||||
return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
|
||||
output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str)
|
||||
if output_budget is not None:
|
||||
context_window, requested_output_tokens = (int(group) for group in output_budget.groups())
|
||||
if requested_output_tokens * 2 > context_window:
|
||||
return error_str
|
||||
return (
|
||||
f"prompt is too long: the prompt and {requested_output_tokens} requested output tokens exceed the "
|
||||
f"{context_window} token context window"
|
||||
)
|
||||
if any(pattern.search(error_str) is not None for pattern in _BEDROCK_MANTLE_MAX_TOKENS_OVERFLOW_PATTERNS):
|
||||
return error_str
|
||||
if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None:
|
||||
return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE
|
||||
if ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
|
||||
return error_str
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -674,6 +674,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
|
|||
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...])
|
||||
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
|
||||
Mapping[str, object] | GenericLiteLLMParams
|
||||
)
|
||||
|
|
@ -883,6 +884,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error
|
|||
return has_generated_content or not error.is_pre_first_chunk
|
||||
|
||||
|
||||
def _anthropic_stream_error_skips_recovery(
|
||||
error: Exception, has_generated_content: bool, holds_for_overflow_only: bool
|
||||
) -> bool:
|
||||
"""
|
||||
A held stream whose only possible recovery is a context window fallback lets every other failure through
|
||||
with its held frames, as it would have reached the client with no hold.
|
||||
"""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
|
||||
has_generated_content, error
|
||||
):
|
||||
return True
|
||||
if not holds_for_overflow_only:
|
||||
return False
|
||||
underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error
|
||||
return not isinstance(underlying, litellm_exceptions.ContextWindowExceededError)
|
||||
|
||||
|
||||
def _stream_raised_error_status(error: Exception) -> int | None:
|
||||
raw_status: Final = getattr(error, "status_code", None)
|
||||
if isinstance(raw_status, int):
|
||||
|
|
@ -896,9 +916,15 @@ def _stream_raised_error_status(error: Exception) -> int | None:
|
|||
def _stream_fallback_error_for_raised(
|
||||
error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic"
|
||||
) -> "MidStreamFallbackError | None":
|
||||
"""The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged."""
|
||||
"""
|
||||
The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain
|
||||
sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates
|
||||
unchanged.
|
||||
"""
|
||||
if has_generated_content:
|
||||
return None
|
||||
if isinstance(error, litellm_exceptions.ContextWindowExceededError):
|
||||
return _stream_pre_content_error(error, model, llm_provider)
|
||||
status_code: Final = _stream_raised_error_status(error)
|
||||
if status_code is None:
|
||||
return _stream_pre_content_error(error, model, llm_provider)
|
||||
|
|
@ -7023,9 +7049,14 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# to take over there is nothing to buffer for, so every frame,
|
||||
# including pings and provider error frames, is forwarded live.
|
||||
model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
|
||||
can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry(
|
||||
initial_kwargs
|
||||
) or self._stream_can_fall_back(model, initial_kwargs)
|
||||
holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow(
|
||||
model, initial_kwargs
|
||||
)
|
||||
has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over
|
||||
self._anthropic_messages_stream_can_retry(initial_kwargs)
|
||||
or self._stream_can_fall_back(model, initial_kwargs)
|
||||
can_retry_or_fall_back or holds_for_overflow_only
|
||||
)
|
||||
buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline
|
||||
try:
|
||||
|
|
@ -7049,7 +7080,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
refusal_stop_details = (
|
||||
parse_anthropic_refusal_stop_details(parse_window)
|
||||
if not has_generated_content and error_event is None
|
||||
if not has_generated_content and error_event is None and not holds_for_overflow_only
|
||||
else None
|
||||
)
|
||||
if (
|
||||
|
|
@ -7087,6 +7118,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
async for item in self._aanthropic_messages_recover_stream_error(
|
||||
stream_error,
|
||||
has_generated_content,
|
||||
holds_for_overflow_only,
|
||||
buffered_lifecycle_chunks,
|
||||
model,
|
||||
initial_kwargs,
|
||||
|
|
@ -7107,6 +7139,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self,
|
||||
stream_error: Exception,
|
||||
has_generated_content: bool,
|
||||
holds_for_overflow_only: bool,
|
||||
buffered_lifecycle_chunks: tuple[bytes, ...],
|
||||
model: str,
|
||||
initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it
|
||||
|
|
@ -7116,12 +7149,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"""Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
|
||||
has_generated_content, stream_error
|
||||
):
|
||||
if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only):
|
||||
for buffered_chunk in buffered_lifecycle_chunks:
|
||||
yield buffered_chunk
|
||||
if stream_error.original_exception is not None:
|
||||
if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None:
|
||||
raise stream_error.original_exception from stream_error
|
||||
raise stream_error
|
||||
fallback_error: Final = (
|
||||
|
|
@ -7267,7 +7298,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
context_window_fallbacks=_request_fallback_list(
|
||||
kwargs,
|
||||
"context_window_fallbacks",
|
||||
cast( # cast-ok: untyped router attribute
|
||||
cast( # cast-ok: list is invariant and the helper takes list[object]
|
||||
"list[object] | None", self._core_context_window_fallbacks
|
||||
),
|
||||
),
|
||||
|
|
@ -7288,6 +7319,16 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
return False
|
||||
return True
|
||||
|
||||
def _stream_overflow_retry_declined(self, trigger: Exception, kwargs: Mapping[str, object]) -> bool:
|
||||
"""
|
||||
An overflow resends the same prompt to the same windows, so only a retry policy that names its error
|
||||
class retries it, never a grant made for the error that opened the retries.
|
||||
"""
|
||||
return (
|
||||
isinstance(trigger, litellm_exceptions.ContextWindowExceededError)
|
||||
and self._anthropic_messages_policy_retries(trigger, kwargs) is None
|
||||
)
|
||||
|
||||
async def _astream_retry_same_group(
|
||||
self,
|
||||
e: "MidStreamFallbackError",
|
||||
|
|
@ -7313,8 +7354,9 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see
|
||||
for attempt in range(attempted_retries_for_request(initial_kwargs), budget):
|
||||
trigger = _mid_stream_retry_trigger(last_error)
|
||||
if not policy_applies and not self._stream_should_retry(
|
||||
trigger, healthy_deployments, all_deployments, initial_kwargs
|
||||
if self._stream_overflow_retry_declined(trigger, initial_kwargs) or (
|
||||
not policy_applies
|
||||
and not self._stream_should_retry(trigger, healthy_deployments, all_deployments, initial_kwargs)
|
||||
):
|
||||
return _StreamRetriesExhausted(last_error)
|
||||
if get_metadata_variable_name_from_kwargs(retry_kwargs) in retry_kwargs:
|
||||
|
|
@ -9223,21 +9265,19 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
lookup_groups=lookup_groups,
|
||||
)
|
||||
)
|
||||
if context_window_fallback_model_group is None:
|
||||
raise original_exception
|
||||
if context_window_fallback_model_group is not None:
|
||||
input_kwargs.update(
|
||||
{
|
||||
"fallback_model_group": context_window_fallback_model_group,
|
||||
"original_model_group": original_model_group,
|
||||
}
|
||||
)
|
||||
|
||||
input_kwargs.update(
|
||||
{
|
||||
"fallback_model_group": context_window_fallback_model_group,
|
||||
"original_model_group": original_model_group,
|
||||
}
|
||||
)
|
||||
|
||||
response = await run_async_fallback(
|
||||
*args,
|
||||
**input_kwargs,
|
||||
)
|
||||
return response
|
||||
response = await run_async_fallback(
|
||||
*args,
|
||||
**input_kwargs,
|
||||
)
|
||||
return response
|
||||
|
||||
else:
|
||||
error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks"
|
||||
|
|
@ -10474,6 +10514,33 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
)
|
||||
|
||||
def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks
|
||||
can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists
|
||||
errs toward True, as _stream_can_fall_back does for non-standard fallbacks."""
|
||||
if fallbacks_disabled_for_request(kwargs):
|
||||
return False
|
||||
if self._stream_can_fall_back(model_group, kwargs):
|
||||
return True
|
||||
configured: Final = _request_fallback_list(
|
||||
kwargs,
|
||||
"context_window_fallbacks",
|
||||
cast( # cast-ok: list is invariant and the helper takes list[object]
|
||||
"list[object] | None", self._core_context_window_fallbacks
|
||||
),
|
||||
)
|
||||
if not configured:
|
||||
return False
|
||||
try:
|
||||
chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured)
|
||||
except ValidationError:
|
||||
return True
|
||||
resolved, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=chain,
|
||||
lookup_groups=fallback_lookup_groups(kwargs, model_group),
|
||||
)
|
||||
return has_unattempted_fallback_target(resolved, kwargs)
|
||||
|
||||
def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
|
||||
"""
|
||||
Whether async_function_with_fallbacks_common_utils could still route a
|
||||
|
|
|
|||
|
|
@ -622,7 +622,7 @@ def get_fallback_model_group(
|
|||
prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys)
|
||||
## check for specific model group-specific fallbacks
|
||||
for idx, item in enumerate(fallbacks):
|
||||
if isinstance(item, dict):
|
||||
if isinstance(item, dict) and item:
|
||||
fallback_key = next(iter(item))
|
||||
if fallback_key == model_group: # check exact match
|
||||
fallback_model_group = item[model_group]
|
||||
|
|
|
|||
|
|
@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet
|
|||
_assert_fell_back(cwf_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None:
|
||||
def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None:
|
||||
prompt: Final = _prompt("cwf messages stream")
|
||||
response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True))
|
||||
assert response.status_code == 200, text
|
||||
error: Final = _only_error(text)
|
||||
assert error["type"] == "invalid_request_error", text
|
||||
assert _GENERIC in str(error["message"]), text
|
||||
_assert_no_fallback(cwf_rig, "overflow", prompt)
|
||||
_assert_fell_back(cwf_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_context_window_fallback_fires_on_streamed_responses(cwf_rig: Rig) -> None:
|
||||
|
|
@ -454,16 +451,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r
|
|||
_assert_fell_back(fb_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back(
|
||||
fb_rig: Rig,
|
||||
) -> None:
|
||||
def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None:
|
||||
prompt: Final = _prompt("fb messages stream")
|
||||
response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True))
|
||||
assert response.status_code == 400, text
|
||||
error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"])
|
||||
assert error["type"] == "invalid_request_error", text
|
||||
assert _GENERIC in str(error["message"]), text
|
||||
_assert_no_fallback(fb_rig, "overflow", prompt)
|
||||
assert response.status_code == 200, text
|
||||
_assert_fell_back(fb_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_plain_fallback_fires_on_streamed_responses(fb_rig: Rig) -> None:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
|
|||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
from _pytest.mark.structures import ParameterSet
|
||||
from fastapi import HTTPException
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
|
||||
|
|
@ -1754,6 +1755,208 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
|
|||
assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message
|
||||
|
||||
|
||||
_GPT_OSS_OVERFLOW_EVENT: Final = (
|
||||
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), '
|
||||
'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"original_exception",
|
||||
[
|
||||
pytest.param(
|
||||
BaseLLMException(
|
||||
status_code=400,
|
||||
message=json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": _GPT_OSS_OVERFLOW_EVENT,
|
||||
"param": None,
|
||||
"type": "invalid_request_error",
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
id="400-body",
|
||||
),
|
||||
pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"),
|
||||
pytest.param(
|
||||
BaseLLMException(
|
||||
status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}"
|
||||
),
|
||||
id="python-repr",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(
|
||||
original_exception: BaseLLMException,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="openai.gpt-oss-20b",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert excinfo.value.response.status_code == 400
|
||||
assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message
|
||||
|
||||
|
||||
def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None:
|
||||
from litellm.responses.streaming_iterator import map_stream_error_to_exception
|
||||
|
||||
mapped_exception: Final = map_stream_error_to_exception(
|
||||
{
|
||||
"code": "invalid_prompt",
|
||||
"message": (
|
||||
'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: '
|
||||
'"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", '
|
||||
'param: Some("input") } }'
|
||||
),
|
||||
},
|
||||
model="openai.gpt-oss-20b",
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert isinstance(mapped_exception, litellm.ContextWindowExceededError)
|
||||
assert mapped_exception.status_code == 400
|
||||
assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message
|
||||
|
||||
|
||||
def _qwen_overflow_event(requested_output_tokens: int) -> str:
|
||||
counted_input_tokens: Final = 32768 - requested_output_tokens + 1
|
||||
return (
|
||||
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum '
|
||||
f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your "
|
||||
f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please "
|
||||
"reduce the length of the input prompt or the number of requested output tokens. "
|
||||
f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}'
|
||||
)
|
||||
|
||||
|
||||
def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]:
|
||||
return (
|
||||
pytest.param(
|
||||
BaseLLMException(
|
||||
status_code=400,
|
||||
message=json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": event,
|
||||
"param": None,
|
||||
"type": "invalid_request_error",
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
id="400-body",
|
||||
),
|
||||
pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"),
|
||||
pytest.param(
|
||||
BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"),
|
||||
id="python-repr",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"original_exception",
|
||||
[
|
||||
*_mantle_overflow_serializations(_qwen_overflow_event(16)),
|
||||
*_mantle_overflow_serializations(_qwen_overflow_event(16384)),
|
||||
],
|
||||
)
|
||||
def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded(
|
||||
original_exception: BaseLLMException,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="qwen.qwen3-32b",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert excinfo.value.response.status_code == 400
|
||||
assert "prompt is too long: the prompt and" in excinfo.value.message
|
||||
assert "requested output tokens exceed the 32768 token context window" in excinfo.value.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"original_exception",
|
||||
[
|
||||
*_mantle_overflow_serializations(_qwen_overflow_event(16385)),
|
||||
*_mantle_overflow_serializations(_qwen_overflow_event(32000)),
|
||||
*_mantle_overflow_serializations(_qwen_overflow_event(32768)),
|
||||
],
|
||||
)
|
||||
def test_bedrock_mantle_qwen_output_budget_over_half_the_window_keeps_mantles_message(
|
||||
original_exception: BaseLLMException,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="qwen.qwen3-32b",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert "prompt is too long" not in excinfo.value.message
|
||||
assert "output tokens and your prompt contains at least" in excinfo.value.message
|
||||
|
||||
|
||||
_ANTHROPIC_MAX_TOKENS_OVERFLOW: Final = (
|
||||
"input length and `max_tokens` exceed context limit: 197202 + 21333 > 200000, decrease input length or "
|
||||
"`max_tokens` and try again"
|
||||
)
|
||||
_VLLM_MAX_TOKENS_OVERFLOW: Final = (
|
||||
"'max_tokens' or 'max_completion_tokens' is too large: 200000. This model's maximum context length is 131072 "
|
||||
"tokens and your request has 83 input tokens (200000 > 131072 - 83)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"original_exception",
|
||||
(
|
||||
*_mantle_overflow_serializations(_ANTHROPIC_MAX_TOKENS_OVERFLOW),
|
||||
*_mantle_overflow_serializations(_VLLM_MAX_TOKENS_OVERFLOW),
|
||||
),
|
||||
)
|
||||
def test_bedrock_mantle_max_tokens_overflow_keeps_mantles_message(original_exception: BaseLLMException) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="anthropic.claude-haiku-4-5",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert "prompt is too long" not in excinfo.value.message
|
||||
assert "max_tokens" in excinfo.value.message
|
||||
assert "200000" in excinfo.value.message
|
||||
|
||||
|
||||
def test_bedrock_mantle_overflow_only_the_shared_list_knows_keeps_mantles_message() -> None:
|
||||
vllm_total_overflow: Final = (
|
||||
"This model's maximum context length is 32768 tokens. However, you requested 40000 tokens (100 in the "
|
||||
"messages, 39900 in the completion). Please reduce the length of the messages or completion."
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="qwen.qwen3-32b",
|
||||
original_exception=BaseLLMException(status_code=400, message=vllm_total_overflow),
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert "prompt is too long" not in excinfo.value.message
|
||||
assert "39900 in the completion" in excinfo.value.message
|
||||
|
||||
|
||||
def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -355,26 +355,30 @@ async def test_an_error_specific_chain_wins_over_the_general_chain(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("error", "field", "raised"),
|
||||
[
|
||||
(context_window, "context_window_fallbacks", litellm.ContextWindowExceededError),
|
||||
(content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError),
|
||||
],
|
||||
ids=["context-window", "content-policy"],
|
||||
)
|
||||
async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain(
|
||||
make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception]
|
||||
async def test_a_content_policy_chain_without_the_group_raises_without_trying_the_general_chain(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]})
|
||||
host.answer("openai/a", error())
|
||||
core = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}])
|
||||
host.answer("openai/a", content_policy())
|
||||
|
||||
with pytest.raises(raised):
|
||||
with pytest.raises(litellm.ContentPolicyViolationError):
|
||||
await complete(core)
|
||||
|
||||
assert host.deployments_called("acompletion") == ["a"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_context_window_chain_without_the_group_falls_back_through_the_general_chain(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core = build(make_core, fallbacks=[{"a": ["b"]}], context_window_fallbacks=[{"other": ["c"]}])
|
||||
host.answer("openai/a", context_window())
|
||||
|
||||
await complete(core)
|
||||
|
||||
assert host.deployments_called("acompletion") == ["a", "b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", [context_window, content_policy], ids=["context-window", "content-policy"])
|
||||
async def test_without_an_error_specific_chain_the_general_chain_applies(
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.router import (
|
|||
FallbackAwareAnthropicMessagesStream,
|
||||
_stream_commits_now,
|
||||
_anthropic_stream_error_is_gateway_verdict,
|
||||
_anthropic_stream_error_skips_recovery,
|
||||
_stream_fallback_error_for_raised,
|
||||
_anthropic_stream_forwards_ping_live,
|
||||
_stream_raised_error_status,
|
||||
|
|
@ -18571,6 +18572,53 @@ def test_anthropic_stream_should_decline_fallback_direct_call():
|
|||
assert _anthropic_stream_should_decline_fallback(False, post_first_chunk_error) is True
|
||||
|
||||
|
||||
def _pre_first_chunk_stream_error(original_exception: Exception) -> MidStreamFallbackError:
|
||||
return MidStreamFallbackError(
|
||||
message="stream failed",
|
||||
model="glm",
|
||||
llm_provider="bedrock_mantle",
|
||||
original_exception=original_exception,
|
||||
is_pre_first_chunk=True,
|
||||
)
|
||||
|
||||
|
||||
_STREAM_OVERFLOW: Final = litellm.ContextWindowExceededError(
|
||||
message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm"
|
||||
)
|
||||
_STREAM_UNAVAILABLE: Final = litellm.ServiceUnavailableError(
|
||||
message="unavailable", llm_provider="bedrock_mantle", model="glm"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "has_generated_content", "holds_for_overflow_only", "skips"),
|
||||
[
|
||||
pytest.param(_STREAM_UNAVAILABLE, False, False, False, id="full-hold-recovers-any-error"),
|
||||
pytest.param(_STREAM_OVERFLOW, False, True, False, id="overflow-only-hold-recovers-an-overflow"),
|
||||
pytest.param(_STREAM_UNAVAILABLE, False, True, True, id="overflow-only-hold-passes-a-503-through"),
|
||||
pytest.param(
|
||||
_pre_first_chunk_stream_error(_STREAM_OVERFLOW),
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
id="overflow-only-hold-recovers-a-wrapped-overflow",
|
||||
),
|
||||
pytest.param(
|
||||
_pre_first_chunk_stream_error(_STREAM_UNAVAILABLE),
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
id="overflow-only-hold-passes-a-wrapped-503-through",
|
||||
),
|
||||
pytest.param(_pre_first_chunk_stream_error(_STREAM_OVERFLOW), True, False, True, id="content-already-sent"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_stream_error_skips_recovery_direct_call(
|
||||
error: Exception, has_generated_content: bool, holds_for_overflow_only: bool, skips: bool
|
||||
) -> None:
|
||||
assert _anthropic_stream_error_skips_recovery(error, has_generated_content, holds_for_overflow_only) is skips
|
||||
|
||||
|
||||
def test_anthropic_stream_commits_now_direct_call():
|
||||
content = _anthropic_messages_content_chunk("hi")
|
||||
lifecycle_chunk = _anthropic_messages_message_start_chunk()
|
||||
|
|
@ -19265,6 +19313,180 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the
|
|||
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
|
||||
|
||||
|
||||
def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream:
|
||||
return _AnthropicMessagesRaisingByteStream(
|
||||
[_anthropic_messages_message_start_chunk()],
|
||||
litellm.ContextWindowExceededError(
|
||||
message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
_anthropic_messages_overflow_before_content,
|
||||
lambda: _AnthropicMessagesFakeByteStream(
|
||||
[_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")]
|
||||
),
|
||||
)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
|
||||
body: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")]
|
||||
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_stream_overflow_on_a_group_missing_from_the_context_window_chain_takes_its_fallbacks() -> (
|
||||
None
|
||||
):
|
||||
router: Final = _anthropic_messages_retry_router(
|
||||
num_retries=0, fallbacks=[{"glm": ["fb"]}], context_window_fallbacks=[{"other": ["cw"]}]
|
||||
)
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
_anthropic_messages_overflow_before_content,
|
||||
lambda: _AnthropicMessagesFakeByteStream(
|
||||
[_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")]
|
||||
),
|
||||
)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
|
||||
body: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from fb")]
|
||||
assert [model in _ANTHROPIC_MESSAGES_RETRY_GROUP for model, _, _ in provider.calls] == [True, False]
|
||||
assert [model for model, _, _ in provider.calls][-1] == "anthropic/fb-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("context_window_fallbacks", "request_kwargs"),
|
||||
[
|
||||
pytest.param(None, {}, id="no-fallback-configured"),
|
||||
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over(
|
||||
context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object]
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks)
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs)
|
||||
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
|
||||
|
||||
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
|
||||
assert isinstance(outcome[-1], litellm.ContextWindowExceededError)
|
||||
assert len(provider.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("raised", "expected_error"),
|
||||
[
|
||||
pytest.param(
|
||||
litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"),
|
||||
litellm.ServiceUnavailableError,
|
||||
id="raised-503",
|
||||
),
|
||||
pytest.param(
|
||||
litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"),
|
||||
litellm.BadRequestError,
|
||||
id="raised-400",
|
||||
),
|
||||
pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start(
|
||||
raised: Exception, expected_error: type[Exception]
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised)
|
||||
)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
|
||||
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
|
||||
|
||||
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
|
||||
assert isinstance(outcome[-1], expected_error)
|
||||
assert len(provider.calls) == 1
|
||||
|
||||
|
||||
def _anthropic_messages_refusal_chunk() -> bytes:
|
||||
return (
|
||||
b'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "refusal", '
|
||||
b'"stop_details": {"type": "refusal", "category": "cyber"}}, "usage": {"output_tokens": 0}}\n\n'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_stream_context_window_only_group_passes_a_refusal_through() -> None:
|
||||
router: Final = _anthropic_messages_retry_router(
|
||||
num_retries=0, fallbacks=[{"*": ["fb"]}], context_window_fallbacks=[{"glm": ["cw"]}]
|
||||
)
|
||||
refusal_stream: Final = (_anthropic_messages_message_start_chunk(), _anthropic_messages_refusal_chunk())
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(lambda: _AnthropicMessagesFakeByteStream(list(refusal_stream)))
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider, fallbacks=[])
|
||||
body: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert body == list(refusal_stream)
|
||||
assert len(provider.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("retry_policy", "expected_calls"),
|
||||
[
|
||||
(RetryPolicy(InternalServerErrorRetries=3), 2),
|
||||
(RetryPolicy(InternalServerErrorRetries=2, BadRequestErrorRetries=2), 3),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_policy_retry_retries_an_overflow_only_when_the_policy_names_it(
|
||||
retry_policy: RetryPolicy, expected_calls: int
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, retry_policy=retry_policy)
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
lambda: _AnthropicMessagesRaisingByteStream(
|
||||
[_anthropic_messages_message_start_chunk()],
|
||||
litellm.InternalServerError(message="upstream reset", llm_provider="bedrock_mantle", model="glm"),
|
||||
),
|
||||
_anthropic_messages_raise_context_window_error,
|
||||
_anthropic_messages_raise_context_window_error,
|
||||
_anthropic_messages_raise_context_window_error,
|
||||
)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
[chunk async for chunk in stream]
|
||||
|
||||
assert len(provider.calls) == expected_calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("context_window_fallbacks", "request_kwargs", "expected"),
|
||||
[
|
||||
pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"),
|
||||
pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"),
|
||||
pytest.param(None, {}, False, id="no-fallback-configured"),
|
||||
pytest.param(None, {"fallbacks": [{"glm": ["fb"]}]}, True, id="request-plain-fallback"),
|
||||
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"),
|
||||
pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"),
|
||||
pytest.param(None, {"context_window_fallbacks": [{}]}, False, id="request-chain-of-an-empty-entry"),
|
||||
],
|
||||
)
|
||||
def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks(
|
||||
context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks)
|
||||
|
||||
assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected
|
||||
|
||||
|
||||
def test_anthropic_messages_retry_budget_precedence_direct_call():
|
||||
"""A retry policy naming the error class outranks the request's num_retries, which outranks the routed
|
||||
deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too."""
|
||||
|
|
@ -20144,7 +20366,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_
|
|||
|
||||
with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt:
|
||||
recovery = router._aanthropic_messages_recover_stream_error(
|
||||
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
declined,
|
||||
True,
|
||||
False,
|
||||
buffered,
|
||||
"primary",
|
||||
{"model": "primary"},
|
||||
_anthropic_messages_make_wrapper(),
|
||||
AsyncMock(),
|
||||
)
|
||||
with pytest.raises(BedrockError) as exc_info:
|
||||
await drain(recovery)
|
||||
|
|
@ -20165,7 +20394,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to
|
|||
|
||||
with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt):
|
||||
recovery = router._aanthropic_messages_recover_stream_error(
|
||||
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
)
|
||||
collected = [chunk async for chunk in recovery]
|
||||
assert collected == [b"fallback"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue