mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
Merge 9de1f570f8 into a392a39627
This commit is contained in:
simonp9
GitHub
commit
f9c312aa23
5 changed files with 340 additions and 31 deletions
|
|
@ -931,7 +931,14 @@ def _map_openai_like_exception(
|
|||
)
|
||||
|
||||
|
||||
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
|
||||
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = (
|
||||
re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"),
|
||||
re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"),
|
||||
re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"),
|
||||
)
|
||||
_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile(
|
||||
r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens"
|
||||
)
|
||||
_BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile(
|
||||
r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']"""
|
||||
)
|
||||
|
|
@ -960,16 +967,26 @@ def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> b
|
|||
|
||||
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
|
||||
"""
|
||||
Mantle reports context overflow as a validation_error carrying the token counts, or as
|
||||
OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event
|
||||
with no error type. Clients such as Claude Code only treat "prompt is too long" as
|
||||
overflow (litellm/litellm#36546).
|
||||
Mantle reports context overflow as a validation_error carrying the token counts (worded
|
||||
differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded
|
||||
code, or in a wording the other providers share, either in a 400 body or in a streamed error
|
||||
event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow
|
||||
(litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested
|
||||
output, so only the output budget says whether a shorter prompt could fit at all; when it cannot,
|
||||
the overflow keeps Mantle's own message so the client is not told to shorten the prompt.
|
||||
"""
|
||||
match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
|
||||
matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS)
|
||||
match: Final = next((found for found in matches if found is not None), None)
|
||||
if match is not None:
|
||||
prompt_tokens, max_tokens = match.groups()
|
||||
return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
|
||||
if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None:
|
||||
output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str)
|
||||
if output_budget is not None:
|
||||
context_window, requested_output_tokens = (int(group) for group in output_budget.groups())
|
||||
return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str
|
||||
if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(
|
||||
error_str
|
||||
) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
|
||||
return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -680,6 +680,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
|
|||
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...])
|
||||
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
|
||||
Mapping[str, object] | GenericLiteLLMParams
|
||||
)
|
||||
|
|
@ -860,6 +861,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error
|
|||
return has_generated_content or not error.is_pre_first_chunk
|
||||
|
||||
|
||||
def _anthropic_stream_error_skips_recovery(
|
||||
error: Exception, has_generated_content: bool, holds_for_overflow_only: bool
|
||||
) -> bool:
|
||||
"""
|
||||
A held stream whose only possible recovery is a context window fallback lets every other failure through
|
||||
with its held frames, as it would have reached the client with no hold.
|
||||
"""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
|
||||
has_generated_content, error
|
||||
):
|
||||
return True
|
||||
if not holds_for_overflow_only:
|
||||
return False
|
||||
underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error
|
||||
return not isinstance(underlying, litellm.ContextWindowExceededError)
|
||||
|
||||
|
||||
def _stream_raised_error_status(error: Exception) -> int | None:
|
||||
raw_status: Final = getattr(error, "status_code", None)
|
||||
if isinstance(raw_status, int):
|
||||
|
|
@ -873,9 +893,15 @@ def _stream_raised_error_status(error: Exception) -> int | None:
|
|||
def _stream_fallback_error_for_raised(
|
||||
error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic"
|
||||
) -> "MidStreamFallbackError | None":
|
||||
"""The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged."""
|
||||
"""
|
||||
The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain
|
||||
sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates
|
||||
unchanged.
|
||||
"""
|
||||
if has_generated_content:
|
||||
return None
|
||||
if isinstance(error, litellm.ContextWindowExceededError):
|
||||
return _stream_pre_content_error(error, model, llm_provider)
|
||||
status_code: Final = _stream_raised_error_status(error)
|
||||
if status_code is None:
|
||||
return _stream_pre_content_error(error, model, llm_provider)
|
||||
|
|
@ -1848,7 +1874,9 @@ class Router(DeprecatedRouterMembers):
|
|||
else:
|
||||
self._core_fallbacks = [{"*": _fallbacks}]
|
||||
|
||||
self._core_context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks
|
||||
self._core_context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolver contract
|
||||
context_window_fallbacks or litellm.context_window_fallbacks
|
||||
)
|
||||
|
||||
_content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks
|
||||
self.validate_fallbacks(fallback_param=_content_policy_fallbacks)
|
||||
|
|
@ -6927,9 +6955,14 @@ class Router(DeprecatedRouterMembers):
|
|||
# to take over there is nothing to buffer for, so every frame,
|
||||
# including pings and provider error frames, is forwarded live.
|
||||
model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
|
||||
can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry(
|
||||
initial_kwargs
|
||||
) or self._stream_can_fall_back(model, initial_kwargs)
|
||||
holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow(
|
||||
model, initial_kwargs
|
||||
)
|
||||
has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over
|
||||
self._anthropic_messages_stream_can_retry(initial_kwargs)
|
||||
or self._stream_can_fall_back(model, initial_kwargs)
|
||||
can_retry_or_fall_back or holds_for_overflow_only
|
||||
)
|
||||
buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline
|
||||
try:
|
||||
|
|
@ -6991,6 +7024,7 @@ class Router(DeprecatedRouterMembers):
|
|||
async for item in self._aanthropic_messages_recover_stream_error(
|
||||
stream_error,
|
||||
has_generated_content,
|
||||
holds_for_overflow_only,
|
||||
buffered_lifecycle_chunks,
|
||||
model,
|
||||
initial_kwargs,
|
||||
|
|
@ -7011,6 +7045,7 @@ class Router(DeprecatedRouterMembers):
|
|||
self,
|
||||
stream_error: Exception,
|
||||
has_generated_content: bool,
|
||||
holds_for_overflow_only: bool,
|
||||
buffered_lifecycle_chunks: tuple[bytes, ...],
|
||||
model: str,
|
||||
initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it
|
||||
|
|
@ -7020,12 +7055,10 @@ class Router(DeprecatedRouterMembers):
|
|||
"""Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
|
||||
has_generated_content, stream_error
|
||||
):
|
||||
if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only):
|
||||
for buffered_chunk in buffered_lifecycle_chunks:
|
||||
yield buffered_chunk
|
||||
if stream_error.original_exception is not None:
|
||||
if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None:
|
||||
raise stream_error.original_exception from stream_error
|
||||
raise stream_error
|
||||
fallback_error: Final = (
|
||||
|
|
@ -7171,7 +7204,7 @@ class Router(DeprecatedRouterMembers):
|
|||
context_window_fallbacks=_request_fallback_list(
|
||||
kwargs,
|
||||
"context_window_fallbacks",
|
||||
cast( # cast-ok: untyped router attribute
|
||||
cast( # cast-ok: list is invariant and the helper takes list[object]
|
||||
"list[object] | None", self._core_context_window_fallbacks
|
||||
),
|
||||
),
|
||||
|
|
@ -10375,6 +10408,27 @@ class Router(DeprecatedRouterMembers):
|
|||
)
|
||||
)
|
||||
|
||||
def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks
|
||||
can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists
|
||||
errs toward True, as _stream_can_fall_back does for non-standard fallbacks."""
|
||||
if fallbacks_disabled_for_request(kwargs):
|
||||
return False
|
||||
if self._stream_can_fall_back(model_group, kwargs):
|
||||
return True
|
||||
configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks)
|
||||
if not configured:
|
||||
return False
|
||||
try:
|
||||
chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured)
|
||||
except ValidationError:
|
||||
return True
|
||||
resolved, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=chain,
|
||||
lookup_groups=fallback_lookup_groups(kwargs, model_group),
|
||||
)
|
||||
return has_unattempted_fallback_target(resolved, kwargs)
|
||||
|
||||
def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
|
||||
"""
|
||||
Whether async_function_with_fallbacks_common_utils could still route a
|
||||
|
|
|
|||
|
|
@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet
|
|||
_assert_fell_back(cwf_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None:
|
||||
def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None:
|
||||
prompt: Final = _prompt("cwf messages stream")
|
||||
response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True))
|
||||
assert response.status_code == 200, text
|
||||
error: Final = _only_error(text)
|
||||
assert error["type"] == "invalid_request_error", text
|
||||
assert _GENERIC in str(error["message"]), text
|
||||
_assert_no_fallback(cwf_rig, "overflow", prompt)
|
||||
_assert_fell_back(cwf_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_context_window_fallback_fires_on_streamed_responses(cwf_rig: Rig) -> None:
|
||||
|
|
@ -454,16 +451,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r
|
|||
_assert_fell_back(fb_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back(
|
||||
fb_rig: Rig,
|
||||
) -> None:
|
||||
def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None:
|
||||
prompt: Final = _prompt("fb messages stream")
|
||||
response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True))
|
||||
assert response.status_code == 400, text
|
||||
error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"])
|
||||
assert error["type"] == "invalid_request_error", text
|
||||
assert _GENERIC in str(error["message"]), text
|
||||
_assert_no_fallback(fb_rig, "overflow", prompt)
|
||||
assert response.status_code == 200, text
|
||||
_assert_fell_back(fb_rig, "overflow", prompt, text)
|
||||
|
||||
|
||||
def test_plain_fallback_fires_on_streamed_responses(fb_rig: Rig) -> None:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
|
|||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
from _pytest.mark.structures import ParameterSet
|
||||
from fastapi import HTTPException
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
|
||||
|
|
@ -1754,6 +1755,145 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
|
|||
assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message
|
||||
|
||||
|
||||
_GPT_OSS_OVERFLOW_EVENT: Final = (
|
||||
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), '
|
||||
'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"original_exception",
|
||||
[
|
||||
pytest.param(
|
||||
BaseLLMException(
|
||||
status_code=400,
|
||||
message=json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": _GPT_OSS_OVERFLOW_EVENT,
|
||||
"param": None,
|
||||
"type": "invalid_request_error",
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
id="400-body",
|
||||
),
|
||||
pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"),
|
||||
pytest.param(
|
||||
BaseLLMException(
|
||||
status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}"
|
||||
),
|
||||
id="python-repr",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded(
|
||||
original_exception: BaseLLMException,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="openai.gpt-oss-20b",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert excinfo.value.response.status_code == 400
|
||||
assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message
|
||||
|
||||
|
||||
def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None:
|
||||
from litellm.responses.streaming_iterator import map_stream_error_to_exception
|
||||
|
||||
mapped_exception: Final = map_stream_error_to_exception(
|
||||
{
|
||||
"code": "invalid_prompt",
|
||||
"message": (
|
||||
'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: '
|
||||
'"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", '
|
||||
'param: Some("input") } }'
|
||||
),
|
||||
},
|
||||
model="openai.gpt-oss-20b",
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert isinstance(mapped_exception, litellm.ContextWindowExceededError)
|
||||
assert mapped_exception.status_code == 400
|
||||
assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message
|
||||
|
||||
|
||||
def _qwen_overflow_event(requested_output_tokens: int) -> str:
|
||||
counted_input_tokens: Final = 32768 - requested_output_tokens + 1
|
||||
return (
|
||||
'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum '
|
||||
f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your "
|
||||
f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please "
|
||||
"reduce the length of the input prompt or the number of requested output tokens. "
|
||||
f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}'
|
||||
)
|
||||
|
||||
|
||||
def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]:
|
||||
return (
|
||||
pytest.param(
|
||||
BaseLLMException(
|
||||
status_code=400,
|
||||
message=json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": event,
|
||||
"param": None,
|
||||
"type": "invalid_request_error",
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
id="400-body",
|
||||
),
|
||||
pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"),
|
||||
pytest.param(
|
||||
BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"),
|
||||
id="python-repr",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16)))
|
||||
def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded(
|
||||
original_exception: BaseLLMException,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="qwen.qwen3-32b",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert excinfo.value.response.status_code == 400
|
||||
assert "prompt is too long" in excinfo.value.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768)))
|
||||
def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message(
|
||||
original_exception: BaseLLMException,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as excinfo:
|
||||
exception_type(
|
||||
model="qwen.qwen3-32b",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert "prompt is too long" not in excinfo.value.message
|
||||
assert "you requested 32768 output tokens" in excinfo.value.message
|
||||
|
||||
|
||||
def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -19265,6 +19265,105 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the
|
|||
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
|
||||
|
||||
|
||||
def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream:
|
||||
return _AnthropicMessagesRaisingByteStream(
|
||||
[_anthropic_messages_message_start_chunk()],
|
||||
litellm.ContextWindowExceededError(
|
||||
message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
_anthropic_messages_overflow_before_content,
|
||||
lambda: _AnthropicMessagesFakeByteStream(
|
||||
[_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")]
|
||||
),
|
||||
)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
|
||||
body: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")]
|
||||
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("context_window_fallbacks", "request_kwargs"),
|
||||
[
|
||||
pytest.param(None, {}, id="no-fallback-configured"),
|
||||
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over(
|
||||
context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object]
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks)
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs)
|
||||
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
|
||||
|
||||
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
|
||||
assert isinstance(outcome[-1], litellm.ContextWindowExceededError)
|
||||
assert len(provider.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("raised", "expected_error"),
|
||||
[
|
||||
pytest.param(
|
||||
litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"),
|
||||
litellm.ServiceUnavailableError,
|
||||
id="raised-503",
|
||||
),
|
||||
pytest.param(
|
||||
litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"),
|
||||
litellm.BadRequestError,
|
||||
id="raised-400",
|
||||
),
|
||||
pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start(
|
||||
raised: Exception, expected_error: type[Exception]
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
|
||||
provider: Final = _AnthropicMessagesScriptedProvider(
|
||||
lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised)
|
||||
)
|
||||
|
||||
stream: Final = await _anthropic_messages_stream_through_router(router, provider)
|
||||
outcome: Final = [chunk async for chunk in _events_until_error(stream)]
|
||||
|
||||
assert outcome[:-1] == [_anthropic_messages_message_start_chunk()]
|
||||
assert isinstance(outcome[-1], expected_error)
|
||||
assert len(provider.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("context_window_fallbacks", "request_kwargs", "expected"),
|
||||
[
|
||||
pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"),
|
||||
pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"),
|
||||
pytest.param(None, {}, False, id="no-fallback-configured"),
|
||||
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"),
|
||||
pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"),
|
||||
],
|
||||
)
|
||||
def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks(
|
||||
context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool
|
||||
) -> None:
|
||||
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks)
|
||||
|
||||
assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected
|
||||
|
||||
|
||||
def test_anthropic_messages_retry_budget_precedence_direct_call():
|
||||
"""A retry policy naming the error class outranks the request's num_retries, which outranks the routed
|
||||
deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too."""
|
||||
|
|
@ -20144,7 +20243,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_
|
|||
|
||||
with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt:
|
||||
recovery = router._aanthropic_messages_recover_stream_error(
|
||||
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
declined,
|
||||
True,
|
||||
False,
|
||||
buffered,
|
||||
"primary",
|
||||
{"model": "primary"},
|
||||
_anthropic_messages_make_wrapper(),
|
||||
AsyncMock(),
|
||||
)
|
||||
with pytest.raises(BedrockError) as exc_info:
|
||||
await drain(recovery)
|
||||
|
|
@ -20165,7 +20271,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to
|
|||
|
||||
with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt):
|
||||
recovery = router._aanthropic_messages_recover_stream_error(
|
||||
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
)
|
||||
collected = [chunk async for chunk in recovery]
|
||||
assert collected == [b"fallback"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue