mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge f160efa7bd into 98acc01e0d
This commit is contained in:
commit
0c09eb7fb7
4 changed files with 175 additions and 5 deletions
|
|
@ -237,6 +237,28 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
|
|||
return status_code is not None and 400 <= status_code < 500 and status_code != 429
|
||||
|
||||
|
||||
def _mid_stream_error_is_terminal(
|
||||
mapped_exception: Exception,
|
||||
mapped_status_code: int | None,
|
||||
original_status_code: int | None,
|
||||
*,
|
||||
router_fallback_available: bool = False,
|
||||
) -> bool:
|
||||
"""Whether a mid-stream failure is surfaced instead of retried on another deployment.
|
||||
|
||||
A 4xx is terminal, with two exceptions: 429 is transient, and a context-window overflow
|
||||
is satisfiable by a deployment with a larger window (#48887). That second exemption is
|
||||
the Router's, not the exception's: a direct SDK call has no other deployment to try, so
|
||||
it keeps receiving the 400 it got (#15785).
|
||||
"""
|
||||
if (
|
||||
isinstance(mapped_exception, litellm.ContextWindowExceededError)
|
||||
and router_fallback_available
|
||||
):
|
||||
return False
|
||||
return _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
|
||||
|
||||
|
||||
def is_async_iterable(obj: object) -> bool:
|
||||
"""
|
||||
Check if an object is an async iterable (can be used with 'async for').
|
||||
|
|
@ -473,6 +495,9 @@ class CustomStreamWrapper:
|
|||
self.response_uptil_now = ""
|
||||
self._emitted_disqualifying_content = False
|
||||
_model_info: Final[dict] = litellm_params.model_info or {}
|
||||
# Set by the Router when it resolved a deployment. A direct SDK call has none, which is
|
||||
# what separates "try another deployment" from "this 400 is the answer".
|
||||
self._has_router_model_info: bool = bool(_model_info)
|
||||
|
||||
_api_base: Final = get_api_base(
|
||||
model=model or "",
|
||||
|
|
@ -2930,8 +2955,11 @@ class CustomStreamWrapper:
|
|||
# Raise non-retriable client errors directly (skip fallback).
|
||||
# Exception: 429 (rate-limit) IS retriable/transient — allow it
|
||||
# through so the Router can switch to a different model group.
|
||||
if not _is_transport_timeout(e) and (
|
||||
_skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
|
||||
if not _is_transport_timeout(e) and _mid_stream_error_is_terminal(
|
||||
mapped_exception,
|
||||
mapped_status_code,
|
||||
original_status_code,
|
||||
router_fallback_available=self._has_router_model_info,
|
||||
):
|
||||
raise mapped_exception
|
||||
|
||||
|
|
|
|||
|
|
@ -373,7 +373,26 @@ def _error_event_error_object(event: object) -> object:
|
|||
return {field: value for field in ("message", "code") if _is_error_field_value(value := _event_field(event, field))}
|
||||
|
||||
|
||||
def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
|
||||
def _has_router_deployment(litellm_metadata: Mapping[str, object] | None) -> bool:
|
||||
"""Whether the Router is the caller, and so has another deployment to try.
|
||||
|
||||
`model_info` is the deployment the Router resolved before the call. A direct SDK
|
||||
call has no deployment, and therefore nothing to fall back to.
|
||||
"""
|
||||
model_info: Final = litellm_metadata.get("model_info") if litellm_metadata else None
|
||||
return _is_json_object(model_info) and bool(model_info)
|
||||
|
||||
|
||||
def _mid_stream_fallback_eligible(
|
||||
mapped_exception: Exception,
|
||||
litellm_metadata: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
# A context-window overflow arrives as a 400, but another deployment with a larger
|
||||
# window can still satisfy it, so it has to reach the Router's fallback chain (#48887).
|
||||
# Only when the Router is actually the caller: a direct SDK call has no other deployment
|
||||
# to consult, and keeps the 400 it got (#15785).
|
||||
if isinstance(mapped_exception, litellm.ContextWindowExceededError):
|
||||
return _has_router_deployment(litellm_metadata)
|
||||
if isinstance(mapped_exception, litellm.ContentPolicyViolationError):
|
||||
return True
|
||||
status_code: Final = getattr(mapped_exception, "status_code", None)
|
||||
|
|
@ -803,7 +822,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
)
|
||||
|
||||
mapped_exception: Final = self._map_error_event_exception(error_obj)
|
||||
if not _mid_stream_fallback_eligible(mapped_exception):
|
||||
if not _mid_stream_fallback_eligible(mapped_exception, self.litellm_metadata):
|
||||
raise mapped_exception
|
||||
mapped_status: Final = getattr(mapped_exception, "status_code", None)
|
||||
raise MidStreamFallbackError(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
"""Regression tests for #48887.
|
||||
|
||||
A context-window overflow reported *inside* a stream arrives after HTTP 200, so it is not
|
||||
covered by any status-code-based fallback gate. It is still satisfiable by a deployment
|
||||
with a larger window, so both the Responses iterator and the chat stream wrapper must let
|
||||
it through to the Router instead of raising it straight to the caller.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.exceptions import BadRequestError, ContextWindowExceededError
|
||||
from litellm.litellm_core_utils.streaming_handler import _mid_stream_error_is_terminal
|
||||
from litellm.responses.streaming_iterator import _mid_stream_fallback_eligible
|
||||
|
||||
|
||||
def _context_window_error() -> ContextWindowExceededError:
|
||||
return ContextWindowExceededError(
|
||||
message="Your input exceeds the context window of this model.",
|
||||
model="gpt-6-luna",
|
||||
llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
def _bad_request(status_code: int | None = 400) -> BadRequestError:
|
||||
error = BadRequestError(
|
||||
message="Invalid value for 'temperature'.",
|
||||
model="gpt-6-luna",
|
||||
llm_provider="openai",
|
||||
)
|
||||
error.status_code = status_code # type: ignore[misc]
|
||||
return error
|
||||
|
||||
|
||||
# --- Responses API streams (litellm/responses/streaming_iterator.py) -------------------
|
||||
|
||||
|
||||
ROUTER_METADATA = {"model_info": {"id": "deployment-1"}}
|
||||
|
||||
|
||||
def test_context_window_error_is_fallback_eligible_on_responses_stream() -> None:
|
||||
assert _mid_stream_fallback_eligible(_context_window_error(), ROUTER_METADATA) is True
|
||||
|
||||
|
||||
def test_context_window_error_is_not_eligible_for_a_direct_sdk_call() -> None:
|
||||
"""No deployment means no larger window to try; the caller keeps the 400 (#15785)."""
|
||||
assert _mid_stream_fallback_eligible(_context_window_error()) is False
|
||||
assert _mid_stream_fallback_eligible(_context_window_error(), {"model_info": {}}) is False
|
||||
|
||||
|
||||
def test_plain_bad_request_still_skips_responses_fallback() -> None:
|
||||
assert _mid_stream_fallback_eligible(_bad_request()) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [None, 429, 500, 503])
|
||||
def test_transient_and_server_errors_stay_fallback_eligible(status_code: int | None) -> None:
|
||||
assert _mid_stream_fallback_eligible(_bad_request(status_code)) is True
|
||||
|
||||
|
||||
# --- Chat completion streams (litellm/litellm_core_utils/streaming_handler.py) ---------
|
||||
|
||||
|
||||
def test_context_window_error_is_not_terminal_on_chat_stream() -> None:
|
||||
error = _context_window_error()
|
||||
assert _mid_stream_error_is_terminal(
|
||||
error, error.status_code, error.status_code, router_fallback_available=True
|
||||
) is False
|
||||
|
||||
|
||||
def test_context_window_error_is_terminal_for_a_direct_sdk_call() -> None:
|
||||
error = _context_window_error()
|
||||
assert _mid_stream_error_is_terminal(error, error.status_code, error.status_code) is True
|
||||
|
||||
|
||||
def test_context_window_error_is_not_terminal_even_when_envelope_reports_400() -> None:
|
||||
"""The original exception can be the only one carrying the context-window type."""
|
||||
assert _mid_stream_error_is_terminal(_bad_request(400), 400, 400) is True
|
||||
assert _mid_stream_error_is_terminal(
|
||||
_context_window_error(), 400, 400, router_fallback_available=True
|
||||
) is False
|
||||
|
||||
|
||||
def test_plain_bad_request_is_terminal_on_chat_stream() -> None:
|
||||
assert _mid_stream_error_is_terminal(_bad_request(), 400, 400) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [429, 500, 503])
|
||||
def test_retriable_status_codes_are_not_terminal_on_chat_stream(status_code: int) -> None:
|
||||
error = _bad_request(status_code)
|
||||
assert _mid_stream_error_is_terminal(error, status_code, status_code) is False
|
||||
|
|
@ -56,7 +56,7 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
|
||||
|
||||
def _make_iterator() -> BaseResponsesAPIStreamingIterator:
|
||||
def _make_iterator(litellm_metadata: Mapping[str, object] | None = None) -> BaseResponsesAPIStreamingIterator:
|
||||
mock_logging_obj = Mock(spec=LiteLLMLoggingObj)
|
||||
mock_logging_obj.model_call_details = {"litellm_params": {}}
|
||||
mock_config = Mock(spec=BaseResponsesAPIConfig)
|
||||
|
|
@ -68,6 +68,7 @@ def _make_iterator() -> BaseResponsesAPIStreamingIterator:
|
|||
responses_api_provider_config=mock_config,
|
||||
logging_obj=mock_logging_obj,
|
||||
custom_llm_provider="openai",
|
||||
litellm_metadata=litellm_metadata,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -959,3 +960,34 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se
|
|||
|
||||
assert failure is not None
|
||||
_assert_names_the_azure_deployment(failure, reported_api_base)
|
||||
|
||||
|
||||
def test_context_window_error_reaches_fallback_only_under_the_router() -> None:
|
||||
"""The exemption is the Router's, not the exception's.
|
||||
|
||||
The case above pins the direct-SDK contract (#15785): no deployment means the 400 is
|
||||
the answer. When the Router resolved a deployment, the same event must be wrapped so
|
||||
`context_window_fallbacks` can route it to a larger window (#48887).
|
||||
"""
|
||||
iterator: Final = _make_iterator(litellm_metadata={"model_info": {"id": "deployment-1"}})
|
||||
chunk: Final = _make_error_chunk(
|
||||
"invalid_request_error",
|
||||
"context_length_exceeded",
|
||||
"Your input exceeds the context window of this model.",
|
||||
)
|
||||
with pytest.raises(MidStreamFallbackError) as exc_info:
|
||||
iterator._maybe_raise_for_error_event(chunk)
|
||||
assert isinstance(exc_info.value.original_exception, litellm.ContextWindowExceededError)
|
||||
|
||||
|
||||
def test_empty_model_info_does_not_count_as_a_router_deployment() -> None:
|
||||
"""`model_info` present but empty is what a caller can pass by hand; it selects nothing."""
|
||||
iterator: Final = _make_iterator(litellm_metadata={"model_info": {}})
|
||||
chunk: Final = _make_error_chunk(
|
||||
"invalid_request_error",
|
||||
"context_length_exceeded",
|
||||
"Your input exceeds the context window of this model.",
|
||||
)
|
||||
with pytest.raises(litellm.ContextWindowExceededError) as exc_info:
|
||||
iterator._maybe_raise_for_error_event(chunk)
|
||||
assert not isinstance(exc_info.value, MidStreamFallbackError)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue