scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 128

fix(router): validate the context_window_fallbacks chain in the streamed overflow hold check

Browse source
This commit is contained in:
mateo-berri 2026-10-30 22:29:48 -07:00
parent 43dd711f99
commit 4545a0c785
2 changed files with 14 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/router.py
View file

@ -680,6 +680,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...])
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
Mapping[str, object] | GenericLiteLLMParams
)
@ -10383,18 +10384,23 @@ class Router(DeprecatedRouterMembers):
)
)
def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool:
"""Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks
can serve as well as every route _stream_can_fall_back counts."""
can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists
errs toward True, as _stream_can_fall_back does for non-standard fallbacks."""
if fallbacks_disabled_for_request(kwargs):
return False
if self._stream_can_fall_back(model_group, kwargs):
return True
context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks)
if not context_window_fallbacks:
configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks)
if not configured:
return False
try:
chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured)
except ValidationError:
return True
resolved, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=context_window_fallbacks,
fallbacks=chain,
lookup_groups=fallback_lookup_groups(kwargs, model_group),
)
return has_unattempted_fallback_target(resolved, kwargs)

5
tests/unit/test_router/test_router.py
View file

@ -19265,7 +19265,7 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the
assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model"
def _anthropic_messages_overflow_before_content():
def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream:
return _AnthropicMessagesRaisingByteStream(
[_anthropic_messages_message_start_chunk()],
litellm.ContextWindowExceededError(
@ -19275,7 +19275,7 @@ def _anthropic_messages_overflow_before_content():
@pytest.mark.asyncio
async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback():
async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None:
router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}])
provider: Final = _AnthropicMessagesScriptedProvider(
_anthropic_messages_overflow_before_content,
@ -19320,6 +19320,7 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_
pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"),
pytest.param(None, {}, False, id="no-fallback-configured"),
pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"),
pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"),
],
)
def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks(