diff --git a/litellm/router.py b/litellm/router.py index fcd7d89a2b3..0c3975ebf61 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -680,6 +680,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams ) @@ -10383,18 +10384,23 @@ class Router(DeprecatedRouterMembers): ) ) - def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool: """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks - can serve as well as every route _stream_can_fall_back counts.""" + can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists + errs toward True, as _stream_can_fall_back does for non-standard fallbacks.""" if fallbacks_disabled_for_request(kwargs): return False if self._stream_can_fall_back(model_group, kwargs): return True - context_window_fallbacks: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) - if not context_window_fallbacks: + configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) + if not configured: return False + try: + chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured) + except ValidationError: + return True resolved, _ = get_fallback_model_group_for_lookup_groups( - fallbacks=context_window_fallbacks, + fallbacks=chain, lookup_groups=fallback_lookup_groups(kwargs, model_group), ) return has_unattempted_fallback_target(resolved, kwargs) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 2d91f2a02c0..bbf706d079d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19265,7 +19265,7 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" -def _anthropic_messages_overflow_before_content(): +def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream: return _AnthropicMessagesRaisingByteStream( [_anthropic_messages_message_start_chunk()], litellm.ContextWindowExceededError( @@ -19275,7 +19275,7 @@ def _anthropic_messages_overflow_before_content(): @pytest.mark.asyncio -async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback(): +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None: router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) provider: Final = _AnthropicMessagesScriptedProvider( _anthropic_messages_overflow_before_content, @@ -19320,6 +19320,7 @@ async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_ pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), pytest.param(None, {}, False, id="no-fallback-configured"), pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"), ], ) def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks(