From f0e87f2457011f01eecfbd3d07ddc94cd99bdf94 Mon Sep 17 00:00:00 2001 From: shivam Date: Mon, 21 Sep 2026 19:50:07 +0000 Subject: [PATCH] fix(router): keep refusal gates closed once every fallback entry was tried Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 9 +++--- .../router_utils/fallback_event_handlers.py | 12 +++++++ .../test_fallback_event_handlers.py | 19 ++++++++++- tests/test_litellm/test_router.py | 32 +++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 98c7c319eaa..d9dd48a2c6a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -191,6 +191,7 @@ from litellm.router_utils.fallback_event_handlers import ( fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, get_pre_routing_selection, + has_unattempted_fallback_target, record_disable_fallbacks, record_pre_routing_selection, run_async_fallback, @@ -8338,12 +8339,12 @@ class Router: """ content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) if content_policy_fallbacks is not None: - return ( + return has_unattempted_fallback_target( self._get_fallback_model_group_for_lookup_groups( fallbacks=content_policy_fallbacks, lookup_groups=fallback_lookup_groups(kwargs, model_group), - ) - is not None + ), + kwargs, ) if self._has_default_fallbacks(): return True @@ -8375,7 +8376,7 @@ class Router: fallbacks=fallbacks, lookup_groups=fallback_lookup_groups(kwargs, model_group), ) - return resolved is not None + return has_unattempted_fallback_target(resolved, kwargs) def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool: """ diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 156195f0587..08b9246e562 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -199,6 +199,18 @@ class AttemptedFallbackTargets: self.keys = self.keys | frozenset((key,)) +def has_unattempted_fallback_target( + fallback_model_group: Sequence[object] | None, kwargs: Mapping[str, object] +) -> bool: + """Whether a resolved chain still holds an entry this request has not tried.""" + if fallback_model_group is None: + return False + attempted: Final = kwargs.get("attempted_targets") + if not isinstance(attempted, AttemptedFallbackTargets): + return True + return any((key := fallback_attempt_key(target)) is None or key not in attempted for target in fallback_model_group) + + def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: """ Handles wildcard routing scenario diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 69772561172..b6ab21dfcef 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -1,6 +1,6 @@ import json from datetime import datetime, timedelta -from typing import NoReturn +from typing import Final, NoReturn from unittest.mock import MagicMock, patch import httpx @@ -1323,3 +1323,20 @@ class TestOrderedFallbackLookupGroups: assert get_fallback_model_group_for_lookup_groups(fallbacks, ("tier9", "smart-router")) == (["backup-b"], None) assert get_fallback_model_group_for_lookup_groups(fallbacks, ("tier9", "no-such")) == (["backup-c"], 2) assert get_fallback_model_group_for_lookup_groups([{"tier1": ["backup-a"]}], ("no", "nope")) == (None, None) + + +class TestHasUnattemptedFallbackTarget: + def test_exhausted_chain_is_not_recoverable_but_a_fresh_entry_is(self): + from litellm.router_utils.fallback_event_handlers import ( + has_unattempted_fallback_target, + ) + + attempted: Final = AttemptedFallbackTargets() + attempted.record("primary") + attempted.record("fb1") + attempted.record("fb2") + + assert has_unattempted_fallback_target(["fb1", "fb2"], {"attempted_targets": attempted}) is False + assert has_unattempted_fallback_target(["fb1", "fb3"], {"attempted_targets": attempted}) is True + assert has_unattempted_fallback_target(["fb1"], {}) is True + assert has_unattempted_fallback_target(None, {}) is False diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cf143744067..e31316009fa 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3522,6 +3522,38 @@ async def test_acompletion_mid_stream_fallback_walks_every_entry_of_the_configur assert attempted_model_groups == ["primary", "fb1", "fb2"] +def test_refusal_on_the_last_fallback_hop_is_returned_instead_of_raised(): + """LIT-7400 follow-up: a refusal on the final hop of an exhausted list passes through.""" + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + + router = litellm.Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/primary-model", "api_key": "fake-key"}}, + {"model_name": "fb1", "litellm_params": {"model": "openai/fb1-model", "api_key": "fake-key"}}, + {"model_name": "fb2", "litellm_params": {"model": "openai/fb2-model", "api_key": "fake-key"}}, + ], + fallbacks=[{"primary": ["fb1", "fb2"]}], + num_retries=0, + ) + + attempted: Final = AttemptedFallbackTargets() + attempted.record("primary") + attempted.record("fb1") + attempted.record("fb2") + kwargs: Final = { + "attempted_targets": attempted, + "metadata": {"model_group": "fb2", "original_model_group": "primary"}, + } + + assert router._refusal_fallback_available("fb2", kwargs) is False + assert ( + router._refusal_fallback_available( + "fb1", {"metadata": {"model_group": "fb1", "original_model_group": "primary"}} + ) + is True + ) + + def test_completion_streaming_iterator_adopts_fallback_response_headers(): """LIT-6767, sync counterpart of the fallback-adoption test.""" from unittest.mock import MagicMock, patch