diff --git a/litellm/router.py b/litellm/router.py index 53301979b2f..4b58629e38b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -475,6 +475,15 @@ _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) +def _untried_fallback_target_exists(chain: Sequence[object] | None, kwargs: Mapping[str, Any]) -> bool: + """ + Whether a resolved fallback chain still names a target this request can try. + `has_unattempted_fallback_target` reports an unattempted target for an empty chain + whenever the request carries no attempt record yet, so guard the empty case here. + """ + return bool(chain) and has_unattempted_fallback_target(chain, kwargs) + + def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () @@ -8514,7 +8523,7 @@ class Router: fallbacks=fallbacks, lookup_groups=fallback_lookup_groups(kwargs, model_group), ) - return has_unattempted_fallback_target(resolved, kwargs) + return _untried_fallback_target_exists(resolved, kwargs) def _anthropic_messages_order_levels(self, model_group: str, kwargs: Mapping[str, Any]) -> tuple[int, ...]: """ @@ -8583,7 +8592,7 @@ class Router: if model_group is None or fallbacks_disabled_for_request(kwargs): return False if _check_non_standard_fallback_format(fallbacks=fallbacks): - return has_unattempted_fallback_target(fallbacks, kwargs) + return _untried_fallback_target_exists(fallbacks, kwargs) dedicated_fallbacks: Final = ( context_window_fallbacks if isinstance(error, litellm.ContextWindowExceededError) @@ -8592,7 +8601,7 @@ class Router: else None ) if dedicated_fallbacks is not None: - return has_unattempted_fallback_target( + return _untried_fallback_target_exists( self._get_fallback_model_group_for_lookup_groups( fallbacks=dedicated_fallbacks, lookup_groups=fallback_lookup_groups(kwargs, model_group), diff --git a/tests/unit/test_router_retry_backoff_headers.py b/tests/unit/test_router_retry_backoff_headers.py index 75413ca633c..487fac08172 100644 --- a/tests/unit/test_router_retry_backoff_headers.py +++ b/tests/unit/test_router_retry_backoff_headers.py @@ -12,6 +12,7 @@ import pytest import litellm from litellm import Router from litellm.constants import MAX_RETRY_DELAY +from litellm.router import _untried_fallback_target_exists from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets, record_disable_fallbacks from litellm.types.router import RetryPolicy @@ -78,6 +79,25 @@ async def test_fallback_configured_for_another_group_keeps_retry_backoff(): ) +@pytest.mark.asyncio +async def test_empty_fallback_chain_keeps_retry_backoff(): + """A chain configured as an empty list names no target, so the request has nothing to + fall back to and the retries must still space themselves out.""" + router: Final = _router_with_single_failing_deployment(fallbacks=[{"primary": []}]) + + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for( + router.acompletion(model="primary", messages=[{"role": "user", "content": "Hello"}]), + timeout=_BACKOFF_DETECTION_TIMEOUT, + ) + + +def test_untried_fallback_target_exists_is_false_for_a_chain_with_no_entries(): + assert _untried_fallback_target_exists(["backup"], {}) is True + assert _untried_fallback_target_exists([], {}) is False + assert _untried_fallback_target_exists(None, {}) is False + + @pytest.mark.asyncio async def test_client_side_fallback_list_does_not_back_off_before_falling_back(): router: Final = _router_with_single_failing_deployment(fallbacks=[])