fix(router): report fallbacks skipped for access or budget instead of calling them failed (#46291)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/router.py

@@ -219,6 +219,7 @@ from litellm.router_utils.cooldown_handlers import (
 from litellm.router_utils.fallback_event_handlers import (
     MID_STREAM_FALLBACK_CONTROLS_KEY,
     AttemptedFallbackTargets,
+    attempted_fallback_targets_from,
     attempted_retries_for_request,
     carry_over_pre_routing_selection,
     carry_over_routed_deployment,
@@ -7723,6 +7724,7 @@ class Router:
         original_exception: Final = e
         fallback_model_group = None
         original_model_group: Final[str | None] = kwargs.get("model")
+        fallback_walk: Final = attempted_fallback_targets_from(kwargs)  # pyright: ignore[reportUnknownArgumentType]  # keep legacy kwargs untyped
         # A pre-routing hook (complexity / auto / adaptive / quality routers) picks a tier
         # behind the router name, and fallbacks are configured per tier, not per router.
         lookup_groups: Final[tuple[str, ...]] = fallback_lookup_groups(kwargs, model_group)
@@ -7746,6 +7748,7 @@ class Router:
             "litellm_router": self,
             "original_exception": original_exception,
             **kwargs,
+            "attempted_targets": fallback_walk,
         }
 
         if "max_fallbacks" not in input_kwargs:
@@ -7975,7 +7978,10 @@ class Router:
             and not nested_fallback_hop
         ):
             original_exception.message += format_fallback_outcome_message(
-                model_group, attempted_fallback_group, fallback_failure_exception_str
+                model_group,
+                attempted_fallback_group,
+                fallback_failure_exception_str,
+                skipped=fallback_walk.skipped,
             )
 
         raise original_exception

litellm/router_utils/common_utils.py

@@ -15,7 +15,7 @@ from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
 from litellm.exceptions import BadRequestError
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
-from litellm.types.router import CredentialLiteLLMParams
+from litellm.types.router import CredentialLiteLLMParams, FallbackSkipReason, SkippedFallbackTarget
 from litellm.types.utils import LlmProviders
 
 _V = TypeVar("_V")
@@ -134,15 +134,61 @@ def format_fallback_outcome_message(
     model_group: str | None,
     fallback_model_group: Sequence[object] | None,
     fallback_failure_detail: str,
+    skipped: Sequence[SkippedFallbackTarget] = (),
 ) -> str:
     """User-facing explanation appended when the fallback orchestrator gives up and re-raises the primary error."""
     lead: Final = f"\n\nLiteLLM: model group '{model_group}' failed with the error above."
     if not fallback_model_group:
         return f"{lead} No fallback was attempted."
+
+    def fallback_target_model_group(target: object) -> str | None:
+        if isinstance(target, str):
+            return target
+        if not isinstance(target, Mapping):
+            return None
+        model: Final = target.get("model")
+        return model if isinstance(model, str) else None
+
+    skipped_targets: Final = frozenset(target.target for target in skipped)
+    attempted_targets: Final = tuple(
+        target for target in fallback_model_group if fallback_target_model_group(target) not in skipped_targets
+    )
+    not_authorized_targets: Final = tuple(
+        dict.fromkeys(target.target for target in skipped if target.reason is FallbackSkipReason.NOT_AUTHORIZED)
+    )
+    over_budget_targets: Final = tuple(
+        dict.fromkeys(target.target for target in skipped if target.reason is FallbackSkipReason.OVER_BUDGET)
+    )
+    attempted_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in attempted_targets)
+    not_authorized_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in not_authorized_targets)
+    over_budget_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in over_budget_targets)
+    attempted_failure: Final = (
+        (f"Fallback to {attempted_text} also failed: {fallback_failure_detail}",)
+        if fallback_failure_detail and attempted_targets
+        else ()
+    )
+    not_authorized_sentence: Final = (
+        (
+            f"Fallback to {not_authorized_text} was skipped because the caller is not authorized to call "
+            f"{'it' if len(not_authorized_targets) == 1 else 'them'}.",
+        )
+        if not_authorized_targets
+        else ()
+    )
+    over_budget_sentence: Final = (
+        (
+            f"Fallback to {over_budget_text} was skipped because the caller is over budget for "
+            f"{'it' if len(over_budget_targets) == 1 else 'them'}.",
+        )
+        if over_budget_targets
+        else ()
+    )
+    sentences: Final = attempted_failure + not_authorized_sentence + over_budget_sentence
+    if sentences:
+        return f"{lead} {' '.join(sentences)}"
+
     targets: Final = ", ".join(str(mask_sensitive_structure(target)) for target in fallback_model_group)
-    if not fallback_failure_detail:
-        return f"{lead} Fallback model group(s) configured: {targets}."
-    return f"{lead} Fallback to {targets} also failed: {fallback_failure_detail}"
+    return f"{lead} Fallback model group(s) configured: {targets}."
 
 
 def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:

litellm/router_utils/fallback_event_handlers.py

@@ -1,7 +1,7 @@
 import hashlib
 import json
 from collections.abc import Callable, Mapping, Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
 from types import MappingProxyType
@@ -29,7 +29,12 @@ from litellm.router_utils.cooldown_handlers import (
 from litellm.router_utils.router_callbacks.track_deployment_metrics import (
     increment_deployment_failures_for_current_minute,
 )
-from litellm.types.router import LiteLLMParamsTypedDict, reject_server_owned_wif_params
+from litellm.types.router import (
+    FallbackSkipReason,
+    LiteLLMParamsTypedDict,
+    SkippedFallbackTarget,
+    reject_server_owned_wif_params,
+)
 
 if TYPE_CHECKING:
     from litellm.router import Router as _Router
@@ -202,6 +207,7 @@ class AttemptedFallbackTargets:
     """
 
     keys: frozenset[str] = frozenset()
+    _skip_reasons: dict[str, FallbackSkipReason] = field(default_factory=dict)  # mutable-ok: O(1) per caller-sized list
 
     def __contains__(self, key: str) -> bool:
         return key in self.keys
@@ -209,6 +215,20 @@ class AttemptedFallbackTargets:
     def record(self, key: str) -> None:
         self.keys = self.keys | frozenset((key,))
 
+    @property
+    def skipped(self) -> tuple[SkippedFallbackTarget, ...]:
+        return tuple(
+            SkippedFallbackTarget(target=target, reason=reason) for target, reason in self._skip_reasons.items()
+        )
+
+    def record_skip(self, target: str, reason: FallbackSkipReason) -> None:
+        self._skip_reasons.setdefault(target, reason)
+
+
+def attempted_fallback_targets_from(kwargs: Mapping[str, object]) -> AttemptedFallbackTargets:
+    carried_targets: Final[object] = kwargs.get("attempted_targets")
+    return carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets()
+
 
 def has_unattempted_fallback_target(
     fallback_model_group: Sequence[object] | None, kwargs: Mapping[str, object]
@@ -620,6 +640,8 @@ async def _is_fallback_target_authorized(
     fallback_entry: str | Mapping[str, object],
     original_model_group: str,
     kwargs: Mapping[str, object],
+    *,
+    attempted: AttemptedFallbackTargets,
 ) -> bool:
     access_check: Final = litellm_router.fallback_access_check
     target: Final = _get_fallback_target_model_group(fallback_entry)
@@ -636,6 +658,7 @@ async def _is_fallback_target_authorized(
         "Skipping fallback to model_group = %s: caller is not authorized to call it",
         mask_sensitive_structure(fallback_entry),
     )
+    attempted.record_skip(target, FallbackSkipReason.NOT_AUTHORIZED)
     return False
 
 
@@ -644,6 +667,8 @@ async def _is_fallback_target_within_budget(
     fallback_entry: str | Mapping[str, object],
     original_model_group: str,
     kwargs: Mapping[str, object],
+    *,
+    attempted: AttemptedFallbackTargets,
 ) -> bool:
     budget_check: Final = litellm_router.fallback_budget_check
     target: Final = _get_fallback_target_model_group(fallback_entry)
@@ -655,6 +680,7 @@ async def _is_fallback_target_within_budget(
         "Skipping fallback to model_group = %s: caller is over budget",
         mask_sensitive_structure(fallback_entry),
     )
+    attempted.record_skip(target, FallbackSkipReason.OVER_BUDGET)
     return False
 
 
@@ -743,13 +769,7 @@ async def run_async_fallback(
     same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource(
         kwargs
     )
-    # Read out of kwargs and narrowed here rather than declared as a parameter: every caller
-    # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter
-    # would carry an annotation that no call site can actually be checked against.
-    carried_targets: Final = kwargs.get("attempted_targets")
-    attempted: Final = (
-        carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets()
-    )
+    attempted: Final = attempted_fallback_targets_from(kwargs)  # pyright: ignore[reportUnknownArgumentType]  # keep legacy kwargs untyped
     failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group
     attempted.record(failed_model_group)
 
@@ -771,9 +791,13 @@ async def run_async_fallback(
                 original_model_group,
             )
             continue
-        if not await _is_fallback_target_authorized(litellm_router, mg, original_model_group, kwargs):
+        if not await _is_fallback_target_authorized(
+            litellm_router, mg, original_model_group, kwargs, attempted=attempted
+        ):
             continue
-        if not await _is_fallback_target_within_budget(litellm_router, mg, original_model_group, kwargs):
+        if not await _is_fallback_target_within_budget(
+            litellm_router, mg, original_model_group, kwargs, attempted=attempted
+        ):
             continue
         attempt_key = fallback_attempt_key(mg)
         if attempt_key is not None:

litellm/types/router.py

@@ -56,6 +56,17 @@ from .utils import (
 AllowedModelRegion: TypeAlias = Literal["eu", "us"]
 
 
+class FallbackSkipReason(str, enum.Enum):
+    NOT_AUTHORIZED = "not_authorized"
+    OVER_BUDGET = "over_budget"
+
+
+@dataclass(frozen=True, slots=True)
+class SkippedFallbackTarget:
+    target: str
+    reason: FallbackSkipReason
+
+
 class ConfigurableClientsideParamsCustomAuth(TypedDict):
     api_base: str
 

tests/unit/proxy/auth/test_fallback_budget.py

@@ -429,7 +429,11 @@ async def test_check_fails_closed_when_the_spend_lookup_breaks(monkeypatch):
 
 @pytest.mark.asyncio
 async def test_router_skips_the_paid_fallback_target_when_over_budget():
-    from litellm.router_utils.fallback_event_handlers import _is_fallback_target_within_budget
+    from litellm.router_utils.fallback_event_handlers import (
+        AttemptedFallbackTargets,
+        _is_fallback_target_within_budget,
+    )
+    from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget
 
     router = Router(
         model_list=[FREE_MODEL, PAID_MODEL],
@@ -438,19 +442,34 @@ async def test_router_skips_the_paid_fallback_target_when_over_budget():
     )
     over = {"metadata": {"user_api_key_auth": _token(user_spend=1900.0, user_max_budget=50.0)}}
     under = {"metadata": {"user_api_key_auth": _token(user_spend=1.0, user_max_budget=50.0)}}
+    over_attempted: Final = AttemptedFallbackTargets()
+    under_attempted: Final = AttemptedFallbackTargets()
 
-    assert await _is_fallback_target_within_budget(router, "paid-model", "free-model", over) is False
-    assert await _is_fallback_target_within_budget(router, "paid-model", "free-model", under) is True
+    assert (
+        await _is_fallback_target_within_budget(router, "paid-model", "free-model", over, attempted=over_attempted)
+        is False
+    )
+    assert (
+        await _is_fallback_target_within_budget(
+            router, "paid-model", "free-model", under, attempted=under_attempted
+        )
+        is True
+    )
+    assert over_attempted.skipped == (SkippedFallbackTarget("paid-model", FallbackSkipReason.OVER_BUDGET),)
 
 
 @pytest.mark.asyncio
 async def test_router_without_a_budget_check_attempts_every_fallback():
-    from litellm.router_utils.fallback_event_handlers import _is_fallback_target_within_budget
+    from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets, _is_fallback_target_within_budget
 
     router = _router()  # fallback_budget_check defaults to None
     over = {"metadata": {"user_api_key_auth": _token(user_spend=1900.0, user_max_budget=50.0)}}
+    attempted: Final = AttemptedFallbackTargets()
 
-    assert await _is_fallback_target_within_budget(router, "paid-model", "free-model", over) is True
+    assert (
+        await _is_fallback_target_within_budget(router, "paid-model", "free-model", over, attempted=attempted)
+        is True
+    )
 
 
 @pytest.mark.asyncio

tests/unit/router_utils/test_fallback_event_handlers.py

@@ -39,6 +39,7 @@ from litellm.router_utils.fallback_event_handlers import (
     routed_deployment_id,
     run_async_fallback,
 )
+from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget
 from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
 from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE
 from typing import Dict
@@ -1418,6 +1419,19 @@ class TestHasUnattemptedFallbackTarget:
         assert has_unattempted_fallback_target(None, {}) is False
 
 
+def test_attempted_fallback_targets_deduplicates_skips_in_order_and_keeps_first_reason() -> None:
+    attempted: Final = AttemptedFallbackTargets()
+    attempted.record_skip("a", FallbackSkipReason.NOT_AUTHORIZED)
+    attempted.record_skip("a", FallbackSkipReason.NOT_AUTHORIZED)
+    attempted.record_skip("b", FallbackSkipReason.OVER_BUDGET)
+    attempted.record_skip("a", FallbackSkipReason.OVER_BUDGET)
+
+    assert attempted.skipped == (
+        SkippedFallbackTarget("a", FallbackSkipReason.NOT_AUTHORIZED),
+        SkippedFallbackTarget("b", FallbackSkipReason.OVER_BUDGET),
+    )
+
+
 def test_get_fallback_model_group_matches_provider_prefixed_key():
     """A bare model group routed via a wildcard (e.g. "gpt-4o" through
     "openai/*") must match a fallback keyed on the provider-prefixed name,

tests/unit/router_utils/test_router_utils_common_utils.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, List, Optional, Union
+from typing import Dict, Final, List, Optional, Union
 from unittest.mock import Mock
 
 import pytest
@@ -12,6 +12,7 @@ from litellm.router_utils.common_utils import (
     add_model_file_id_mappings,
     filter_team_based_models,
     filter_web_search_deployments,
+    format_fallback_outcome_message,
     provider_for_generic_call,
     resolve_model_group_alias,
     resolve_served_model,
@@ -19,6 +20,7 @@ from litellm.router_utils.common_utils import (
     PROVIDER_SCOPED_CREDENTIAL_PARAMS,
     warn_on_provider_credential_mismatch,
 )
+from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget
 
 
 class TestFilterTeamBasedModels:
@@ -610,6 +612,56 @@ class TestTruncateFallbackErrorDetail:
         assert len(truncate_fallback_error_detail("x" * 1_000_000)) < 3_000
 
 
+def test_format_fallback_outcome_reports_unauthorized_skips_without_failure_detail() -> None:
+    message: Final = format_fallback_outcome_message(
+        "primary",
+        ("restricted-model",),
+        "",
+        skipped=(SkippedFallbackTarget("restricted-model", FallbackSkipReason.NOT_AUTHORIZED),),
+    )
+
+    assert "Fallback to restricted-model was skipped because the caller is not authorized to call it." in message
+    assert "also failed" not in message
+
+
+def test_format_fallback_outcome_separates_attempted_failures_from_skipped_targets() -> None:
+    message: Final = format_fallback_outcome_message(
+        "primary",
+        ("restricted-model", "dead-allowed"),
+        "boom",
+        skipped=(SkippedFallbackTarget("restricted-model", FallbackSkipReason.NOT_AUTHORIZED),),
+    )
+    failed_sentence: Final = message.split("Fallback to dead-allowed also failed:", 1)[1].split(
+        " Fallback to restricted-model was skipped", 1
+    )[0]
+
+    assert "Fallback to dead-allowed also failed: boom" in message
+    assert "Fallback to restricted-model was skipped because the caller is not authorized" in message
+    assert "restricted-model" not in failed_sentence
+
+
+def test_format_fallback_outcome_uses_plural_for_multiple_over_budget_skips() -> None:
+    message: Final = format_fallback_outcome_message(
+        "primary",
+        ("paid-one", "paid-two"),
+        "",
+        skipped=(
+            SkippedFallbackTarget("paid-one", FallbackSkipReason.OVER_BUDGET),
+            SkippedFallbackTarget("paid-two", FallbackSkipReason.OVER_BUDGET),
+        ),
+    )
+
+    assert "Fallback to paid-one, paid-two was skipped because the caller is over budget for them." in message
+
+
+def test_format_fallback_outcome_preserves_attempted_failure_wording_without_skips() -> None:
+    assert (
+        format_fallback_outcome_message("primary", ("fallback",), "detail")
+        == "\n\nLiteLLM: model group 'primary' failed with the error above. "
+        "Fallback to fallback also failed: detail"
+    )
+
+
 class TestWarnOnProviderCredentialMismatch:
     """A deployment that carries one provider's credentials while resolving to
     another is silently broken: litellm ignores the credentials and sends the

tests/unit/test_router/test_router.py

@@ -17801,6 +17801,48 @@ def _router_with_failing_primary(fallback_access_check) -> Router:
     )
 
 
+class _DenyFallbackBudgetCheck:
+    async def __call__(
+        self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router
+    ) -> bool:
+        return False
+
+
+def _router_with_failing_fallback_chain(
+    fallback_targets: tuple[str, ...],
+    *,
+    fallback_access_check: _AllowlistFallbackAccessCheck | None = None,
+    fallback_budget_check: _DenyFallbackBudgetCheck | None = None,
+) -> Router:
+    return Router(
+        model_list=[
+            {
+                "model_name": "primary",
+                "litellm_params": {
+                    "model": "openai/primary",
+                    "api_key": "k",
+                    "mock_response": Exception("primary is down"),
+                },
+            },
+            *(
+                {
+                    "model_name": target,
+                    "litellm_params": {
+                        "model": f"openai/{target}",
+                        "api_key": "k",
+                        "mock_response": Exception(f"{target} is down"),
+                    },
+                }
+                for target in fallback_targets
+            ),
+        ],
+        fallbacks=[{"primary": list(fallback_targets)}],
+        num_retries=0,
+        fallback_access_check=fallback_access_check,
+        fallback_budget_check=fallback_budget_check,
+    )
+
+
 @pytest.mark.asyncio
 async def test_fallback_access_check_blocks_config_fallback_the_caller_cannot_use():
     access_check = _AllowlistFallbackAccessCheck(allowed_models=frozenset())
@@ -17812,6 +17854,59 @@ async def test_fallback_access_check_blocks_config_fallback_the_caller_cannot_us
     assert access_check.checked_models == ["secret-fallback"]
 
 
+@pytest.mark.asyncio
+async def test_fallback_access_check_reports_denied_fallback_as_skipped(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True)
+    access_check: Final = _AllowlistFallbackAccessCheck(allowed_models=frozenset())
+    router: Final = _router_with_failing_primary(access_check)
+    primary_only_router: Final = Router(model_list=router.model_list, fallbacks=[], num_retries=0)
+
+    with pytest.raises(litellm.InternalServerError) as primary_exception:
+        await primary_only_router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}])
+    with pytest.raises(litellm.InternalServerError) as fallback_exception:
+        await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}])
+
+    message: Final = str(fallback_exception.value)
+    assert type(fallback_exception.value) is type(primary_exception.value)
+    assert fallback_exception.value.status_code == primary_exception.value.status_code
+    assert message.count("primary is down") == 1
+    assert "Fallback to secret-fallback was skipped because the caller is not authorized to call it." in message
+    assert "also failed" not in message
+
+
+@pytest.mark.asyncio
+async def test_fallback_access_check_separates_denied_and_failed_targets(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True)
+    access_check: Final = _AllowlistFallbackAccessCheck(allowed_models=frozenset({"allowed-fallback"}))
+    router: Final = _router_with_failing_fallback_chain(
+        ("secret-fallback", "allowed-fallback"),
+        fallback_access_check=access_check,
+    )
+
+    with pytest.raises(litellm.InternalServerError) as exception:
+        await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}])
+
+    message: Final = str(exception.value)
+    assert "Fallback to allowed-fallback also failed:" in message
+    assert "Fallback to secret-fallback was skipped because the caller is not authorized" in message
+
+
+@pytest.mark.asyncio
+async def test_fallback_budget_check_reports_skipped_targets(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True)
+    router: Final = _router_with_failing_fallback_chain(
+        ("paid-fallback",),
+        fallback_budget_check=_DenyFallbackBudgetCheck(),
+    )
+
+    with pytest.raises(litellm.InternalServerError) as exception:
+        await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}])
+
+    message: Final = str(exception.value)
+    assert "Fallback to paid-fallback was skipped because the caller is over budget for it." in message
+    assert "also failed" not in message
+
+
 @pytest.mark.asyncio
 async def test_fallback_access_check_lets_an_authorized_config_fallback_through():
     router = _router_with_failing_primary(_AllowlistFallbackAccessCheck(allowed_models=frozenset({"secret-fallback"})))