diff --git a/litellm/router.py b/litellm/router.py index b38b78b60d9..9066311451b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -219,6 +219,7 @@ from litellm.router_utils.cooldown_handlers import ( from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, AttemptedFallbackTargets, + attempted_fallback_targets_from, attempted_retries_for_request, carry_over_pre_routing_selection, carry_over_routed_deployment, @@ -7723,6 +7724,7 @@ class Router: original_exception: Final = e fallback_model_group = None original_model_group: Final[str | None] = kwargs.get("model") + fallback_walk: Final = attempted_fallback_targets_from(kwargs) # pyright: ignore[reportUnknownArgumentType] # keep legacy kwargs untyped # A pre-routing hook (complexity / auto / adaptive / quality routers) picks a tier # behind the router name, and fallbacks are configured per tier, not per router. lookup_groups: Final[tuple[str, ...]] = fallback_lookup_groups(kwargs, model_group) @@ -7746,6 +7748,7 @@ class Router: "litellm_router": self, "original_exception": original_exception, **kwargs, + "attempted_targets": fallback_walk, } if "max_fallbacks" not in input_kwargs: @@ -7975,7 +7978,10 @@ class Router: and not nested_fallback_hop ): original_exception.message += format_fallback_outcome_message( - model_group, attempted_fallback_group, fallback_failure_exception_str + model_group, + attempted_fallback_group, + fallback_failure_exception_str, + skipped=fallback_walk.skipped, ) raise original_exception diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 6976a64ef39..7623a2bd343 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -15,7 +15,7 @@ from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure -from litellm.types.router import CredentialLiteLLMParams +from litellm.types.router import CredentialLiteLLMParams, FallbackSkipReason, SkippedFallbackTarget from litellm.types.utils import LlmProviders _V = TypeVar("_V") @@ -134,15 +134,61 @@ def format_fallback_outcome_message( model_group: str | None, fallback_model_group: Sequence[object] | None, fallback_failure_detail: str, + skipped: Sequence[SkippedFallbackTarget] = (), ) -> str: """User-facing explanation appended when the fallback orchestrator gives up and re-raises the primary error.""" lead: Final = f"\n\nLiteLLM: model group '{model_group}' failed with the error above." if not fallback_model_group: return f"{lead} No fallback was attempted." + + def fallback_target_model_group(target: object) -> str | None: + if isinstance(target, str): + return target + if not isinstance(target, Mapping): + return None + model: Final = target.get("model") + return model if isinstance(model, str) else None + + skipped_targets: Final = frozenset(target.target for target in skipped) + attempted_targets: Final = tuple( + target for target in fallback_model_group if fallback_target_model_group(target) not in skipped_targets + ) + not_authorized_targets: Final = tuple( + dict.fromkeys(target.target for target in skipped if target.reason is FallbackSkipReason.NOT_AUTHORIZED) + ) + over_budget_targets: Final = tuple( + dict.fromkeys(target.target for target in skipped if target.reason is FallbackSkipReason.OVER_BUDGET) + ) + attempted_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in attempted_targets) + not_authorized_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in not_authorized_targets) + over_budget_text: Final = ", ".join(str(mask_sensitive_structure(target)) for target in over_budget_targets) + attempted_failure: Final = ( + (f"Fallback to {attempted_text} also failed: {fallback_failure_detail}",) + if fallback_failure_detail and attempted_targets + else () + ) + not_authorized_sentence: Final = ( + ( + f"Fallback to {not_authorized_text} was skipped because the caller is not authorized to call " + f"{'it' if len(not_authorized_targets) == 1 else 'them'}.", + ) + if not_authorized_targets + else () + ) + over_budget_sentence: Final = ( + ( + f"Fallback to {over_budget_text} was skipped because the caller is over budget for " + f"{'it' if len(over_budget_targets) == 1 else 'them'}.", + ) + if over_budget_targets + else () + ) + sentences: Final = attempted_failure + not_authorized_sentence + over_budget_sentence + if sentences: + return f"{lead} {' '.join(sentences)}" + targets: Final = ", ".join(str(mask_sensitive_structure(target)) for target in fallback_model_group) - if not fallback_failure_detail: - return f"{lead} Fallback model group(s) configured: {targets}." - return f"{lead} Fallback to {targets} also failed: {fallback_failure_detail}" + return f"{lead} Fallback model group(s) configured: {targets}." def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str: diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 4c393a7a93b..4a2ec6ed1f8 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -1,7 +1,7 @@ import hashlib import json from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from enum import Enum from types import MappingProxyType @@ -29,7 +29,12 @@ from litellm.router_utils.cooldown_handlers import ( from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_failures_for_current_minute, ) -from litellm.types.router import LiteLLMParamsTypedDict, reject_server_owned_wif_params +from litellm.types.router import ( + FallbackSkipReason, + LiteLLMParamsTypedDict, + SkippedFallbackTarget, + reject_server_owned_wif_params, +) if TYPE_CHECKING: from litellm.router import Router as _Router @@ -202,6 +207,7 @@ class AttemptedFallbackTargets: """ keys: frozenset[str] = frozenset() + _skip_reasons: dict[str, FallbackSkipReason] = field(default_factory=dict) # mutable-ok: O(1) per caller-sized list def __contains__(self, key: str) -> bool: return key in self.keys @@ -209,6 +215,20 @@ class AttemptedFallbackTargets: def record(self, key: str) -> None: self.keys = self.keys | frozenset((key,)) + @property + def skipped(self) -> tuple[SkippedFallbackTarget, ...]: + return tuple( + SkippedFallbackTarget(target=target, reason=reason) for target, reason in self._skip_reasons.items() + ) + + def record_skip(self, target: str, reason: FallbackSkipReason) -> None: + self._skip_reasons.setdefault(target, reason) + + +def attempted_fallback_targets_from(kwargs: Mapping[str, object]) -> AttemptedFallbackTargets: + carried_targets: Final[object] = kwargs.get("attempted_targets") + return carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets() + def has_unattempted_fallback_target( fallback_model_group: Sequence[object] | None, kwargs: Mapping[str, object] @@ -620,6 +640,8 @@ async def _is_fallback_target_authorized( fallback_entry: str | Mapping[str, object], original_model_group: str, kwargs: Mapping[str, object], + *, + attempted: AttemptedFallbackTargets, ) -> bool: access_check: Final = litellm_router.fallback_access_check target: Final = _get_fallback_target_model_group(fallback_entry) @@ -636,6 +658,7 @@ async def _is_fallback_target_authorized( "Skipping fallback to model_group = %s: caller is not authorized to call it", mask_sensitive_structure(fallback_entry), ) + attempted.record_skip(target, FallbackSkipReason.NOT_AUTHORIZED) return False @@ -644,6 +667,8 @@ async def _is_fallback_target_within_budget( fallback_entry: str | Mapping[str, object], original_model_group: str, kwargs: Mapping[str, object], + *, + attempted: AttemptedFallbackTargets, ) -> bool: budget_check: Final = litellm_router.fallback_budget_check target: Final = _get_fallback_target_model_group(fallback_entry) @@ -655,6 +680,7 @@ async def _is_fallback_target_within_budget( "Skipping fallback to model_group = %s: caller is over budget", mask_sensitive_structure(fallback_entry), ) + attempted.record_skip(target, FallbackSkipReason.OVER_BUDGET) return False @@ -743,13 +769,7 @@ async def run_async_fallback( same_model_group_only: Final = references_provider_scoped_resource(kwargs) or creates_provider_scoped_resource( kwargs ) - # Read out of kwargs and narrowed here rather than declared as a parameter: every caller - # reaches this function by spreading a loosely-typed kwargs dict, so a declared parameter - # would carry an annotation that no call site can actually be checked against. - carried_targets: Final = kwargs.get("attempted_targets") - attempted: Final = ( - carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets() - ) + attempted: Final = attempted_fallback_targets_from(kwargs) # pyright: ignore[reportUnknownArgumentType] # keep legacy kwargs untyped failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group attempted.record(failed_model_group) @@ -771,9 +791,13 @@ async def run_async_fallback( original_model_group, ) continue - if not await _is_fallback_target_authorized(litellm_router, mg, original_model_group, kwargs): + if not await _is_fallback_target_authorized( + litellm_router, mg, original_model_group, kwargs, attempted=attempted + ): continue - if not await _is_fallback_target_within_budget(litellm_router, mg, original_model_group, kwargs): + if not await _is_fallback_target_within_budget( + litellm_router, mg, original_model_group, kwargs, attempted=attempted + ): continue attempt_key = fallback_attempt_key(mg) if attempt_key is not None: diff --git a/litellm/types/router.py b/litellm/types/router.py index 59b6798bfdc..de993d46e44 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -56,6 +56,17 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class FallbackSkipReason(str, enum.Enum): + NOT_AUTHORIZED = "not_authorized" + OVER_BUDGET = "over_budget" + + +@dataclass(frozen=True, slots=True) +class SkippedFallbackTarget: + target: str + reason: FallbackSkipReason + + class ConfigurableClientsideParamsCustomAuth(TypedDict): api_base: str diff --git a/tests/unit/proxy/auth/test_fallback_budget.py b/tests/unit/proxy/auth/test_fallback_budget.py index 3d31f1e0c78..1117ca43657 100644 --- a/tests/unit/proxy/auth/test_fallback_budget.py +++ b/tests/unit/proxy/auth/test_fallback_budget.py @@ -429,7 +429,11 @@ async def test_check_fails_closed_when_the_spend_lookup_breaks(monkeypatch): @pytest.mark.asyncio async def test_router_skips_the_paid_fallback_target_when_over_budget(): - from litellm.router_utils.fallback_event_handlers import _is_fallback_target_within_budget + from litellm.router_utils.fallback_event_handlers import ( + AttemptedFallbackTargets, + _is_fallback_target_within_budget, + ) + from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget router = Router( model_list=[FREE_MODEL, PAID_MODEL], @@ -438,19 +442,34 @@ async def test_router_skips_the_paid_fallback_target_when_over_budget(): ) over = {"metadata": {"user_api_key_auth": _token(user_spend=1900.0, user_max_budget=50.0)}} under = {"metadata": {"user_api_key_auth": _token(user_spend=1.0, user_max_budget=50.0)}} + over_attempted: Final = AttemptedFallbackTargets() + under_attempted: Final = AttemptedFallbackTargets() - assert await _is_fallback_target_within_budget(router, "paid-model", "free-model", over) is False - assert await _is_fallback_target_within_budget(router, "paid-model", "free-model", under) is True + assert ( + await _is_fallback_target_within_budget(router, "paid-model", "free-model", over, attempted=over_attempted) + is False + ) + assert ( + await _is_fallback_target_within_budget( + router, "paid-model", "free-model", under, attempted=under_attempted + ) + is True + ) + assert over_attempted.skipped == (SkippedFallbackTarget("paid-model", FallbackSkipReason.OVER_BUDGET),) @pytest.mark.asyncio async def test_router_without_a_budget_check_attempts_every_fallback(): - from litellm.router_utils.fallback_event_handlers import _is_fallback_target_within_budget + from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets, _is_fallback_target_within_budget router = _router() # fallback_budget_check defaults to None over = {"metadata": {"user_api_key_auth": _token(user_spend=1900.0, user_max_budget=50.0)}} + attempted: Final = AttemptedFallbackTargets() - assert await _is_fallback_target_within_budget(router, "paid-model", "free-model", over) is True + assert ( + await _is_fallback_target_within_budget(router, "paid-model", "free-model", over, attempted=attempted) + is True + ) @pytest.mark.asyncio diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index b353967df3a..56184fa299a 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -39,6 +39,7 @@ from litellm.router_utils.fallback_event_handlers import ( routed_deployment_id, run_async_fallback, ) +from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE from typing import Dict @@ -1418,6 +1419,19 @@ class TestHasUnattemptedFallbackTarget: assert has_unattempted_fallback_target(None, {}) is False +def test_attempted_fallback_targets_deduplicates_skips_in_order_and_keeps_first_reason() -> None: + attempted: Final = AttemptedFallbackTargets() + attempted.record_skip("a", FallbackSkipReason.NOT_AUTHORIZED) + attempted.record_skip("a", FallbackSkipReason.NOT_AUTHORIZED) + attempted.record_skip("b", FallbackSkipReason.OVER_BUDGET) + attempted.record_skip("a", FallbackSkipReason.OVER_BUDGET) + + assert attempted.skipped == ( + SkippedFallbackTarget("a", FallbackSkipReason.NOT_AUTHORIZED), + SkippedFallbackTarget("b", FallbackSkipReason.OVER_BUDGET), + ) + + def test_get_fallback_model_group_matches_provider_prefixed_key(): """A bare model group routed via a wildcard (e.g. "gpt-4o" through "openai/*") must match a fallback keyed on the provider-prefixed name, diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 3694ecea4ed..d532ce186dd 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, List, Optional, Union +from typing import Dict, Final, List, Optional, Union from unittest.mock import Mock import pytest @@ -12,6 +12,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + format_fallback_outcome_message, provider_for_generic_call, resolve_model_group_alias, resolve_served_model, @@ -19,6 +20,7 @@ from litellm.router_utils.common_utils import ( PROVIDER_SCOPED_CREDENTIAL_PARAMS, warn_on_provider_credential_mismatch, ) +from litellm.types.router import FallbackSkipReason, SkippedFallbackTarget class TestFilterTeamBasedModels: @@ -610,6 +612,56 @@ class TestTruncateFallbackErrorDetail: assert len(truncate_fallback_error_detail("x" * 1_000_000)) < 3_000 +def test_format_fallback_outcome_reports_unauthorized_skips_without_failure_detail() -> None: + message: Final = format_fallback_outcome_message( + "primary", + ("restricted-model",), + "", + skipped=(SkippedFallbackTarget("restricted-model", FallbackSkipReason.NOT_AUTHORIZED),), + ) + + assert "Fallback to restricted-model was skipped because the caller is not authorized to call it." in message + assert "also failed" not in message + + +def test_format_fallback_outcome_separates_attempted_failures_from_skipped_targets() -> None: + message: Final = format_fallback_outcome_message( + "primary", + ("restricted-model", "dead-allowed"), + "boom", + skipped=(SkippedFallbackTarget("restricted-model", FallbackSkipReason.NOT_AUTHORIZED),), + ) + failed_sentence: Final = message.split("Fallback to dead-allowed also failed:", 1)[1].split( + " Fallback to restricted-model was skipped", 1 + )[0] + + assert "Fallback to dead-allowed also failed: boom" in message + assert "Fallback to restricted-model was skipped because the caller is not authorized" in message + assert "restricted-model" not in failed_sentence + + +def test_format_fallback_outcome_uses_plural_for_multiple_over_budget_skips() -> None: + message: Final = format_fallback_outcome_message( + "primary", + ("paid-one", "paid-two"), + "", + skipped=( + SkippedFallbackTarget("paid-one", FallbackSkipReason.OVER_BUDGET), + SkippedFallbackTarget("paid-two", FallbackSkipReason.OVER_BUDGET), + ), + ) + + assert "Fallback to paid-one, paid-two was skipped because the caller is over budget for them." in message + + +def test_format_fallback_outcome_preserves_attempted_failure_wording_without_skips() -> None: + assert ( + format_fallback_outcome_message("primary", ("fallback",), "detail") + == "\n\nLiteLLM: model group 'primary' failed with the error above. " + "Fallback to fallback also failed: detail" + ) + + class TestWarnOnProviderCredentialMismatch: """A deployment that carries one provider's credentials while resolving to another is silently broken: litellm ignores the credentials and sends the diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 5faea8d6c1d..0a795ff2a0f 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -17801,6 +17801,48 @@ def _router_with_failing_primary(fallback_access_check) -> Router: ) +class _DenyFallbackBudgetCheck: + async def __call__( + self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router + ) -> bool: + return False + + +def _router_with_failing_fallback_chain( + fallback_targets: tuple[str, ...], + *, + fallback_access_check: _AllowlistFallbackAccessCheck | None = None, + fallback_budget_check: _DenyFallbackBudgetCheck | None = None, +) -> Router: + return Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/primary", + "api_key": "k", + "mock_response": Exception("primary is down"), + }, + }, + *( + { + "model_name": target, + "litellm_params": { + "model": f"openai/{target}", + "api_key": "k", + "mock_response": Exception(f"{target} is down"), + }, + } + for target in fallback_targets + ), + ], + fallbacks=[{"primary": list(fallback_targets)}], + num_retries=0, + fallback_access_check=fallback_access_check, + fallback_budget_check=fallback_budget_check, + ) + + @pytest.mark.asyncio async def test_fallback_access_check_blocks_config_fallback_the_caller_cannot_use(): access_check = _AllowlistFallbackAccessCheck(allowed_models=frozenset()) @@ -17812,6 +17854,59 @@ async def test_fallback_access_check_blocks_config_fallback_the_caller_cannot_us assert access_check.checked_models == ["secret-fallback"] +@pytest.mark.asyncio +async def test_fallback_access_check_reports_denied_fallback_as_skipped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) + access_check: Final = _AllowlistFallbackAccessCheck(allowed_models=frozenset()) + router: Final = _router_with_failing_primary(access_check) + primary_only_router: Final = Router(model_list=router.model_list, fallbacks=[], num_retries=0) + + with pytest.raises(litellm.InternalServerError) as primary_exception: + await primary_only_router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + with pytest.raises(litellm.InternalServerError) as fallback_exception: + await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + + message: Final = str(fallback_exception.value) + assert type(fallback_exception.value) is type(primary_exception.value) + assert fallback_exception.value.status_code == primary_exception.value.status_code + assert message.count("primary is down") == 1 + assert "Fallback to secret-fallback was skipped because the caller is not authorized to call it." in message + assert "also failed" not in message + + +@pytest.mark.asyncio +async def test_fallback_access_check_separates_denied_and_failed_targets(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) + access_check: Final = _AllowlistFallbackAccessCheck(allowed_models=frozenset({"allowed-fallback"})) + router: Final = _router_with_failing_fallback_chain( + ("secret-fallback", "allowed-fallback"), + fallback_access_check=access_check, + ) + + with pytest.raises(litellm.InternalServerError) as exception: + await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + + message: Final = str(exception.value) + assert "Fallback to allowed-fallback also failed:" in message + assert "Fallback to secret-fallback was skipped because the caller is not authorized" in message + + +@pytest.mark.asyncio +async def test_fallback_budget_check_reports_skipped_targets(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) + router: Final = _router_with_failing_fallback_chain( + ("paid-fallback",), + fallback_budget_check=_DenyFallbackBudgetCheck(), + ) + + with pytest.raises(litellm.InternalServerError) as exception: + await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}]) + + message: Final = str(exception.value) + assert "Fallback to paid-fallback was skipped because the caller is over budget for it." in message + assert "also failed" not in message + + @pytest.mark.asyncio async def test_fallback_access_check_lets_an_authorized_config_fallback_through(): router = _router_with_failing_primary(_AllowlistFallbackAccessCheck(allowed_models=frozenset({"secret-fallback"})))