diff --git a/litellm/constants.py b/litellm/constants.py index cd7c0176428..cae7db018e2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -55,6 +55,7 @@ AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPON AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 +ROUTER_FALLBACK_GROUP_LISTING_MAX_GROUPS: Final = 5 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( { "routing_strategy_args", diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index a06860e7786..5d2ee86555e 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -2,7 +2,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar, cast from pydantic import TypeAdapter, ValidationError @@ -11,7 +11,7 @@ if TYPE_CHECKING: import litellm from litellm._logging import verbose_logger, verbose_router_logger -from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS +from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS, ROUTER_FALLBACK_GROUP_LISTING_MAX_GROUPS from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap @@ -184,13 +184,35 @@ def truncate_fallback_error_detail(detail: str) -> str: return f"{detail[:ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS]}... [truncated {dropped} characters]" +def compact_fallback_error_detail(detail: str) -> str: + """Reduce a nested hop failure to one line for the user-facing message. + + The detail of a failed hop is the whole message of the hop below it, which already ends + with that hop's own fallback explanation. Carrying that verbatim into this hop makes the + text repeat itself once per hop and buries the first, most useful failure. + """ + first_line: Final = detail.strip().split("\n", 1)[0].strip() + return truncate_fallback_error_detail(first_line) + + +def _configured_groups_text(configured: Sequence[str]) -> str: + """Name a few configured model groups, never the whole config. + + This is one sentence of an error a caller reads; listing every group makes the message + grow with the size of the router config instead of with the request. + """ + if not configured: + return " No fallbacks are configured." + listed: Final = configured[:ROUTER_FALLBACK_GROUP_LISTING_MAX_GROUPS] + remainder: Final = len(configured) - len(listed) + return f" Fallbacks are configured for: {', '.join(listed)}" + (f" and {remainder} more." if remainder else ".") + + def format_no_fallback_group_message(lookup_groups: Sequence[str], fallbacks: Sequence[Mapping[str, object]]) -> str: """User-facing explanation appended when a request fails and no fallback chain matches its model group.""" requested: Final = " -> ".join(lookup_groups) configured: Final = tuple(dict.fromkeys(key for entry in fallbacks for key in entry)) - configured_text: Final = ( - f" Fallbacks are configured for: {', '.join(configured)}." if configured else " No fallbacks are configured." - ) + configured_text: Final = _configured_groups_text(configured) return ( f"\n\nLiteLLM: model group '{requested}' failed with the error above and no fallback model group was found " f"for it, so the request was not retried on another model.{configured_text}" @@ -260,16 +282,6 @@ def format_fallback_outcome_message( return f"{lead} Fallback model group(s) configured: {targets}." -def format_content_policy_fallback_hint( - model_group: str | None, content_policy_fallbacks: object, fallbacks: object -) -> str: - return ( - f"model={model_group}. content_policy_fallback={mask_sensitive_structure(content_policy_fallbacks)}. " - f"fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'content_policy_fallback' - " - "https://docs.litellm.ai/docs/routing#fallbacks" - ) - - def _as_fallback_entries(fallbacks: object) -> tuple[Mapping[str, object], ...] | None: try: return _FALLBACK_ENTRIES_ADAPTER.validate_python(fallbacks) @@ -277,6 +289,61 @@ def _as_fallback_entries(fallbacks: object) -> tuple[Mapping[str, object], ...] return None +def format_chain_for_lookup_groups(config_name: str, configured: object, lookup_groups: Sequence[str]) -> str: + """Name only the chain a request would follow, never the whole fallback config. + + Every fallback hop appends its own hint, so dumping each configured group repeats the + entire config per hop and pushes the actual failure out of the message. + """ + if configured is None: + return f"{config_name} is not set" + entries: Final = _as_fallback_entries(configured) + if entries is None or not lookup_groups: + return f"{config_name}={mask_sensitive_structure(configured)}" + from litellm.router_utils.fallback_event_handlers import get_fallback_chain_for_lookup_groups + + resolved: Final = get_fallback_chain_for_lookup_groups( + fallbacks=cast("Sequence[Mapping[str, list[str]]]", entries), # cast-ok: only keys and values are read + lookup_groups=tuple(lookup_groups), + ) + if resolved is None: + others: Final = len(tuple(key for entry in entries for key in entry if key not in lookup_groups)) + searched: Final = " -> ".join(repr(group) for group in lookup_groups) + return f"{config_name} has no chain for {searched}" + ( + f" ({others} other model groups are configured)" if others else "" + ) + owner, chain = resolved + return f"{config_name}: {owner!r} -> {mask_sensitive_structure(chain)}" + + +def format_content_policy_fallback_hint( + model_group: str | None, + content_policy_fallbacks: object, + lookup_groups: Sequence[str], +) -> str: + chain: Final = format_chain_for_lookup_groups("content_policy_fallbacks", content_policy_fallbacks, lookup_groups) + return ( + f"model={model_group}. {chain}. " + "To retry content-policy errors on another model, set content_policy_fallbacks for this model group - " + "https://docs.litellm.ai/docs/routing#fallbacks" + ) + + +def format_context_window_fallback_hint( + model_group: str | None, + context_window_fallbacks: object, + fallbacks: object, + lookup_groups: Sequence[str], +) -> str: + return ( + f"model={model_group}. " + f"{format_chain_for_lookup_groups('context_window_fallbacks', context_window_fallbacks, lookup_groups)}. " + f"{format_chain_for_lookup_groups('fallbacks', fallbacks, lookup_groups)}. " + "To give context-window errors their own chain, set context_window_fallbacks - " + "https://docs.litellm.ai/docs/routing#fallbacks" + ) + + def format_content_policy_no_fallback_note( model_group: str, lookup_groups: tuple[str, ...], @@ -287,7 +354,7 @@ def format_content_policy_no_fallback_note( outcome: Final = "" if nested_fallback_hop else format_fallback_outcome_message(model_group, None, "") if content_policy_fallbacks is not None: return outcome - hint: Final = f"\n{format_content_policy_fallback_hint(model_group, content_policy_fallbacks, fallbacks)}" + hint: Final = f"\n{format_content_policy_fallback_hint(model_group, content_policy_fallbacks, lookup_groups)}" fallback_entries: Final = _as_fallback_entries(fallbacks) if fallbacks is not None else None if fallback_entries is None or not lookup_groups or nested_fallback_hop: return hint + outcome diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 30b05456987..d9198ee39c7 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -185,12 +185,14 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( from litellm.litellm_core_utils.routing_common_utils import ( MID_STREAM_CONTINUATION_KWARG, as_retry_skipped_deployment_ids, + compact_fallback_error_detail, deployment_model_id_pin, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, format_content_policy_fallback_hint, format_content_policy_no_fallback_note, + format_context_window_fallback_hint, format_fallback_outcome_message, format_no_fallback_group_message, get_request_team_id, @@ -9240,7 +9242,9 @@ class PythonCore(DeprecatedRouterMembers): return response else: - error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" + error_message = format_context_window_fallback_hint( + model_group, context_window_fallbacks, fallbacks, lookup_groups + ) verbose_router_logger.info( msg=f"Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \ to fallbacks, if available.{error_message}" @@ -9273,7 +9277,7 @@ class PythonCore(DeprecatedRouterMembers): return response else: error_message = format_content_policy_fallback_hint( - model_group, content_policy_fallbacks, fallbacks + model_group, content_policy_fallbacks, lookup_groups ) verbose_router_logger.info( msg=f"Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \ @@ -9349,7 +9353,7 @@ class PythonCore(DeprecatedRouterMembers): original_exception.message += format_fallback_outcome_message( model_group, attempted_fallback_group, - fallback_failure_exception_str, + compact_fallback_error_detail(fallback_failure_exception_str), skipped=fallback_walk.skipped, ) @@ -15188,7 +15192,7 @@ class PythonCore(DeprecatedRouterMembers): _model_info.get("base_model") or _litellm_params.get("base_model") or _litellm_params.get("model") ) _potential_error_str += ( - f"Model={deployment_model}, Max Input Tokens={max_input_tokens}, Got={input_tokens}" + f"\n {deployment_model}: max_input_tokens={max_input_tokens}, got {input_tokens} input tokens" ) continue diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..4bcb6524497 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -585,6 +585,28 @@ def _resolved_a_specific_chain( return generic_idx is None or resolved is not fallbacks[generic_idx]["*"] +def get_fallback_chain_for_lookup_groups( + fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape + lookup_groups: tuple[str, ...], +) -> tuple[str, list[str]] | None: # mutable-ok: mirrors get_fallback_model_group's contract + """The chain a request follows, with the lookup group that owns it. + + The owner is "*" when the chain comes from the generic entry, which applies to every + model group rather than to one of the lookup groups. A pre-routing hook makes the tier + the first lookup group, so the owner is not always the group the caller requested. + """ + results: Final = tuple( + (group, get_fallback_model_group(fallbacks=fallbacks, model_group=group)) for group in lookup_groups + ) + specific: Final = next((entry for entry in results if _resolved_a_specific_chain(fallbacks, entry[1])), None) + if specific is not None and specific[1][0] is not None: + return specific[0], specific[1][0] + generic: Final = next((entry for entry in results if entry[1][0] is not None), None) + if generic is None or generic[1][0] is None: + return None + return "*", generic[1][0] + + def get_fallback_model_group_for_lookup_groups( fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape lookup_groups: tuple[str, ...], diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 6ec72868b6e..29b39b223d0 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -15,8 +15,10 @@ from litellm.litellm_core_utils.routing_common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + format_chain_for_lookup_groups, format_content_policy_no_fallback_note, format_fallback_outcome_message, + format_no_fallback_group_message, provider_for_generic_call, resolve_model_group_alias, resolve_served_model, @@ -683,11 +685,63 @@ def test_format_fallback_outcome_preserves_attempted_failure_wording_without_ski def test_content_policy_note_explains_the_failure_when_fallbacks_are_a_plain_model_list() -> None: note: Final = format_content_policy_no_fallback_note("gpt-bg", ("gpt-bg",), None, ["backup"], False) - assert "content_policy_fallback=None" in note + assert "content_policy_fallbacks is not set" in note assert "LiteLLM: model group 'gpt-bg' failed with the error above. No fallback was attempted." in note assert "Fallbacks are configured for" not in note +def test_no_fallback_group_message_lists_every_group_when_the_config_is_small() -> None: + message: Final = format_no_fallback_group_message(("primary",), ({"a": ["x"]}, {"b": ["y"]})) + + assert "Fallbacks are configured for: a, b." in message + + +def test_no_fallback_group_message_caps_the_listing_at_the_configured_group_count() -> None: + """The listing is one line of an error a caller reads, so it must not grow with the config.""" + small: Final = format_no_fallback_group_message(("primary",), tuple({f"g{i}": ["x"]} for i in range(5))) + huge: Final = format_no_fallback_group_message(("primary",), tuple({f"g{i}": ["x"]} for i in range(200))) + + assert "and 195 more." in huge + assert "g5" not in huge + assert len(huge) - len(small) < len(" and 195 more.") + 1 + + +def test_chain_names_the_lookup_group_that_owns_it_not_the_first_one_searched() -> None: + """A pre-routing hook puts the tier first, and the chain usually belongs to the group + behind it; naming the tier would point a reader at the wrong config entry.""" + chain: Final = format_chain_for_lookup_groups( + "fallbacks", [{"cheap-tier": ["tier-backup"]}, {"requested-model": ["requested-backup"]}], ("cheap-tier", "requested-model") + ) + behind_the_tier: Final = format_chain_for_lookup_groups( + "fallbacks", [{"requested-model": ["requested-backup"]}], ("cheap-tier", "requested-model") + ) + + assert chain == "fallbacks: 'cheap-tier' -> ['tier-backup']" + assert behind_the_tier == "fallbacks: 'requested-model' -> ['requested-backup']" + + +def test_chain_attributes_a_generic_entry_to_the_wildcard_not_to_a_lookup_group() -> None: + chain: Final = format_chain_for_lookup_groups("fallbacks", [{"*": ["backup"]}], ("tier", "requested-model")) + + assert chain == "fallbacks: '*' -> ['backup']" + + +def test_chain_renders_fallback_entries_that_override_request_params_instead_of_dumping_the_config() -> None: + chain: Final = format_chain_for_lookup_groups( + "fallbacks", [{"a": [{"model": "b", "api_key": "sk-secret"}]}, {"unrelated": ["c"]}], ("a",) + ) + + assert chain.startswith("fallbacks: 'a' -> [{'model': 'b'") + assert "sk-secret" not in chain + assert "unrelated" not in chain + + +def test_chain_says_which_lookup_groups_it_searched_when_none_of_them_has_one() -> None: + chain: Final = format_chain_for_lookup_groups("fallbacks", [{"x": ["y"]}, {"z": ["w"]}], ("tier", "requested-model")) + + assert chain == "fallbacks has no chain for 'tier' -> 'requested-model' (2 other model groups are configured)" + + class TestWarnOnProviderCredentialMismatch: """A deployment that carries one provider's credentials while resolving to another is silently broken: litellm ignores the credentials and sends the diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index e4e5c3120c8..ec8ad81359a 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -1,6 +1,7 @@ """Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised.""" from collections.abc import Callable, Mapping +from typing import Final import pytest @@ -686,6 +687,58 @@ async def test_router_debug_on_an_unmatched_context_window_chain_says_no_fallbac ) +@pytest.mark.asyncio +async def test_router_debug_names_only_the_chain_that_ran_not_the_whole_config( + make_core: MakeCore, host: ScriptedHost +) -> None: + unrelated: Final = [{f"unrelated-{index}": ["x"]} for index in range(50)] + core = build(make_core, fallbacks=[{"a": ["b"]}, *unrelated]) + show_router_debug(host) + host.answer("openai/a", context_window()) + host.answer("openai/b", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + message: Final = str(raised.value) + assert "fallbacks: 'a' -> ['b']" in message + assert "fallbacks: 'b' -> " not in message + for entry in unrelated: + assert next(iter(entry)) not in message + + +@pytest.mark.asyncio +async def test_router_debug_states_how_many_groups_the_config_covers_when_the_chain_does_not_match( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"b": ["c"]}, {"c": ["a"]}]) + show_router_debug(host) + host.answer("openai/a", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + assert "fallbacks has no chain for 'a' (2 other model groups are configured)" in str(raised.value) + + +@pytest.mark.asyncio +async def test_router_debug_reports_a_failed_hop_as_one_line_without_repeating_its_own_hint( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}, {"c": ["a"]}]) + show_router_debug(host) + host.answer("openai/a", context_window()) + host.answer("openai/b", context_window()) + host.answer("openai/c", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + message: Final = str(raised.value) + assert message.count("LiteLLM: model group") == 1 + assert message.count("context_window_fallbacks is not set") == 1 + + @pytest.mark.asyncio async def test_router_debug_on_content_policy_without_any_chain_hints_at_content_policy_fallbacks( make_core: MakeCore, host: ScriptedHost @@ -698,7 +751,7 @@ async def test_router_debug_on_content_policy_without_any_chain_hints_at_content await complete(core) message = str(raised.value) - assert "model=a. content_policy_fallback=None. fallbacks=None." in message + assert "model=a. content_policy_fallbacks is not set." in message assert message.endswith("LiteLLM: model group 'a' failed with the error above. No fallback was attempted.") diff --git a/tests/unit/test_router_exception_redaction.py b/tests/unit/test_router_exception_redaction.py index 5f9a5ba06a8..a43f82f3827 100644 --- a/tests/unit/test_router_exception_redaction.py +++ b/tests/unit/test_router_exception_redaction.py @@ -47,7 +47,7 @@ from litellm import Router _RECEIVED_MODEL_GROUP_PHRASE = "failed with the error above" _AVAILABLE_FALLBACKS_PHRASE = "No fallback was attempted" _NO_FALLBACK_GROUP_PHRASE = "no fallback model group was found" -_CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks=" +_CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks" _INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal" _FALLBACK_CREDENTIAL = "sk-INLINEFALLBACKSECRET1234567890" @@ -360,7 +360,7 @@ async def test_flag_off_does_not_leak_content_policy_fallback_hint(monkeypatch: mock_response=_content_policy_error(), ) msg = excinfo.value.message - assert "content_policy_fallback=" not in msg, msg + assert "content_policy_fallbacks" not in msg, msg assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg @@ -375,7 +375,7 @@ async def test_flag_on_shows_content_policy_fallback_hint(monkeypatch: pytest.Mo mock_response=_content_policy_error(), ) msg = excinfo.value.message - assert "content_policy_fallback=" in msg, msg + assert "content_policy_fallbacks" in msg, msg assert _INTERNAL_MODEL_GROUP_NAME in msg, msg