From d72d4c311c4bb60300f5c7020d91fd45e1bcd036 Mon Sep 17 00:00:00 2001 From: Rick <26716961+Bytechoreographer@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:17:07 +0800 Subject: [PATCH] fix(router): describe the fallback route, not the whole config, in errors Fallback errors dumped every configured fallback entry instead of the one chain the request resolved to. With a real deployment config that is tens of unrelated groups, repeated once per fallback hop, and the 2000-char truncation then cut off the actual downstream failure. The message that explains why a request was not retried listed every configured group too, so it grew with the router config: 200 groups produced a 3.2k error. Name the chain that ran and the lookup group that owns it, report each failed hop on one line with the full detail left in the log, and list at most five configured groups before counting the rest. Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> --- litellm/constants.py | 1 + .../routing_common_utils.py | 99 ++++++++++++++++--- litellm/router_python_core.py | 12 ++- .../router_utils/fallback_event_handlers.py | 22 +++++ .../test_router_utils_common_utils.py | 56 ++++++++++- tests/unit/test_router/core/test_fallbacks.py | 55 ++++++++++- tests/unit/test_router_exception_redaction.py | 6 +- 7 files changed, 226 insertions(+), 25 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index cd7c0176428..cae7db018e2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -55,6 +55,7 @@ AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPON AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 +ROUTER_FALLBACK_GROUP_LISTING_MAX_GROUPS: Final = 5 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( { "routing_strategy_args", diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index a06860e7786..5d2ee86555e 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -2,7 +2,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar, cast from pydantic import TypeAdapter, ValidationError @@ -11,7 +11,7 @@ if TYPE_CHECKING: import litellm from litellm._logging import verbose_logger, verbose_router_logger -from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS +from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS, ROUTER_FALLBACK_GROUP_LISTING_MAX_GROUPS from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap @@ -184,13 +184,35 @@ def truncate_fallback_error_detail(detail: str) -> str: return f"{detail[:ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS]}... [truncated {dropped} characters]" +def compact_fallback_error_detail(detail: str) -> str: + """Reduce a nested hop failure to one line for the user-facing message. + + The detail of a failed hop is the whole message of the hop below it, which already ends + with that hop's own fallback explanation. Carrying that verbatim into this hop makes the + text repeat itself once per hop and buries the first, most useful failure. + """ + first_line: Final = detail.strip().split("\n", 1)[0].strip() + return truncate_fallback_error_detail(first_line) + + +def _configured_groups_text(configured: Sequence[str]) -> str: + """Name a few configured model groups, never the whole config. + + This is one sentence of an error a caller reads; listing every group makes the message + grow with the size of the router config instead of with the request. + """ + if not configured: + return " No fallbacks are configured." + listed: Final = configured[:ROUTER_FALLBACK_GROUP_LISTING_MAX_GROUPS] + remainder: Final = len(configured) - len(listed) + return f" Fallbacks are configured for: {', '.join(listed)}" + (f" and {remainder} more." if remainder else ".") + + def format_no_fallback_group_message(lookup_groups: Sequence[str], fallbacks: Sequence[Mapping[str, object]]) -> str: """User-facing explanation appended when a request fails and no fallback chain matches its model group.""" requested: Final = " -> ".join(lookup_groups) configured: Final = tuple(dict.fromkeys(key for entry in fallbacks for key in entry)) - configured_text: Final = ( - f" Fallbacks are configured for: {', '.join(configured)}." if configured else " No fallbacks are configured." - ) + configured_text: Final = _configured_groups_text(configured) return ( f"\n\nLiteLLM: model group '{requested}' failed with the error above and no fallback model group was found " f"for it, so the request was not retried on another model.{configured_text}" @@ -260,16 +282,6 @@ def format_fallback_outcome_message( return f"{lead} Fallback model group(s) configured: {targets}." -def format_content_policy_fallback_hint( - model_group: str | None, content_policy_fallbacks: object, fallbacks: object -) -> str: - return ( - f"model={model_group}. content_policy_fallback={mask_sensitive_structure(content_policy_fallbacks)}. " - f"fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'content_policy_fallback' - " - "https://docs.litellm.ai/docs/routing#fallbacks" - ) - - def _as_fallback_entries(fallbacks: object) -> tuple[Mapping[str, object], ...] | None: try: return _FALLBACK_ENTRIES_ADAPTER.validate_python(fallbacks) @@ -277,6 +289,61 @@ def _as_fallback_entries(fallbacks: object) -> tuple[Mapping[str, object], ...] return None +def format_chain_for_lookup_groups(config_name: str, configured: object, lookup_groups: Sequence[str]) -> str: + """Name only the chain a request would follow, never the whole fallback config. + + Every fallback hop appends its own hint, so dumping each configured group repeats the + entire config per hop and pushes the actual failure out of the message. + """ + if configured is None: + return f"{config_name} is not set" + entries: Final = _as_fallback_entries(configured) + if entries is None or not lookup_groups: + return f"{config_name}={mask_sensitive_structure(configured)}" + from litellm.router_utils.fallback_event_handlers import get_fallback_chain_for_lookup_groups + + resolved: Final = get_fallback_chain_for_lookup_groups( + fallbacks=cast("Sequence[Mapping[str, list[str]]]", entries), # cast-ok: only keys and values are read + lookup_groups=tuple(lookup_groups), + ) + if resolved is None: + others: Final = len(tuple(key for entry in entries for key in entry if key not in lookup_groups)) + searched: Final = " -> ".join(repr(group) for group in lookup_groups) + return f"{config_name} has no chain for {searched}" + ( + f" ({others} other model groups are configured)" if others else "" + ) + owner, chain = resolved + return f"{config_name}: {owner!r} -> {mask_sensitive_structure(chain)}" + + +def format_content_policy_fallback_hint( + model_group: str | None, + content_policy_fallbacks: object, + lookup_groups: Sequence[str], +) -> str: + chain: Final = format_chain_for_lookup_groups("content_policy_fallbacks", content_policy_fallbacks, lookup_groups) + return ( + f"model={model_group}. {chain}. " + "To retry content-policy errors on another model, set content_policy_fallbacks for this model group - " + "https://docs.litellm.ai/docs/routing#fallbacks" + ) + + +def format_context_window_fallback_hint( + model_group: str | None, + context_window_fallbacks: object, + fallbacks: object, + lookup_groups: Sequence[str], +) -> str: + return ( + f"model={model_group}. " + f"{format_chain_for_lookup_groups('context_window_fallbacks', context_window_fallbacks, lookup_groups)}. " + f"{format_chain_for_lookup_groups('fallbacks', fallbacks, lookup_groups)}. " + "To give context-window errors their own chain, set context_window_fallbacks - " + "https://docs.litellm.ai/docs/routing#fallbacks" + ) + + def format_content_policy_no_fallback_note( model_group: str, lookup_groups: tuple[str, ...], @@ -287,7 +354,7 @@ def format_content_policy_no_fallback_note( outcome: Final = "" if nested_fallback_hop else format_fallback_outcome_message(model_group, None, "") if content_policy_fallbacks is not None: return outcome - hint: Final = f"\n{format_content_policy_fallback_hint(model_group, content_policy_fallbacks, fallbacks)}" + hint: Final = f"\n{format_content_policy_fallback_hint(model_group, content_policy_fallbacks, lookup_groups)}" fallback_entries: Final = _as_fallback_entries(fallbacks) if fallbacks is not None else None if fallback_entries is None or not lookup_groups or nested_fallback_hop: return hint + outcome diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 30b05456987..d9198ee39c7 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -185,12 +185,14 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( from litellm.litellm_core_utils.routing_common_utils import ( MID_STREAM_CONTINUATION_KWARG, as_retry_skipped_deployment_ids, + compact_fallback_error_detail, deployment_model_id_pin, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, format_content_policy_fallback_hint, format_content_policy_no_fallback_note, + format_context_window_fallback_hint, format_fallback_outcome_message, format_no_fallback_group_message, get_request_team_id, @@ -9240,7 +9242,9 @@ class PythonCore(DeprecatedRouterMembers): return response else: - error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" + error_message = format_context_window_fallback_hint( + model_group, context_window_fallbacks, fallbacks, lookup_groups + ) verbose_router_logger.info( msg=f"Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \ to fallbacks, if available.{error_message}" @@ -9273,7 +9277,7 @@ class PythonCore(DeprecatedRouterMembers): return response else: error_message = format_content_policy_fallback_hint( - model_group, content_policy_fallbacks, fallbacks + model_group, content_policy_fallbacks, lookup_groups ) verbose_router_logger.info( msg=f"Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \ @@ -9349,7 +9353,7 @@ class PythonCore(DeprecatedRouterMembers): original_exception.message += format_fallback_outcome_message( model_group, attempted_fallback_group, - fallback_failure_exception_str, + compact_fallback_error_detail(fallback_failure_exception_str), skipped=fallback_walk.skipped, ) @@ -15188,7 +15192,7 @@ class PythonCore(DeprecatedRouterMembers): _model_info.get("base_model") or _litellm_params.get("base_model") or _litellm_params.get("model") ) _potential_error_str += ( - f"Model={deployment_model}, Max Input Tokens={max_input_tokens}, Got={input_tokens}" + f"\n {deployment_model}: max_input_tokens={max_input_tokens}, got {input_tokens} input tokens" ) continue diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..4bcb6524497 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -585,6 +585,28 @@ def _resolved_a_specific_chain( return generic_idx is None or resolved is not fallbacks[generic_idx]["*"] +def get_fallback_chain_for_lookup_groups( + fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape + lookup_groups: tuple[str, ...], +) -> tuple[str, list[str]] | None: # mutable-ok: mirrors get_fallback_model_group's contract + """The chain a request follows, with the lookup group that owns it. + + The owner is "*" when the chain comes from the generic entry, which applies to every + model group rather than to one of the lookup groups. A pre-routing hook makes the tier + the first lookup group, so the owner is not always the group the caller requested. + """ + results: Final = tuple( + (group, get_fallback_model_group(fallbacks=fallbacks, model_group=group)) for group in lookup_groups + ) + specific: Final = next((entry for entry in results if _resolved_a_specific_chain(fallbacks, entry[1])), None) + if specific is not None and specific[1][0] is not None: + return specific[0], specific[1][0] + generic: Final = next((entry for entry in results if entry[1][0] is not None), None) + if generic is None or generic[1][0] is None: + return None + return "*", generic[1][0] + + def get_fallback_model_group_for_lookup_groups( fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape lookup_groups: tuple[str, ...], diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 6ec72868b6e..29b39b223d0 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -15,8 +15,10 @@ from litellm.litellm_core_utils.routing_common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + format_chain_for_lookup_groups, format_content_policy_no_fallback_note, format_fallback_outcome_message, + format_no_fallback_group_message, provider_for_generic_call, resolve_model_group_alias, resolve_served_model, @@ -683,11 +685,63 @@ def test_format_fallback_outcome_preserves_attempted_failure_wording_without_ski def test_content_policy_note_explains_the_failure_when_fallbacks_are_a_plain_model_list() -> None: note: Final = format_content_policy_no_fallback_note("gpt-bg", ("gpt-bg",), None, ["backup"], False) - assert "content_policy_fallback=None" in note + assert "content_policy_fallbacks is not set" in note assert "LiteLLM: model group 'gpt-bg' failed with the error above. No fallback was attempted." in note assert "Fallbacks are configured for" not in note +def test_no_fallback_group_message_lists_every_group_when_the_config_is_small() -> None: + message: Final = format_no_fallback_group_message(("primary",), ({"a": ["x"]}, {"b": ["y"]})) + + assert "Fallbacks are configured for: a, b." in message + + +def test_no_fallback_group_message_caps_the_listing_at_the_configured_group_count() -> None: + """The listing is one line of an error a caller reads, so it must not grow with the config.""" + small: Final = format_no_fallback_group_message(("primary",), tuple({f"g{i}": ["x"]} for i in range(5))) + huge: Final = format_no_fallback_group_message(("primary",), tuple({f"g{i}": ["x"]} for i in range(200))) + + assert "and 195 more." in huge + assert "g5" not in huge + assert len(huge) - len(small) < len(" and 195 more.") + 1 + + +def test_chain_names_the_lookup_group_that_owns_it_not_the_first_one_searched() -> None: + """A pre-routing hook puts the tier first, and the chain usually belongs to the group + behind it; naming the tier would point a reader at the wrong config entry.""" + chain: Final = format_chain_for_lookup_groups( + "fallbacks", [{"cheap-tier": ["tier-backup"]}, {"requested-model": ["requested-backup"]}], ("cheap-tier", "requested-model") + ) + behind_the_tier: Final = format_chain_for_lookup_groups( + "fallbacks", [{"requested-model": ["requested-backup"]}], ("cheap-tier", "requested-model") + ) + + assert chain == "fallbacks: 'cheap-tier' -> ['tier-backup']" + assert behind_the_tier == "fallbacks: 'requested-model' -> ['requested-backup']" + + +def test_chain_attributes_a_generic_entry_to_the_wildcard_not_to_a_lookup_group() -> None: + chain: Final = format_chain_for_lookup_groups("fallbacks", [{"*": ["backup"]}], ("tier", "requested-model")) + + assert chain == "fallbacks: '*' -> ['backup']" + + +def test_chain_renders_fallback_entries_that_override_request_params_instead_of_dumping_the_config() -> None: + chain: Final = format_chain_for_lookup_groups( + "fallbacks", [{"a": [{"model": "b", "api_key": "sk-secret"}]}, {"unrelated": ["c"]}], ("a",) + ) + + assert chain.startswith("fallbacks: 'a' -> [{'model': 'b'") + assert "sk-secret" not in chain + assert "unrelated" not in chain + + +def test_chain_says_which_lookup_groups_it_searched_when_none_of_them_has_one() -> None: + chain: Final = format_chain_for_lookup_groups("fallbacks", [{"x": ["y"]}, {"z": ["w"]}], ("tier", "requested-model")) + + assert chain == "fallbacks has no chain for 'tier' -> 'requested-model' (2 other model groups are configured)" + + class TestWarnOnProviderCredentialMismatch: """A deployment that carries one provider's credentials while resolving to another is silently broken: litellm ignores the credentials and sends the diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index e4e5c3120c8..ec8ad81359a 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -1,6 +1,7 @@ """Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised.""" from collections.abc import Callable, Mapping +from typing import Final import pytest @@ -686,6 +687,58 @@ async def test_router_debug_on_an_unmatched_context_window_chain_says_no_fallbac ) +@pytest.mark.asyncio +async def test_router_debug_names_only_the_chain_that_ran_not_the_whole_config( + make_core: MakeCore, host: ScriptedHost +) -> None: + unrelated: Final = [{f"unrelated-{index}": ["x"]} for index in range(50)] + core = build(make_core, fallbacks=[{"a": ["b"]}, *unrelated]) + show_router_debug(host) + host.answer("openai/a", context_window()) + host.answer("openai/b", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + message: Final = str(raised.value) + assert "fallbacks: 'a' -> ['b']" in message + assert "fallbacks: 'b' -> " not in message + for entry in unrelated: + assert next(iter(entry)) not in message + + +@pytest.mark.asyncio +async def test_router_debug_states_how_many_groups_the_config_covers_when_the_chain_does_not_match( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"b": ["c"]}, {"c": ["a"]}]) + show_router_debug(host) + host.answer("openai/a", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + assert "fallbacks has no chain for 'a' (2 other model groups are configured)" in str(raised.value) + + +@pytest.mark.asyncio +async def test_router_debug_reports_a_failed_hop_as_one_line_without_repeating_its_own_hint( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}, {"c": ["a"]}]) + show_router_debug(host) + host.answer("openai/a", context_window()) + host.answer("openai/b", context_window()) + host.answer("openai/c", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + message: Final = str(raised.value) + assert message.count("LiteLLM: model group") == 1 + assert message.count("context_window_fallbacks is not set") == 1 + + @pytest.mark.asyncio async def test_router_debug_on_content_policy_without_any_chain_hints_at_content_policy_fallbacks( make_core: MakeCore, host: ScriptedHost @@ -698,7 +751,7 @@ async def test_router_debug_on_content_policy_without_any_chain_hints_at_content await complete(core) message = str(raised.value) - assert "model=a. content_policy_fallback=None. fallbacks=None." in message + assert "model=a. content_policy_fallbacks is not set." in message assert message.endswith("LiteLLM: model group 'a' failed with the error above. No fallback was attempted.") diff --git a/tests/unit/test_router_exception_redaction.py b/tests/unit/test_router_exception_redaction.py index 5f9a5ba06a8..a43f82f3827 100644 --- a/tests/unit/test_router_exception_redaction.py +++ b/tests/unit/test_router_exception_redaction.py @@ -47,7 +47,7 @@ from litellm import Router _RECEIVED_MODEL_GROUP_PHRASE = "failed with the error above" _AVAILABLE_FALLBACKS_PHRASE = "No fallback was attempted" _NO_FALLBACK_GROUP_PHRASE = "no fallback model group was found" -_CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks=" +_CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks" _INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal" _FALLBACK_CREDENTIAL = "sk-INLINEFALLBACKSECRET1234567890" @@ -360,7 +360,7 @@ async def test_flag_off_does_not_leak_content_policy_fallback_hint(monkeypatch: mock_response=_content_policy_error(), ) msg = excinfo.value.message - assert "content_policy_fallback=" not in msg, msg + assert "content_policy_fallbacks" not in msg, msg assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg @@ -375,7 +375,7 @@ async def test_flag_on_shows_content_policy_fallback_hint(monkeypatch: pytest.Mo mock_response=_content_policy_error(), ) msg = excinfo.value.message - assert "content_policy_fallback=" in msg, msg + assert "content_policy_fallbacks" in msg, msg assert _INTERNAL_MODEL_GROUP_NAME in msg, msg