diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 17a8aba2d0c..4c8688a2d0b 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -40,6 +40,38 @@ def _prefill_explicitly_unsupported(model: str) -> bool: return False +def _resolve_dict_fallback_targets( + model_group: str, fallbacks: list[FallbackEntry] +) -> tuple[str, ...]: + """Dict-format fallback targets configured for this group, mirroring + get_fallback_model_group's exact > stripped > wildcard priority. It iterates + directly rather than calling that helper because the helper pops string + entries mid-iteration, which skips the following entry and can drop a dict + target when a bare string precedes it in the list. + """ + dict_entries = tuple( + (key, entry[key]) + for entry in fallbacks + if isinstance(entry, dict) and entry + for key in (next(iter(entry)),) + ) + exact = next((t for k, t in dict_entries if k == model_group), None) + if exact is not None: + return tuple(exact) + stripped = next( + ( + t + for k, t in dict_entries + if _check_stripped_model_group(model_group=model_group, fallback_key=k) + ), + None, + ) + if stripped is not None: + return tuple(stripped) + wildcard = next((t for k, t in dict_entries if k == "*"), None) + return tuple(wildcard or ()) + + def _candidate_model_groups( model_group: str | None, fallbacks: list[FallbackEntry] | None ) -> tuple[str, ...]: @@ -54,15 +86,8 @@ def _candidate_model_groups( if not fallbacks: return (model_group,) bare_strings = tuple(f for f in fallbacks if isinstance(f, str)) - try: - # Copy: get_fallback_model_group pops string entries while iterating, - # which would mutate the caller's live fallbacks list. - dict_targets, _ = get_fallback_model_group( - fallbacks=list(fallbacks), model_group=model_group - ) - except Exception: - dict_targets = None - return (model_group, *bare_strings, *(dict_targets or ())) + dict_targets = _resolve_dict_fallback_targets(model_group, fallbacks) + return (model_group, *bare_strings, *dict_targets) def _resolved_registry_model( diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 6a2b9b92363..f856c601d6b 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -243,6 +243,51 @@ def test_resolver_returning_prefill_supporting_model_keeps_legacy(): _assert_legacy_prefill(result) +def test_string_before_dict_fallback_does_not_skip_dict_target(): + """A bare string preceding a dict entry must not hide the dict's + prefill-rejecting target. get_fallback_model_group pops string entries while + iterating, which skips the next entry; candidate collection iterates dict + entries directly so the target is still classified.""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="gpt-4", + fallbacks=["gpt-4o", {"gpt-4": ["claude-sonnet-4-6"]}], + ) + assert len(result) == 2 + assert result[1]["role"] == "user" + assert PARTIAL in result[1]["content"] + assert all(m.get("prefix") is not True for m in result) + + +def test_stripped_model_group_fallback_is_classified(): + """Wildcard routing matches a provider-stripped group name against the + fallback key (e.g. "openai/gpt-3.5-turbo" -> "gpt-3.5-turbo"); a + prefill-rejecting target behind such a key must still flip the continuation.""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="openai/gpt-3.5-turbo", + fallbacks=[{"gpt-3.5-turbo": ["claude-sonnet-4-6"]}], + ) + assert len(result) == 2 + assert result[1]["role"] == "user" + assert PARTIAL in result[1]["content"] + + +def test_exact_group_match_takes_priority_over_wildcard(): + """When both an exact group match and a wildcard fallback are configured, + only the exact target is tried, so a prefill-rejecting model behind the + wildcard must not flip a prefill-supporting exact chain.""" + result = build_mid_stream_continuation_messages( + messages=MESSAGES, + generated_content=PARTIAL, + model_group="gpt-4", + fallbacks=[{"gpt-4": ["gpt-4o"]}, {"*": ["claude-sonnet-4-6"]}], + ) + _assert_legacy_prefill(result) + + def test_malformed_fallback_entry_does_not_crash(): """A malformed fallback entry (e.g. an empty dict) makes the underlying fallback-group resolution raise; the capability check must swallow it and