diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index d0abaed4d3a..156195f0587 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -307,13 +307,19 @@ def fallbacks_disabled_for_request(kwargs: Mapping[str, Any]) -> bool: def fallback_lookup_groups(kwargs: Mapping[str, object], model_group: str | None) -> tuple[str, ...]: """ Ordered keys for resolving a fallback chain: the tier a pre-routing hook selected wins, - then the routed group, then the requested group. The routed group differs when Claude Code - session affinity remaps a subagent's concrete model to its bound router. + then the routed group, then the requested group, then the group the request was + originally for. The routed group differs when Claude Code session affinity remaps a + subagent's concrete model to its bound router. The original group differs on a fallback + hop that fails after `run_async_fallback` already returned its stream: the hop has no + chain of its own, so it resumes the original group's chain, and `attempted_targets` keeps + the entries already tried from being repeated. """ metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs)) routed_group_value: Final = metadata.get("model_group") if isinstance(metadata, Mapping) else None routed_group: Final = routed_group_value if isinstance(routed_group_value, str) else None - ordered: Final = (get_pre_routing_selection(kwargs), routed_group, model_group) + original_group_value: Final = metadata.get("original_model_group") if isinstance(metadata, Mapping) else None + original_group: Final = original_group_value if isinstance(original_group_value, str) else None + ordered: Final = (get_pre_routing_selection(kwargs), routed_group, model_group, original_group) return tuple(dict.fromkeys(group for group in ordered if group)) diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index dfe06bffd09..69772561172 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -1305,6 +1305,14 @@ class TestOrderedFallbackLookupGroups: "requested-model", ) + def test_fallback_hop_resumes_the_original_groups_chain_last(self): + from litellm.router_utils.fallback_event_handlers import fallback_lookup_groups + + kwargs = {"metadata": {"model_group": "fb1", "original_model_group": "primary"}} + + assert fallback_lookup_groups(kwargs, "fb1") == ("fb1", "primary") + assert fallback_lookup_groups({"metadata": {"original_model_group": 42}}, "fb1") == ("fb1",) + def test_first_resolving_group_wins_and_generic_idx_survives_a_miss(self): from litellm.router_utils.fallback_event_handlers import ( get_fallback_model_group_for_lookup_groups, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 8310d30d90e..cf143744067 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3447,6 +3447,81 @@ def test_completion_streaming_iterator_adopts_the_deployment_that_served_a_neste assert result._hidden_params["model_id"] == "served-deployment" +@pytest.mark.asyncio +async def test_acompletion_mid_stream_fallback_walks_every_entry_of_the_configured_list(): + """LIT-7400: fallbacks=[{primary: [fb1, fb2]}] must reach fb2 when fb1 dies before its first chunk. + + run_async_fallback returns as soon as fb1's stream wrapper exists, so fb1's failure surfaces + inside the streaming iterator, where the lookup is keyed by fb1. That key has no chain of its + own, so the iterator has to resume the chain of the group the request was originally for. + """ + from unittest.mock import MagicMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + attempted_model_groups: list[str] = [] + + class FailingStream(CustomStreamWrapper): + def __init__(self, model: str): + super().__init__( + completion_stream=object(), model=model, custom_llm_provider="openai", logging_obj=MagicMock() + ) + + def __aiter__(self): + return self + + async def __anext__(self): + raise MidStreamFallbackError( + message=f"provider 500 from {self.model}", + model=self.model, + llm_provider="openai", + generated_content="", + is_pre_first_chunk=True, + original_exception=litellm.InternalServerError( + message=f"provider 500 from {self.model}", model=self.model, llm_provider="openai" + ), + ) + + class OkStream(FailingStream): + def __init__(self, model: str): + super().__init__(model) + self._chunks = iter( + [litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": f"ok-from-{model}"}}])] + ) + + async def __anext__(self): + try: + return next(self._chunks) + except StopIteration: + raise StopAsyncIteration from None + + async def fake_acompletion(**kwargs): + attempted_model_groups.append(kwargs["metadata"]["model_group"]) + if "fb2" in kwargs["model"]: + return OkStream(kwargs["model"]) + return FailingStream(kwargs["model"]) + + router = litellm.Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/primary-model", "api_key": "fake-key"}}, + {"model_name": "fb1", "litellm_params": {"model": "openai/fb1-model", "api_key": "fake-key"}}, + {"model_name": "fb2", "litellm_params": {"model": "openai/fb2-model", "api_key": "fake-key"}}, + ], + fallbacks=[{"primary": ["fb1", "fb2"]}], + num_retries=0, + ) + + with patch("litellm.acompletion", side_effect=fake_acompletion): + response = await router.acompletion(model="primary", messages=[{"role": "user", "content": "hi"}], stream=True) + content: Final = "".join( + [chunk.choices[0].delta.content or "" async for chunk in response if chunk is not None] + ) + + assert content == "ok-from-openai/fb2-model" + assert attempted_model_groups == ["primary", "fb1", "fb2"] + + def test_completion_streaming_iterator_adopts_fallback_response_headers(): """LIT-6767, sync counterpart of the fallback-adoption test.""" from unittest.mock import MagicMock, patch