diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py index d8b4f87501a..9286e9aa06c 100644 --- a/litellm/router_rust/invoke/streaming.py +++ b/litellm/router_rust/invoke/streaming.py @@ -684,7 +684,10 @@ def _mid_stream_continuation_eligible( lookup_groups: Final = fallback_lookup_groups( request, requested_group if isinstance(requested_group, str) else None ) - targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them + targets: Final = continuation_fallback_targets( + request.get("fallbacks", core.settings().fallbacks), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the request's list replaces the router's + lookup_groups, + ) if any(fallback_target_overrides_request_params(target) for target in targets): return False if any(_names_deployment_that_cannot_continue(core, target) for target in targets): diff --git a/tests/unit/test_router/core/test_streaming_characterization_chat.py b/tests/unit/test_router/core/test_streaming_characterization_chat.py index 260ad8534cf..590fafede8e 100644 --- a/tests/unit/test_router/core/test_streaming_characterization_chat.py +++ b/tests/unit/test_router/core/test_streaming_characterization_chat.py @@ -516,6 +516,18 @@ async def test_continuation_declines_a_fallback_target_that_overrides_request_pa assert host.deployments_called() == ["a"] +async def test_continuation_reads_the_request_fallbacks_over_the_routers( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="request override"))) + host.answer(PREFILL_MODEL, chat_stream(" world")) + core = continuation_core(make_core, fallbacks=[{"a": ["b"]}]) + stream = await open_stream(core, fallbacks=[{"model": "b", "messages": [{"role": "user", "content": "again"}]}]) + with pytest.raises(litellm.RateLimitError, match="request override"): + await collect_texts(stream) + assert host.deployments_called() == ["a"] + + async def test_continuation_runs_a_plain_fallback_for_an_error_before_content( make_core: MakeCore, host: ScriptedHost ) -> None: