scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(router): honor request fallbacks in the mid-stream continuation check (#50260)

Browse source
This commit is contained in:
Nate Armstrong 2026-11-09 17:56:41 -08:00 • committed by GitHub
parent cd9f98d095
commit 7ed28ba3fe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 16 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/router_rust/invoke/streaming.py
View file

@ -684,7 +684,10 @@ def _mid_stream_continuation_eligible(
lookup_groups: Final = fallback_lookup_groups(
request, requested_group if isinstance(requested_group, str) else None
)
targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them
targets: Final = continuation_fallback_targets(
request.get("fallbacks", core.settings().fallbacks), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the request's list replaces the router's
lookup_groups,
)
if any(fallback_target_overrides_request_params(target) for target in targets):
return False
if any(_names_deployment_that_cannot_continue(core, target) for target in targets):

12
tests/unit/test_router/core/test_streaming_characterization_chat.py
View file

@ -516,6 +516,18 @@ async def test_continuation_declines_a_fallback_target_that_overrides_request_pa
assert host.deployments_called() == ["a"]
async def test_continuation_reads_the_request_fallbacks_over_the_routers(
make_core: MakeCore, host: ScriptedHost
) -> None:
host.answer("openai/a", chat_stream("he", "llo", error=rate_limit(message="request override")))
host.answer(PREFILL_MODEL, chat_stream(" world"))
core = continuation_core(make_core, fallbacks=[{"a": ["b"]}])
stream = await open_stream(core, fallbacks=[{"model": "b", "messages": [{"role": "user", "content": "again"}]}])
with pytest.raises(litellm.RateLimitError, match="request override"):
await collect_texts(stream)
assert host.deployments_called() == ["a"]
async def test_continuation_runs_a_plain_fallback_for_an_error_before_content(
make_core: MakeCore, host: ScriptedHost
) -> None: