From 5752fc35edb54ce8a5a6d39dde6c714348bd00e2 Mon Sep 17 00:00:00 2001 From: MarkXian Date: Fri, 28 Aug 2026 11:59:52 +0800 Subject: [PATCH 1/4] fix(proxy): hold fallback lifecycle frames --- litellm/router.py | 12 +++++++++++- tests/test_litellm/test_router.py | 32 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index c93c1753f0e..8b1487222cc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4984,6 +4984,7 @@ class Router: self, response: AsyncIterator[bytes], initial_kwargs: dict[str, Any], # mutable-ok: mutated in-place before re-entering the fallback chain + allow_fallback: bool = True, ) -> AsyncIterator[bytes]: """ Wrap an anthropic_messages (/v1/messages) streaming response so a @@ -5100,6 +5101,10 @@ class Router: for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk except Exception as stream_error: # noqa: BLE001 # any raised provider error must reach the fallback gate + if not allow_fallback: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: + raise stream_error.original_exception from stream_error + raise async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, @@ -5201,7 +5206,12 @@ class Router: wrapper.merge_fallback_hidden_params(fallback_hidden_params, fallback_headers) wrapper.adopt_fallback_source(fallback_response) if hasattr(fallback_response, "__aiter__"): - async for fallback_item in fallback_response: + fallback_stream: Final = await self._aanthropic_messages_streaming_iterator( + response=cast("AsyncIterator[bytes]", fallback_response), + initial_kwargs=initial_kwargs, + allow_fallback=False, + ) + async for fallback_item in fallback_stream: yield fallback_item else: # A fallback can resolve to a complete AnthropicMessagesResponse diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 97286017ffe..ea475c3277f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10686,6 +10686,38 @@ async def test_anthropic_messages_fallback_also_failing_raises_original_exceptio assert exc_info.value is original_exception +@pytest.mark.asyncio +async def test_anthropic_messages_double_failure_holds_back_fallback_lifecycle_frames(): + router = _anthropic_messages_make_router() + primary_stream = _AnthropicMessagesFakeByteStream([_anthropic_messages_overloaded_error_chunk()]) + fallback_error = litellm.APIError( + status_code=503, + message="fallback also overloaded", + llm_provider="bedrock", + model="fallback", + ) + fallback_stream = _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], fallback_error + ) + + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=fallback_stream), + ): + wrapped = await router._aanthropic_messages_streaming_iterator( + response=primary_stream, + initial_kwargs={"model": "primary"}, + ) + collected = [] + with pytest.raises(litellm.APIError) as exc_info: + async for chunk in wrapped: + collected.append(chunk) + + assert collected == [] + assert exc_info.value is fallback_error + + # -------- _dispatch_generic_call_type -------- From d64971888b0d48d37d1961acf12d3021f4c06640 Mon Sep 17 00:00:00 2001 From: MarkXian Date: Fri, 28 Aug 2026 12:09:00 +0800 Subject: [PATCH 2/4] chore: rerun CI against staging base From bf37cd221a9dc110361fbde6c334acf09d58b8f8 Mon Sep 17 00:00:00 2001 From: MarkXian Date: Fri, 28 Aug 2026 14:40:21 +0800 Subject: [PATCH 3/4] test: satisfy pytest raises lint rule --- tests/test_litellm/test_router.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index ea475c3277f..d61bd5f64f2 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10709,11 +10709,16 @@ async def test_anthropic_messages_double_failure_holds_back_fallback_lifecycle_f response=primary_stream, initial_kwargs={"model": "primary"}, ) + collected = [] - with pytest.raises(litellm.APIError) as exc_info: + + async def collect_chunks(): async for chunk in wrapped: collected.append(chunk) + with pytest.raises(litellm.APIError) as exc_info: + await collect_chunks() + assert collected == [] assert exc_info.value is fallback_error From d569e2b91ff5160bc857afb1e9ebef4feac505b9 Mon Sep 17 00:00:00 2001 From: MarkXian Date: Sat, 29 Aug 2026 11:13:59 +0800 Subject: [PATCH 4/4] fix(router): justify fallback stream cast --- litellm/router.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 8b1487222cc..a2e12f85388 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5207,7 +5207,9 @@ class Router: wrapper.adopt_fallback_source(fallback_response) if hasattr(fallback_response, "__aiter__"): fallback_stream: Final = await self._aanthropic_messages_streaming_iterator( - response=cast("AsyncIterator[bytes]", fallback_response), + response=cast( # cast-ok: __aiter__ check established the streaming fallback shape + "AsyncIterator[bytes]", fallback_response + ), initial_kwargs=initial_kwargs, allow_fallback=False, )