diff --git a/litellm/router.py b/litellm/router.py index c93c1753f0e..8b1487222cc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4984,6 +4984,7 @@ class Router: self, response: AsyncIterator[bytes], initial_kwargs: dict[str, Any], # mutable-ok: mutated in-place before re-entering the fallback chain + allow_fallback: bool = True, ) -> AsyncIterator[bytes]: """ Wrap an anthropic_messages (/v1/messages) streaming response so a @@ -5100,6 +5101,10 @@ class Router: for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk except Exception as stream_error: # noqa: BLE001 # any raised provider error must reach the fallback gate + if not allow_fallback: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: + raise stream_error.original_exception from stream_error + raise async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, @@ -5201,7 +5206,12 @@ class Router: wrapper.merge_fallback_hidden_params(fallback_hidden_params, fallback_headers) wrapper.adopt_fallback_source(fallback_response) if hasattr(fallback_response, "__aiter__"): - async for fallback_item in fallback_response: + fallback_stream: Final = await self._aanthropic_messages_streaming_iterator( + response=cast("AsyncIterator[bytes]", fallback_response), + initial_kwargs=initial_kwargs, + allow_fallback=False, + ) + async for fallback_item in fallback_stream: yield fallback_item else: # A fallback can resolve to a complete AnthropicMessagesResponse diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 97286017ffe..ea475c3277f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10686,6 +10686,38 @@ async def test_anthropic_messages_fallback_also_failing_raises_original_exceptio assert exc_info.value is original_exception +@pytest.mark.asyncio +async def test_anthropic_messages_double_failure_holds_back_fallback_lifecycle_frames(): + router = _anthropic_messages_make_router() + primary_stream = _AnthropicMessagesFakeByteStream([_anthropic_messages_overloaded_error_chunk()]) + fallback_error = litellm.APIError( + status_code=503, + message="fallback also overloaded", + llm_provider="bedrock", + model="fallback", + ) + fallback_stream = _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], fallback_error + ) + + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=fallback_stream), + ): + wrapped = await router._aanthropic_messages_streaming_iterator( + response=primary_stream, + initial_kwargs={"model": "primary"}, + ) + collected = [] + with pytest.raises(litellm.APIError) as exc_info: + async for chunk in wrapped: + collected.append(chunk) + + assert collected == [] + assert exc_info.value is fallback_error + + # -------- _dispatch_generic_call_type --------