From b3fe12ecbfc20cc4c9702e30a12f345be6611a7f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:42:37 -0700 Subject: [PATCH] fix(router): renumber Responses stream events only after a fallback takes over A stream that never falls back now keeps its provider's own sequence numbers, repeats included, so a starting_after resume still matches them. After a fallback the wrapper continues above the highest number the client saw --- litellm/router.py | 3 ++ tests/unit/test_router/test_router.py | 66 +++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 9a2dcb4aa8f..7c3f0314f10 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4416,6 +4416,9 @@ class Router(DeprecatedRouterMembers): sequence_number: Final = getattr(chunk, "sequence_number", None) if not isinstance(sequence_number, int): return chunk + if not self.fallback_headers_adopted: + self._last_sequence_number = max(self._last_sequence_number, sequence_number) + return chunk self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number) if self._last_sequence_number == sequence_number: return chunk diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b8ff0e8cd3f..1f0bf2b419e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -6854,6 +6854,72 @@ async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumb ("e", 8), ] + +@pytest.mark.asyncio +async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchanged(): + from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents + + def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + sequence_number=sequence_number, + ) + + router: Final = _InjectedFallbackRouter(_AsyncList([])) + primary: Final = _make_responses_iterator( + chunks=[delta(0, "a"), delta(1, "b"), delta(1, "c"), delta(0, "d")], model="primary" + ) + + wrapped: Final = await router._aresponses_streaming_iterator( + response=primary, + initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses}, + ) + events: Final = [event async for event in wrapped] + + assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 1), ("c", 1), ("d", 0)] + + +@pytest.mark.asyncio +async def test_aresponses_stream_fallback_continues_above_the_highest_number_the_primary_sent(): + from litellm.exceptions import MidStreamFallbackError + from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents + + def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + sequence_number=sequence_number, + ) + + router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "d")])) + primary: Final = _make_responses_iterator( + chunks=[delta(0, "a"), delta(3, "b"), delta(1, "c")], + error=MidStreamFallbackError( + message="primary died", + model="primary", + llm_provider="anthropic", + is_pre_first_chunk=False, + generated_content="abc", + ), + model="primary", + ) + + wrapped: Final = await router._aresponses_streaming_iterator( + response=primary, + initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses}, + ) + events: Final = [event async for event in wrapped] + + assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 3), ("c", 1), ("d", 4)] + + @pytest.mark.asyncio async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_fallback_dispatch(): """The fallback dispatch matches on the trigger's own type, so the wrapper must hand it the