fix(router): renumber Responses stream events only after a fallback takes over

A stream that never falls back now keeps its provider's own sequence numbers, repeats included, so a starting_after resume still matches them. After a fallback the wrapper continues above the highest number the client saw
This commit is contained in:
mateo-berri 2026-10-31 01:42:37 -07:00
parent af590f01ec
commit b3fe12ecbf
2 changed files with 69 additions and 0 deletions

View file

@ -4416,6 +4416,9 @@ class Router(DeprecatedRouterMembers):
sequence_number: Final = getattr(chunk, "sequence_number", None)
if not isinstance(sequence_number, int):
return chunk
if not self.fallback_headers_adopted:
self._last_sequence_number = max(self._last_sequence_number, sequence_number)
return chunk
self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number)
if self._last_sequence_number == sequence_number:
return chunk

View file

@ -6854,6 +6854,72 @@ async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumb
("e", 8),
]
@pytest.mark.asyncio
async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchanged():
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent:
return OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
item_id="msg_1",
output_index=0,
content_index=0,
delta=text,
sequence_number=sequence_number,
)
router: Final = _InjectedFallbackRouter(_AsyncList([]))
primary: Final = _make_responses_iterator(
chunks=[delta(0, "a"), delta(1, "b"), delta(1, "c"), delta(0, "d")], model="primary"
)
wrapped: Final = await router._aresponses_streaming_iterator(
response=primary,
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses},
)
events: Final = [event async for event in wrapped]
assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 1), ("c", 1), ("d", 0)]
@pytest.mark.asyncio
async def test_aresponses_stream_fallback_continues_above_the_highest_number_the_primary_sent():
from litellm.exceptions import MidStreamFallbackError
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent:
return OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
item_id="msg_1",
output_index=0,
content_index=0,
delta=text,
sequence_number=sequence_number,
)
router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "d")]))
primary: Final = _make_responses_iterator(
chunks=[delta(0, "a"), delta(3, "b"), delta(1, "c")],
error=MidStreamFallbackError(
message="primary died",
model="primary",
llm_provider="anthropic",
is_pre_first_chunk=False,
generated_content="abc",
),
model="primary",
)
wrapped: Final = await router._aresponses_streaming_iterator(
response=primary,
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses},
)
events: Final = [event async for event in wrapped]
assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 3), ("c", 1), ("d", 4)]
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_fallback_dispatch():
"""The fallback dispatch matches on the trigger's own type, so the wrapper must hand it the