mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): renumber Responses stream events only after a fallback takes over
A stream that never falls back now keeps its provider's own sequence numbers, repeats included, so a starting_after resume still matches them. After a fallback the wrapper continues above the highest number the client saw
This commit is contained in:
parent
af590f01ec
commit
b3fe12ecbf
2 changed files with 69 additions and 0 deletions
|
|
@ -4416,6 +4416,9 @@ class Router(DeprecatedRouterMembers):
|
|||
sequence_number: Final = getattr(chunk, "sequence_number", None)
|
||||
if not isinstance(sequence_number, int):
|
||||
return chunk
|
||||
if not self.fallback_headers_adopted:
|
||||
self._last_sequence_number = max(self._last_sequence_number, sequence_number)
|
||||
return chunk
|
||||
self._last_sequence_number = max(self._last_sequence_number + 1, sequence_number)
|
||||
if self._last_sequence_number == sequence_number:
|
||||
return chunk
|
||||
|
|
|
|||
|
|
@ -6854,6 +6854,72 @@ async def test_aresponses_stream_fallback_keeps_higher_numbers_and_passes_unnumb
|
|||
("e", 8),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_stream_without_fallback_passes_upstream_numbers_unchanged():
|
||||
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
|
||||
|
||||
def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent:
|
||||
return OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
|
||||
item_id="msg_1",
|
||||
output_index=0,
|
||||
content_index=0,
|
||||
delta=text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
router: Final = _InjectedFallbackRouter(_AsyncList([]))
|
||||
primary: Final = _make_responses_iterator(
|
||||
chunks=[delta(0, "a"), delta(1, "b"), delta(1, "c"), delta(0, "d")], model="primary"
|
||||
)
|
||||
|
||||
wrapped: Final = await router._aresponses_streaming_iterator(
|
||||
response=primary,
|
||||
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses},
|
||||
)
|
||||
events: Final = [event async for event in wrapped]
|
||||
|
||||
assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 1), ("c", 1), ("d", 0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_stream_fallback_continues_above_the_highest_number_the_primary_sent():
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
|
||||
|
||||
def delta(sequence_number: int, text: str) -> OutputTextDeltaEvent:
|
||||
return OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
|
||||
item_id="msg_1",
|
||||
output_index=0,
|
||||
content_index=0,
|
||||
delta=text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
router: Final = _InjectedFallbackRouter(_AsyncList([delta(0, "d")]))
|
||||
primary: Final = _make_responses_iterator(
|
||||
chunks=[delta(0, "a"), delta(3, "b"), delta(1, "c")],
|
||||
error=MidStreamFallbackError(
|
||||
message="primary died",
|
||||
model="primary",
|
||||
llm_provider="anthropic",
|
||||
is_pre_first_chunk=False,
|
||||
generated_content="abc",
|
||||
),
|
||||
model="primary",
|
||||
)
|
||||
|
||||
wrapped: Final = await router._aresponses_streaming_iterator(
|
||||
response=primary,
|
||||
initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": litellm.aresponses},
|
||||
)
|
||||
events: Final = [event async for event in wrapped]
|
||||
|
||||
assert [(event.delta, event.sequence_number) for event in events] == [("a", 0), ("b", 3), ("c", 1), ("d", 4)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_unwraps_content_policy_trigger_for_fallback_dispatch():
|
||||
"""The fallback dispatch matches on the trigger's own type, so the wrapper must hand it the
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue