From 0dee2a8a0d093f402134a86b0dac0b2be221a781 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sun, 15 Nov 2026 12:19:58 -0800 Subject: [PATCH] fix(responses): retry a stream that breaks before output in its own model group (#51953) --- litellm/router_rust/invoke/streaming.py | 104 +++++++++++++++++- .../test_responses_stream_same_group_retry.py | 48 ++++++++ ...st_streaming_characterization_responses.py | 95 ++++++++++++++++ 3 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 tests/integration/routing/test_responses_stream_same_group_retry.py diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py index 797f74e0b26..6234c9783b0 100644 --- a/litellm/router_rust/invoke/streaming.py +++ b/litellm/router_rust/invoke/streaming.py @@ -1575,6 +1575,7 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own self._async_generator: Final = async_generator + self.attempt: Final = attempt self._billing_source: object = source_iterator self._last_sequence_number = -1 self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source @@ -1699,10 +1700,12 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p if response_id_exposed: _raise_released_stream_error(attempt, e) async with contextlib.aclosing( - _responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held) - ) as fallback_stream: - async for fallback_item in fallback_stream: - yield fallback_item + _responses_recover_stream_error( + attempt, e, source_iterator, wrapper, wrapper.adopt_fallback_headers, held + ) + ) as recovered_stream: + async for recovered_item in recovered_stream: + yield recovered_item except Exception: for held_event in held: yield held_event @@ -1791,6 +1794,99 @@ def _responses_continuation_request( } +def _responses_stream_retries( + attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator" +) -> bool: + """Whether a broken Responses stream retries in its own group first: it broke before any output and + before a tool call, with an error the retry rule takes (408, 409, 429, 5xx, or no status), as a + ``/v1/messages`` stream does. After output the continuation fallback takes over instead.""" + if not _mid_stream_error_before_output(e) or source_iterator.started_tool_call: + return False + status_code: Final = _stream_raised_error_status(_mid_stream_retry_trigger(e)) + return status_code is None or attempt.should_retry_status(status_code) + + +async def _responses_recover_stream_error( + attempt: StreamAttempt, + e: MidStreamFallbackError, + source_iterator: "BaseResponsesAPIStreamingIterator", + wrapper: object, + adopt_headers: Callable[[object], _Prepared], + held: tuple[object, ...], +) -> AsyncGenerator[object, None]: + """A Responses stream that broke before any output is retried in its own group, then falls back + with the newest failure once the retries are spent. The primary's held lifecycle events are dropped + when a retry or a fallback answers.""" + last_error: MidStreamFallbackError = e # rebind-ok: the newest failure once a retry also failed + if _responses_stream_retries(attempt, e, source_iterator): + if hasattr(source_iterator, "aclose"): + with anyio.CancelScope(shield=True), contextlib.suppress(Exception): + await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + await attempt.failure_recorded() + trigger: Final = _mid_stream_retry_trigger(e) + retried: Final = await attempt.retry(trigger) + if not isinstance(retried, BaseException): + async for retried_item in _responses_yield_retried(retried, wrapper, adopt_headers): + yield retried_item + return + if retried is not trigger: + last_error = _stream_pre_content_error( # rebind-ok: the retry failed again, so its failure is the newest + retried if isinstance(retried, Exception) else e, e.model or "", e.llm_provider + ) + async with contextlib.aclosing( + _responses_fallback_attempt(attempt, last_error, source_iterator, adopt_headers, held) + ) as fallback_stream: + async for fallback_item in fallback_stream: + yield fallback_item + + +async def _responses_yield_retried( + retried: tuple[object, int, int], wrapper: object, adopt_headers: Callable[[object], _Prepared] +) -> AsyncGenerator[object, None]: + """Hands a same-group retry's stream to the client through the wrapper, with the retry's headers, and + closes it after. A retried stream that breaks again counts on from the retries already made, and when + it recovers onto another source (its own fallback) the wrapper and the events follow that source.""" + response, attempted, budget = retried + inner_attempt: Final = getattr(response, "attempt", None) + if isinstance(inner_attempt, StreamAttempt): + inner_attempt.attempted_retries = attempted + inner_attempt.committed_budget = budget + + served_as: Final = _prepare_fallback_hidden_params(wrapper)[1].get("x-litellm-model-group") + + def follow_source() -> _Prepared: + prepared: Final = adopt_headers(response) + if served_as is not None: + _ = prepared[1].setdefault("x-litellm-model-group", served_as) + newest: Final = prepared[1].get("x-litellm-attempted-retries") + _ = add_retry_headers_to_response( + response=wrapper, + attempted_retries=max(attempted, newest) if isinstance(newest, int) else attempted, + max_retries=budget, + ) + return prepared + + followed: _Prepared = follow_source() # rebind-ok: refreshed when the retried stream changes its source + source: object = getattr(response, "_billing_source", None) # rebind-ok: the retried stream's current source + try: + if hasattr(response, "__aiter__"): + async for retried_item in cast("AsyncIterator[object]", response): # cast-ok: __aiter__ checked above + if (current_source := getattr(response, "_billing_source", None)) is not source: + source = current_source + followed = follow_source() + _apply_fallback_hidden_params_to_item(retried_item, followed) + yield retried_item + else: + yield response + finally: + if hasattr(response, "aclose"): + with anyio.CancelScope(shield=True): + try: + await response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above + except Exception as exc: # noqa: BLE001 # closing is best-effort + verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing retry: %s", exc) + + async def _responses_fallback_attempt( attempt: StreamAttempt, e: MidStreamFallbackError, diff --git a/tests/integration/routing/test_responses_stream_same_group_retry.py b/tests/integration/routing/test_responses_stream_same_group_retry.py new file mode 100644 index 00000000000..e3a12fb4578 --- /dev/null +++ b/tests/integration/routing/test_responses_stream_same_group_retry.py @@ -0,0 +1,48 @@ +import itertools +import uuid +from collections.abc import Callable +from typing import Final + +from integration._support.client import Gateway +from integration._support.responses_stream import ( + AZURE_TARGET, + healthy_stream, + rate_limited_stream, + sse_events, +) +from integration._support.wire import Reply, Request, wire_server + +_RETRY_TEXT: Final = "retry answer" + + +def _throttle_first_request_only(identity: str) -> Callable[[Request], Reply]: + """Shared by both regions: whichever the router picks first is out of quota, the other one answers.""" + arrivals: Final = itertools.count() + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target.startswith(AZURE_TARGET), request.target + stream: Final = rate_limited_stream(identity) if next(arrivals) == 0 else healthy_stream(identity, _RETRY_TEXT) + return Reply(content_type="text/event-stream", chunks=stream) + + return respond + + +def test_streamed_responses_in_stream_rate_limit_is_retried_in_its_model_group(gateway: Gateway) -> None: + identity: Final = "resp_" + uuid.uuid4().hex + respond: Final = _throttle_first_request_only(identity) + with wire_server(respond) as region_a, wire_server(respond) as region_b, gateway.scenario() as scenario: + group: Final = f"responses-stream-retry-{uuid.uuid4().hex}" + for region in (region_a, region_b): + scenario.model(name=group, model="azure/gpt-6", api_base=region.url, api_key="synthetic-azure-key") + response: Final = gateway.request( + "POST", + "/v1/responses", + {"model": group, "input": identity, "stream": True, "num_retries": 2, "cache": {"no-cache": True}}, + ) + assert response.status_code == 200, response.text + events: Final = sse_events(response.text) + assert [name for name, _ in events][-1] == "response.completed", response.text + assert _RETRY_TEXT in response.text, response.text + calls_per_region: Final = (len(region_a.drain()), len(region_b.drain())) + assert sum(calls_per_region) == 2, f"one throttled call and one retry: {calls_per_region}" + assert response.headers["x-litellm-attempted-retries"] == "1", dict(response.headers) diff --git a/tests/unit/test_router/core/test_streaming_characterization_responses.py b/tests/unit/test_router/core/test_streaming_characterization_responses.py index 4b601705324..76c77d7cd8b 100644 --- a/tests/unit/test_router/core/test_streaming_characterization_responses.py +++ b/tests/unit/test_router/core/test_streaming_characterization_responses.py @@ -25,6 +25,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) +from litellm.types.router import AllowedFailsPolicy, RetryPolicy from tests.unit.test_router.core.scripted_host import ( ScriptedHost, content_policy, @@ -456,3 +457,97 @@ async def test_the_primary_and_fallback_streams_are_closed(make_core: MakeCore, core = build(make_core) await collect_event_types(await open_stream(core)) assert [stream.closed for stream in opened] == [True, True] + + +GROUP_A_TWICE: Final = [deployment("a", "a1"), deployment("a", "a2")] + + +def retried_ok() -> Callable[..., ScriptedResponsesStream]: + return responses_stream( + Event(CREATED), Event(TEXT_DELTA), Event(COMPLETED), hidden_params={"model_id": "retry-stream"} + ) + + +async def test_a_rate_limit_before_output_is_retried_on_the_other_deployment_of_the_group( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", responses_stream(Event(CREATED), Event(IN_PROGRESS), error=cut()), retried_ok()) + core = make_core(model_list=GROUP_A_TWICE, num_retries=1) + stream = await open_stream(core) + assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED] + first, retried = host.deployments_called() + assert retried != first + assert hidden(stream)["model_id"] == "retry-stream" + assert hidden(stream)["additional_headers"]["x-litellm-attempted-retries"] == 1 + + +async def test_a_rate_limit_retry_policy_retries_a_responses_stream_when_num_retries_is_zero( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.answer("*", responses_stream(Event(CREATED), error=cut()), retried_ok()) + core = make_core(model_list=GROUP_A_TWICE, num_retries=0, retry_policy=RetryPolicy(RateLimitErrorRetries=2)) + stream = await open_stream(core) + assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED] + assert len(host.deployments_called()) == 2 + assert hidden(stream)["additional_headers"]["x-litellm-max-retries"] == 2 + + +async def test_once_the_stream_retries_are_spent_the_fallback_answers(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("openai/a", responses_stream(Event(CREATED), error=cut())) + host.answer("openai/b", fallback_ok()) + core = build(make_core, num_retries=1) + stream = await open_stream(core) + events = [event async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream + assert [event.type for event in events] == [CREATED, TEXT_DELTA, COMPLETED] + called = host.deployments_called() + assert called[-1] == "b" + assert called.count("a") == 2 + assert hidden(stream)["model_id"] == "b-stream" + assert all(event._hidden_params["model_id"] == "b-stream" for event in events) + + +async def test_after_text_the_stream_is_not_retried_in_the_group(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("*", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("Hel")), retried_ok()) + core = make_core(model_list=GROUP_A_TWICE, num_retries=1) + stream = await open_stream(core) + seen: list[object] = [] + with pytest.raises(litellm.RateLimitError, match="stream cut"): + await drain_into(stream, seen) + assert [getattr(event, "type") for event in seen] == [CREATED, TEXT_DELTA] + assert len(host.deployments_called()) == 1 + + +async def test_a_second_failed_stream_counts_as_a_second_retry(make_core: MakeCore, host: ScriptedHost) -> None: + broken: Final = responses_stream(Event(CREATED), error=cut()) + host.answer("*", broken, broken, retried_ok()) + core = make_core( + model_list=[*GROUP_A_TWICE, deployment("a", "a3")], + num_retries=2, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1000), + ) + stream = await open_stream(core) + assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED] + assert len(host.deployments_called()) == 3 + assert hidden(stream)["additional_headers"]["x-litellm-attempted-retries"] == 2 + + +async def test_a_retried_stream_keeps_the_model_group_header(make_core: MakeCore, host: ScriptedHost) -> None: + host.answer("*", responses_stream(Event(CREATED), error=cut()), retried_ok()) + stream = await open_stream(make_core(model_list=GROUP_A_TWICE, num_retries=1)) + served_as: Final = hidden(stream)["additional_headers"]["x-litellm-model-group"] + assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED] + assert hidden(stream)["additional_headers"]["x-litellm-model-group"] == served_as + + +async def test_the_failed_stream_is_closed_before_the_retry_is_opened(make_core: MakeCore, host: ScriptedHost) -> None: + opened: list[ScriptedResponsesStream] = [] + closed_when_retry_opens: list[bool] = [] + + def retry(**request: object) -> ScriptedResponsesStream: # kwargs-ok: the host passes the request kwargs + closed_when_retry_opens.append(opened[0].closed) + return retried_ok()(**request) + + host.answer("*", responses_stream(Event(CREATED), error=cut(), opened=opened), retry) + core = make_core(model_list=GROUP_A_TWICE, num_retries=1) + assert await collect_event_types(await open_stream(core)) == [CREATED, TEXT_DELTA, COMPLETED] + assert closed_when_retry_opens == [True]