fix(responses): retry a stream that breaks before output in its own model group (#51953)

This commit is contained in:
Kerry Lu 2026-11-15 12:19:58 -08:00 • committed by GitHub
parent e6b673e6e0
commit 0dee2a8a0d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 243 additions and 4 deletions

View file

@ -1575,6 +1575,7 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own
self._async_generator: Final = async_generator
self.attempt: Final = attempt
self._billing_source: object = source_iterator
self._last_sequence_number = -1
self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source
@ -1699,10 +1700,12 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
if response_id_exposed:
_raise_released_stream_error(attempt, e)
async with contextlib.aclosing(
_responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held)
) as fallback_stream:
async for fallback_item in fallback_stream:
yield fallback_item
_responses_recover_stream_error(
attempt, e, source_iterator, wrapper, wrapper.adopt_fallback_headers, held
)
) as recovered_stream:
async for recovered_item in recovered_stream:
yield recovered_item
except Exception:
for held_event in held:
yield held_event
@ -1791,6 +1794,99 @@ def _responses_continuation_request(
}
def _responses_stream_retries(
attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator"
) -> bool:
"""Whether a broken Responses stream retries in its own group first: it broke before any output and
before a tool call, with an error the retry rule takes (408, 409, 429, 5xx, or no status), as a
``/v1/messages`` stream does. After output the continuation fallback takes over instead."""
if not _mid_stream_error_before_output(e) or source_iterator.started_tool_call:
return False
status_code: Final = _stream_raised_error_status(_mid_stream_retry_trigger(e))
return status_code is None or attempt.should_retry_status(status_code)
async def _responses_recover_stream_error(
attempt: StreamAttempt,
e: MidStreamFallbackError,
source_iterator: "BaseResponsesAPIStreamingIterator",
wrapper: object,
adopt_headers: Callable[[object], _Prepared],
held: tuple[object, ...],
) -> AsyncGenerator[object, None]:
"""A Responses stream that broke before any output is retried in its own group, then falls back
with the newest failure once the retries are spent. The primary's held lifecycle events are dropped
when a retry or a fallback answers."""
last_error: MidStreamFallbackError = e # rebind-ok: the newest failure once a retry also failed
if _responses_stream_retries(attempt, e, source_iterator):
if hasattr(source_iterator, "aclose"):
with anyio.CancelScope(shield=True), contextlib.suppress(Exception):
await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
await attempt.failure_recorded()
trigger: Final = _mid_stream_retry_trigger(e)
retried: Final = await attempt.retry(trigger)
if not isinstance(retried, BaseException):
async for retried_item in _responses_yield_retried(retried, wrapper, adopt_headers):
yield retried_item
return
if retried is not trigger:
last_error = _stream_pre_content_error( # rebind-ok: the retry failed again, so its failure is the newest
retried if isinstance(retried, Exception) else e, e.model or "", e.llm_provider
)
async with contextlib.aclosing(
_responses_fallback_attempt(attempt, last_error, source_iterator, adopt_headers, held)
) as fallback_stream:
async for fallback_item in fallback_stream:
yield fallback_item
async def _responses_yield_retried(
retried: tuple[object, int, int], wrapper: object, adopt_headers: Callable[[object], _Prepared]
) -> AsyncGenerator[object, None]:
"""Hands a same-group retry's stream to the client through the wrapper, with the retry's headers, and
closes it after. A retried stream that breaks again counts on from the retries already made, and when
it recovers onto another source (its own fallback) the wrapper and the events follow that source."""
response, attempted, budget = retried
inner_attempt: Final = getattr(response, "attempt", None)
if isinstance(inner_attempt, StreamAttempt):
inner_attempt.attempted_retries = attempted
inner_attempt.committed_budget = budget
served_as: Final = _prepare_fallback_hidden_params(wrapper)[1].get("x-litellm-model-group")
def follow_source() -> _Prepared:
prepared: Final = adopt_headers(response)
if served_as is not None:
_ = prepared[1].setdefault("x-litellm-model-group", served_as)
newest: Final = prepared[1].get("x-litellm-attempted-retries")
_ = add_retry_headers_to_response(
response=wrapper,
attempted_retries=max(attempted, newest) if isinstance(newest, int) else attempted,
max_retries=budget,
)
return prepared
followed: _Prepared = follow_source() # rebind-ok: refreshed when the retried stream changes its source
source: object = getattr(response, "_billing_source", None) # rebind-ok: the retried stream's current source
try:
if hasattr(response, "__aiter__"):
async for retried_item in cast("AsyncIterator[object]", response): # cast-ok: __aiter__ checked above
if (current_source := getattr(response, "_billing_source", None)) is not source:
source = current_source
followed = follow_source()
_apply_fallback_hidden_params_to_item(retried_item, followed)
yield retried_item
else:
yield response
finally:
if hasattr(response, "aclose"):
with anyio.CancelScope(shield=True):
try:
await response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
except Exception as exc: # noqa: BLE001 # closing is best-effort
verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing retry: %s", exc)
async def _responses_fallback_attempt(
attempt: StreamAttempt,
e: MidStreamFallbackError,

View file

@ -0,0 +1,48 @@
import itertools
import uuid
from collections.abc import Callable
from typing import Final
from integration._support.client import Gateway
from integration._support.responses_stream import (
AZURE_TARGET,
healthy_stream,
rate_limited_stream,
sse_events,
)
from integration._support.wire import Reply, Request, wire_server
_RETRY_TEXT: Final = "retry answer"
def _throttle_first_request_only(identity: str) -> Callable[[Request], Reply]:
"""Shared by both regions: whichever the router picks first is out of quota, the other one answers."""
arrivals: Final = itertools.count()
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target.startswith(AZURE_TARGET), request.target
stream: Final = rate_limited_stream(identity) if next(arrivals) == 0 else healthy_stream(identity, _RETRY_TEXT)
return Reply(content_type="text/event-stream", chunks=stream)
return respond
def test_streamed_responses_in_stream_rate_limit_is_retried_in_its_model_group(gateway: Gateway) -> None:
identity: Final = "resp_" + uuid.uuid4().hex
respond: Final = _throttle_first_request_only(identity)
with wire_server(respond) as region_a, wire_server(respond) as region_b, gateway.scenario() as scenario:
group: Final = f"responses-stream-retry-{uuid.uuid4().hex}"
for region in (region_a, region_b):
scenario.model(name=group, model="azure/gpt-6", api_base=region.url, api_key="synthetic-azure-key")
response: Final = gateway.request(
"POST",
"/v1/responses",
{"model": group, "input": identity, "stream": True, "num_retries": 2, "cache": {"no-cache": True}},
)
assert response.status_code == 200, response.text
events: Final = sse_events(response.text)
assert [name for name, _ in events][-1] == "response.completed", response.text
assert _RETRY_TEXT in response.text, response.text
calls_per_region: Final = (len(region_a.drain()), len(region_b.drain()))
assert sum(calls_per_region) == 2, f"one throttled call and one retry: {calls_per_region}"
assert response.headers["x-litellm-attempted-retries"] == "1", dict(response.headers)

View file

@ -25,6 +25,7 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.router import AllowedFailsPolicy, RetryPolicy
from tests.unit.test_router.core.scripted_host import (
ScriptedHost,
content_policy,
@ -456,3 +457,97 @@ async def test_the_primary_and_fallback_streams_are_closed(make_core: MakeCore,
core = build(make_core)
await collect_event_types(await open_stream(core))
assert [stream.closed for stream in opened] == [True, True]
GROUP_A_TWICE: Final = [deployment("a", "a1"), deployment("a", "a2")]
def retried_ok() -> Callable[..., ScriptedResponsesStream]:
return responses_stream(
Event(CREATED), Event(TEXT_DELTA), Event(COMPLETED), hidden_params={"model_id": "retry-stream"}
)
async def test_a_rate_limit_before_output_is_retried_on_the_other_deployment_of_the_group(
make_core: MakeCore, host: ScriptedHost
) -> None:
host.answer("*", responses_stream(Event(CREATED), Event(IN_PROGRESS), error=cut()), retried_ok())
core = make_core(model_list=GROUP_A_TWICE, num_retries=1)
stream = await open_stream(core)
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
first, retried = host.deployments_called()
assert retried != first
assert hidden(stream)["model_id"] == "retry-stream"
assert hidden(stream)["additional_headers"]["x-litellm-attempted-retries"] == 1
async def test_a_rate_limit_retry_policy_retries_a_responses_stream_when_num_retries_is_zero(
make_core: MakeCore, host: ScriptedHost
) -> None:
host.answer("*", responses_stream(Event(CREATED), error=cut()), retried_ok())
core = make_core(model_list=GROUP_A_TWICE, num_retries=0, retry_policy=RetryPolicy(RateLimitErrorRetries=2))
stream = await open_stream(core)
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
assert len(host.deployments_called()) == 2
assert hidden(stream)["additional_headers"]["x-litellm-max-retries"] == 2
async def test_once_the_stream_retries_are_spent_the_fallback_answers(make_core: MakeCore, host: ScriptedHost) -> None:
host.answer("openai/a", responses_stream(Event(CREATED), error=cut()))
host.answer("openai/b", fallback_ok())
core = build(make_core, num_retries=1)
stream = await open_stream(core)
events = [event async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream
assert [event.type for event in events] == [CREATED, TEXT_DELTA, COMPLETED]
called = host.deployments_called()
assert called[-1] == "b"
assert called.count("a") == 2
assert hidden(stream)["model_id"] == "b-stream"
assert all(event._hidden_params["model_id"] == "b-stream" for event in events)
async def test_after_text_the_stream_is_not_retried_in_the_group(make_core: MakeCore, host: ScriptedHost) -> None:
host.answer("*", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("Hel")), retried_ok())
core = make_core(model_list=GROUP_A_TWICE, num_retries=1)
stream = await open_stream(core)
seen: list[object] = []
with pytest.raises(litellm.RateLimitError, match="stream cut"):
await drain_into(stream, seen)
assert [getattr(event, "type") for event in seen] == [CREATED, TEXT_DELTA]
assert len(host.deployments_called()) == 1
async def test_a_second_failed_stream_counts_as_a_second_retry(make_core: MakeCore, host: ScriptedHost) -> None:
broken: Final = responses_stream(Event(CREATED), error=cut())
host.answer("*", broken, broken, retried_ok())
core = make_core(
model_list=[*GROUP_A_TWICE, deployment("a", "a3")],
num_retries=2,
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1000),
)
stream = await open_stream(core)
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
assert len(host.deployments_called()) == 3
assert hidden(stream)["additional_headers"]["x-litellm-attempted-retries"] == 2
async def test_a_retried_stream_keeps_the_model_group_header(make_core: MakeCore, host: ScriptedHost) -> None:
host.answer("*", responses_stream(Event(CREATED), error=cut()), retried_ok())
stream = await open_stream(make_core(model_list=GROUP_A_TWICE, num_retries=1))
served_as: Final = hidden(stream)["additional_headers"]["x-litellm-model-group"]
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
assert hidden(stream)["additional_headers"]["x-litellm-model-group"] == served_as
async def test_the_failed_stream_is_closed_before_the_retry_is_opened(make_core: MakeCore, host: ScriptedHost) -> None:
opened: list[ScriptedResponsesStream] = []
closed_when_retry_opens: list[bool] = []
def retry(**request: object) -> ScriptedResponsesStream: # kwargs-ok: the host passes the request kwargs
closed_when_retry_opens.append(opened[0].closed)
return retried_ok()(**request)
host.answer("*", responses_stream(Event(CREATED), error=cut(), opened=opened), retry)
core = make_core(model_list=GROUP_A_TWICE, num_retries=1)
assert await collect_event_types(await open_stream(core)) == [CREATED, TEXT_DELTA, COMPLETED]
assert closed_when_retry_opens == [True]