mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(responses): retry a stream that breaks before output in its own model group (#51953)
This commit is contained in:
parent
e6b673e6e0
commit
0dee2a8a0d
3 changed files with 243 additions and 4 deletions
|
|
@ -1575,6 +1575,7 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
|
|||
|
||||
def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own
|
||||
self._async_generator: Final = async_generator
|
||||
self.attempt: Final = attempt
|
||||
self._billing_source: object = source_iterator
|
||||
self._last_sequence_number = -1
|
||||
self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source
|
||||
|
|
@ -1699,10 +1700,12 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
|
|||
if response_id_exposed:
|
||||
_raise_released_stream_error(attempt, e)
|
||||
async with contextlib.aclosing(
|
||||
_responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held)
|
||||
) as fallback_stream:
|
||||
async for fallback_item in fallback_stream:
|
||||
yield fallback_item
|
||||
_responses_recover_stream_error(
|
||||
attempt, e, source_iterator, wrapper, wrapper.adopt_fallback_headers, held
|
||||
)
|
||||
) as recovered_stream:
|
||||
async for recovered_item in recovered_stream:
|
||||
yield recovered_item
|
||||
except Exception:
|
||||
for held_event in held:
|
||||
yield held_event
|
||||
|
|
@ -1791,6 +1794,99 @@ def _responses_continuation_request(
|
|||
}
|
||||
|
||||
|
||||
def _responses_stream_retries(
|
||||
attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator"
|
||||
) -> bool:
|
||||
"""Whether a broken Responses stream retries in its own group first: it broke before any output and
|
||||
before a tool call, with an error the retry rule takes (408, 409, 429, 5xx, or no status), as a
|
||||
``/v1/messages`` stream does. After output the continuation fallback takes over instead."""
|
||||
if not _mid_stream_error_before_output(e) or source_iterator.started_tool_call:
|
||||
return False
|
||||
status_code: Final = _stream_raised_error_status(_mid_stream_retry_trigger(e))
|
||||
return status_code is None or attempt.should_retry_status(status_code)
|
||||
|
||||
|
||||
async def _responses_recover_stream_error(
|
||||
attempt: StreamAttempt,
|
||||
e: MidStreamFallbackError,
|
||||
source_iterator: "BaseResponsesAPIStreamingIterator",
|
||||
wrapper: object,
|
||||
adopt_headers: Callable[[object], _Prepared],
|
||||
held: tuple[object, ...],
|
||||
) -> AsyncGenerator[object, None]:
|
||||
"""A Responses stream that broke before any output is retried in its own group, then falls back
|
||||
with the newest failure once the retries are spent. The primary's held lifecycle events are dropped
|
||||
when a retry or a fallback answers."""
|
||||
last_error: MidStreamFallbackError = e # rebind-ok: the newest failure once a retry also failed
|
||||
if _responses_stream_retries(attempt, e, source_iterator):
|
||||
if hasattr(source_iterator, "aclose"):
|
||||
with anyio.CancelScope(shield=True), contextlib.suppress(Exception):
|
||||
await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
|
||||
await attempt.failure_recorded()
|
||||
trigger: Final = _mid_stream_retry_trigger(e)
|
||||
retried: Final = await attempt.retry(trigger)
|
||||
if not isinstance(retried, BaseException):
|
||||
async for retried_item in _responses_yield_retried(retried, wrapper, adopt_headers):
|
||||
yield retried_item
|
||||
return
|
||||
if retried is not trigger:
|
||||
last_error = _stream_pre_content_error( # rebind-ok: the retry failed again, so its failure is the newest
|
||||
retried if isinstance(retried, Exception) else e, e.model or "", e.llm_provider
|
||||
)
|
||||
async with contextlib.aclosing(
|
||||
_responses_fallback_attempt(attempt, last_error, source_iterator, adopt_headers, held)
|
||||
) as fallback_stream:
|
||||
async for fallback_item in fallback_stream:
|
||||
yield fallback_item
|
||||
|
||||
|
||||
async def _responses_yield_retried(
|
||||
retried: tuple[object, int, int], wrapper: object, adopt_headers: Callable[[object], _Prepared]
|
||||
) -> AsyncGenerator[object, None]:
|
||||
"""Hands a same-group retry's stream to the client through the wrapper, with the retry's headers, and
|
||||
closes it after. A retried stream that breaks again counts on from the retries already made, and when
|
||||
it recovers onto another source (its own fallback) the wrapper and the events follow that source."""
|
||||
response, attempted, budget = retried
|
||||
inner_attempt: Final = getattr(response, "attempt", None)
|
||||
if isinstance(inner_attempt, StreamAttempt):
|
||||
inner_attempt.attempted_retries = attempted
|
||||
inner_attempt.committed_budget = budget
|
||||
|
||||
served_as: Final = _prepare_fallback_hidden_params(wrapper)[1].get("x-litellm-model-group")
|
||||
|
||||
def follow_source() -> _Prepared:
|
||||
prepared: Final = adopt_headers(response)
|
||||
if served_as is not None:
|
||||
_ = prepared[1].setdefault("x-litellm-model-group", served_as)
|
||||
newest: Final = prepared[1].get("x-litellm-attempted-retries")
|
||||
_ = add_retry_headers_to_response(
|
||||
response=wrapper,
|
||||
attempted_retries=max(attempted, newest) if isinstance(newest, int) else attempted,
|
||||
max_retries=budget,
|
||||
)
|
||||
return prepared
|
||||
|
||||
followed: _Prepared = follow_source() # rebind-ok: refreshed when the retried stream changes its source
|
||||
source: object = getattr(response, "_billing_source", None) # rebind-ok: the retried stream's current source
|
||||
try:
|
||||
if hasattr(response, "__aiter__"):
|
||||
async for retried_item in cast("AsyncIterator[object]", response): # cast-ok: __aiter__ checked above
|
||||
if (current_source := getattr(response, "_billing_source", None)) is not source:
|
||||
source = current_source
|
||||
followed = follow_source()
|
||||
_apply_fallback_hidden_params_to_item(retried_item, followed)
|
||||
yield retried_item
|
||||
else:
|
||||
yield response
|
||||
finally:
|
||||
if hasattr(response, "aclose"):
|
||||
with anyio.CancelScope(shield=True):
|
||||
try:
|
||||
await response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
|
||||
except Exception as exc: # noqa: BLE001 # closing is best-effort
|
||||
verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing retry: %s", exc)
|
||||
|
||||
|
||||
async def _responses_fallback_attempt(
|
||||
attempt: StreamAttempt,
|
||||
e: MidStreamFallbackError,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import itertools
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.responses_stream import (
|
||||
AZURE_TARGET,
|
||||
healthy_stream,
|
||||
rate_limited_stream,
|
||||
sse_events,
|
||||
)
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
_RETRY_TEXT: Final = "retry answer"
|
||||
|
||||
|
||||
def _throttle_first_request_only(identity: str) -> Callable[[Request], Reply]:
|
||||
"""Shared by both regions: whichever the router picks first is out of quota, the other one answers."""
|
||||
arrivals: Final = itertools.count()
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target.startswith(AZURE_TARGET), request.target
|
||||
stream: Final = rate_limited_stream(identity) if next(arrivals) == 0 else healthy_stream(identity, _RETRY_TEXT)
|
||||
return Reply(content_type="text/event-stream", chunks=stream)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def test_streamed_responses_in_stream_rate_limit_is_retried_in_its_model_group(gateway: Gateway) -> None:
|
||||
identity: Final = "resp_" + uuid.uuid4().hex
|
||||
respond: Final = _throttle_first_request_only(identity)
|
||||
with wire_server(respond) as region_a, wire_server(respond) as region_b, gateway.scenario() as scenario:
|
||||
group: Final = f"responses-stream-retry-{uuid.uuid4().hex}"
|
||||
for region in (region_a, region_b):
|
||||
scenario.model(name=group, model="azure/gpt-6", api_base=region.url, api_key="synthetic-azure-key")
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{"model": group, "input": identity, "stream": True, "num_retries": 2, "cache": {"no-cache": True}},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = sse_events(response.text)
|
||||
assert [name for name, _ in events][-1] == "response.completed", response.text
|
||||
assert _RETRY_TEXT in response.text, response.text
|
||||
calls_per_region: Final = (len(region_a.drain()), len(region_b.drain()))
|
||||
assert sum(calls_per_region) == 2, f"one throttled call and one retry: {calls_per_region}"
|
||||
assert response.headers["x-litellm-attempted-retries"] == "1", dict(response.headers)
|
||||
|
|
@ -25,6 +25,7 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.router import AllowedFailsPolicy, RetryPolicy
|
||||
from tests.unit.test_router.core.scripted_host import (
|
||||
ScriptedHost,
|
||||
content_policy,
|
||||
|
|
@ -456,3 +457,97 @@ async def test_the_primary_and_fallback_streams_are_closed(make_core: MakeCore,
|
|||
core = build(make_core)
|
||||
await collect_event_types(await open_stream(core))
|
||||
assert [stream.closed for stream in opened] == [True, True]
|
||||
|
||||
|
||||
GROUP_A_TWICE: Final = [deployment("a", "a1"), deployment("a", "a2")]
|
||||
|
||||
|
||||
def retried_ok() -> Callable[..., ScriptedResponsesStream]:
|
||||
return responses_stream(
|
||||
Event(CREATED), Event(TEXT_DELTA), Event(COMPLETED), hidden_params={"model_id": "retry-stream"}
|
||||
)
|
||||
|
||||
|
||||
async def test_a_rate_limit_before_output_is_retried_on_the_other_deployment_of_the_group(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
host.answer("*", responses_stream(Event(CREATED), Event(IN_PROGRESS), error=cut()), retried_ok())
|
||||
core = make_core(model_list=GROUP_A_TWICE, num_retries=1)
|
||||
stream = await open_stream(core)
|
||||
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
|
||||
first, retried = host.deployments_called()
|
||||
assert retried != first
|
||||
assert hidden(stream)["model_id"] == "retry-stream"
|
||||
assert hidden(stream)["additional_headers"]["x-litellm-attempted-retries"] == 1
|
||||
|
||||
|
||||
async def test_a_rate_limit_retry_policy_retries_a_responses_stream_when_num_retries_is_zero(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
host.answer("*", responses_stream(Event(CREATED), error=cut()), retried_ok())
|
||||
core = make_core(model_list=GROUP_A_TWICE, num_retries=0, retry_policy=RetryPolicy(RateLimitErrorRetries=2))
|
||||
stream = await open_stream(core)
|
||||
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
|
||||
assert len(host.deployments_called()) == 2
|
||||
assert hidden(stream)["additional_headers"]["x-litellm-max-retries"] == 2
|
||||
|
||||
|
||||
async def test_once_the_stream_retries_are_spent_the_fallback_answers(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
host.answer("openai/a", responses_stream(Event(CREATED), error=cut()))
|
||||
host.answer("openai/b", fallback_ok())
|
||||
core = build(make_core, num_retries=1)
|
||||
stream = await open_stream(core)
|
||||
events = [event async for event in stream] # pyright: ignore[reportGeneralTypeIssues] # an async stream
|
||||
assert [event.type for event in events] == [CREATED, TEXT_DELTA, COMPLETED]
|
||||
called = host.deployments_called()
|
||||
assert called[-1] == "b"
|
||||
assert called.count("a") == 2
|
||||
assert hidden(stream)["model_id"] == "b-stream"
|
||||
assert all(event._hidden_params["model_id"] == "b-stream" for event in events)
|
||||
|
||||
|
||||
async def test_after_text_the_stream_is_not_retried_in_the_group(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
host.answer("*", responses_stream(Event(CREATED), Event(TEXT_DELTA), error=cut("Hel")), retried_ok())
|
||||
core = make_core(model_list=GROUP_A_TWICE, num_retries=1)
|
||||
stream = await open_stream(core)
|
||||
seen: list[object] = []
|
||||
with pytest.raises(litellm.RateLimitError, match="stream cut"):
|
||||
await drain_into(stream, seen)
|
||||
assert [getattr(event, "type") for event in seen] == [CREATED, TEXT_DELTA]
|
||||
assert len(host.deployments_called()) == 1
|
||||
|
||||
|
||||
async def test_a_second_failed_stream_counts_as_a_second_retry(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
broken: Final = responses_stream(Event(CREATED), error=cut())
|
||||
host.answer("*", broken, broken, retried_ok())
|
||||
core = make_core(
|
||||
model_list=[*GROUP_A_TWICE, deployment("a", "a3")],
|
||||
num_retries=2,
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1000),
|
||||
)
|
||||
stream = await open_stream(core)
|
||||
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
|
||||
assert len(host.deployments_called()) == 3
|
||||
assert hidden(stream)["additional_headers"]["x-litellm-attempted-retries"] == 2
|
||||
|
||||
|
||||
async def test_a_retried_stream_keeps_the_model_group_header(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
host.answer("*", responses_stream(Event(CREATED), error=cut()), retried_ok())
|
||||
stream = await open_stream(make_core(model_list=GROUP_A_TWICE, num_retries=1))
|
||||
served_as: Final = hidden(stream)["additional_headers"]["x-litellm-model-group"]
|
||||
assert await collect_event_types(stream) == [CREATED, TEXT_DELTA, COMPLETED]
|
||||
assert hidden(stream)["additional_headers"]["x-litellm-model-group"] == served_as
|
||||
|
||||
|
||||
async def test_the_failed_stream_is_closed_before_the_retry_is_opened(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
opened: list[ScriptedResponsesStream] = []
|
||||
closed_when_retry_opens: list[bool] = []
|
||||
|
||||
def retry(**request: object) -> ScriptedResponsesStream: # kwargs-ok: the host passes the request kwargs
|
||||
closed_when_retry_opens.append(opened[0].closed)
|
||||
return retried_ok()(**request)
|
||||
|
||||
host.answer("*", responses_stream(Event(CREATED), error=cut(), opened=opened), retry)
|
||||
core = make_core(model_list=GROUP_A_TWICE, num_retries=1)
|
||||
assert await collect_event_types(await open_stream(core)) == [CREATED, TEXT_DELTA, COMPLETED]
|
||||
assert closed_when_retry_opens == [True]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue