mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(responses): end a stream cleanly when the read times out after its terminal event
This commit is contained in:
parent
1399ea95f2
commit
acaa8e34b0
3 changed files with 15 additions and 6 deletions
|
|
@ -1255,6 +1255,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
raise StopAsyncIteration from e
|
||||
self._raise_for_transport_error(e)
|
||||
except httpx.TimeoutException as e:
|
||||
if self.completed_response is not None:
|
||||
self.finished = True
|
||||
raise StopAsyncIteration from e
|
||||
self._raise_for_transport_timeout(e)
|
||||
except httpx.HTTPError as e:
|
||||
# Handle HTTP errors
|
||||
|
|
@ -1340,6 +1343,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
raise StopIteration from e
|
||||
self._raise_for_transport_error(e)
|
||||
except httpx.TimeoutException as e:
|
||||
if self.completed_response is not None:
|
||||
self.finished = True
|
||||
raise StopIteration from e
|
||||
self._raise_for_transport_timeout(e)
|
||||
except httpx.HTTPError as e:
|
||||
# Handle HTTP errors
|
||||
|
|
|
|||
|
|
@ -485,7 +485,7 @@ class _UpstreamThatCompletesThenTimesOut:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_after_completed_response_does_not_estimate_partial_usage(
|
||||
async def test_timeout_after_completed_response_ends_the_stream_and_bills_it_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
recorder: Final = FailureRecorder()
|
||||
|
|
@ -501,17 +501,18 @@ async def test_failure_after_completed_response_does_not_estimate_partial_usage(
|
|||
)
|
||||
iterator: Final = _live_iterator(logging_obj, upstream=_UpstreamThatCompletesThenTimesOut())
|
||||
|
||||
with pytest.raises(litellm.Timeout, match="Timeout on reading data from socket"):
|
||||
await _ChunkRecorder().drain(iterator)
|
||||
chunks: Final = _ChunkRecorder()
|
||||
await chunks.drain(iterator)
|
||||
await iterator._await_pending_logging()
|
||||
|
||||
assert chunks.chunks[-1].type == "response.completed"
|
||||
assert len(recorder.success_payloads) == 1
|
||||
success_payload: Final = recorder.success_payloads[0]
|
||||
response_payload: Final = success_payload["response"]
|
||||
assert isinstance(response_payload, dict)
|
||||
usage: Final = cast(dict[str, object], response_payload["usage"])
|
||||
assert (usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) == (5, 7, 12)
|
||||
assert len(recorder.failure_payloads) == 1
|
||||
assert recorder.failure_payloads == ()
|
||||
assert logging_obj.model_call_details.get("combined_usage_object") is None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -210,9 +210,11 @@ _TRAILING_ERRORS = [
|
|||
httpx.RemoteProtocolError("peer closed connection without sending complete message body"),
|
||||
]
|
||||
|
||||
_TRAILING_ERRORS_AFTER_COMPLETED = [*_TRAILING_ERRORS, httpx.ReadTimeout("Timeout on reading data from socket")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type)
|
||||
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS_AFTER_COMPLETED, ids=type)
|
||||
async def test_transport_error_after_completed_event_ends_stream_cleanly(trailing_error):
|
||||
"""A sloppy connection close after `response.completed` must not turn a
|
||||
complete stream into an error (regression guard for the transport no longer
|
||||
|
|
@ -243,7 +245,7 @@ async def test_transport_error_before_completed_event_raises():
|
|||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type)
|
||||
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS_AFTER_COMPLETED, ids=type)
|
||||
def test_sync_transport_error_after_completed_event_ends_stream_cleanly(trailing_error):
|
||||
iterator = _make_sync_iterator(
|
||||
sse_events=_COMPLETE_STREAM_EVENTS,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue