fix(responses): end a stream cleanly when the read times out after its terminal event

This commit is contained in:
mateo-berri 2026-10-28 02:05:53 -07:00
parent 1399ea95f2
commit acaa8e34b0
3 changed files with 15 additions and 6 deletions

View file

@ -1255,6 +1255,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
raise StopAsyncIteration from e
self._raise_for_transport_error(e)
except httpx.TimeoutException as e:
if self.completed_response is not None:
self.finished = True
raise StopAsyncIteration from e
self._raise_for_transport_timeout(e)
except httpx.HTTPError as e:
# Handle HTTP errors
@ -1340,6 +1343,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
raise StopIteration from e
self._raise_for_transport_error(e)
except httpx.TimeoutException as e:
if self.completed_response is not None:
self.finished = True
raise StopIteration from e
self._raise_for_transport_timeout(e)
except httpx.HTTPError as e:
# Handle HTTP errors

View file

@ -485,7 +485,7 @@ class _UpstreamThatCompletesThenTimesOut:
@pytest.mark.asyncio
async def test_failure_after_completed_response_does_not_estimate_partial_usage(
async def test_timeout_after_completed_response_ends_the_stream_and_bills_it_once(
monkeypatch: pytest.MonkeyPatch,
) -> None:
recorder: Final = FailureRecorder()
@ -501,17 +501,18 @@ async def test_failure_after_completed_response_does_not_estimate_partial_usage(
)
iterator: Final = _live_iterator(logging_obj, upstream=_UpstreamThatCompletesThenTimesOut())
with pytest.raises(litellm.Timeout, match="Timeout on reading data from socket"):
await _ChunkRecorder().drain(iterator)
chunks: Final = _ChunkRecorder()
await chunks.drain(iterator)
await iterator._await_pending_logging()
assert chunks.chunks[-1].type == "response.completed"
assert len(recorder.success_payloads) == 1
success_payload: Final = recorder.success_payloads[0]
response_payload: Final = success_payload["response"]
assert isinstance(response_payload, dict)
usage: Final = cast(dict[str, object], response_payload["usage"])
assert (usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) == (5, 7, 12)
assert len(recorder.failure_payloads) == 1
assert recorder.failure_payloads == ()
assert logging_obj.model_call_details.get("combined_usage_object") is None

View file

@ -210,9 +210,11 @@ _TRAILING_ERRORS = [
httpx.RemoteProtocolError("peer closed connection without sending complete message body"),
]
_TRAILING_ERRORS_AFTER_COMPLETED = [*_TRAILING_ERRORS, httpx.ReadTimeout("Timeout on reading data from socket")]
@pytest.mark.asyncio
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type)
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS_AFTER_COMPLETED, ids=type)
async def test_transport_error_after_completed_event_ends_stream_cleanly(trailing_error):
"""A sloppy connection close after `response.completed` must not turn a
complete stream into an error (regression guard for the transport no longer
@ -243,7 +245,7 @@ async def test_transport_error_before_completed_event_raises():
pass
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type)
@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS_AFTER_COMPLETED, ids=type)
def test_sync_transport_error_after_completed_event_ends_stream_cleanly(trailing_error):
iterator = _make_sync_iterator(
sse_events=_COMPLETE_STREAM_EVENTS,