diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a1978e944d6..fe0b34aff02 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1255,6 +1255,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): raise StopAsyncIteration from e self._raise_for_transport_error(e) except httpx.TimeoutException as e: + if self.completed_response is not None: + self.finished = True + raise StopAsyncIteration from e self._raise_for_transport_timeout(e) except httpx.HTTPError as e: # Handle HTTP errors @@ -1340,6 +1343,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): raise StopIteration from e self._raise_for_transport_error(e) except httpx.TimeoutException as e: + if self.completed_response is not None: + self.finished = True + raise StopIteration from e self._raise_for_transport_timeout(e) except httpx.HTTPError as e: # Handle HTTP errors diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 997d031870a..ae688ff9173 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -485,7 +485,7 @@ class _UpstreamThatCompletesThenTimesOut: @pytest.mark.asyncio -async def test_failure_after_completed_response_does_not_estimate_partial_usage( +async def test_timeout_after_completed_response_ends_the_stream_and_bills_it_once( monkeypatch: pytest.MonkeyPatch, ) -> None: recorder: Final = FailureRecorder() @@ -501,17 +501,18 @@ async def test_failure_after_completed_response_does_not_estimate_partial_usage( ) iterator: Final = _live_iterator(logging_obj, upstream=_UpstreamThatCompletesThenTimesOut()) - with pytest.raises(litellm.Timeout, match="Timeout on reading data from socket"): - await _ChunkRecorder().drain(iterator) + chunks: Final = _ChunkRecorder() + await chunks.drain(iterator) await iterator._await_pending_logging() + assert chunks.chunks[-1].type == "response.completed" assert len(recorder.success_payloads) == 1 success_payload: Final = recorder.success_payloads[0] response_payload: Final = success_payload["response"] assert isinstance(response_payload, dict) usage: Final = cast(dict[str, object], response_payload["usage"]) assert (usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) == (5, 7, 12) - assert len(recorder.failure_payloads) == 1 + assert recorder.failure_payloads == () assert logging_obj.model_call_details.get("combined_usage_object") is None diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index ecaf77447e1..664f4dba163 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -210,9 +210,11 @@ _TRAILING_ERRORS = [ httpx.RemoteProtocolError("peer closed connection without sending complete message body"), ] +_TRAILING_ERRORS_AFTER_COMPLETED = [*_TRAILING_ERRORS, httpx.ReadTimeout("Timeout on reading data from socket")] + @pytest.mark.asyncio -@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type) +@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS_AFTER_COMPLETED, ids=type) async def test_transport_error_after_completed_event_ends_stream_cleanly(trailing_error): """A sloppy connection close after `response.completed` must not turn a complete stream into an error (regression guard for the transport no longer @@ -243,7 +245,7 @@ async def test_transport_error_before_completed_event_raises(): pass -@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS, ids=type) +@pytest.mark.parametrize("trailing_error", _TRAILING_ERRORS_AFTER_COMPLETED, ids=type) def test_sync_transport_error_after_completed_event_ends_stream_cleanly(trailing_error): iterator = _make_sync_iterator( sse_events=_COMPLETE_STREAM_EVENTS,