diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2df470edc59..dc043055454 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1928,6 +1928,13 @@ class CustomStreamWrapper: model_response.choices[0].finish_reason = "tool_calls" return model_response + def _missing_required_finish_reason_error(self) -> httpx.RemoteProtocolError | None: + if self.custom_llm_provider != "hosted_vllm": + return None + if self.received_finish_reason is not None or self.intermittent_finish_reason is not None: + return None + return httpx.RemoteProtocolError("Upstream hosted_vllm stream ended before a finish_reason was received") + def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None: """ Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a @@ -2104,6 +2111,16 @@ class CustomStreamWrapper: return response except StopIteration: + incomplete_stream_error: Final = self._missing_required_finish_reason_error() + if incomplete_stream_error is not None: + incomplete_traceback: Final = traceback.format_exc() + if self.logging_obj is not None: + self._record_partial_usage_for_failure() + threading.Thread( + target=self.logging_obj.failure_handler, + args=(incomplete_stream_error, incomplete_traceback), + ).start() + self._handle_stream_fallback_error(incomplete_stream_error) if self.sent_last_chunk is True: complete_streaming_response: Final = self._build_complete_streaming_response() response = self._new_usage_chunk() @@ -2307,6 +2324,9 @@ class CustomStreamWrapper: continue return usage_free_chunk except (StopAsyncIteration, StopIteration): + incomplete_stream_error: Final = self._missing_required_finish_reason_error() + if incomplete_stream_error is not None: + self._log_stream_failure_and_raise(incomplete_stream_error) return await self._finalize_completed_stream(cache_hit=cache_hit) except httpx.TimeoutException as e: # if httpx read timeout error occues traceback_exception = traceback.format_exc() diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index cc1fa1b8803..e122b1d1e70 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -4851,6 +4851,36 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log assert fabricated_finish_reasons == [] +async def _drain_stream(response: CustomStreamWrapper, sync_mode: bool) -> None: + if sync_mode: + tuple(response) + return + async for _chunk in response: + pass + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_hosted_vllm_eof_before_finish_reason_raises( + logging_obj: Logging, + sync_mode: bool, +) -> None: + from litellm.exceptions import MidStreamFallbackError + + partial_chunk: Final = _reset_test_chunk(content="unfinished") + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=[partial_chunk]), + model="hosted_vllm/stub-model", + custom_llm_provider="hosted_vllm", + logging_obj=logging_obj, + ) + + with pytest.raises(MidStreamFallbackError, match="finish_reason") as exc_info: + await _drain_stream(response, sync_mode) + + assert exc_info.value.generated_content == "unfinished" + + def test_openai_custom_tool_call_stream_deltas_survive_conversion(logging_obj: Logging): """ Regression test: OpenAI chat completions custom tool calls stream as