fix(streaming): reject incomplete hosted vllm streams

This commit is contained in:
Sherif Waly 2026-10-01 13:49:04 +01:00
parent b4bb2a77a2
commit 4f1e2b542e
2 changed files with 50 additions and 0 deletions

View file

@ -1729,6 +1729,13 @@ class CustomStreamWrapper:
model_response.choices[0].finish_reason = "tool_calls"
return model_response
def _missing_required_finish_reason_error(self) -> httpx.RemoteProtocolError | None:
if self.custom_llm_provider != "hosted_vllm":
return None
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
return None
return httpx.RemoteProtocolError("Upstream hosted_vllm stream ended before a finish_reason was received")
def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
"""
Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
@ -1848,6 +1855,16 @@ class CustomStreamWrapper:
return response
except StopIteration:
incomplete_stream_error: Final = self._missing_required_finish_reason_error()
if incomplete_stream_error is not None:
incomplete_traceback: Final = traceback.format_exc()
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
threading.Thread(
target=self.logging_obj.failure_handler,
args=(incomplete_stream_error, incomplete_traceback),
).start()
self._handle_stream_fallback_error(incomplete_stream_error)
if self.sent_last_chunk is True:
try:
complete_streaming_response = litellm.stream_chunk_builder(
@ -2081,6 +2098,9 @@ class CustomStreamWrapper:
self.chunks.append(processed_chunk)
return processed_chunk
except (StopAsyncIteration, StopIteration):
incomplete_stream_error: Final = self._missing_required_finish_reason_error()
if incomplete_stream_error is not None:
self._log_stream_failure_and_raise(incomplete_stream_error)
return await self._finalize_completed_stream(cache_hit=cache_hit)
except httpx.TimeoutException as e: # if httpx read timeout error occues
traceback_exception = traceback.format_exc()

View file

@ -3884,6 +3884,36 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log
assert fabricated_finish_reasons == []
async def _drain_stream(response: CustomStreamWrapper, sync_mode: bool) -> None:
if sync_mode:
tuple(response)
return
async for _chunk in response:
pass
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_hosted_vllm_eof_before_finish_reason_raises(
logging_obj: Logging,
sync_mode: bool,
) -> None:
from litellm.exceptions import MidStreamFallbackError
partial_chunk: Final = _reset_test_chunk(content="unfinished")
response: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(model_responses=[partial_chunk]),
model="hosted_vllm/stub-model",
custom_llm_provider="hosted_vllm",
logging_obj=logging_obj,
)
with pytest.raises(MidStreamFallbackError, match="finish_reason") as exc_info:
await _drain_stream(response, sync_mode)
assert exc_info.value.generated_content == "unfinished"
def test_openai_custom_tool_call_stream_deltas_survive_conversion(logging_obj: Logging):
"""
Regression test: OpenAI chat completions custom tool calls stream as