mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 4f1e2b542e into e16d79d635
This commit is contained in:
commit
7ed32d6c1b
2 changed files with 50 additions and 0 deletions
|
|
@ -1928,6 +1928,13 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].finish_reason = "tool_calls"
|
||||
return model_response
|
||||
|
||||
def _missing_required_finish_reason_error(self) -> httpx.RemoteProtocolError | None:
|
||||
if self.custom_llm_provider != "hosted_vllm":
|
||||
return None
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
return None
|
||||
return httpx.RemoteProtocolError("Upstream hosted_vllm stream ended before a finish_reason was received")
|
||||
|
||||
def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
|
||||
"""
|
||||
Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
|
||||
|
|
@ -2104,6 +2111,16 @@ class CustomStreamWrapper:
|
|||
return response
|
||||
|
||||
except StopIteration:
|
||||
incomplete_stream_error: Final = self._missing_required_finish_reason_error()
|
||||
if incomplete_stream_error is not None:
|
||||
incomplete_traceback: Final = traceback.format_exc()
|
||||
if self.logging_obj is not None:
|
||||
self._record_partial_usage_for_failure()
|
||||
threading.Thread(
|
||||
target=self.logging_obj.failure_handler,
|
||||
args=(incomplete_stream_error, incomplete_traceback),
|
||||
).start()
|
||||
self._handle_stream_fallback_error(incomplete_stream_error)
|
||||
if self.sent_last_chunk is True:
|
||||
complete_streaming_response: Final = self._build_complete_streaming_response()
|
||||
response = self._new_usage_chunk()
|
||||
|
|
@ -2307,6 +2324,9 @@ class CustomStreamWrapper:
|
|||
continue
|
||||
return usage_free_chunk
|
||||
except (StopAsyncIteration, StopIteration):
|
||||
incomplete_stream_error: Final = self._missing_required_finish_reason_error()
|
||||
if incomplete_stream_error is not None:
|
||||
self._log_stream_failure_and_raise(incomplete_stream_error)
|
||||
return await self._finalize_completed_stream(cache_hit=cache_hit)
|
||||
except httpx.TimeoutException as e: # if httpx read timeout error occues
|
||||
traceback_exception = traceback.format_exc()
|
||||
|
|
|
|||
|
|
@ -4851,6 +4851,36 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log
|
|||
assert fabricated_finish_reasons == []
|
||||
|
||||
|
||||
async def _drain_stream(response: CustomStreamWrapper, sync_mode: bool) -> None:
|
||||
if sync_mode:
|
||||
tuple(response)
|
||||
return
|
||||
async for _chunk in response:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_hosted_vllm_eof_before_finish_reason_raises(
|
||||
logging_obj: Logging,
|
||||
sync_mode: bool,
|
||||
) -> None:
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
partial_chunk: Final = _reset_test_chunk(content="unfinished")
|
||||
response: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(model_responses=[partial_chunk]),
|
||||
model="hosted_vllm/stub-model",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
with pytest.raises(MidStreamFallbackError, match="finish_reason") as exc_info:
|
||||
await _drain_stream(response, sync_mode)
|
||||
|
||||
assert exc_info.value.generated_content == "unfinished"
|
||||
|
||||
|
||||
def test_openai_custom_tool_call_stream_deltas_survive_conversion(logging_obj: Logging):
|
||||
"""
|
||||
Regression test: OpenAI chat completions custom tool calls stream as
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue