mirror of
https://github.com/HKUDS/OpenSpace.git
synced 2026-08-28 05:15:00 +00:00
feat(llm): add stream mode support for proxies that only return content via streaming
Some OpenAI-compatible proxies (e.g. jarodfund.xyz) return empty content in non-streaming mode. When stream=True is passed via OPENSPACE_LLM_CONFIG, use litellm's stream_chunk_builder to collect streaming chunks into a standard response.
This commit is contained in:
parent
79a98abda7
commit
486ee37630
1 changed files with 18 additions and 5 deletions
|
|
@ -548,14 +548,27 @@ class LLMClient:
|
|||
- Total max time: timeout * max_retries + sum(retry_delays)
|
||||
"""
|
||||
last_exception = None
|
||||
use_stream = completion_kwargs.pop("stream", False)
|
||||
if use_stream:
|
||||
completion_kwargs["stream_options"] = {"include_usage": True}
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
# Add timeout to the completion call
|
||||
response = await asyncio.wait_for(
|
||||
litellm.acompletion(**completion_kwargs),
|
||||
timeout=self.timeout
|
||||
)
|
||||
if use_stream:
|
||||
chunks = []
|
||||
async for chunk in await litellm.acompletion(stream=True, **completion_kwargs):
|
||||
chunks.append(chunk)
|
||||
if not chunks:
|
||||
raise ValueError("Stream returned no chunks")
|
||||
response = await asyncio.wait_for(
|
||||
asyncio.to_thread(litellm.stream_chunk_builder, chunks, completion_kwargs.get("messages", [])),
|
||||
timeout=self.timeout,
|
||||
)
|
||||
else:
|
||||
response = await asyncio.wait_for(
|
||||
litellm.acompletion(**completion_kwargs),
|
||||
timeout=self.timeout
|
||||
)
|
||||
return response
|
||||
except asyncio.TimeoutError:
|
||||
self._logger.error(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue