feat(llm): add stream mode support for proxies that only return content via streaming

Some OpenAI-compatible proxies (e.g. jarodfund.xyz) return empty content
in non-streaming mode. When stream=True is passed via OPENSPACE_LLM_CONFIG,
use litellm's stream_chunk_builder to collect streaming chunks into a
standard response.
This commit is contained in:
kobeng 2026-04-12 18:05:52 +08:00
parent 79a98abda7
commit 486ee37630

View file

@ -548,14 +548,27 @@ class LLMClient:
- Total max time: timeout * max_retries + sum(retry_delays)
"""
last_exception = None
use_stream = completion_kwargs.pop("stream", False)
if use_stream:
completion_kwargs["stream_options"] = {"include_usage": True}
for attempt in range(self.max_retries):
try:
# Add timeout to the completion call
response = await asyncio.wait_for(
litellm.acompletion(**completion_kwargs),
timeout=self.timeout
)
if use_stream:
chunks = []
async for chunk in await litellm.acompletion(stream=True, **completion_kwargs):
chunks.append(chunk)
if not chunks:
raise ValueError("Stream returned no chunks")
response = await asyncio.wait_for(
asyncio.to_thread(litellm.stream_chunk_builder, chunks, completion_kwargs.get("messages", [])),
timeout=self.timeout,
)
else:
response = await asyncio.wait_for(
litellm.acompletion(**completion_kwargs),
timeout=self.timeout
)
return response
except asyncio.TimeoutError:
self._logger.error(