scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(responses): eliminate per-chunk thread spawning in async streaming path

Browse source 
_process_chunk() called run_async_function() on every SSE chunk, which
when invoked from an async context spawns a thread + event loop per call.

Move the hook call out of _process_chunk into the callers: async __anext__
directly awaits it, sync __next__ uses run_async_function.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ishaan Jaffer 2026-02-20 15:38:08 -08:00
parent 24e806bdc8
commit 841989c22d
1 changed files with 10 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/responses/streaming_iterator.py
View file

@ -123,12 +123,6 @@ class BaseResponsesAPIStreamingIterator:
)
setattr(openai_responses_api_chunk, "response", response)
# Allow callbacks to modify chunk before returning
openai_responses_api_chunk = run_async_function(
async_function=self._call_post_streaming_deployment_hook,
chunk=openai_responses_api_chunk,
)
# Store the completed response
if (
openai_responses_api_chunk
@ -376,6 +370,11 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
if self.finished:
raise StopAsyncIteration
elif result is not None:
# Await hook directly instead of run_async_function
# (which spawns a thread + event loop per call)
result = await self._call_post_streaming_deployment_hook(
chunk=result,
)
return result
# If result is None, continue the loop to get the next chunk
@ -474,6 +473,11 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
if self.finished:
raise StopIteration
elif result is not None:
# Sync path: use run_async_function for the hook
result = run_async_function(
async_function=self._call_post_streaming_deployment_hook,
chunk=result,
)
return result
# If result is None, continue the loop to get the next chunk