From 841989c22dccf5b225877bd738c1415e19ade11f Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Fri, 20 Feb 2026 15:38:08 -0800 Subject: [PATCH] fix(responses): eliminate per-chunk thread spawning in async streaming path _process_chunk() called run_async_function() on every SSE chunk, which when invoked from an async context spawns a thread + event loop per call. Move the hook call out of _process_chunk into the callers: async __anext__ directly awaits it, sync __next__ uses run_async_function. Co-Authored-By: Claude Opus 4.6 --- litellm/responses/streaming_iterator.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6d0c4abac81..edcbb0d11b8 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -123,12 +123,6 @@ class BaseResponsesAPIStreamingIterator: ) setattr(openai_responses_api_chunk, "response", response) - # Allow callbacks to modify chunk before returning - openai_responses_api_chunk = run_async_function( - async_function=self._call_post_streaming_deployment_hook, - chunk=openai_responses_api_chunk, - ) - # Store the completed response if ( openai_responses_api_chunk @@ -376,6 +370,11 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): if self.finished: raise StopAsyncIteration elif result is not None: + # Await hook directly instead of run_async_function + # (which spawns a thread + event loop per call) + result = await self._call_post_streaming_deployment_hook( + chunk=result, + ) return result # If result is None, continue the loop to get the next chunk @@ -474,6 +473,11 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): if self.finished: raise StopIteration elif result is not None: + # Sync path: use run_async_function for the hook + result = run_async_function( + async_function=self._call_post_streaming_deployment_hook, + chunk=result, + ) return result # If result is None, continue the loop to get the next chunk