scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

Optimize streaming response accumulation

Browse source 
Refactor async_data_generator to build streamed text via list accumulation and ''.join() instead of repeated string concatenation. This improves performance for long responses without changing streaming behavior.
This commit is contained in:
AlexsanderHamir 2025-11-15 17:30:58 -08:00
parent 348d28d871
commit 8ea0e31678
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/proxy/proxy_server.py
View file

@ -4017,7 +4017,8 @@ async def async_data_generator(
):
verbose_proxy_logger.debug("inside generator")
try:
str_so_far = ""
# Use a list to accumulate response segments to avoid O(n^2) string concatenation
str_so_far_parts: list[str] = []
error_message: Optional[str] = None
async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
@ -4033,12 +4034,12 @@ async def async_data_generator(
user_api_key_dict=user_api_key_dict,
response=chunk,
data=request_data,
str_so_far=str_so_far,
str_so_far="".join(str_so_far_parts),
)
if isinstance(chunk, (ModelResponse, ModelResponseStream)):
response_str = litellm.get_response_string(response_obj=chunk)
str_so_far += response_str
str_so_far_parts.append(response_str)
if isinstance(chunk, BaseModel):
chunk = chunk.model_dump_json(exclude_none=True, exclude_unset=True)