scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

perf: replace list+join O(n²) with str+= O(n) in async_data_generator

Browse source
This commit is contained in:
Ishaan Jaffer 2026-02-27 20:28:19 -08:00
parent a923d27e70
commit ebc8ec81ec
1 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/proxy_server.py
View file

@ -5302,13 +5302,15 @@ async def async_data_generator(
):
verbose_proxy_logger.debug("inside generator")
try:
# Use a list to accumulate response segments to avoid O(n^2) string concatenation
str_so_far_parts: list[str] = []
error_message: Optional[str] = None
requested_model_from_client = _get_client_requested_model_for_streaming(
request_data=request_data
)
model_mismatch_logged = False
# Use a running string instead of list + join to avoid O(n^2) overhead.
# Previously "".join(str_so_far_parts) was called every chunk, re-joining
# the entire accumulated response. String += is O(n) amortized total.
_str_so_far: str = ""
async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
response=response,
@ -5319,12 +5321,12 @@ async def async_data_generator(
user_api_key_dict=user_api_key_dict,
response=chunk,
data=request_data,
str_so_far="".join(str_so_far_parts),
str_so_far=_str_so_far if _str_so_far else None,
)
if isinstance(chunk, (ModelResponse, ModelResponseStream)):
response_str = litellm.get_response_string(response_obj=chunk)
str_so_far_parts.append(response_str)
_str_so_far += response_str
chunk, model_mismatch_logged = _restamp_streaming_chunk_model(
chunk=chunk,