scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-17 04:34:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 173

docs(proxy): streaming loop comment names the 50 ms hold

Browse source
This commit is contained in:
devin-ai-integration[bot] 2026-10-16 17:11:19 +00:00
parent d118f901f8
commit cbf1371873
1 changed files with 3 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/proxy/proxy_server.py
View file

@ -10290,8 +10290,9 @@ async def async_data_generator(
loop_released_at = time.monotonic() # rebind-ok: last time this stream let other tasks run
async for item in stream_source:
# Unblock the event loop: a stream whose chunks are already buffered never awaits
# between chunks, so without this yield one long stream holds the loop for its whole body
# Unblock the event loop: a stream whose chunks are already buffered never awaits between
# chunks, so without this one long stream holds the loop for its whole body. Yield once this
# stream has held the loop for 50 ms (_STREAM_LOOP_HOLD_SECONDS), never on a stream that waits
if time.monotonic() - loop_released_at >= _STREAM_LOOP_HOLD_SECONDS:
await asyncio.sleep(0)
loop_released_at = time.monotonic()