From cbf1371873c0540b83fdcf4e9a2bafb69b72f460 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:11:19 +0000 Subject: [PATCH] docs(proxy): streaming loop comment names the 50 ms hold --- litellm/proxy/proxy_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f8902f6276a..7d9db0caebb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10290,8 +10290,9 @@ async def async_data_generator( loop_released_at = time.monotonic() # rebind-ok: last time this stream let other tasks run async for item in stream_source: - # Unblock the event loop: a stream whose chunks are already buffered never awaits - # between chunks, so without this yield one long stream holds the loop for its whole body + # Unblock the event loop: a stream whose chunks are already buffered never awaits between + # chunks, so without this one long stream holds the loop for its whole body. Yield once this + # stream has held the loop for 50 ms (_STREAM_LOOP_HOLD_SECONDS), never on a stream that waits if time.monotonic() - loop_released_at >= _STREAM_LOOP_HOLD_SECONDS: await asyncio.sleep(0) loop_released_at = time.monotonic()