scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 255

refactor(proxy): streaming loop hold budget is _STREAM_LOOP_HOLD_MS = 50

Browse source
This commit is contained in:
devin-ai-integration[bot] 2026-10-16 17:13:10 +00:00
parent 2f318eadde
commit 356454def6
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/proxy_server.py
View file

@ -10012,7 +10012,7 @@ def _pop_complete_sse_frame(buffer: str) -> tuple[str | None, str]:
_STREAM_KEEPALIVE: Final = object()
_KEEPALIVE_MIN_SECONDS: Final = 1.0
_STREAM_LOOP_HOLD_SECONDS: Final = 0.05
_STREAM_LOOP_HOLD_MS: Final = 50
_KEEPALIVE_MAX_SECONDS: Final = 300.0
_EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({})
_EMPTY_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
@ -10292,7 +10292,7 @@ async def async_data_generator(
async for item in stream_source:
# Unblock the event loop: a stream whose chunks are already buffered never awaits
# between chunks, so without this yield one long stream holds the loop for its whole body
if time.monotonic() - loop_released_at >= _STREAM_LOOP_HOLD_SECONDS:
if (time.monotonic() - loop_released_at) * 1000 >= _STREAM_LOOP_HOLD_MS:
await asyncio.sleep(0)
loop_released_at = time.monotonic()
if item is _STREAM_KEEPALIVE: