diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f8902f6276a..97d2f8a3386 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10012,7 +10012,7 @@ def _pop_complete_sse_frame(buffer: str) -> tuple[str | None, str]: _STREAM_KEEPALIVE: Final = object() _KEEPALIVE_MIN_SECONDS: Final = 1.0 -_STREAM_LOOP_HOLD_SECONDS: Final = 0.05 +_STREAM_LOOP_HOLD_MS: Final = 50 _KEEPALIVE_MAX_SECONDS: Final = 300.0 _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) @@ -10292,7 +10292,7 @@ async def async_data_generator( async for item in stream_source: # Unblock the event loop: a stream whose chunks are already buffered never awaits # between chunks, so without this yield one long stream holds the loop for its whole body - if time.monotonic() - loop_released_at >= _STREAM_LOOP_HOLD_SECONDS: + if (time.monotonic() - loop_released_at) * 1000 >= _STREAM_LOOP_HOLD_MS: await asyncio.sleep(0) loop_released_at = time.monotonic() if item is _STREAM_KEEPALIVE: