scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

add add LITELLM_MAX_STREAMING_DURATION_SECONDS

Browse source
This commit is contained in:
Ishaan Jaffer 2026-02-26 11:34:37 -08:00
parent e9169fa137
commit b143b407e8
4 changed files with 20 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/constants.py
View file

@ -53,7 +53,7 @@ DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
# Streams exceeding this duration are terminated with a Timeout error.
# None (default) = no limit. Set env var to a number of seconds to enable globally.
_max_stream_duration_env = os.getenv("LITELLM_MAX_STREAMING_DURATION_SECONDS", None)
MAX_STREAMING_DURATION_S = (
LITELLM_MAX_STREAMING_DURATION_SECONDS = (
float(_max_stream_duration_env) if _max_stream_duration_env is not None else None
)

10
litellm/litellm_core_utils/streaming_handler.py
View file

@ -163,15 +163,15 @@ class CustomStreamWrapper:
self.created: Optional[int] = None
def _check_max_streaming_duration(self) -> None:
"""Raise litellm.Timeout if the stream has exceeded MAX_STREAMING_DURATION_S."""
from litellm.constants import MAX_STREAMING_DURATION_S
"""Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS."""
from litellm.constants import LITELLM_MAX_STREAMING_DURATION_SECONDS
if MAX_STREAMING_DURATION_S is None:
if LITELLM_MAX_STREAMING_DURATION_SECONDS is None:
return
elapsed = time.time() - self._stream_created_time
if elapsed > MAX_STREAMING_DURATION_S:
if elapsed > LITELLM_MAX_STREAMING_DURATION_SECONDS:
raise litellm.Timeout(
message=f"Stream exceeded max streaming duration of {MAX_STREAMING_DURATION_S}s (elapsed {elapsed:.1f}s)",
message=f"Stream exceeded max streaming duration of {LITELLM_MAX_STREAMING_DURATION_SECONDS}s (elapsed {elapsed:.1f}s)",
model=self.model or "",
llm_provider=self.custom_llm_provider or "",
)

10
litellm/responses/streaming_iterator.py
View file

@ -8,7 +8,7 @@ from typing import Any, Dict, Optional
import httpx
import litellm
from litellm.constants import MAX_STREAMING_DURATION_S, STREAM_SSE_DONE_STRING
from litellm.constants import LITELLM_MAX_STREAMING_DURATION_SECONDS, STREAM_SSE_DONE_STRING
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -85,13 +85,13 @@ class BaseResponsesAPIStreamingIterator:
) # GUARANTEE OPENAI HEADERS IN RESPONSE
def _check_max_streaming_duration(self) -> None:
"""Raise litellm.Timeout if the stream has exceeded MAX_STREAMING_DURATION_S."""
if MAX_STREAMING_DURATION_S is None:
"""Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS."""
if LITELLM_MAX_STREAMING_DURATION_SECONDS is None:
return
elapsed = time.time() - self._stream_created_time
if elapsed > MAX_STREAMING_DURATION_S:
if elapsed > LITELLM_MAX_STREAMING_DURATION_SECONDS:
raise litellm.Timeout(
message=f"Stream exceeded max streaming duration of {MAX_STREAMING_DURATION_S}s (elapsed {elapsed:.1f}s)",
message=f"Stream exceeded max streaming duration of {LITELLM_MAX_STREAMING_DURATION_SECONDS}s (elapsed {elapsed:.1f}s)",
model=self.model or "",
llm_provider=self.custom_llm_provider or "",
)

18
tests/test_litellm/litellm_core_utils/test_max_streaming_duration.py
View file

@ -1,5 +1,5 @@
"""
Tests for MAX_STREAMING_DURATION_S — the global cap on streaming response wall-clock time.
Tests for LITELLM_MAX_STREAMING_DURATION_SECONDS — the global cap on streaming response wall-clock time.
Covers:
- CustomStreamWrapper (chat/completions) sync + async
@ -41,20 +41,20 @@ class TestCustomStreamWrapperMaxDuration:
def test_should_not_raise_when_duration_is_none(self):
"""No limit configured → never raises."""
wrapper = _make_custom_stream_wrapper()
with patch("litellm.constants.MAX_STREAMING_DURATION_S", None):
with patch("litellm.constants.LITELLM_MAX_STREAMING_DURATION_SECONDS", None):
wrapper._check_max_streaming_duration() # should not raise
def test_should_not_raise_when_under_limit(self):
"""Stream is under the limit → no error."""
wrapper = _make_custom_stream_wrapper()
with patch("litellm.constants.MAX_STREAMING_DURATION_S", 60.0):
with patch("litellm.constants.LITELLM_MAX_STREAMING_DURATION_SECONDS", 60.0):
wrapper._check_max_streaming_duration() # should not raise
def test_should_raise_timeout_when_exceeded(self):
"""Stream exceeded the limit → litellm.Timeout."""
wrapper = _make_custom_stream_wrapper()
wrapper._stream_created_time = time.time() - 20 # simulate 20s elapsed
with patch("litellm.constants.MAX_STREAMING_DURATION_S", 10.0):
with patch("litellm.constants.LITELLM_MAX_STREAMING_DURATION_SECONDS", 10.0):
with pytest.raises(litellm.Timeout, match="max streaming duration"):
wrapper._check_max_streaming_duration()
@ -62,7 +62,7 @@ class TestCustomStreamWrapperMaxDuration:
"""__next__ should check the limit before iterating."""
wrapper = _make_custom_stream_wrapper()
wrapper._stream_created_time = time.time() - 20
with patch("litellm.constants.MAX_STREAMING_DURATION_S", 10.0):
with patch("litellm.constants.LITELLM_MAX_STREAMING_DURATION_SECONDS", 10.0):
with pytest.raises(litellm.Timeout):
wrapper.__next__()
@ -71,7 +71,7 @@ class TestCustomStreamWrapperMaxDuration:
"""__anext__ should check the limit before iterating."""
wrapper = _make_custom_stream_wrapper()
wrapper._stream_created_time = time.time() - 20
with patch("litellm.constants.MAX_STREAMING_DURATION_S", 10.0):
with patch("litellm.constants.LITELLM_MAX_STREAMING_DURATION_SECONDS", 10.0):
with pytest.raises(litellm.Timeout):
await wrapper.__anext__()
@ -105,14 +105,14 @@ class TestResponsesStreamingIteratorMaxDuration:
def test_should_not_raise_when_duration_is_none(self):
it = self._make_base_iterator()
with patch(
"litellm.responses.streaming_iterator.MAX_STREAMING_DURATION_S", None
"litellm.responses.streaming_iterator.LITELLM_MAX_STREAMING_DURATION_SECONDS", None
):
it._check_max_streaming_duration()
def test_should_not_raise_when_under_limit(self):
it = self._make_base_iterator()
with patch(
"litellm.responses.streaming_iterator.MAX_STREAMING_DURATION_S", 60.0
"litellm.responses.streaming_iterator.LITELLM_MAX_STREAMING_DURATION_SECONDS", 60.0
):
it._check_max_streaming_duration()
@ -120,7 +120,7 @@ class TestResponsesStreamingIteratorMaxDuration:
it = self._make_base_iterator()
it._stream_created_time = time.time() - 20
with patch(
"litellm.responses.streaming_iterator.MAX_STREAMING_DURATION_S", 10.0
"litellm.responses.streaming_iterator.LITELLM_MAX_STREAMING_DURATION_SECONDS", 10.0
):
with pytest.raises(litellm.Timeout, match="max streaming duration"):
it._check_max_streaming_duration()