scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

fix(proxy): give the remaining CLI clients a request timeout

Browse source 
The keys, credentials, models, model groups, and chat clients still sent
requests with no timeout, so a proxy that accepts the connection and
never answers pinned the caller forever. They now default to the same
30 seconds as their teams and users siblings, with chat on the OpenAI
SDK's 600 second default, and Client wires its timeout through to all of
them. S113 cannot see Session methods, so each client gets a
hanging-server regression test instead.
This commit is contained in:
mateo-berri 2026-08-29 13:32:39 -07:00
parent 8aba6e9203
commit 4d5205c355
13 changed files with 140 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/proxy/client/chat.py
View file

@ -8,16 +8,19 @@ from .exceptions import UnauthorizedError
class ChatClient:
def __init__(self, base_url: str, api_key: str | None = None):
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 600):
"""
Initialize the ChatClient.
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout (int): Request timeout in seconds (default: 600, the OpenAI SDK default, since a completion
can legitimately take minutes)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
self._api_key = api_key
self._timeout = timeout
def _get_headers(self) -> dict[str, str]:
"""
@ -96,7 +99,7 @@ class ChatClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -161,7 +164,9 @@ class ChatClient:
# Make streaming request
session: Final = requests.Session()
try:
response: Final = session.post(url, headers=self._get_headers(), json=data, stream=True)
response: Final = session.post(
url, headers=self._get_headers(), json=data, stream=True, timeout=self._timeout
)
response.raise_for_status()
# Parse SSE stream

11
litellm/proxy/client/client.py
View file

@ -24,7 +24,8 @@ class Client:
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:4000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout: Request timeout in seconds (default: 30)
timeout: Request timeout in seconds for management calls (default: 30). Chat completions keep
ChatClient's own 600 second default, since a completion can legitimately take minutes
"""
self._base_url = base_url.rstrip("/")
# Only use the stored CLI key when it was issued for this server.
@ -33,9 +34,9 @@ class Client:
# Initialize resource clients
self.http = HTTPClient(base_url=base_url, api_key=self._api_key, timeout=timeout)
self.models = ModelsManagementClient(base_url=self._base_url, api_key=self._api_key)
self.model_groups = ModelGroupsManagementClient(base_url=self._base_url, api_key=self._api_key)
self.models = ModelsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
self.model_groups = ModelGroupsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
self.chat = ChatClient(base_url=self._base_url, api_key=self._api_key)
self.keys = KeysManagementClient(base_url=self._base_url, api_key=self._api_key)
self.credentials = CredentialsManagementClient(base_url=self._base_url, api_key=self._api_key)
self.keys = KeysManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
self.credentials = CredentialsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)
self.teams = TeamsManagementClient(base_url=self._base_url, api_key=self._api_key, timeout=timeout)

12
litellm/proxy/client/credentials.py
View file

@ -6,16 +6,18 @@ from .exceptions import UnauthorizedError
class CredentialsManagementClient:
def __init__(self, base_url: str, api_key: str | None = None):
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
"""
Initialize the CredentialsManagementClient.
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout (int): Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
self._api_key = api_key
self._timeout = timeout
def _get_headers(self) -> dict[str, str]:
"""
@ -56,7 +58,7 @@ class CredentialsManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -103,7 +105,7 @@ class CredentialsManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -140,7 +142,7 @@ class CredentialsManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -177,7 +179,7 @@ class CredentialsManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:

14
litellm/proxy/client/keys.py
View file

@ -9,16 +9,18 @@ from .exceptions import UnauthorizedError
class KeysManagementClient:
def __init__(self, base_url: str, api_key: str | None = None):
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
"""
Initialize the KeysManagementClient.
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout (int): Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
self._api_key = api_key
self._timeout = timeout
def _get_headers(self) -> dict[str, str]:
"""
@ -99,7 +101,7 @@ class KeysManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -174,7 +176,7 @@ class KeysManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -218,7 +220,7 @@ class KeysManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -279,7 +281,7 @@ class KeysManagementClient:
session: Final = requests.Session()
response_text: str | None = None
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response_text = response.text
response.raise_for_status()
return response.json()
@ -309,7 +311,7 @@ class KeysManagementClient:
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:

6
litellm/proxy/client/model_groups.py
View file

@ -6,16 +6,18 @@ from .exceptions import UnauthorizedError
class ModelGroupsManagementClient:
def __init__(self, base_url: str, api_key: str | None = None):
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
"""
Initialize the ModelGroupsManagementClient.
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout (int): Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
self._api_key = api_key
self._timeout = timeout
def _get_headers(self) -> dict[str, str]:
"""
@ -53,7 +55,7 @@ class ModelGroupsManagementClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()["data"]
except requests.exceptions.HTTPError as e:

14
litellm/proxy/client/models.py
View file

@ -7,16 +7,18 @@ from .exceptions import NotFoundError, UnauthorizedError
class ModelsManagementClient:
def __init__(self, base_url: str, api_key: str | None = None):
def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
"""
Initialize the ModelsManagementClient.
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout (int): Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
self._api_key = api_key
self._timeout = timeout
def _get_headers(self) -> dict[str, str]:
"""
@ -55,7 +57,7 @@ class ModelsManagementClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()["data"]
except requests.exceptions.HTTPError as e:
@ -104,7 +106,7 @@ class ModelsManagementClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -140,7 +142,7 @@ class ModelsManagementClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:
@ -232,7 +234,7 @@ class ModelsManagementClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()["data"]
except requests.exceptions.HTTPError as e:
@ -282,7 +284,7 @@ class ModelsManagementClient:
# Prepare and send the request
session: Final = requests.Session()
try:
response: Final = session.send(request.prepare())
response: Final = session.send(request.prepare(), timeout=self._timeout)
response.raise_for_status()
return response.json()
except requests.exceptions.HTTPError as e:

29
tests/test_litellm/proxy/client/test_chat.py
View file

@ -1,6 +1,7 @@
import importlib
import importlib.util
from importlib.machinery import PathFinder
import time
import site
import sys
@ -227,3 +228,31 @@ def test_completions_other_errors(client, sample_messages):
with pytest.raises(requests.exceptions.HTTPError) as exc_info:
client.completions(model="gpt-4", messages=sample_messages)
assert exc_info.value.response.status_code == 500
def test_completions_gives_up_at_the_timeout_instead_of_hanging(hanging_server):
"""
A proxy that accepts the connection but never answers used to pin the caller's
process forever, since the request carried no timeout at all.
"""
client = ChatClient(base_url=hanging_server, api_key="sk-test", timeout=1)
started = time.monotonic()
with pytest.raises(requests.exceptions.Timeout):
client.completions(model="gpt-5.4", messages=[{"role": "user", "content": "hi"}])
assert time.monotonic() - started < 10
def test_completions_stream_gives_up_at_the_timeout_instead_of_hanging(hanging_server):
"""
The streaming call opens the response before reading chunks, so a proxy that never
sends its headers used to hang here forever too.
"""
client = ChatClient(base_url=hanging_server, api_key="sk-test", timeout=1)
started = time.monotonic()
with pytest.raises(requests.exceptions.Timeout):
next(client.completions_stream(model="gpt-5.4", messages=[{"role": "user", "content": "hi"}]))
assert time.monotonic() - started < 10

8
tests/test_litellm/proxy/client/test_client.py
View file

@ -82,6 +82,12 @@ def test_client_initialization():
assert client.http._base_url == "http://localhost:4000"
assert client.http._api_key == "test-key"
assert client.http._timeout == 60
assert client.teams._timeout == 60
assert client.keys._timeout == 60
assert client.credentials._timeout == 60
assert client.models._timeout == 60
assert client.model_groups._timeout == 60
assert client.chat._timeout == 600
def test_client_default_timeout():
@ -92,6 +98,8 @@ def test_client_default_timeout():
)
assert client.http._timeout == 30
assert client.keys._timeout == 30
assert client.chat._timeout == 600
def test_client_without_api_key():

15
tests/test_litellm/proxy/client/test_credentials.py
View file

@ -1,4 +1,5 @@
import time
import pytest
import requests
@ -276,3 +277,17 @@ def test_encrypt_credential_values_does_not_mutate_original(monkeypatch):
assert encrypted.credential_values["api_key"] != "sk-123"
assert credential.credential_values["api_key"] == "sk-123"
assert encrypted.credential_name == credential.credential_name
def test_list_gives_up_at_the_timeout_instead_of_hanging(hanging_server):
"""
A proxy that accepts the connection but never answers used to pin the caller's
process forever, since the request carried no timeout at all.
"""
client = CredentialsManagementClient(base_url=hanging_server, api_key="sk-test", timeout=1)
started = time.monotonic()
with pytest.raises(requests.exceptions.Timeout):
client.list()
assert time.monotonic() - started < 10

15
tests/test_litellm/proxy/client/test_keys.py
View file

@ -1,3 +1,4 @@
import time
import traceback
import pytest
@ -509,3 +510,17 @@ def test_not_found_error_redacts_wrapped_key():
assert "REDACTED" in str(wrapped)
assert LEAKY_KEY not in str(wrapped.orig_exception)
assert wrapped.orig_exception.response.status_code == 404
def test_list_gives_up_at_the_timeout_instead_of_hanging(hanging_server):
"""
A proxy that accepts the connection but never answers used to pin the caller's
process forever, since the request carried no timeout at all.
"""
client = KeysManagementClient(base_url=hanging_server, api_key="sk-test", timeout=1)
started = time.monotonic()
with pytest.raises(requests.exceptions.Timeout):
client.list()
assert time.monotonic() - started < 10

15
tests/test_litellm/proxy/client/test_model_groups.py
View file

@ -1,4 +1,5 @@
import time
import pytest
import requests
@ -172,3 +173,17 @@ def test_client_initialization_without_api_key(base_url):
assert client._api_key is None
assert client.model_groups._api_key is None
def test_info_gives_up_at_the_timeout_instead_of_hanging(hanging_server):
"""
A proxy that accepts the connection but never answers used to pin the caller's
process forever, since the request carried no timeout at all.
"""
client = ModelGroupsManagementClient(base_url=hanging_server, api_key="sk-test", timeout=1)
started = time.monotonic()
with pytest.raises(requests.exceptions.Timeout):
client.info()
assert time.monotonic() - started < 10

15
tests/test_litellm/proxy/client/test_models.py
View file

@ -1,4 +1,5 @@
import time
import pytest
import requests
@ -732,3 +733,17 @@ def test_update_other_errors(client):
with pytest.raises(requests.exceptions.HTTPError) as exc_info:
client.update(model_id=model_id, model_params=model_params)
assert exc_info.value.response.status_code == 500
def test_list_gives_up_at_the_timeout_instead_of_hanging(hanging_server):
"""
A proxy that accepts the connection but never answers used to pin the caller's
process forever, since the request carried no timeout at all.
"""
client = ModelsManagementClient(base_url=hanging_server, api_key="sk-test", timeout=1)
started = time.monotonic()
with pytest.raises(requests.exceptions.Timeout):
client.list()
assert time.monotonic() - started < 10

4
type-discipline-budget.json
View file

@ -1,6 +1,6 @@
{
"LIT001": {
"limit": 22705
"limit": 22704
},
"LIT002": {
"limit": 26854
@ -33,6 +33,6 @@
"limit": 5577
},
"LIT012": {
"limit": 4508
"limit": 4506
}
}