mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
fix: don't close HTTP/SDK clients on LLMClientCache eviction
Removing the _remove_key override that eagerly called aclose()/close()
on evicted clients. Evicted clients may still be held by in-flight
streaming requests; closing them causes:
RuntimeError: Cannot send a request, as the client has been closed.
This is a regression from commit fb72979432. Clients that are no longer
referenced will be garbage-collected naturally. Explicit shutdown cleanup
happens via close_litellm_async_clients().
Fixes production crashes after the 1-hour cache TTL expires.
This commit is contained in:
parent
a8cf646850
commit
cd1d8a1ae4
1 changed files with 9 additions and 24 deletions
|
|
@ -3,36 +3,21 @@ Add the event loop to the cache key, to prevent event loop closed errors.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Set
|
||||
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
class LLMClientCache(InMemoryCache):
|
||||
# Background tasks must be stored to prevent garbage collection, which would
|
||||
# trigger "coroutine was never awaited" warnings. See:
|
||||
# https://docs.python.org/3/library/asyncio-task.html#creating-tasks
|
||||
# Intentionally shared across all instances as a global task registry.
|
||||
_background_tasks: Set[asyncio.Task] = set()
|
||||
"""Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
|
||||
|
||||
def _remove_key(self, key: str) -> None:
|
||||
"""Close async clients before evicting them to prevent connection pool leaks."""
|
||||
value = self.cache_dict.get(key)
|
||||
super()._remove_key(key)
|
||||
if value is not None:
|
||||
close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
|
||||
if close_fn and asyncio.iscoroutinefunction(close_fn):
|
||||
try:
|
||||
task = asyncio.get_running_loop().create_task(close_fn())
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._background_tasks.discard)
|
||||
except RuntimeError:
|
||||
pass
|
||||
elif close_fn and callable(close_fn):
|
||||
try:
|
||||
close_fn()
|
||||
except Exception:
|
||||
pass
|
||||
IMPORTANT: This cache intentionally does NOT close clients on eviction.
|
||||
Evicted clients may still be in use by in-flight requests. Closing them
|
||||
eagerly causes ``RuntimeError: Cannot send a request, as the client has
|
||||
been closed.`` errors in production after the TTL (1 hour) expires.
|
||||
|
||||
Clients that are no longer referenced will be garbage-collected normally.
|
||||
For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
|
||||
"""
|
||||
|
||||
def update_cache_key_with_event_loop(self, key):
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue