diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index 331aa8f51cd..c2274713bb9 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -3,36 +3,21 @@ Add the event loop to the cache key, to prevent event loop closed errors. """ import asyncio -from typing import Set from .in_memory_cache import InMemoryCache class LLMClientCache(InMemoryCache): - # Background tasks must be stored to prevent garbage collection, which would - # trigger "coroutine was never awaited" warnings. See: - # https://docs.python.org/3/library/asyncio-task.html#creating-tasks - # Intentionally shared across all instances as a global task registry. - _background_tasks: Set[asyncio.Task] = set() + """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.). - def _remove_key(self, key: str) -> None: - """Close async clients before evicting them to prevent connection pool leaks.""" - value = self.cache_dict.get(key) - super()._remove_key(key) - if value is not None: - close_fn = getattr(value, "aclose", None) or getattr(value, "close", None) - if close_fn and asyncio.iscoroutinefunction(close_fn): - try: - task = asyncio.get_running_loop().create_task(close_fn()) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) - except RuntimeError: - pass - elif close_fn and callable(close_fn): - try: - close_fn() - except Exception: - pass + IMPORTANT: This cache intentionally does NOT close clients on eviction. + Evicted clients may still be in use by in-flight requests. Closing them + eagerly causes ``RuntimeError: Cannot send a request, as the client has + been closed.`` errors in production after the TTL (1 hour) expires. + + Clients that are no longer referenced will be garbage-collected normally. + For explicit shutdown cleanup, use ``close_litellm_async_clients()``. + """ def update_cache_key_with_event_loop(self, key): """