From cd1d8a1ae47bda82bd48e9f0105348588f6ee3f3 Mon Sep 17 00:00:00 2001
From: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
Date: Thu, 5 Mar 2026 11:57:02 -0800
Subject: [PATCH] fix: don't close HTTP/SDK clients on LLMClientCache eviction

Removing the _remove_key override that eagerly called aclose()/close()
on evicted clients. Evicted clients may still be held by in-flight
streaming requests; closing them causes:

  RuntimeError: Cannot send a request, as the client has been closed.

This is a regression from commit fb72979432. Clients that are no longer
referenced will be garbage-collected naturally. Explicit shutdown cleanup
happens via close_litellm_async_clients().

Fixes production crashes after the 1-hour cache TTL expires.
---
 litellm/caching/llm_caching_handler.py | 33 +++++++-------------------
 1 file changed, 9 insertions(+), 24 deletions(-)

diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py
index 331aa8f51cd..c2274713bb9 100644
--- a/litellm/caching/llm_caching_handler.py
+++ b/litellm/caching/llm_caching_handler.py
@@ -3,36 +3,21 @@ Add the event loop to the cache key, to prevent event loop closed errors.
 """
 
 import asyncio
-from typing import Set
 
 from .in_memory_cache import InMemoryCache
 
 
 class LLMClientCache(InMemoryCache):
-    # Background tasks must be stored to prevent garbage collection, which would
-    # trigger "coroutine was never awaited" warnings. See:
-    # https://docs.python.org/3/library/asyncio-task.html#creating-tasks
-    # Intentionally shared across all instances as a global task registry.
-    _background_tasks: Set[asyncio.Task] = set()
+    """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
 
-    def _remove_key(self, key: str) -> None:
-        """Close async clients before evicting them to prevent connection pool leaks."""
-        value = self.cache_dict.get(key)
-        super()._remove_key(key)
-        if value is not None:
-            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
-            if close_fn and asyncio.iscoroutinefunction(close_fn):
-                try:
-                    task = asyncio.get_running_loop().create_task(close_fn())
-                    self._background_tasks.add(task)
-                    task.add_done_callback(self._background_tasks.discard)
-                except RuntimeError:
-                    pass
-            elif close_fn and callable(close_fn):
-                try:
-                    close_fn()
-                except Exception:
-                    pass
+    IMPORTANT: This cache intentionally does NOT close clients on eviction.
+    Evicted clients may still be in use by in-flight requests. Closing them
+    eagerly causes ``RuntimeError: Cannot send a request, as the client has
+    been closed.`` errors in production after the TTL (1 hour) expires.
+
+    Clients that are no longer referenced will be garbage-collected normally.
+    For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
+    """
 
     def update_cache_key_with_event_loop(self, key):
         """