From da8769284b6b272abfddada0f123a747cc1ac72d Mon Sep 17 00:00:00 2001 From: Oxygenesis Date: Mon, 28 Sep 2026 11:23:28 +0700 Subject: [PATCH 1/4] fix(cache): isolate mutable in-memory reads --- litellm/caching/in_memory_cache.py | 26 +++++++++++- litellm/caching/llm_caching_handler.py | 4 +- tests/unit/caching/test_in_memory_cache.py | 40 +++++++++++++++++-- .../unit/caching/test_llm_caching_handler.py | 9 +++++ 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 56c9147e066..9a970e64039 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -8,6 +8,7 @@ Has 4 methods: - async_get_cache """ +import copy import heapq import json import sys @@ -208,7 +209,25 @@ class InMemoryCache(BaseCache): return True return False - def get_cache(self, key, **kwargs): + @staticmethod + def _copy_cached_value(value: Any) -> Any: + """Return a read-isolated value without making the cache brittle. + + In-memory cache entries can contain Pydantic models and mutable + containers. Live resources such as SDK clients are deliberately left + alone; attempting to deepcopy them can create partially initialized + transports before failing. A failed deepcopy should not turn a cache + hit into a request failure, so retain the existing value as a last + resort. + """ + if not isinstance(value, (dict, list, set, tuple, frozenset, bytearray, BaseModel)): + return value + try: + return copy.deepcopy(value) + except Exception: # noqa: BLE001 - cache reads must tolerate non-copyable values + return value + + def _get_cache_value(self, key, *, copy_value: bool) -> Any: if key in self.cache_dict: if self.evict_element_if_expired(key): return None @@ -217,9 +236,12 @@ class InMemoryCache(BaseCache): cached_response = json.loads(original_cached_response) except Exception: cached_response = original_cached_response - return cached_response + return self._copy_cached_value(cached_response) if copy_value else cached_response return None + def get_cache(self, key, **kwargs): + return self._get_cache_value(key=key, copy_value=True) + def batch_get_cache(self, keys: list, **kwargs): return_val: Final = [] for k in keys: diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index a89e43b78b4..9441fa3b157 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -72,7 +72,9 @@ class LLMClientCache(InMemoryCache): key = self.update_cache_key_with_event_loop(key) self.evicted_client_closer.reap() - return super().get_cache(key, **kwargs) + # SDK/httpx clients are live resources. Returning a deepcopy would + # detach the caller from the client tracked by the eviction closer. + return super()._get_cache_value(key=key, copy_value=False) async def async_get_cache(self, key, **kwargs): key = self.update_cache_key_with_event_loop(key) diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index 40ad4f0c6f0..c9135825e59 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -3,14 +3,13 @@ import json import threading import time from concurrent.futures import ThreadPoolExecutor -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx from fastapi.testclient import TestClient - -from unittest.mock import AsyncMock +from pydantic import BaseModel from litellm.caching.in_memory_cache import InMemoryCache @@ -45,6 +44,41 @@ async def test_async_increment_delegates_to_locked_sync_path(): assert cache.get_cache("counter") == 5 +def test_in_memory_cache_returns_read_isolated_mutable_values(): + cache = InMemoryCache() + cache.set_cache("user", {"spend": 1.0, "metadata": {"region": "us"}}) + + cached = cache.get_cache("user") + assert cached == {"spend": 1.0, "metadata": {"region": "us"}} + assert cached is not cache.cache_dict["user"] + + cached["spend"] = 50 + cached["metadata"]["region"] = "eu" + + assert cache.get_cache("user") == {"spend": 1.0, "metadata": {"region": "us"}} + + +def test_in_memory_cache_returns_read_isolated_pydantic_models(): + class CachedBudget(BaseModel): + spend: float + metadata: dict[str, str] + + cache = InMemoryCache() + cache.set_cache("budget", CachedBudget(spend=1.0, metadata={"region": "us"})) + + cached = cache.get_cache("budget") + assert isinstance(cached, CachedBudget) + assert cached is not cache.cache_dict["budget"] + + cached.spend = 50 + cached.metadata["region"] = "eu" + + stored = cache.get_cache("budget") + assert isinstance(stored, CachedBudget) + assert stored.spend == 1.0 + assert stored.metadata == {"region": "us"} + + def test_in_memory_openai_obj_cache(): from openai import OpenAI diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index dd81b877c0e..2a1660a6459 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -38,6 +38,15 @@ class MockSyncClient: self.closed = True +def test_client_cache_returns_live_client_reference(): + cache = LLMClientCache() + client = MockSyncClient() + + cache.set_cache("client", client) + + assert cache.get_cache("client") is client + + @pytest.mark.asyncio async def test_remove_key_does_not_close_async_client(): """ From 332e2ece440327acfa8addd0eefb00d0bc2154d7 Mon Sep 17 00:00:00 2001 From: Oxygenesis Date: Tue, 29 Sep 2026 23:13:38 +0700 Subject: [PATCH 2/4] fix(cache): satisfy strict typing budget --- litellm/caching/in_memory_cache.py | 4 ++-- tests/unit/caching/test_in_memory_cache.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 9a970e64039..2aa2b868600 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -210,7 +210,7 @@ class InMemoryCache(BaseCache): return False @staticmethod - def _copy_cached_value(value: Any) -> Any: + def _copy_cached_value(value: object) -> object: """Return a read-isolated value without making the cache brittle. In-memory cache entries can contain Pydantic models and mutable @@ -227,7 +227,7 @@ class InMemoryCache(BaseCache): except Exception: # noqa: BLE001 - cache reads must tolerate non-copyable values return value - def _get_cache_value(self, key, *, copy_value: bool) -> Any: + def _get_cache_value(self, key, *, copy_value: bool) -> object: if key in self.cache_dict: if self.evict_element_if_expired(key): return None diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index c9135825e59..3ef302fa889 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -79,6 +79,19 @@ def test_in_memory_cache_returns_read_isolated_pydantic_models(): assert stored.metadata == {"region": "us"} +def test_in_memory_cache_preserves_non_copyable_mutable_values(): + cache = InMemoryCache() + cache.set_cache("payload", ["unchanged"]) + + with patch( + "litellm.caching.in_memory_cache.copy.deepcopy", + side_effect=RuntimeError("not copyable"), + ): + cached = cache.get_cache("payload") + + assert cached is cache.cache_dict["payload"] + + def test_in_memory_openai_obj_cache(): from openai import OpenAI From d4d676bdc57683a9f469aeb07235ac75483993c6 Mon Sep 17 00:00:00 2001 From: Oxygenesis Date: Tue, 29 Sep 2026 23:31:59 +0700 Subject: [PATCH 3/4] fix(cache): preserve dynamic cache value types --- litellm/caching/in_memory_cache.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 2aa2b868600..ca3d3546d72 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -15,7 +15,7 @@ import sys import threading import time from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypeVar, cast if TYPE_CHECKING: from litellm.types.caching import RedisPipelineIncrementOperation @@ -27,6 +27,7 @@ from litellm.constants import MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB from .base_cache import BaseCache DEFAULT_MAX_SIZE_IN_MEMORY: Final = 200 +_T = TypeVar("_T") class InMemoryCache(BaseCache): @@ -210,7 +211,7 @@ class InMemoryCache(BaseCache): return False @staticmethod - def _copy_cached_value(value: object) -> object: + def _copy_cached_value(value: _T) -> _T: """Return a read-isolated value without making the cache brittle. In-memory cache entries can contain Pydantic models and mutable @@ -223,11 +224,11 @@ class InMemoryCache(BaseCache): if not isinstance(value, (dict, list, set, tuple, frozenset, bytearray, BaseModel)): return value try: - return copy.deepcopy(value) + return cast(_T, copy.deepcopy(value)) except Exception: # noqa: BLE001 - cache reads must tolerate non-copyable values return value - def _get_cache_value(self, key, *, copy_value: bool) -> object: + def _get_cache_value(self, key, *, copy_value: bool): if key in self.cache_dict: if self.evict_element_if_expired(key): return None From d098536b92b74390e6b1984266eb2f47b27c910b Mon Sep 17 00:00:00 2001 From: Oxygenesis Date: Wed, 30 Sep 2026 11:01:11 +0700 Subject: [PATCH 4/4] fix(cache): persist hanging alert state after isolated reads --- .../integrations/SlackAlerting/hanging_request_check.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/integrations/SlackAlerting/hanging_request_check.py b/litellm/integrations/SlackAlerting/hanging_request_check.py index 4d7cbfe8fd1..85670610242 100644 --- a/litellm/integrations/SlackAlerting/hanging_request_check.py +++ b/litellm/integrations/SlackAlerting/hanging_request_check.py @@ -138,6 +138,13 @@ class AlertingHangingRequestCheck: # flag so the entry is skipped on later ticks; one alert per hang, # with the existing TTL still handling cleanup hanging_request_data.alerted = True + # InMemoryCache returns read-isolated mutable values. Persist the + # state transition explicitly so the next checker tick observes + # that this request has already been alerted. + await self.hanging_request_cache.async_set_cache( + key=request_id, + value=hanging_request_data, + ) return