From 5daa555bb059c43d7dba360e0d8671fcc60f637e Mon Sep 17 00:00:00 2001 From: linhongyu510 Date: Thu, 10 Sep 2026 18:14:02 +0800 Subject: [PATCH] fix(cache): measure container contents for item limits --- litellm/caching/in_memory_cache.py | 5 ----- tests/test_litellm/caching/test_in_memory_cache.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 38a9966f9f9..494ee03b47f 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -65,11 +65,6 @@ class InMemoryCache(BaseCache): if isinstance(value, bytes): return sys.getsizeof(value) / 1024 <= self.max_size_per_item - # Handle special types without full conversion when possible - if hasattr(value, "__sizeof__"): # Use __sizeof__ if available - size: Final = value.__sizeof__() / 1024 - return size <= self.max_size_per_item - # Fallback for complex types if isinstance(value, BaseModel) and hasattr(value, "model_dump"): # Pydantic v2 value = value.model_dump() diff --git a/tests/test_litellm/caching/test_in_memory_cache.py b/tests/test_litellm/caching/test_in_memory_cache.py index 85e8308ae91..96e2e38b2be 100644 --- a/tests/test_litellm/caching/test_in_memory_cache.py +++ b/tests/test_litellm/caching/test_in_memory_cache.py @@ -72,6 +72,18 @@ def test_in_memory_cache_max_size_per_item(): assert result is False +def test_in_memory_cache_max_size_per_item_measures_container_contents(): + in_memory_cache = InMemoryCache(max_size_per_item=1) + oversized_value = {"timestamp": 0.0, "response": "a" * 5000} + small_value = {"timestamp": 0.0, "response": "ok"} + + in_memory_cache.set_cache(key="oversized", value=oversized_value) + in_memory_cache.set_cache(key="small", value=small_value) + + assert in_memory_cache.get_cache(key="oversized") is None + assert in_memory_cache.get_cache(key="small") == small_value + + def test_in_memory_cache_ttl(): """ Check that