From c9cf1a6f512d876e53b9bb79187b5d9aa2fafd1c Mon Sep 17 00:00:00 2001 From: linhongyu510 Date: Thu, 10 Sep 2026 18:14:02 +0800 Subject: [PATCH 1/3] fix(cache): measure container contents for item limits --- litellm/caching/in_memory_cache.py | 5 ----- tests/unit/caching/test_in_memory_cache.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 1cc3e106aa0..6738b29ce65 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -71,11 +71,6 @@ class InMemoryCache(BaseCache): if isinstance(value, bytes): return sys.getsizeof(value) / 1024 <= self.max_size_per_item - # Handle special types without full conversion when possible - if hasattr(value, "__sizeof__"): # Use __sizeof__ if available - size: Final = value.__sizeof__() / 1024 - return size <= self.max_size_per_item - # Fallback for complex types if isinstance(value, BaseModel) and hasattr(value, "model_dump"): # Pydantic v2 value = value.model_dump() diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index 75232476ecd..d617316e0bc 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -117,6 +117,18 @@ def test_in_memory_cache_max_size_per_item(): assert result is False +def test_in_memory_cache_max_size_per_item_measures_container_contents(): + in_memory_cache = InMemoryCache(max_size_per_item=1) + oversized_value = {"timestamp": 0.0, "response": "a" * 5000} + small_value = {"timestamp": 0.0, "response": "ok"} + + in_memory_cache.set_cache(key="oversized", value=oversized_value) + in_memory_cache.set_cache(key="small", value=small_value) + + assert in_memory_cache.get_cache(key="oversized") is None + assert in_memory_cache.get_cache(key="small") == small_value + + def test_in_memory_cache_ttl(): """ Check that From 1f150b9ac4745ddd3506fa8b0721f7d21710c49a Mon Sep 17 00:00:00 2001 From: linhongyu510 Date: Fri, 11 Sep 2026 12:28:34 +0800 Subject: [PATCH 2/3] fix(cache): preserve non-JSON container entries --- litellm/caching/in_memory_cache.py | 5 ++++- tests/unit/caching/test_in_memory_cache.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 6738b29ce65..22e3a3610f2 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -79,7 +79,10 @@ class InMemoryCache(BaseCache): # Only convert to JSON if absolutely necessary if not isinstance(value, (str, bytes)): - value = json.dumps(value, default=str) + try: + value = json.dumps(value, default=str) + except (TypeError, ValueError): + value = repr(value) return sys.getsizeof(value) / 1024 <= self.max_size_per_item diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index d617316e0bc..38eac1ede32 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -129,6 +129,15 @@ def test_in_memory_cache_max_size_per_item_measures_container_contents(): assert in_memory_cache.get_cache(key="small") == small_value +def test_in_memory_cache_accepts_small_container_with_non_json_keys(): + in_memory_cache = InMemoryCache(max_size_per_item=1) + value = {("key", "key-hash"): ("job-1",)} + + in_memory_cache.set_cache(key="active-jobs", value=value) + + assert in_memory_cache.get_cache(key="active-jobs") == value + + def test_in_memory_cache_ttl(): """ Check that From 2c3c7b534791287e8709f7079b9670618ee16a70 Mon Sep 17 00:00:00 2001 From: linhongyu510 Date: Mon, 26 Oct 2026 02:38:37 +0800 Subject: [PATCH 3/3] fix(cache): return size directly to avoid parameter rebind The LIT011 type-discipline gate counts parameter rebinding; returning sys.getsizeof(...) directly from each branch avoids reassigning the 'value' parameter while preserving the same size-limit behavior. --- litellm/caching/in_memory_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 22e3a3610f2..de46c3976ad 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -80,9 +80,9 @@ class InMemoryCache(BaseCache): # Only convert to JSON if absolutely necessary if not isinstance(value, (str, bytes)): try: - value = json.dumps(value, default=str) + return sys.getsizeof(json.dumps(value, default=str)) / 1024 <= self.max_size_per_item except (TypeError, ValueError): - value = repr(value) + return sys.getsizeof(repr(value)) / 1024 <= self.max_size_per_item return sys.getsizeof(value) / 1024 <= self.max_size_per_item