diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 56c9147e066..ae58152c9cb 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -70,11 +70,6 @@ class InMemoryCache(BaseCache): if isinstance(value, bytes): return sys.getsizeof(value) / 1024 <= self.max_size_per_item - # Handle special types without full conversion when possible - if hasattr(value, "__sizeof__"): # Use __sizeof__ if available - size: Final = value.__sizeof__() / 1024 - return size <= self.max_size_per_item - # Fallback for complex types if isinstance(value, BaseModel) and hasattr(value, "model_dump"): # Pydantic v2 value = value.model_dump() @@ -83,7 +78,10 @@ class InMemoryCache(BaseCache): # Only convert to JSON if absolutely necessary if not isinstance(value, (str, bytes)): - value = json.dumps(value, default=str) + try: + value = json.dumps(value, default=str) + except (TypeError, ValueError): + value = repr(value) return sys.getsizeof(value) / 1024 <= self.max_size_per_item diff --git a/tests/test_litellm/caching/test_in_memory_cache.py b/tests/test_litellm/caching/test_in_memory_cache.py index 40ad4f0c6f0..7a8ae17407f 100644 --- a/tests/test_litellm/caching/test_in_memory_cache.py +++ b/tests/test_litellm/caching/test_in_memory_cache.py @@ -72,6 +72,27 @@ def test_in_memory_cache_max_size_per_item(): assert result is False +def test_in_memory_cache_max_size_per_item_measures_container_contents(): + in_memory_cache = InMemoryCache(max_size_per_item=1) + oversized_value = {"timestamp": 0.0, "response": "a" * 5000} + small_value = {"timestamp": 0.0, "response": "ok"} + + in_memory_cache.set_cache(key="oversized", value=oversized_value) + in_memory_cache.set_cache(key="small", value=small_value) + + assert in_memory_cache.get_cache(key="oversized") is None + assert in_memory_cache.get_cache(key="small") == small_value + + +def test_in_memory_cache_accepts_small_container_with_non_json_keys(): + in_memory_cache = InMemoryCache(max_size_per_item=1) + value = {("key", "key-hash"): ("job-1",)} + + in_memory_cache.set_cache(key="active-jobs", value=value) + + assert in_memory_cache.get_cache(key="active-jobs") == value + + def test_in_memory_cache_ttl(): """ Check that