Merge 785ae73276 into 5028f9ec59

litellm/caching/in_memory_cache.py

@@ -70,11 +70,6 @@ class InMemoryCache(BaseCache):
             if isinstance(value, bytes):
                 return sys.getsizeof(value) / 1024 <= self.max_size_per_item
 
-            # Handle special types without full conversion when possible
-            if hasattr(value, "__sizeof__"):  # Use __sizeof__ if available
-                size: Final = value.__sizeof__() / 1024
-                return size <= self.max_size_per_item
-
             # Fallback for complex types
             if isinstance(value, BaseModel) and hasattr(value, "model_dump"):  # Pydantic v2
                 value = value.model_dump()
@@ -83,7 +78,10 @@ class InMemoryCache(BaseCache):
 
             # Only convert to JSON if absolutely necessary
             if not isinstance(value, (str, bytes)):
-                value = json.dumps(value, default=str)
+                try:
+                    value = json.dumps(value, default=str)
+                except (TypeError, ValueError):
+                    value = repr(value)
 
             return sys.getsizeof(value) / 1024 <= self.max_size_per_item
 

tests/test_litellm/caching/test_in_memory_cache.py

@@ -72,6 +72,27 @@ def test_in_memory_cache_max_size_per_item():
     assert result is False
 
 
+def test_in_memory_cache_max_size_per_item_measures_container_contents():
+    in_memory_cache = InMemoryCache(max_size_per_item=1)
+    oversized_value = {"timestamp": 0.0, "response": "a" * 5000}
+    small_value = {"timestamp": 0.0, "response": "ok"}
+
+    in_memory_cache.set_cache(key="oversized", value=oversized_value)
+    in_memory_cache.set_cache(key="small", value=small_value)
+
+    assert in_memory_cache.get_cache(key="oversized") is None
+    assert in_memory_cache.get_cache(key="small") == small_value
+
+
+def test_in_memory_cache_accepts_small_container_with_non_json_keys():
+    in_memory_cache = InMemoryCache(max_size_per_item=1)
+    value = {("key", "key-hash"): ("job-1",)}
+
+    in_memory_cache.set_cache(key="active-jobs", value=value)
+
+    assert in_memory_cache.get_cache(key="active-jobs") == value
+
+
 def test_in_memory_cache_ttl():
     """
     Check that