Merge 2c3c7b5347 into e9b1c11098

litellm/caching/in_memory_cache.py

@@ -71,11 +71,6 @@ class InMemoryCache(BaseCache):
             if isinstance(value, bytes):
                 return sys.getsizeof(value) / 1024 <= self.max_size_per_item
 
-            # Handle special types without full conversion when possible
-            if hasattr(value, "__sizeof__"):  # Use __sizeof__ if available
-                size: Final = value.__sizeof__() / 1024
-                return size <= self.max_size_per_item
-
             # Fallback for complex types
             if isinstance(value, BaseModel) and hasattr(value, "model_dump"):  # Pydantic v2
                 value = value.model_dump()
@@ -84,7 +79,10 @@ class InMemoryCache(BaseCache):
 
             # Only convert to JSON if absolutely necessary
             if not isinstance(value, (str, bytes)):
-                value = json.dumps(value, default=str)
+                try:
+                    return sys.getsizeof(json.dumps(value, default=str)) / 1024 <= self.max_size_per_item
+                except (TypeError, ValueError):
+                    return sys.getsizeof(repr(value)) / 1024 <= self.max_size_per_item
 
             return sys.getsizeof(value) / 1024 <= self.max_size_per_item
 

tests/unit/caching/test_in_memory_cache.py

@@ -117,6 +117,27 @@ def test_in_memory_cache_max_size_per_item():
     assert result is False
 
 
+def test_in_memory_cache_max_size_per_item_measures_container_contents():
+    in_memory_cache = InMemoryCache(max_size_per_item=1)
+    oversized_value = {"timestamp": 0.0, "response": "a" * 5000}
+    small_value = {"timestamp": 0.0, "response": "ok"}
+
+    in_memory_cache.set_cache(key="oversized", value=oversized_value)
+    in_memory_cache.set_cache(key="small", value=small_value)
+
+    assert in_memory_cache.get_cache(key="oversized") is None
+    assert in_memory_cache.get_cache(key="small") == small_value
+
+
+def test_in_memory_cache_accepts_small_container_with_non_json_keys():
+    in_memory_cache = InMemoryCache(max_size_per_item=1)
+    value = {("key", "key-hash"): ("job-1",)}
+
+    in_memory_cache.set_cache(key="active-jobs", value=value)
+
+    assert in_memory_cache.get_cache(key="active-jobs") == value
+
+
 def test_in_memory_cache_ttl():
     """
     Check that