scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 165

fix(cache): measure container contents for item limits

Browse source
This commit is contained in:
linhongyu510 2026-09-10 18:14:02 +08:00
parent e5da59336d
commit 5daa555bb0
2 changed files with 12 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/caching/in_memory_cache.py
View file

@ -65,11 +65,6 @@ class InMemoryCache(BaseCache):
if isinstance(value, bytes):
return sys.getsizeof(value) / 1024 <= self.max_size_per_item
# Handle special types without full conversion when possible
if hasattr(value, "__sizeof__"): # Use __sizeof__ if available
size: Final = value.__sizeof__() / 1024
return size <= self.max_size_per_item
# Fallback for complex types
if isinstance(value, BaseModel) and hasattr(value, "model_dump"): # Pydantic v2
value = value.model_dump()

12
tests/test_litellm/caching/test_in_memory_cache.py
View file

@ -72,6 +72,18 @@ def test_in_memory_cache_max_size_per_item():
assert result is False
def test_in_memory_cache_max_size_per_item_measures_container_contents():
in_memory_cache = InMemoryCache(max_size_per_item=1)
oversized_value = {"timestamp": 0.0, "response": "a" * 5000}
small_value = {"timestamp": 0.0, "response": "ok"}
in_memory_cache.set_cache(key="oversized", value=oversized_value)
in_memory_cache.set_cache(key="small", value=small_value)
assert in_memory_cache.get_cache(key="oversized") is None
assert in_memory_cache.get_cache(key="small") == small_value
def test_in_memory_cache_ttl():
"""
Check that