scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(cache): enforce byte-size limits for cached strings

Browse source
This commit is contained in:
moyai-berriai[bot] 2026-10-20 15:07:31 +00:00 • committed by GitHub
parent b88f64e65f
commit 5ceaddd0b9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 24 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/caching/in_memory_cache.py
View file

@ -62,13 +62,12 @@ class InMemoryCache(BaseCache):
try:
# Fast path for common primitive types that are typically small
if (
isinstance(value, (bool, int, float, str))
isinstance(value, (bool, int, float))
and len(str(value)) < self.max_size_per_item * MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB
): # Conservative estimate
return True
# Direct size check for bytes objects
if isinstance(value, bytes):
if isinstance(value, (str, bytes)):
return sys.getsizeof(value) / 1024 <= self.max_size_per_item
# Handle special types without full conversion when possible

22
tests/unit/caching/test_in_memory_cache.py
View file

@ -76,6 +76,28 @@ def test_in_memory_cache_max_size_per_item():
assert result is False
@pytest.mark.parametrize(
("character", "count", "should_cache"),
[
("a", 100, True),
("\u00e9", 100, True),
("\u6f22", 100, True),
("\U0001f600", 100, True),
("a", 1023, False),
("\u00e9", 1023, False),
("\u6f22", 600, False),
("\U0001f600", 300, False),
],
)
def test_string_cache_admission_respects_byte_size(character: str, count: int, should_cache: bool) -> None:
cache: Final = InMemoryCache(max_size_per_item=1, clock=lambda: 1000.0)
value: Final = character * count
cache.set_cache("text", value)
assert cache.get_cache("text") == (value if should_cache else None)
def test_in_memory_cache_ttl():
"""
Check that