diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index 6b40c48c4ef..785bc5a68ec 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -62,13 +62,12 @@ class InMemoryCache(BaseCache): try: # Fast path for common primitive types that are typically small if ( - isinstance(value, (bool, int, float, str)) + isinstance(value, (bool, int, float)) and len(str(value)) < self.max_size_per_item * MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB ): # Conservative estimate return True - # Direct size check for bytes objects - if isinstance(value, bytes): + if isinstance(value, (str, bytes)): return sys.getsizeof(value) / 1024 <= self.max_size_per_item # Handle special types without full conversion when possible diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index 901832937b5..c62b1828cd1 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -117,6 +117,28 @@ def test_in_memory_cache_max_size_per_item(): assert result is False +@pytest.mark.parametrize( + ("character", "count", "should_cache"), + [ + ("a", 100, True), + ("\u00e9", 100, True), + ("\u6f22", 100, True), + ("\U0001f600", 100, True), + ("a", 1023, False), + ("\u00e9", 1023, False), + ("\u6f22", 600, False), + ("\U0001f600", 300, False), + ], +) +def test_string_cache_admission_respects_byte_size(character: str, count: int, should_cache: bool) -> None: + cache: Final = InMemoryCache(max_size_per_item=1, clock=lambda: 1000.0) + value: Final = character * count + + cache.set_cache("text", value) + + assert cache.get_cache("text") == (value if should_cache else None) + + def test_in_memory_cache_ttl(): """ Check that