From 4053c7aeb33193cc75436715a77b92382f1c6fa1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 24 Jun 2024 17:15:53 -0700 Subject: [PATCH] use lru cache --- litellm/caching.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 705b5fc13ec..ceb8e70b16c 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -17,7 +17,7 @@ import traceback from datetime import timedelta from typing import Any, BinaryIO, List, Literal, Optional, Union -from cachetools import Cache as CachetoolsCache +from cachetools import LRUCache from openai._models import BaseModel as OpenAIObject import litellm @@ -71,10 +71,9 @@ class InMemoryCache(BaseCache): this is done to prevent overuse of System RAM """ # if users don't provider one, use the default litellm cache - self.cache_dict: CachetoolsCache = CachetoolsCache( - maxsize=1000, - ) - self.ttl_dict: dict = {} + max_size_in_memory = 1000 + self.cache_dict: LRUCache = LRUCache(maxsize=max_size_in_memory) + self.ttl_dict: LRUCache = LRUCache(maxsize=max_size_in_memory) self.default_ttl = default_ttl or 120.0 def set_cache(self, key, value, **kwargs):