From 8a66e074ce7c92b3cced842095d0d0afc0270418 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 22 Jun 2024 19:51:43 -0700 Subject: [PATCH] fix in mem cache tests --- litellm/caching.py | 4 ++-- litellm/router.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 4fe9ace07fe..e77d71dd8b1 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -64,7 +64,7 @@ class BaseCache: class InMemoryCache(BaseCache): - def __init__(self, default_ttl: Optional[float] = 60.0): + def __init__(self, default_ttl: Optional[float] = 120.0): """ default_ttl [float]: If default_ttl is 6 seconds, every 6 seconds the cache will be set to {} this is done to prevent overuse of System RAM @@ -72,7 +72,7 @@ class InMemoryCache(BaseCache): # if users don't provider one, use the default litellm cache self.cache_dict: dict = {} self.ttl_dict: dict = {} - self.default_ttl = default_ttl or 60.0 + self.default_ttl = default_ttl or 120.0 self.last_cleaned = 0 # since this is in memory we need to periodically clean it up to not overuse the machines RAM def set_cache(self, key, value, **kwargs): diff --git a/litellm/router.py b/litellm/router.py index df783eab829..8c05a7e8be9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -282,7 +282,7 @@ class Router: litellm.cache = litellm.Cache(type=cache_type, **cache_config) # type: ignore self.cache_responses = cache_responses self.cache = DualCache( - redis_cache=redis_cache, in_memory_cache=InMemoryCache() + redis_cache=redis_cache, in_memory_cache=InMemoryCache(default_ttl=86400) ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. ### SCHEDULER ###