scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

(fix) undo changes from other branches

Browse source
This commit is contained in:
Ishaan Jaff 2024-03-26 09:22:19 -07:00
parent b8af946fb9
commit 098a03facc
2 changed files with 11 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/caching.py
View file

@ -119,9 +119,6 @@ class RedisCache(BaseCache):
# for high traffic, we store the redis results in memory and then batch write to redis
self.redis_batch_writing_buffer = []
self.redis_batch_reading_buffer = []
self.redis_last_updated_read_buffer = None
self.redis_fetch_interval = 1 # fetch from redis every 1 second
self.redis_flush_size = redis_flush_size
self.redis_version = "Unknown"
try:
@ -256,24 +253,11 @@ class RedisCache(BaseCache):
traceback.print_exc()
logging.debug("LiteLLM Caching: get() - Got exception from REDIS: ", e)
def _should_fetch_from_redis(self):
if self.redis_last_updated_read_buffer is None:
return True
if (
time.time() - self.redis_last_updated_read_buffer
> self.redis_fetch_interval
):
return True
return False
async def async_get_cache(self, key, **kwargs):
_redis_client = self.init_async_client()
async with _redis_client as redis_client:
try:
print_verbose(f"Get Async Redis Cache: key: {key}")
if self._should_fetch_from_redis():
self.redis_last_updated_read_buffer = time.time()
cached_response = await redis_client.get(key)
print_verbose(
f"Got Async Redis Cache: key: {key}, cached_response {cached_response}"

25
litellm/utils.py
View file

@ -2814,21 +2814,18 @@ def client(original_function):
)
# if caching is false, don't run this
final_embedding_cached_response = None
cache_controls = kwargs.get("cache", None)
# Check if user has opted out of caching
_opted_out_with_cache_controls = (
cache_controls and cache_controls.get("no-cache", False) == True
)
_opted_out_with_caching_param = kwargs.get("caching", True) == False
# cache is not None and user has not opted out
if (
litellm.cache is not None
and (not _opted_out_with_cache_controls)
and (not _opted_out_with_caching_param)
):
# allow users to control returning cached responses from the completion function
(
kwargs.get("caching", None) is None
and kwargs.get("cache", None) is None
and litellm.cache is not None
)
or kwargs.get("caching", False) == True
or (
kwargs.get("cache", None) is not None
and kwargs.get("cache").get("no-cache", False) != True
)
): # allow users to control returning cached responses from the completion function
# checking cache
print_verbose(f"INSIDE CHECKING CACHE")
if (