scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

fix cooldown message time

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-14 16:20:27 -07:00
parent 04f2117277
commit 9a5337c503
1 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/router_utils/cooldown_cache.py
View file

@ -83,7 +83,7 @@ class CooldownCache:
keys = [f"deployment:{model_id}:cooldown" for model_id in model_ids]
# Retrieve the values for the keys using mget
results = await self.cache.async_batch_get_cache(keys=keys)
results = await self.cache.async_batch_get_cache(keys=keys) or []
active_cooldowns = []
# Process the results
@ -101,7 +101,7 @@ class CooldownCache:
keys = [f"deployment:{model_id}:cooldown" for model_id in model_ids]
# Retrieve the values for the keys using mget
results = self.cache.batch_get_cache(keys=keys)
results = self.cache.batch_get_cache(keys=keys) or []
active_cooldowns = []
# Process the results
@ -119,17 +119,19 @@ class CooldownCache:
keys = [f"deployment:{model_id}:cooldown" for model_id in model_ids]
# Retrieve the values for the keys using mget
results = self.cache.batch_get_cache(keys=keys)
results = self.cache.batch_get_cache(keys=keys) or []
min_cooldown_time = self.default_cooldown_time
min_cooldown_time: Optional[float] = None
# Process the results
for model_id, result in zip(model_ids, results):
if result and isinstance(result, dict):
cooldown_cache_value = CooldownCacheValue(**result) # type: ignore
if cooldown_cache_value["cooldown_time"] < min_cooldown_time:
if min_cooldown_time is None:
min_cooldown_time = cooldown_cache_value["cooldown_time"]
elif cooldown_cache_value["cooldown_time"] < min_cooldown_time:
min_cooldown_time = cooldown_cache_value["cooldown_time"]
return min_cooldown_time
return min_cooldown_time or self.default_cooldown_time
# Usage example: