From f542011076723ee9250f8f6dfceeac02f5ac256a Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 21 Nov 2025 19:11:17 -0800 Subject: [PATCH] fix: cache cooldown key (#16954) There's no need to generate a key multiple times for the same model, cache it with a max limit. --- litellm/router_utils/cooldown_cache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index e92f114dd52..edbcacca277 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -2,6 +2,7 @@ Wrapper around router cache. Meant to handle model cooldown logic """ +import functools import time from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union @@ -44,7 +45,7 @@ class CooldownCache: ) -> Tuple[str, CooldownCacheValue]: try: current_time = time.time() - cooldown_key = f"deployment:{model_id}:cooldown" + cooldown_key = CooldownCache.get_cooldown_cache_key(model_id) # Store the cooldown information for the deployment separately cooldown_data = CooldownCacheValue( @@ -104,8 +105,9 @@ class CooldownCache: raise e @staticmethod + @functools.lru_cache(maxsize=1024) def get_cooldown_cache_key(model_id: str) -> str: - return f"deployment:{model_id}:cooldown" + return "deployment:" + model_id + ":cooldown" async def async_get_active_cooldowns( self, model_ids: List[str], parent_otel_span: Optional[Span] @@ -140,7 +142,7 @@ class CooldownCache: self, model_ids: List[str], parent_otel_span: Optional[Span] ) -> List[Tuple[str, CooldownCacheValue]]: # Generate the keys for the deployments - keys = [f"deployment:{model_id}:cooldown" for model_id in model_ids] + keys = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids] # Retrieve the values for the keys using mget results = ( self.cache.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span)