From 732dffa6b19e944923b1ea7816aa91224d8647e2 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 30 May 2025 23:50:15 -0700 Subject: [PATCH] fix(base_routing_strategy.py): catch the delta change during redis sync ensures values are kept in sync --- .../router_strategy/base_routing_strategy.py | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/litellm/router_strategy/base_routing_strategy.py b/litellm/router_strategy/base_routing_strategy.py index a20f7bb6508..997288813d9 100644 --- a/litellm/router_strategy/base_routing_strategy.py +++ b/litellm/router_strategy/base_routing_strategy.py @@ -180,15 +180,9 @@ class BaseRoutingStrategy(ABC): # 2. Fetch all current provider spend from Redis to update in-memory cache pattern = self.get_key_pattern_to_sync() cache_keys: Optional[Union[Set[str], List[str]]] = None - if pattern: - cache_keys = await self.dual_cache.redis_cache.async_scan_iter( - pattern=pattern - ) - - if cache_keys is None: - cache_keys = ( - self.get_in_memory_keys_to_update() - ) # if no pattern OR redis cache does not support scan_iter, use in-memory keys + cache_keys = ( + self.get_in_memory_keys_to_update() + ) # if no pattern OR redis cache does not support scan_iter, use in-memory keys if isinstance(cache_keys, set): cache_keys_list = list(cache_keys) @@ -226,27 +220,15 @@ class BaseRoutingStrategy(ABC): before = float(in_memory_before_dict.get(key, 0) or 0) after = float(in_memory_after_dict.get(key, 0) or 0) delta = after - before + if delta > 0: + await self._increment_value_in_current_window( + key=key, value=delta, ttl=60 + ) merged = redis_val + delta await self.dual_cache.in_memory_cache.async_set_cache( key=key, value=merged ) - # # Batch fetch current spend values from Redis - # redis_values = await self.dual_cache.redis_cache.async_batch_get_cache( - # key_list=cache_keys_list - # ) - - # # Update in-memory cache with Redis values - # if isinstance(redis_values, dict): # Check if redis_values is a dictionary - # for key, value in redis_values.items(): - # if value is not None: - # await self.dual_cache.in_memory_cache.async_set_cache( - # key=key, value=float(value) - # ) - # # verbose_router_logger.debug( - # # f"Updated in-memory cache for {key}: {value}" - # # ) - self.reset_in_memory_keys_to_update() except Exception as e: verbose_router_logger.exception(