scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

fix(base_routing_strategy.py): catch the delta change during redis sync

Browse source 
ensures values are kept in sync
This commit is contained in:
Krrish Dholakia 2025-05-30 23:50:15 -07:00
parent 4a9908eaba
commit 732dffa6b1
1 changed files with 7 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
litellm/router_strategy/base_routing_strategy.py
View file

@ -180,15 +180,9 @@ class BaseRoutingStrategy(ABC):
# 2. Fetch all current provider spend from Redis to update in-memory cache
pattern = self.get_key_pattern_to_sync()
cache_keys: Optional[Union[Set[str], List[str]]] = None
if pattern:
cache_keys = await self.dual_cache.redis_cache.async_scan_iter(
pattern=pattern
)
if cache_keys is None:
cache_keys = (
self.get_in_memory_keys_to_update()
) # if no pattern OR redis cache does not support scan_iter, use in-memory keys
cache_keys = (
self.get_in_memory_keys_to_update()
) # if no pattern OR redis cache does not support scan_iter, use in-memory keys
if isinstance(cache_keys, set):
cache_keys_list = list(cache_keys)
@ -226,27 +220,15 @@ class BaseRoutingStrategy(ABC):
before = float(in_memory_before_dict.get(key, 0) or 0)
after = float(in_memory_after_dict.get(key, 0) or 0)
delta = after - before
if delta > 0:
await self._increment_value_in_current_window(
key=key, value=delta, ttl=60
)
merged = redis_val + delta
await self.dual_cache.in_memory_cache.async_set_cache(
key=key, value=merged
)
# # Batch fetch current spend values from Redis
# redis_values = await self.dual_cache.redis_cache.async_batch_get_cache(
# key_list=cache_keys_list
# )
# # Update in-memory cache with Redis values
# if isinstance(redis_values, dict): # Check if redis_values is a dictionary
# for key, value in redis_values.items():
# if value is not None:
# await self.dual_cache.in_memory_cache.async_set_cache(
# key=key, value=float(value)
# )
# # verbose_router_logger.debug(
# # f"Updated in-memory cache for {key}: {value}"
# # )
self.reset_in_memory_keys_to_update()
except Exception as e:
verbose_router_logger.exception(