scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

fix(base_routing_strategy.py): fix merge from redis - preserve values in in-memory cache during gap b/w push to redis and read from redis

Browse source
This commit is contained in:
Krrish Dholakia 2025-05-30 18:15:48 -07:00
parent 0f363f7b48
commit 4a9908eaba
1 changed files with 47 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
litellm/router_strategy/base_routing_strategy.py
View file

@ -195,21 +195,57 @@ class BaseRoutingStrategy(ABC):
else:
cache_keys_list = cache_keys
# Batch fetch current spend values from Redis
# 1. Snapshot in-memory before
in_memory_before_dict = {}
in_memory_before = (
await self.dual_cache.in_memory_cache.async_batch_get_cache(
keys=cache_keys_list
)
)
for k, v in zip(cache_keys_list, in_memory_before):
in_memory_before_dict[k] = v
# 2. Fetch from Redis
redis_values = await self.dual_cache.redis_cache.async_batch_get_cache(
key_list=cache_keys_list
)
# Update in-memory cache with Redis values
if isinstance(redis_values, dict): # Check if redis_values is a dictionary
for key, value in redis_values.items():
if value is not None:
await self.dual_cache.in_memory_cache.async_set_cache(
key=key, value=float(value)
)
# verbose_router_logger.debug(
# f"Updated in-memory cache for {key}: {value}"
# )
# 3. Snapshot in-memory after
in_memory_after = (
await self.dual_cache.in_memory_cache.async_batch_get_cache(
keys=cache_keys_list
)
)
in_memory_after_dict = {}
for k, v in zip(cache_keys_list, in_memory_after):
in_memory_after_dict[k] = v
# 4. Merge
for key in cache_keys_list:
redis_val = float(redis_values.get(key, 0) or 0)
before = float(in_memory_before_dict.get(key, 0) or 0)
after = float(in_memory_after_dict.get(key, 0) or 0)
delta = after - before
merged = redis_val + delta
await self.dual_cache.in_memory_cache.async_set_cache(
key=key, value=merged
)
# # Batch fetch current spend values from Redis
# redis_values = await self.dual_cache.redis_cache.async_batch_get_cache(
# key_list=cache_keys_list
# )
# # Update in-memory cache with Redis values
# if isinstance(redis_values, dict): # Check if redis_values is a dictionary
# for key, value in redis_values.items():
# if value is not None:
# await self.dual_cache.in_memory_cache.async_set_cache(
# key=key, value=float(value)
# )
# # verbose_router_logger.debug(
# # f"Updated in-memory cache for {key}: {value}"
# # )
self.reset_in_memory_keys_to_update()
except Exception as e: