mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(complexity_router): add fallback to local cache when redis fails
Signed-off-by: Himanshu Joshi <himanshu.zyx7@gmail.com>
This commit is contained in:
parent
5c0138ae5a
commit
abbf811b09
2 changed files with 38 additions and 2 deletions
|
|
@ -3975,10 +3975,18 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
if cache_key is not None and pin_replay_allowed:
|
||||
cache = self.litellm_router_instance.cache
|
||||
pinned_value: Optional[Any] = None
|
||||
if getattr(cache, "redis_cache", None) is not None:
|
||||
pinned_value: Final = await cache.redis_cache.async_get_cache(key=cache_key)
|
||||
try:
|
||||
pinned_value = await cache.redis_cache.async_get_cache(key=cache_key)
|
||||
except Exception as e:
|
||||
verbose_router_logger.debug(
|
||||
"ComplexityRouter: Failed to read session affinity from redis_cache, falling back to cache. Error: %s",
|
||||
e,
|
||||
)
|
||||
pinned_value = await cache.async_get_cache(key=cache_key)
|
||||
else:
|
||||
pinned_value: Final = await cache.async_get_cache(key=cache_key)
|
||||
pinned_value = await cache.async_get_cache(key=cache_key)
|
||||
pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names())
|
||||
if pinned_pin is not None:
|
||||
user_message: Final = _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None
|
||||
|
|
|
|||
|
|
@ -6876,6 +6876,34 @@ class TestSessionAffinity:
|
|||
dual_cache.async_get_cache.assert_not_called()
|
||||
assert response.model == "o1-preview"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_affinity_pin_falls_back_to_local_cache_when_redis_fails(
|
||||
self, mock_router_instance, basic_config
|
||||
):
|
||||
"""When redis_cache raises an exception (e.g. outage or connection error),
|
||||
gracefully fall back to local cache so routing is not interrupted."""
|
||||
redis_cache = AsyncMock()
|
||||
redis_cache.async_get_cache = AsyncMock(side_effect=Exception("Redis connection error"))
|
||||
dual_cache = AsyncMock()
|
||||
dual_cache.redis_cache = redis_cache
|
||||
dual_cache.async_get_cache = AsyncMock(return_value="gpt-4o-mini")
|
||||
mock_router_instance.cache = dual_cache
|
||||
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={
|
||||
**basic_config,
|
||||
"session_affinity": True,
|
||||
},
|
||||
)
|
||||
response = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=self._request_kwargs("session-1"), messages=self.SIMPLE_MESSAGE
|
||||
)
|
||||
redis_cache.async_get_cache.assert_called_once()
|
||||
dual_cache.async_get_cache.assert_called_once()
|
||||
assert response.model == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_api_keys_do_not_share_pin(self, mock_router_instance, session_affinity_config):
|
||||
"""A session_id is client-supplied and unauthenticated; two different callers
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue