fix(complexity_router): read session affinity pin from redis to avoid stale memory cache (#41225)

This commit is contained in:
Himanshu Joshi 2026-09-16 21:33:27 +05:30
parent 9cd787386e
commit 5c0138ae5a
2 changed files with 33 additions and 1 deletions

View file

@ -3974,7 +3974,11 @@ class ComplexityRouter(CustomLogger):
)
if cache_key is not None and pin_replay_allowed:
pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
cache = self.litellm_router_instance.cache
if getattr(cache, "redis_cache", None) is not None:
pinned_value: Final = await cache.redis_cache.async_get_cache(key=cache_key)
else:
pinned_value: Final = await cache.async_get_cache(key=cache_key)
pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names())
if pinned_pin is not None:
user_message: Final = _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None

View file

@ -6848,6 +6848,34 @@ class TestSessionAffinity:
assert call_kwargs["value"] == {"model": "o1-preview", "tier": "REASONING"}
assert call_kwargs["ttl"] == 90
@pytest.mark.asyncio
async def test_session_affinity_pin_reads_from_redis_cache_when_available(
self, mock_router_instance, basic_config
):
"""Regression (#41225): In multi-replica deployments, ComplexityRouter must read
session affinity pin directly from redis_cache to prevent in-memory cache from serving stale pin."""
redis_cache = AsyncMock()
redis_cache.async_get_cache = AsyncMock(return_value="o1-preview")
dual_cache = AsyncMock()
dual_cache.redis_cache = redis_cache
dual_cache.async_get_cache = AsyncMock(return_value="gpt-4o-mini")
mock_router_instance.cache = dual_cache
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={
**basic_config,
"session_affinity": True,
},
)
response = await router.async_pre_routing_hook(
model="test-model", request_kwargs=self._request_kwargs("session-1"), messages=self.SIMPLE_MESSAGE
)
redis_cache.async_get_cache.assert_called_once()
dual_cache.async_get_cache.assert_not_called()
assert response.model == "o1-preview"
@pytest.mark.asyncio
async def test_different_api_keys_do_not_share_pin(self, mock_router_instance, session_affinity_config):
"""A session_id is client-supplied and unauthenticated; two different callers