From 5c0138ae5a58ce705d03d943defff7aef97ca03e Mon Sep 17 00:00:00 2001 From: Himanshu Joshi Date: Wed, 16 Sep 2026 21:33:27 +0530 Subject: [PATCH] fix(complexity_router): read session affinity pin from redis to avoid stale memory cache (#41225) --- .../complexity_router/complexity_router.py | 6 +++- .../router_strategy/test_complexity_router.py | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d19cdfaa899..71facd6533b 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3974,7 +3974,11 @@ class ComplexityRouter(CustomLogger): ) if cache_key is not None and pin_replay_allowed: - pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key) + cache = self.litellm_router_instance.cache + if getattr(cache, "redis_cache", None) is not None: + pinned_value: Final = await cache.redis_cache.async_get_cache(key=cache_key) + else: + pinned_value: Final = await cache.async_get_cache(key=cache_key) pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names()) if pinned_pin is not None: user_message: Final = _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 9874028fc62..1e2411f21c4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6848,6 +6848,34 @@ class TestSessionAffinity: assert call_kwargs["value"] == {"model": "o1-preview", "tier": "REASONING"} assert call_kwargs["ttl"] == 90 + @pytest.mark.asyncio + async def test_session_affinity_pin_reads_from_redis_cache_when_available( + self, mock_router_instance, basic_config + ): + """Regression (#41225): In multi-replica deployments, ComplexityRouter must read + session affinity pin directly from redis_cache to prevent in-memory cache from serving stale pin.""" + redis_cache = AsyncMock() + redis_cache.async_get_cache = AsyncMock(return_value="o1-preview") + dual_cache = AsyncMock() + dual_cache.redis_cache = redis_cache + dual_cache.async_get_cache = AsyncMock(return_value="gpt-4o-mini") + mock_router_instance.cache = dual_cache + + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **basic_config, + "session_affinity": True, + }, + ) + response = await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("session-1"), messages=self.SIMPLE_MESSAGE + ) + redis_cache.async_get_cache.assert_called_once() + dual_cache.async_get_cache.assert_not_called() + assert response.model == "o1-preview" + @pytest.mark.asyncio async def test_different_api_keys_do_not_share_pin(self, mock_router_instance, session_affinity_config): """A session_id is client-supplied and unauthenticated; two different callers