From 7132ca2ed91e3916ada170bfe5cee93b087df934 Mon Sep 17 00:00:00 2001 From: Vlad Andreev <15246286+ukw2d@users.noreply.github.com> Date: Thu, 17 Sep 2026 00:11:21 +0300 Subject: [PATCH] feat: allow prompt cache key as affinity identity --- .../complexity_router/complexity_router.py | 36 ++++---- .../complexity_router/config.py | 12 +++ .../router_strategy/test_complexity_router.py | 82 +++++++++++++++++++ 3 files changed, 110 insertions(+), 20 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d19cdfaa899..01110812b52 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3868,32 +3868,28 @@ class ComplexityRouter(CustomLogger): if isinstance(metadata := request_kwargs.get(metadata_key), dict) ] - @staticmethod - def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None: - """Resolve a client-supplied session_id.""" - for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs): + def _get_session_id_from_request_kwargs(self, request_kwargs: dict) -> str | None: + """Resolve explicit session identity, then the opt-in prompt-cache fallback.""" + for metadata in self._iter_metadata_dicts(request_kwargs): session_id = metadata.get("session_id") if session_id is not None and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return str(session_id) - return None - - @staticmethod - def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None: - """Resolve the proxy-derived API key hash, the same trust boundary - DeploymentAffinityCheck uses for its own key-based affinity (not the - client-supplied OpenAI `user` param, which isn't authenticated).""" - for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs): - user_key = metadata.get("user_api_key_hash") - if user_key is not None: - return str(user_key) + if self.config.prompt_cache_key_as_session_id: + prompt_cache_key = request_kwargs.get("prompt_cache_key") + if isinstance(prompt_cache_key, str) and 0 < len(prompt_cache_key) <= 64: + # Namespaced, not hashed: prompt_cache_key is an opaque caller-chosen + # bucket label, not a credential, and explicit session_id is likewise + # used verbatim. The prefix keeps the two identity sources from + # colliding in the affinity cache keyspace. + return f"prompt-cache:{prompt_cache_key}" return None def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str: - # Namespace by the caller's API key hash so two different callers reusing the - # same client-supplied session_id can't poison each other's routing pin. Falls - # back to "unscoped" only when there's no authenticated caller to scope by - # (e.g. direct Router usage without the proxy layer). - caller_scope: Final = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped" + # Namespace by the proxy-derived caller identity so two callers reusing the same + # session identity cannot poison each other's routing pin. The shared resolver + # uses the API-key hash for virtual keys and the authenticated user ID for JWTs; + # it deliberately ignores the client-supplied OpenAI `user` parameter. + caller_scope: Final = DeploymentAffinityCheck.get_user_key_from_request_kwargs(request_kwargs) or "unscoped" return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}" @property diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 370589d7da4..7bad53b2562 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1336,6 +1336,18 @@ class ComplexityRouterConfig(BaseModel): ), ) + prompt_cache_key_as_session_id: bool = Field( + default=False, + description=( + "Use the OpenAI-compatible prompt_cache_key as a fallback session identity when no explicit, " + "client-supplied session_id is available. Disabled by default because a prompt-cache bucket is " + "not universally a conversation identifier. Explicit session metadata/headers always win. The " + "fallback accepts non-empty values up to OpenAI's 64-character limit and namespaces them under a " + "'prompt-cache:' prefix in router cache keys. This enables session_affinity and classification_mode='user_turn' for " + "clients that expose a stable conversation key only through the standard prompt_cache_key field." + ), + ) + # Session affinity: pin the first turn's routed model for the rest of the session session_affinity: bool = Field( default=False, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 9874028fc62..f6070b9264c 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6873,6 +6873,38 @@ class TestSessionAffinity: ) assert result_for_b.model == "gpt-4o-mini" + @pytest.mark.asyncio + async def test_prompt_cache_key_does_not_share_pin_between_jwt_callers( + self, mock_router_instance, session_affinity_config + ): + mock_router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **session_affinity_config, + "prompt_cache_key_as_session_id": True, + }, + ) + caller_a_kwargs = { + "prompt_cache_key": "shared-prompt-bucket", + "metadata": {"user_api_key_user_id": "jwt-user-a"}, + } + caller_b_kwargs = { + "prompt_cache_key": "shared-prompt-bucket", + "metadata": {"user_api_key_user_id": "jwt-user-b"}, + } + + pinned_for_a = await router.async_pre_routing_hook( + model="test-model", request_kwargs=caller_a_kwargs, messages=self.REASONING_MESSAGE + ) + assert pinned_for_a.model == "o1-preview" + + result_for_b = await router.async_pre_routing_hook( + model="test-model", request_kwargs=caller_b_kwargs, messages=self.SIMPLE_MESSAGE + ) + assert result_for_b.model == "gpt-4o-mini" + @pytest.mark.asyncio async def test_no_session_id_falls_back_to_reclassify(self, mock_router_instance, session_affinity_config): cache = AsyncMock() @@ -7046,6 +7078,56 @@ class TestClassificationMode: assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"] assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses) + @pytest.mark.asyncio + async def test_prompt_cache_key_opt_in_holds_tool_loop_decision(self, mock_router_instance, user_turn_config): + """An OpenAI-compatible client may expose only its stable prompt-cache bucket. The + explicit opt-in lets that key hold a continuation decision without trusting generated IDs.""" + router = self._router( + mock_router_instance, + {**user_turn_config, "prompt_cache_key_as_session_id": True}, + ) + request_kwargs = { + "prompt_cache_key": "opaque-thread-123", + "metadata": { + "session_id": "per-request-generated-id", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + } + with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy: + responses = [ + await router.async_pre_routing_hook( + model="test-model", request_kwargs=deepcopy(request_kwargs), messages=turn + ) + for turn in self._tool_loop_turns() + ] + assert spy.call_count == 1 + assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"] + + def test_prompt_cache_key_affinity_is_explicit_bounded_and_lower_precedence( + self, mock_router_instance, user_turn_config + ): + disabled = self._router(mock_router_instance, user_turn_config) + assert disabled._get_session_id_from_request_kwargs({"prompt_cache_key": "opaque-thread-123"}) is None + + enabled = self._router( + mock_router_instance, + {**user_turn_config, "prompt_cache_key_as_session_id": True}, + ) + fallback = enabled._get_session_id_from_request_kwargs({"prompt_cache_key": "opaque-thread-123"}) + assert fallback == "prompt-cache:opaque-thread-123" + assert enabled._get_session_id_from_request_kwargs({"session_id": "opaque-thread-123"}) is None + assert enabled._get_session_id_from_request_kwargs({"prompt_cache_key": ""}) is None + assert enabled._get_session_id_from_request_kwargs({"prompt_cache_key": "x" * 65}) is None + assert ( + enabled._get_session_id_from_request_kwargs( + { + "prompt_cache_key": "opaque-thread-123", + "metadata": {"session_id": "explicit-session"}, + } + ) + == "explicit-session" + ) + @pytest.mark.asyncio async def test_plugins_suppress_user_turn_gate(self, mock_router_instance, basic_config): """A replayed decision would bypass the plugin pipeline, so plugins force every request