feat: allow prompt cache key as affinity identity

This commit is contained in:
Vlad Andreev 2026-09-17 00:11:21 +03:00
parent b3898dfd85
commit 7132ca2ed9
3 changed files with 110 additions and 20 deletions

View file

@ -3868,32 +3868,28 @@ class ComplexityRouter(CustomLogger):
if isinstance(metadata := request_kwargs.get(metadata_key), dict)
]
@staticmethod
def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
"""Resolve a client-supplied session_id."""
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
def _get_session_id_from_request_kwargs(self, request_kwargs: dict) -> str | None:
"""Resolve explicit session identity, then the opt-in prompt-cache fallback."""
for metadata in self._iter_metadata_dicts(request_kwargs):
session_id = metadata.get("session_id")
if session_id is not None and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY):
return str(session_id)
return None
@staticmethod
def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
"""Resolve the proxy-derived API key hash, the same trust boundary
DeploymentAffinityCheck uses for its own key-based affinity (not the
client-supplied OpenAI `user` param, which isn't authenticated)."""
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
user_key = metadata.get("user_api_key_hash")
if user_key is not None:
return str(user_key)
if self.config.prompt_cache_key_as_session_id:
prompt_cache_key = request_kwargs.get("prompt_cache_key")
if isinstance(prompt_cache_key, str) and 0 < len(prompt_cache_key) <= 64:
# Namespaced, not hashed: prompt_cache_key is an opaque caller-chosen
# bucket label, not a credential, and explicit session_id is likewise
# used verbatim. The prefix keeps the two identity sources from
# colliding in the affinity cache keyspace.
return f"prompt-cache:{prompt_cache_key}"
return None
def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str:
# Namespace by the caller's API key hash so two different callers reusing the
# same client-supplied session_id can't poison each other's routing pin. Falls
# back to "unscoped" only when there's no authenticated caller to scope by
# (e.g. direct Router usage without the proxy layer).
caller_scope: Final = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped"
# Namespace by the proxy-derived caller identity so two callers reusing the same
# session identity cannot poison each other's routing pin. The shared resolver
# uses the API-key hash for virtual keys and the authenticated user ID for JWTs;
# it deliberately ignores the client-supplied OpenAI `user` parameter.
caller_scope: Final = DeploymentAffinityCheck.get_user_key_from_request_kwargs(request_kwargs) or "unscoped"
return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}"
@property

View file

@ -1336,6 +1336,18 @@ class ComplexityRouterConfig(BaseModel):
),
)
prompt_cache_key_as_session_id: bool = Field(
default=False,
description=(
"Use the OpenAI-compatible prompt_cache_key as a fallback session identity when no explicit, "
"client-supplied session_id is available. Disabled by default because a prompt-cache bucket is "
"not universally a conversation identifier. Explicit session metadata/headers always win. The "
"fallback accepts non-empty values up to OpenAI's 64-character limit and namespaces them under a "
"'prompt-cache:' prefix in router cache keys. This enables session_affinity and classification_mode='user_turn' for "
"clients that expose a stable conversation key only through the standard prompt_cache_key field."
),
)
# Session affinity: pin the first turn's routed model for the rest of the session
session_affinity: bool = Field(
default=False,

View file

@ -6873,6 +6873,38 @@ class TestSessionAffinity:
)
assert result_for_b.model == "gpt-4o-mini"
@pytest.mark.asyncio
async def test_prompt_cache_key_does_not_share_pin_between_jwt_callers(
self, mock_router_instance, session_affinity_config
):
mock_router_instance.cache = DualCache()
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={
**session_affinity_config,
"prompt_cache_key_as_session_id": True,
},
)
caller_a_kwargs = {
"prompt_cache_key": "shared-prompt-bucket",
"metadata": {"user_api_key_user_id": "jwt-user-a"},
}
caller_b_kwargs = {
"prompt_cache_key": "shared-prompt-bucket",
"metadata": {"user_api_key_user_id": "jwt-user-b"},
}
pinned_for_a = await router.async_pre_routing_hook(
model="test-model", request_kwargs=caller_a_kwargs, messages=self.REASONING_MESSAGE
)
assert pinned_for_a.model == "o1-preview"
result_for_b = await router.async_pre_routing_hook(
model="test-model", request_kwargs=caller_b_kwargs, messages=self.SIMPLE_MESSAGE
)
assert result_for_b.model == "gpt-4o-mini"
@pytest.mark.asyncio
async def test_no_session_id_falls_back_to_reclassify(self, mock_router_instance, session_affinity_config):
cache = AsyncMock()
@ -7046,6 +7078,56 @@ class TestClassificationMode:
assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
@pytest.mark.asyncio
async def test_prompt_cache_key_opt_in_holds_tool_loop_decision(self, mock_router_instance, user_turn_config):
"""An OpenAI-compatible client may expose only its stable prompt-cache bucket. The
explicit opt-in lets that key hold a continuation decision without trusting generated IDs."""
router = self._router(
mock_router_instance,
{**user_turn_config, "prompt_cache_key_as_session_id": True},
)
request_kwargs = {
"prompt_cache_key": "opaque-thread-123",
"metadata": {
"session_id": "per-request-generated-id",
SESSION_ID_GENERATED_METADATA_KEY: True,
},
}
with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
responses = [
await router.async_pre_routing_hook(
model="test-model", request_kwargs=deepcopy(request_kwargs), messages=turn
)
for turn in self._tool_loop_turns()
]
assert spy.call_count == 1
assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
def test_prompt_cache_key_affinity_is_explicit_bounded_and_lower_precedence(
self, mock_router_instance, user_turn_config
):
disabled = self._router(mock_router_instance, user_turn_config)
assert disabled._get_session_id_from_request_kwargs({"prompt_cache_key": "opaque-thread-123"}) is None
enabled = self._router(
mock_router_instance,
{**user_turn_config, "prompt_cache_key_as_session_id": True},
)
fallback = enabled._get_session_id_from_request_kwargs({"prompt_cache_key": "opaque-thread-123"})
assert fallback == "prompt-cache:opaque-thread-123"
assert enabled._get_session_id_from_request_kwargs({"session_id": "opaque-thread-123"}) is None
assert enabled._get_session_id_from_request_kwargs({"prompt_cache_key": ""}) is None
assert enabled._get_session_id_from_request_kwargs({"prompt_cache_key": "x" * 65}) is None
assert (
enabled._get_session_id_from_request_kwargs(
{
"prompt_cache_key": "opaque-thread-123",
"metadata": {"session_id": "explicit-session"},
}
)
== "explicit-session"
)
@pytest.mark.asyncio
async def test_plugins_suppress_user_turn_gate(self, mock_router_instance, basic_config):
"""A replayed decision would bypass the plugin pipeline, so plugins force every request