mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat: allow prompt cache key as affinity identity
This commit is contained in:
parent
b3898dfd85
commit
7132ca2ed9
3 changed files with 110 additions and 20 deletions
|
|
@ -3868,32 +3868,28 @@ class ComplexityRouter(CustomLogger):
|
|||
if isinstance(metadata := request_kwargs.get(metadata_key), dict)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
|
||||
"""Resolve a client-supplied session_id."""
|
||||
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
|
||||
def _get_session_id_from_request_kwargs(self, request_kwargs: dict) -> str | None:
|
||||
"""Resolve explicit session identity, then the opt-in prompt-cache fallback."""
|
||||
for metadata in self._iter_metadata_dicts(request_kwargs):
|
||||
session_id = metadata.get("session_id")
|
||||
if session_id is not None and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY):
|
||||
return str(session_id)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
|
||||
"""Resolve the proxy-derived API key hash, the same trust boundary
|
||||
DeploymentAffinityCheck uses for its own key-based affinity (not the
|
||||
client-supplied OpenAI `user` param, which isn't authenticated)."""
|
||||
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
|
||||
user_key = metadata.get("user_api_key_hash")
|
||||
if user_key is not None:
|
||||
return str(user_key)
|
||||
if self.config.prompt_cache_key_as_session_id:
|
||||
prompt_cache_key = request_kwargs.get("prompt_cache_key")
|
||||
if isinstance(prompt_cache_key, str) and 0 < len(prompt_cache_key) <= 64:
|
||||
# Namespaced, not hashed: prompt_cache_key is an opaque caller-chosen
|
||||
# bucket label, not a credential, and explicit session_id is likewise
|
||||
# used verbatim. The prefix keeps the two identity sources from
|
||||
# colliding in the affinity cache keyspace.
|
||||
return f"prompt-cache:{prompt_cache_key}"
|
||||
return None
|
||||
|
||||
def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str:
|
||||
# Namespace by the caller's API key hash so two different callers reusing the
|
||||
# same client-supplied session_id can't poison each other's routing pin. Falls
|
||||
# back to "unscoped" only when there's no authenticated caller to scope by
|
||||
# (e.g. direct Router usage without the proxy layer).
|
||||
caller_scope: Final = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped"
|
||||
# Namespace by the proxy-derived caller identity so two callers reusing the same
|
||||
# session identity cannot poison each other's routing pin. The shared resolver
|
||||
# uses the API-key hash for virtual keys and the authenticated user ID for JWTs;
|
||||
# it deliberately ignores the client-supplied OpenAI `user` parameter.
|
||||
caller_scope: Final = DeploymentAffinityCheck.get_user_key_from_request_kwargs(request_kwargs) or "unscoped"
|
||||
return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}"
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -1336,6 +1336,18 @@ class ComplexityRouterConfig(BaseModel):
|
|||
),
|
||||
)
|
||||
|
||||
prompt_cache_key_as_session_id: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Use the OpenAI-compatible prompt_cache_key as a fallback session identity when no explicit, "
|
||||
"client-supplied session_id is available. Disabled by default because a prompt-cache bucket is "
|
||||
"not universally a conversation identifier. Explicit session metadata/headers always win. The "
|
||||
"fallback accepts non-empty values up to OpenAI's 64-character limit and namespaces them under a "
|
||||
"'prompt-cache:' prefix in router cache keys. This enables session_affinity and classification_mode='user_turn' for "
|
||||
"clients that expose a stable conversation key only through the standard prompt_cache_key field."
|
||||
),
|
||||
)
|
||||
|
||||
# Session affinity: pin the first turn's routed model for the rest of the session
|
||||
session_affinity: bool = Field(
|
||||
default=False,
|
||||
|
|
|
|||
|
|
@ -6873,6 +6873,38 @@ class TestSessionAffinity:
|
|||
)
|
||||
assert result_for_b.model == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_cache_key_does_not_share_pin_between_jwt_callers(
|
||||
self, mock_router_instance, session_affinity_config
|
||||
):
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={
|
||||
**session_affinity_config,
|
||||
"prompt_cache_key_as_session_id": True,
|
||||
},
|
||||
)
|
||||
caller_a_kwargs = {
|
||||
"prompt_cache_key": "shared-prompt-bucket",
|
||||
"metadata": {"user_api_key_user_id": "jwt-user-a"},
|
||||
}
|
||||
caller_b_kwargs = {
|
||||
"prompt_cache_key": "shared-prompt-bucket",
|
||||
"metadata": {"user_api_key_user_id": "jwt-user-b"},
|
||||
}
|
||||
|
||||
pinned_for_a = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=caller_a_kwargs, messages=self.REASONING_MESSAGE
|
||||
)
|
||||
assert pinned_for_a.model == "o1-preview"
|
||||
|
||||
result_for_b = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=caller_b_kwargs, messages=self.SIMPLE_MESSAGE
|
||||
)
|
||||
assert result_for_b.model == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_session_id_falls_back_to_reclassify(self, mock_router_instance, session_affinity_config):
|
||||
cache = AsyncMock()
|
||||
|
|
@ -7046,6 +7078,56 @@ class TestClassificationMode:
|
|||
assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
|
||||
assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_cache_key_opt_in_holds_tool_loop_decision(self, mock_router_instance, user_turn_config):
|
||||
"""An OpenAI-compatible client may expose only its stable prompt-cache bucket. The
|
||||
explicit opt-in lets that key hold a continuation decision without trusting generated IDs."""
|
||||
router = self._router(
|
||||
mock_router_instance,
|
||||
{**user_turn_config, "prompt_cache_key_as_session_id": True},
|
||||
)
|
||||
request_kwargs = {
|
||||
"prompt_cache_key": "opaque-thread-123",
|
||||
"metadata": {
|
||||
"session_id": "per-request-generated-id",
|
||||
SESSION_ID_GENERATED_METADATA_KEY: True,
|
||||
},
|
||||
}
|
||||
with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
|
||||
responses = [
|
||||
await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=deepcopy(request_kwargs), messages=turn
|
||||
)
|
||||
for turn in self._tool_loop_turns()
|
||||
]
|
||||
assert spy.call_count == 1
|
||||
assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
|
||||
|
||||
def test_prompt_cache_key_affinity_is_explicit_bounded_and_lower_precedence(
|
||||
self, mock_router_instance, user_turn_config
|
||||
):
|
||||
disabled = self._router(mock_router_instance, user_turn_config)
|
||||
assert disabled._get_session_id_from_request_kwargs({"prompt_cache_key": "opaque-thread-123"}) is None
|
||||
|
||||
enabled = self._router(
|
||||
mock_router_instance,
|
||||
{**user_turn_config, "prompt_cache_key_as_session_id": True},
|
||||
)
|
||||
fallback = enabled._get_session_id_from_request_kwargs({"prompt_cache_key": "opaque-thread-123"})
|
||||
assert fallback == "prompt-cache:opaque-thread-123"
|
||||
assert enabled._get_session_id_from_request_kwargs({"session_id": "opaque-thread-123"}) is None
|
||||
assert enabled._get_session_id_from_request_kwargs({"prompt_cache_key": ""}) is None
|
||||
assert enabled._get_session_id_from_request_kwargs({"prompt_cache_key": "x" * 65}) is None
|
||||
assert (
|
||||
enabled._get_session_id_from_request_kwargs(
|
||||
{
|
||||
"prompt_cache_key": "opaque-thread-123",
|
||||
"metadata": {"session_id": "explicit-session"},
|
||||
}
|
||||
)
|
||||
== "explicit-session"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugins_suppress_user_turn_gate(self, mock_router_instance, basic_config):
|
||||
"""A replayed decision would bypass the plugin pipeline, so plugins force every request
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue