feat(prompt-caching): add comments clarifying cache miss and partial match behavior

Add comment explaining cache miss must leave all healthy deployments eligible. Add comment clarifying only exact cached-prefix matches establish deployment affinity while partial matches fall back to normal routing.
This commit is contained in:
nuernber 2026-09-11 13:31:14 -07:00
parent 17fc935ed6
commit 00a9877a5c
2 changed files with 3 additions and 0 deletions

View file

@ -98,6 +98,7 @@ class PromptCachingDeploymentCheck(CustomLogger):
request_kwargs.get("system") if request_kwargs is not None else None,
)
# A cache miss must leave all healthy deployments eligible.
model_id_dict: Final = await prompt_cache.async_get_model_id(
messages=affinity_messages,
tools=(

View file

@ -217,6 +217,8 @@ class PromptCachingCache:
if cacheable_messages is not None and not cacheable_messages:
return None
# Only exact cached-prefix matches can establish deployment affinity.
# Partial matches must fall back to normal routing.
# Use serialize_object for consistent and stable serialization
data_to_hash: Final = {}
if cacheable_messages is not None: