From 00a9877a5c0272f5276b543da953851d8718a7a0 Mon Sep 17 00:00:00 2001 From: nuernber Date: Fri, 11 Sep 2026 13:31:14 -0700 Subject: [PATCH] feat(prompt-caching): add comments clarifying cache miss and partial match behavior Add comment explaining cache miss must leave all healthy deployments eligible. Add comment clarifying only exact cached-prefix matches establish deployment affinity while partial matches fall back to normal routing. --- .../pre_call_checks/prompt_caching_deployment_check.py | 1 + litellm/router_utils/prompt_caching_cache.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index da74788c92b..514fbd18e8c 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -98,6 +98,7 @@ class PromptCachingDeploymentCheck(CustomLogger): request_kwargs.get("system") if request_kwargs is not None else None, ) + # A cache miss must leave all healthy deployments eligible. model_id_dict: Final = await prompt_cache.async_get_model_id( messages=affinity_messages, tools=( diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index e8ef3d0d3fd..5ef5f050deb 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -217,6 +217,8 @@ class PromptCachingCache: if cacheable_messages is not None and not cacheable_messages: return None + # Only exact cached-prefix matches can establish deployment affinity. + # Partial matches must fall back to normal routing. # Use serialize_object for consistent and stable serialization data_to_hash: Final = {} if cacheable_messages is not None: