mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
feat(prompt-caching): add comments clarifying cache miss and partial match behavior
Add comment explaining cache miss must leave all healthy deployments eligible. Add comment clarifying only exact cached-prefix matches establish deployment affinity while partial matches fall back to normal routing.
This commit is contained in:
parent
17fc935ed6
commit
00a9877a5c
2 changed files with 3 additions and 0 deletions
|
|
@ -98,6 +98,7 @@ class PromptCachingDeploymentCheck(CustomLogger):
|
|||
request_kwargs.get("system") if request_kwargs is not None else None,
|
||||
)
|
||||
|
||||
# A cache miss must leave all healthy deployments eligible.
|
||||
model_id_dict: Final = await prompt_cache.async_get_model_id(
|
||||
messages=affinity_messages,
|
||||
tools=(
|
||||
|
|
|
|||
|
|
@ -217,6 +217,8 @@ class PromptCachingCache:
|
|||
if cacheable_messages is not None and not cacheable_messages:
|
||||
return None
|
||||
|
||||
# Only exact cached-prefix matches can establish deployment affinity.
|
||||
# Partial matches must fall back to normal routing.
|
||||
# Use serialize_object for consistent and stable serialization
|
||||
data_to_hash: Final = {}
|
||||
if cacheable_messages is not None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue