mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(caching): key the response cache on params a deployment forwards into the payload
With forward_payload_fields set, a forwarded param such as context changes the agent's answer, but the cache key skipped provider params unless enable_caching_on_provider_specific_optional_params was on. Two requests with the same messages and different context shared one entry, so the second caller got the first caller's answer Provider params now join the cache key whenever the request carries a non-empty forward_payload_fields
This commit is contained in:
parent
28e9968c81
commit
209602939a
2 changed files with 19 additions and 1 deletions
|
|
@ -417,6 +417,10 @@ class Cache:
|
|||
combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params()
|
||||
is_semantic_cache: Final = self._is_semantic_cache()
|
||||
scope_excluded_params: Final = self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS if is_semantic_cache else frozenset()
|
||||
# A deployment that forwards request params into the provider payload makes their values part of the answer
|
||||
keys_provider_params: Final = litellm.enable_caching_on_provider_specific_optional_params is True or bool(
|
||||
kwargs.get("forward_payload_fields")
|
||||
)
|
||||
for param in kwargs:
|
||||
if param in scope_excluded_params:
|
||||
continue
|
||||
|
|
@ -427,7 +431,7 @@ class Cache:
|
|||
if param_value is not None:
|
||||
cache_key += f"{param}: {param_value}"
|
||||
elif not is_litellm_owned_kwarg(param):
|
||||
if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now
|
||||
if keys_provider_params:
|
||||
if kwargs[param] is None:
|
||||
continue # ignore None params
|
||||
cache_key += f"{param}: {kwargs[param]}"
|
||||
|
|
|
|||
|
|
@ -472,6 +472,20 @@ def test_cache_key_tells_apart_requests_that_differ_only_in_continuation_token()
|
|||
assert cache.get_cache_key(**request, continuation_token="") == first_segment_key
|
||||
|
||||
|
||||
def test_cache_key_tells_apart_requests_that_forward_different_payload_fields() -> None:
|
||||
cache: Final = Cache(type=LiteLLMCacheType.LOCAL)
|
||||
request: Final = {
|
||||
"model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:111111111111:runtime/docs_agent",
|
||||
"messages": [{"role": "user", "content": "Summarize the document"}],
|
||||
"forward_payload_fields": ["context"],
|
||||
}
|
||||
|
||||
first_document_key: Final = cache.get_cache_key(**request, context={"document": "Q3 revenue grew 4 percent."})
|
||||
|
||||
assert cache.get_cache_key(**request, context={"document": "Q4 revenue fell 2 percent."}) != first_document_key
|
||||
assert cache.get_cache_key(**request, context={"document": "Q3 revenue grew 4 percent."}) == first_document_key
|
||||
|
||||
|
||||
def test_cache_key_tells_apart_requests_that_ask_for_token_ids() -> None:
|
||||
cache: Final = Cache(type=LiteLLMCacheType.LOCAL)
|
||||
request: Final = {"model": "hosted_vllm/qwen3", "messages": [{"role": "user", "content": "count"}]}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue