fix(caching): key the response cache on params a deployment forwards into the payload

With forward_payload_fields set, a forwarded param such as context changes the
agent's answer, but the cache key skipped provider params unless
enable_caching_on_provider_specific_optional_params was on. Two requests with
the same messages and different context shared one entry, so the second caller
got the first caller's answer

Provider params now join the cache key whenever the request carries a
non-empty forward_payload_fields
This commit is contained in:
Brendan Smith-Elion 2026-11-05 08:31:11 -05:00
parent 28e9968c81
commit 209602939a
2 changed files with 19 additions and 1 deletions

View file

@ -417,6 +417,10 @@ class Cache:
combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params()
is_semantic_cache: Final = self._is_semantic_cache()
scope_excluded_params: Final = self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS if is_semantic_cache else frozenset()
# A deployment that forwards request params into the provider payload makes their values part of the answer
keys_provider_params: Final = litellm.enable_caching_on_provider_specific_optional_params is True or bool(
kwargs.get("forward_payload_fields")
)
for param in kwargs:
if param in scope_excluded_params:
continue
@ -427,7 +431,7 @@ class Cache:
if param_value is not None:
cache_key += f"{param}: {param_value}"
elif not is_litellm_owned_kwarg(param):
if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now
if keys_provider_params:
if kwargs[param] is None:
continue # ignore None params
cache_key += f"{param}: {kwargs[param]}"

View file

@ -472,6 +472,20 @@ def test_cache_key_tells_apart_requests_that_differ_only_in_continuation_token()
assert cache.get_cache_key(**request, continuation_token="") == first_segment_key
def test_cache_key_tells_apart_requests_that_forward_different_payload_fields() -> None:
cache: Final = Cache(type=LiteLLMCacheType.LOCAL)
request: Final = {
"model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:111111111111:runtime/docs_agent",
"messages": [{"role": "user", "content": "Summarize the document"}],
"forward_payload_fields": ["context"],
}
first_document_key: Final = cache.get_cache_key(**request, context={"document": "Q3 revenue grew 4 percent."})
assert cache.get_cache_key(**request, context={"document": "Q4 revenue fell 2 percent."}) != first_document_key
assert cache.get_cache_key(**request, context={"document": "Q3 revenue grew 4 percent."}) == first_document_key
def test_cache_key_tells_apart_requests_that_ask_for_token_ids() -> None:
cache: Final = Cache(type=LiteLLMCacheType.LOCAL)
request: Final = {"model": "hosted_vllm/qwen3", "messages": [{"role": "user", "content": "count"}]}