From 209602939a41fcef0d67f1b7f73fe5202345e193 Mon Sep 17 00:00:00 2001 From: Brendan Smith-Elion Date: Thu, 5 Nov 2026 08:31:11 -0500 Subject: [PATCH] fix(caching): key the response cache on params a deployment forwards into the payload With forward_payload_fields set, a forwarded param such as context changes the agent's answer, but the cache key skipped provider params unless enable_caching_on_provider_specific_optional_params was on. Two requests with the same messages and different context shared one entry, so the second caller got the first caller's answer Provider params now join the cache key whenever the request carries a non-empty forward_payload_fields --- litellm/caching/caching.py | 6 +++++- tests/unit/caching/test_caching.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 3b7ea66ad6b..f8dbc038f58 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -417,6 +417,10 @@ class Cache: combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params() is_semantic_cache: Final = self._is_semantic_cache() scope_excluded_params: Final = self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS if is_semantic_cache else frozenset() + # A deployment that forwards request params into the provider payload makes their values part of the answer + keys_provider_params: Final = litellm.enable_caching_on_provider_specific_optional_params is True or bool( + kwargs.get("forward_payload_fields") + ) for param in kwargs: if param in scope_excluded_params: continue @@ -427,7 +431,7 @@ class Cache: if param_value is not None: cache_key += f"{param}: {param_value}" elif not is_litellm_owned_kwarg(param): - if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now + if keys_provider_params: if kwargs[param] is None: continue # ignore None params cache_key += f"{param}: {kwargs[param]}" diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 0911d5ef888..48543933a62 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -472,6 +472,20 @@ def test_cache_key_tells_apart_requests_that_differ_only_in_continuation_token() assert cache.get_cache_key(**request, continuation_token="") == first_segment_key +def test_cache_key_tells_apart_requests_that_forward_different_payload_fields() -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL) + request: Final = { + "model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:111111111111:runtime/docs_agent", + "messages": [{"role": "user", "content": "Summarize the document"}], + "forward_payload_fields": ["context"], + } + + first_document_key: Final = cache.get_cache_key(**request, context={"document": "Q3 revenue grew 4 percent."}) + + assert cache.get_cache_key(**request, context={"document": "Q4 revenue fell 2 percent."}) != first_document_key + assert cache.get_cache_key(**request, context={"document": "Q3 revenue grew 4 percent."}) == first_document_key + + def test_cache_key_tells_apart_requests_that_ask_for_token_ids() -> None: cache: Final = Cache(type=LiteLLMCacheType.LOCAL) request: Final = {"model": "hosted_vllm/qwen3", "messages": [{"role": "user", "content": "count"}]}