From b5005f8cb6bbb20f1ae8a3bc7bca370b99d09cdd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:23:23 -0700 Subject: [PATCH] fix(proxy): merge a key's cache controls into the request's cache settings instead of replacing them --- litellm/proxy/litellm_pre_call_utils.py | 13 +++-- .../unit/proxy/test_litellm_pre_call_utils.py | 57 +++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 14d941ff62c..5a310542150 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2125,12 +2125,13 @@ class LiteLLMProxyRequestSetup: def add_key_level_controls(key_metadata: dict | None, data: dict, _metadata_variable_name: str): if key_metadata is None: return data - if "cache" in key_metadata: - data["cache"] = {} - if isinstance(key_metadata["cache"], dict): - for k, v in key_metadata["cache"].items(): - if k in SupportedCacheControls: - data["cache"][k] = v + if "cache" in key_metadata and isinstance(key_metadata["cache"], dict): + request_cache: Final = _DECODED_JSON.validate_python(data["cache"]) if "cache" in data else None + key_cache_controls: Final = _request_fields(_DECODED_JSON.validate_python(key_metadata["cache"])) + data["cache"] = { # rebind-ok: data is an out-param + **_request_fields(request_cache), + **{k: v for k, v in key_cache_controls.items() if k in SupportedCacheControls}, + } ## KEY-LEVEL SPEND LOGS / TAGS if "tags" in key_metadata and key_metadata["tags"] is not None: diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 81a196af676..3c97c0a5868 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -17,6 +17,7 @@ from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers import litellm +from litellm.caching.caching import Cache, CacheMode from litellm.constants import ( CALLER_SESSION_ID_METADATA_KEY, COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY, @@ -60,6 +61,7 @@ from litellm.proxy.litellm_pre_call_utils import ( move_guardrails_to_metadata, ) from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.caching import LiteLLMCacheType from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.utils import CredentialItem @@ -7920,6 +7922,61 @@ async def test_key_level_callback_vars_survive_the_strip(): assert updated["dd_site"] == "us5.datadoghq.com" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_cache, request_cache, expected_cache, uses_cache_when_default_off", + [ + pytest.param( + {"ttl": 600}, + {"use-cache": True, "namespace": "team-a"}, + {"use-cache": True, "namespace": "team-a", "ttl": 600}, + True, + id="key-control-added-to-request-opt-in", + ), + pytest.param( + {"no-store": True}, + {"use-cache": True, "no-store": False}, + {"use-cache": True, "no-store": True}, + True, + id="key-control-wins-over-request", + ), + pytest.param( + {"use-cache": True, "namespace": "key-ns"}, + None, + {}, + False, + id="unsupported-key-controls-ignored", + ), + pytest.param(None, {"use-cache": True}, {"use-cache": True}, True, id="malformed-key-cache-keeps-request"), + ], +) +async def test_key_cache_controls_merge_into_request_cache_controls( + key_cache: dict[str, object] | None, + request_cache: dict[str, object] | None, + expected_cache: dict[str, object], + uses_cache_when_default_off: bool, +): + key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"cache": key_cache}) + data: Final = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "hello"}], + **({} if request_cache is None else {"cache": request_cache}), + } + + updated: Final = await add_litellm_data_to_request( + data=data, + request=_callback_credential_request_mock(), + user_api_key_dict=key, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["cache"] == expected_cache + opt_in_cache: Final = Cache(type=LiteLLMCacheType.LOCAL, mode=CacheMode.default_off) + assert opt_in_cache.should_use_cache(**updated) is uses_cache_when_default_off + + class TestPromotedTraceControlFields: """LIT-5137: caller metadata trace fields must reach litellm_metadata."""