fix(proxy): merge a key's cache controls into the request's cache settings instead of replacing them

This commit is contained in:
mateo-berri 2026-10-29 08:23:23 -07:00
parent 67ae3fb527
commit b5005f8cb6
2 changed files with 64 additions and 6 deletions

View file

@ -2125,12 +2125,13 @@ class LiteLLMProxyRequestSetup:
def add_key_level_controls(key_metadata: dict | None, data: dict, _metadata_variable_name: str):
if key_metadata is None:
return data
if "cache" in key_metadata:
data["cache"] = {}
if isinstance(key_metadata["cache"], dict):
for k, v in key_metadata["cache"].items():
if k in SupportedCacheControls:
data["cache"][k] = v
if "cache" in key_metadata and isinstance(key_metadata["cache"], dict):
request_cache: Final = _DECODED_JSON.validate_python(data["cache"]) if "cache" in data else None
key_cache_controls: Final = _request_fields(_DECODED_JSON.validate_python(key_metadata["cache"]))
data["cache"] = { # rebind-ok: data is an out-param
**_request_fields(request_cache),
**{k: v for k, v in key_cache_controls.items() if k in SupportedCacheControls},
}
## KEY-LEVEL SPEND LOGS / TAGS
if "tags" in key_metadata and key_metadata["tags"] is not None:

View file

@ -17,6 +17,7 @@ from pydantic import ValidationError as PydanticValidationError
from starlette.datastructures import Headers
import litellm
from litellm.caching.caching import Cache, CacheMode
from litellm.constants import (
CALLER_SESSION_ID_METADATA_KEY,
COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY,
@ -60,6 +61,7 @@ from litellm.proxy.litellm_pre_call_utils import (
move_guardrails_to_metadata,
)
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.types.caching import LiteLLMCacheType
from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES
from litellm.types.utils import CredentialItem
@ -7920,6 +7922,61 @@ async def test_key_level_callback_vars_survive_the_strip():
assert updated["dd_site"] == "us5.datadoghq.com"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"key_cache, request_cache, expected_cache, uses_cache_when_default_off",
[
pytest.param(
{"ttl": 600},
{"use-cache": True, "namespace": "team-a"},
{"use-cache": True, "namespace": "team-a", "ttl": 600},
True,
id="key-control-added-to-request-opt-in",
),
pytest.param(
{"no-store": True},
{"use-cache": True, "no-store": False},
{"use-cache": True, "no-store": True},
True,
id="key-control-wins-over-request",
),
pytest.param(
{"use-cache": True, "namespace": "key-ns"},
None,
{},
False,
id="unsupported-key-controls-ignored",
),
pytest.param(None, {"use-cache": True}, {"use-cache": True}, True, id="malformed-key-cache-keeps-request"),
],
)
async def test_key_cache_controls_merge_into_request_cache_controls(
key_cache: dict[str, object] | None,
request_cache: dict[str, object] | None,
expected_cache: dict[str, object],
uses_cache_when_default_off: bool,
):
key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"cache": key_cache})
data: Final = {
"model": "gpt-4.1-mini",
"messages": [{"role": "user", "content": "hello"}],
**({} if request_cache is None else {"cache": request_cache}),
}
updated: Final = await add_litellm_data_to_request(
data=data,
request=_callback_credential_request_mock(),
user_api_key_dict=key,
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert updated["cache"] == expected_cache
opt_in_cache: Final = Cache(type=LiteLLMCacheType.LOCAL, mode=CacheMode.default_off)
assert opt_in_cache.should_use_cache(**updated) is uses_cache_when_default_off
class TestPromotedTraceControlFields:
"""LIT-5137: caller metadata trace fields must reach litellm_metadata."""