scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 174

fix(caching): key the response cache by the router's metadata bucket only

Browse source
This commit is contained in:
mateo-berri 2026-10-28 04:25:44 -07:00
parent f7b57750bb
commit b641a0d9b7
3 changed files with 35 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/caching/caching.py
View file

@ -456,10 +456,8 @@ class Cache:
"""
litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
metadata_sources: Final[tuple[dict, ...]] = (
kwargs.get("metadata") or {},
kwargs.get("litellm_metadata") or {},
litellm_params.get("metadata") or {},
litellm_params.get("litellm_metadata") or {},
kwargs.get("litellm_metadata" if "litellm_metadata" in kwargs else "metadata") or {},
litellm_params.get("litellm_metadata") or litellm_params.get("metadata") or {},
)
model_group: Final[str | None] = next(
(source["model_group"] for source in metadata_sources if source.get("model_group")), None

6
tests/integration/caching/test_response_cache_model_group_key.py
View file

@ -21,7 +21,7 @@ from openai import OpenAI
from pydantic import JsonValue
from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, object_value
from tests.integration._support.process import graceful_stop_seconds, owned_proxy_process
from tests.integration._support.process import OwnedState, graceful_stop_seconds, owned_proxy_process
from tests.integration._support.upstream import ScenarioHandle
from tests.integration.caching.response_cache_case import (
CHAT,
@ -175,7 +175,9 @@ def test_groups_in_one_caching_group_on_different_models_share_one_entry(
first: Final = _configured_group(scenario, endpoint)
second: Final = _configured_group(scenario, endpoint)
config: Final = _caching_groups_config(tmp_path, endpoint, first, second)
with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=1) as owned:
with owned_proxy_process(
gateway, tmp_path, {}, config=config, workers=1, state=OwnedState.SHARED_WITH_WORKER
) as owned:
filled: Final = endpoint.ask(owned.gateway, first.name, text, {})
await_entry(store, filled)
assert endpoint.ask(owned.gateway, second.name, text, {}) == filled

29
tests/unit/caching/test_unit_test_caching.py
View file

@ -278,6 +278,35 @@ def test_get_model_param_value_reads_model_group_from_litellm_metadata():
)
def test_get_model_param_value_ignores_a_caller_model_group_outside_the_router_bucket():
cache = Cache()
request = {"model": "openai/gpt-5.6", "input": "search this text"}
assert (
cache._get_model_param_value(
{**request, "metadata": {"model_group": "target"}, "litellm_metadata": {"model_group": "caller"}}
)
== "caller"
)
assert (
cache._get_model_param_value({**request, "metadata": {"model_group": "target"}, "litellm_metadata": {}})
== "openai/gpt-5.6"
)
assert (
cache._get_model_param_value(
{
**request,
"metadata": {"model_group": "target"},
"litellm_metadata": {"user_api_key": "hashed"},
"litellm_params": {
"metadata": {"model_group": "target"},
"litellm_metadata": {"user_api_key": "hashed"},
},
}
)
== "openai/gpt-5.6"
)
def test_preset_cache_key():
"""
Test that the preset cache key is used if it is set in kwargs["litellm_params"]