fix(caching): keep unrestricted calls within cache-safe types

Signed-off-by: Yusuf Efe <120668197+yuefdev@users.noreply.github.com>
This commit is contained in:
Yusuf Efe 2026-10-18 15:29:59 +03:00
parent 900b222f83
commit bc6029d567
2 changed files with 38 additions and 24 deletions

View file

@ -95,13 +95,14 @@ def _request_message_count(kwargs: Mapping[str, object]) -> int:
def is_cache_call_type_supported(call_type: str, supported_call_types: Sequence[str] | None) -> bool:
if supported_call_types is None:
return True
allowed_call_types: Final = (
DEFAULT_CACHING_SUPPORTED_CALL_TYPES if supported_call_types is None else supported_call_types
)
normalised_call_type: Final = call_type.lstrip("_")
covering_call_types: Final = (
("aresponses", "responses") if normalised_call_type == "aresponses" else (normalised_call_type,)
)
return any(name in supported_call_types for name in covering_call_types)
return any(name in allowed_call_types for name in covering_call_types)
class Cache:
@ -205,7 +206,7 @@ class Cache:
gcs_path (str, optional): Folder path inside the bucket to store cache files.
# Common Cache Args
supported_call_types (list, optional): List of call types to cache for. Defaults to cache == on for all call types.
supported_call_types (list, optional): Call types to cache. None uses DEFAULT_CACHING_SUPPORTED_CALL_TYPES.
**kwargs: Additional keyword arguments for redis.Redis() cache
Raises:
@ -981,21 +982,16 @@ class Cache:
If cache is default_off then this is only true when user has opted in to use cache
Always False once the request carries more than `max_messages` messages
"""
if self.supported_call_types is not None:
raw_source: Final = (
kwargs.get("call_type")
if kwargs.get("call_type") is not None
else (
kwargs.get("route_type")
if kwargs.get("route_type") is not None
else kwargs.get("original_function")
)
)
if raw_source is not None:
raw_name: Final[object] = getattr(raw_source, "value", None) or getattr(raw_source, "__name__", None)
call_type_name: Final = raw_name if isinstance(raw_name, str) else str(raw_source)
if not is_cache_call_type_supported(call_type_name, self.supported_call_types):
return False
raw_source: Final = (
kwargs.get("call_type")
if kwargs.get("call_type") is not None
else (kwargs.get("route_type") if kwargs.get("route_type") is not None else kwargs.get("original_function"))
)
if raw_source is not None:
raw_name: Final[object] = getattr(raw_source, "value", None) or getattr(raw_source, "__name__", None)
call_type_name: Final = raw_name if isinstance(raw_name, str) else str(raw_source)
if not is_cache_call_type_supported(call_type_name, self.supported_call_types):
return False
if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages:
return False
if self.mode == CacheMode.default_on:

View file

@ -3279,16 +3279,34 @@ def test_call_type_filter_preserves_responses_alias(monkeypatch: pytest.MonkeyPa
assert handler.should_store_result_in_cache(original_function=litellm.aresponses, kwargs={}) is True
@pytest.mark.parametrize("allow_all", [True, False])
def test_unrestricted_and_empty_call_type_lists(monkeypatch: pytest.MonkeyPatch, allow_all: bool) -> None:
cache: Final = Cache(type="local", supported_call_types=None if allow_all else [])
@pytest.mark.parametrize("allow_inference", [True, False])
def test_none_and_empty_call_type_lists(monkeypatch: pytest.MonkeyPatch, allow_inference: bool) -> None:
cache: Final = Cache(type="local", supported_call_types=None if allow_inference else [])
monkeypatch.setattr(litellm, "cache", cache)
handler: Final = LLMCachingHandler(
original_function=Router._aembedding, request_kwargs={}, start_time=datetime(2026, 1, 1)
)
assert cache.should_use_cache(call_type="_aembedding") is allow_all
assert handler.should_store_result_in_cache(original_function=Router._aembedding, kwargs={}) is allow_all
assert cache.should_use_cache(call_type="_aembedding") is allow_inference
assert handler.should_store_result_in_cache(original_function=Router._aembedding, kwargs={}) is allow_inference
def test_none_call_types_do_not_read_or_store_file_listings(monkeypatch: pytest.MonkeyPatch) -> None:
cache: Final = Cache(type="local", supported_call_types=None)
monkeypatch.setattr(litellm, "cache", cache)
handler: Final = LLMCachingHandler(
original_function=litellm.afile_list, request_kwargs={}, start_time=datetime(2026, 1, 1)
)
response: Final = {"object": "list", "data": [{"id": "private-file", "object": "file"}]}
handler.sync_set_cache(result=response, kwargs={"cache_key": "file-list"})
assert cache.get_cache(cache_key="file-list") is None
cache.add_cache(response, cache_key="existing")
assert cache.get_cache(cache_key="existing", call_type="afile_list") is None
cache.add_cache(response, cache_key="direct-file-list", call_type="afile_list")
assert cache.get_cache(cache_key="direct-file-list") is None
def test_call_type_filter_preserves_message_limit_and_opt_in() -> None: