mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-31 06:56:59 +00:00
fix(caching): keep unrestricted calls within cache-safe types
Signed-off-by: Yusuf Efe <120668197+yuefdev@users.noreply.github.com>
This commit is contained in:
parent
900b222f83
commit
bc6029d567
2 changed files with 38 additions and 24 deletions
|
|
@ -95,13 +95,14 @@ def _request_message_count(kwargs: Mapping[str, object]) -> int:
|
|||
|
||||
|
||||
def is_cache_call_type_supported(call_type: str, supported_call_types: Sequence[str] | None) -> bool:
|
||||
if supported_call_types is None:
|
||||
return True
|
||||
allowed_call_types: Final = (
|
||||
DEFAULT_CACHING_SUPPORTED_CALL_TYPES if supported_call_types is None else supported_call_types
|
||||
)
|
||||
normalised_call_type: Final = call_type.lstrip("_")
|
||||
covering_call_types: Final = (
|
||||
("aresponses", "responses") if normalised_call_type == "aresponses" else (normalised_call_type,)
|
||||
)
|
||||
return any(name in supported_call_types for name in covering_call_types)
|
||||
return any(name in allowed_call_types for name in covering_call_types)
|
||||
|
||||
|
||||
class Cache:
|
||||
|
|
@ -205,7 +206,7 @@ class Cache:
|
|||
gcs_path (str, optional): Folder path inside the bucket to store cache files.
|
||||
|
||||
# Common Cache Args
|
||||
supported_call_types (list, optional): List of call types to cache for. Defaults to cache == on for all call types.
|
||||
supported_call_types (list, optional): Call types to cache. None uses DEFAULT_CACHING_SUPPORTED_CALL_TYPES.
|
||||
**kwargs: Additional keyword arguments for redis.Redis() cache
|
||||
|
||||
Raises:
|
||||
|
|
@ -981,21 +982,16 @@ class Cache:
|
|||
If cache is default_off then this is only true when user has opted in to use cache
|
||||
Always False once the request carries more than `max_messages` messages
|
||||
"""
|
||||
if self.supported_call_types is not None:
|
||||
raw_source: Final = (
|
||||
kwargs.get("call_type")
|
||||
if kwargs.get("call_type") is not None
|
||||
else (
|
||||
kwargs.get("route_type")
|
||||
if kwargs.get("route_type") is not None
|
||||
else kwargs.get("original_function")
|
||||
)
|
||||
)
|
||||
if raw_source is not None:
|
||||
raw_name: Final[object] = getattr(raw_source, "value", None) or getattr(raw_source, "__name__", None)
|
||||
call_type_name: Final = raw_name if isinstance(raw_name, str) else str(raw_source)
|
||||
if not is_cache_call_type_supported(call_type_name, self.supported_call_types):
|
||||
return False
|
||||
raw_source: Final = (
|
||||
kwargs.get("call_type")
|
||||
if kwargs.get("call_type") is not None
|
||||
else (kwargs.get("route_type") if kwargs.get("route_type") is not None else kwargs.get("original_function"))
|
||||
)
|
||||
if raw_source is not None:
|
||||
raw_name: Final[object] = getattr(raw_source, "value", None) or getattr(raw_source, "__name__", None)
|
||||
call_type_name: Final = raw_name if isinstance(raw_name, str) else str(raw_source)
|
||||
if not is_cache_call_type_supported(call_type_name, self.supported_call_types):
|
||||
return False
|
||||
if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages:
|
||||
return False
|
||||
if self.mode == CacheMode.default_on:
|
||||
|
|
|
|||
|
|
@ -3279,16 +3279,34 @@ def test_call_type_filter_preserves_responses_alias(monkeypatch: pytest.MonkeyPa
|
|||
assert handler.should_store_result_in_cache(original_function=litellm.aresponses, kwargs={}) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("allow_all", [True, False])
|
||||
def test_unrestricted_and_empty_call_type_lists(monkeypatch: pytest.MonkeyPatch, allow_all: bool) -> None:
|
||||
cache: Final = Cache(type="local", supported_call_types=None if allow_all else [])
|
||||
@pytest.mark.parametrize("allow_inference", [True, False])
|
||||
def test_none_and_empty_call_type_lists(monkeypatch: pytest.MonkeyPatch, allow_inference: bool) -> None:
|
||||
cache: Final = Cache(type="local", supported_call_types=None if allow_inference else [])
|
||||
monkeypatch.setattr(litellm, "cache", cache)
|
||||
handler: Final = LLMCachingHandler(
|
||||
original_function=Router._aembedding, request_kwargs={}, start_time=datetime(2026, 1, 1)
|
||||
)
|
||||
|
||||
assert cache.should_use_cache(call_type="_aembedding") is allow_all
|
||||
assert handler.should_store_result_in_cache(original_function=Router._aembedding, kwargs={}) is allow_all
|
||||
assert cache.should_use_cache(call_type="_aembedding") is allow_inference
|
||||
assert handler.should_store_result_in_cache(original_function=Router._aembedding, kwargs={}) is allow_inference
|
||||
|
||||
|
||||
def test_none_call_types_do_not_read_or_store_file_listings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
cache: Final = Cache(type="local", supported_call_types=None)
|
||||
monkeypatch.setattr(litellm, "cache", cache)
|
||||
handler: Final = LLMCachingHandler(
|
||||
original_function=litellm.afile_list, request_kwargs={}, start_time=datetime(2026, 1, 1)
|
||||
)
|
||||
response: Final = {"object": "list", "data": [{"id": "private-file", "object": "file"}]}
|
||||
|
||||
handler.sync_set_cache(result=response, kwargs={"cache_key": "file-list"})
|
||||
assert cache.get_cache(cache_key="file-list") is None
|
||||
|
||||
cache.add_cache(response, cache_key="existing")
|
||||
assert cache.get_cache(cache_key="existing", call_type="afile_list") is None
|
||||
|
||||
cache.add_cache(response, cache_key="direct-file-list", call_type="afile_list")
|
||||
assert cache.get_cache(cache_key="direct-file-list") is None
|
||||
|
||||
|
||||
def test_call_type_filter_preserves_message_limit_and_opt_in() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue