diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..f92c6fbc85d 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -13,7 +13,7 @@ import json import logging import time import traceback -from collections.abc import Generator, Mapping +from collections.abc import Generator, Mapping, Sequence from contextlib import contextmanager from enum import Enum from typing import Any, Final, Literal, cast @@ -94,6 +94,17 @@ def _request_message_count(kwargs: Mapping[str, object]) -> int: return sum(1 for item in items if _is_conversation_item(item)) +def is_cache_call_type_supported(call_type: str, supported_call_types: Sequence[str] | None) -> bool: + allowed_call_types: Final = ( + DEFAULT_CACHING_SUPPORTED_CALL_TYPES if supported_call_types is None else supported_call_types + ) + normalised_call_type: Final = call_type.lstrip("_") + covering_call_types: Final = ( + ("aresponses", "responses") if normalised_call_type == "aresponses" else (normalised_call_type,) + ) + return any(name in allowed_call_types for name in covering_call_types) + + class Cache: def __init__( self, @@ -195,7 +206,7 @@ class Cache: gcs_path (str, optional): Folder path inside the bucket to store cache files. # Common Cache Args - supported_call_types (list, optional): List of call types to cache for. Defaults to cache == on for all call types. + supported_call_types (list, optional): Call types to cache. None uses DEFAULT_CACHING_SUPPORTED_CALL_TYPES. **kwargs: Additional keyword arguments for redis.Redis() cache Raises: @@ -963,7 +974,7 @@ class Cache: except Exception as e: self._log_add_cache_failure(e) - def should_use_cache(self, **kwargs): + def should_use_cache(self, **kwargs: object) -> bool: """ Returns true if we should use the cache for LLM API calls @@ -971,6 +982,16 @@ class Cache: If cache is default_off then this is only true when user has opted in to use cache Always False once the request carries more than `max_messages` messages """ + raw_source: Final = ( + kwargs.get("call_type") + if kwargs.get("call_type") is not None + else (kwargs.get("route_type") if kwargs.get("route_type") is not None else kwargs.get("original_function")) + ) + if raw_source is not None: + raw_name: Final[object] = getattr(raw_source, "value", None) or getattr(raw_source, "__name__", None) + call_type_name: Final = raw_name if isinstance(raw_name, str) else str(raw_source) + if not is_cache_call_type_supported(call_type_name, self.supported_call_types): + return False if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages: return False if self.mode == CacheMode.default_on: diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7089ff78835..38c4b9cb391 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -27,7 +27,7 @@ import litellm from litellm._internal_context import is_internal_call, post_response_phase from litellm._logging import print_verbose, verbose_logger from litellm.caching import InMemoryCache -from litellm.caching.caching import S3Cache, response_cache_phase +from litellm.caching.caching import S3Cache, is_cache_call_type_supported, response_cache_phase from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS, RESPONSE_CACHE_EXCLUDED_PROVIDERS from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( @@ -1290,11 +1290,9 @@ class LLMCachingHandler: Returns: bool: True if the call type is supported by the cache, False otherwise. """ - if litellm.cache is None or litellm.cache.supported_call_types is None: + if litellm.cache is None: return False - call_type: Final = str(original_function.__name__) - covering_call_types: Final = ("aresponses", "responses") if call_type == "aresponses" else (call_type,) - return any(name in litellm.cache.supported_call_types for name in covering_call_types) + return is_cache_call_type_supported(original_function.__name__, litellm.cache.supported_call_types) async def add_streaming_response_to_cache(self, processed_chunk: ModelResponse) -> None: """ diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 51754c80dd6..b48400d1960 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -16,7 +16,8 @@ from pydantic import TypeAdapter import litellm from litellm import aembedding, completion, embedding, aresponses, responses from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.caching.caching import Cache +from litellm.caching.caching import Cache, CacheMode +from litellm.router import Router from litellm.responses.streaming_iterator import CachedResponsesAPIStreamingIterator from unittest.mock import AsyncMock, patch, MagicMock @@ -3231,3 +3232,92 @@ async def test_a_cache_hit_inside_an_internal_sub_call_is_free_and_fires_no_call assert async_logging.async_success_handler.called is not internal assert async_logging.handle_sync_success_callbacks_for_async_calls.called is not internal assert sync_logging.handle_sync_cache_hit_success_callbacks.called is not internal + + +@pytest.mark.parametrize( + "call_metadata", + ( + {"call_type": "aembedding", "route_type": "acompletion"}, + {"route_type": "_aembedding"}, + {"original_function": litellm.aembedding}, + {"original_function": Router._aembedding}, + {"call_type": CallTypes.aembedding}, + ), +) +def test_excluded_call_types_neither_read_nor_write_cache(call_metadata: Mapping[str, object]) -> None: + cache: Final = Cache(type="local", supported_call_types=["acompletion"]) + response: Final = {"value": "cached"} + cache.add_cache(response, cache_key="existing") + + cache.add_cache(response, cache_key="excluded", **call_metadata) + + assert cache.get_cache(cache_key="existing", **call_metadata) is None + assert cache.get_cache(cache_key="excluded") is None + assert cache.get_cache(cache_key="existing", call_type="acompletion") == response + + +def test_router_function_names_preserve_supported_cache_calls(monkeypatch: pytest.MonkeyPatch) -> None: + cache: Final = Cache(type="local", supported_call_types=["acompletion"]) + monkeypatch.setattr(litellm, "cache", cache) + handler: Final = LLMCachingHandler( + original_function=Router._aembedding, request_kwargs={}, start_time=datetime(2026, 1, 1) + ) + + assert handler.should_store_result_in_cache(original_function=Router._aembedding, kwargs={}) is False + assert handler.should_store_result_in_cache(original_function=Router._acompletion, kwargs={}) is True + assert cache.should_use_cache(call_type="_acompletion") is True + + +def test_call_type_filter_preserves_responses_alias(monkeypatch: pytest.MonkeyPatch) -> None: + cache: Final = Cache(type="local", supported_call_types=["responses"]) + monkeypatch.setattr(litellm, "cache", cache) + handler: Final = LLMCachingHandler( + original_function=litellm.aresponses, request_kwargs={}, start_time=datetime(2026, 1, 1) + ) + + assert cache.should_use_cache(call_type=CallTypes.aresponses) is True + assert handler.should_store_result_in_cache(original_function=litellm.aresponses, kwargs={}) is True + + +@pytest.mark.parametrize("allow_inference", [True, False]) +def test_none_and_empty_call_type_lists(monkeypatch: pytest.MonkeyPatch, allow_inference: bool) -> None: + cache: Final = Cache(type="local", supported_call_types=None if allow_inference else []) + monkeypatch.setattr(litellm, "cache", cache) + handler: Final = LLMCachingHandler( + original_function=Router._aembedding, request_kwargs={}, start_time=datetime(2026, 1, 1) + ) + + assert cache.should_use_cache(call_type="_aembedding") is allow_inference + assert handler.should_store_result_in_cache(original_function=Router._aembedding, kwargs={}) is allow_inference + + +def test_none_call_types_do_not_read_or_store_file_listings(monkeypatch: pytest.MonkeyPatch) -> None: + cache: Final = Cache(type="local", supported_call_types=None) + monkeypatch.setattr(litellm, "cache", cache) + handler: Final = LLMCachingHandler( + original_function=litellm.afile_list, request_kwargs={}, start_time=datetime(2026, 1, 1) + ) + response: Final = {"object": "list", "data": [{"id": "private-file", "object": "file"}]} + + handler.sync_set_cache(result=response, kwargs={"cache_key": "file-list"}) + assert cache.get_cache(cache_key="file-list") is None + + cache.add_cache(response, cache_key="existing") + assert cache.get_cache(cache_key="existing", call_type="afile_list") is None + + cache.add_cache(response, cache_key="direct-file-list", call_type="afile_list") + assert cache.get_cache(cache_key="direct-file-list") is None + + +def test_call_type_filter_preserves_message_limit_and_opt_in() -> None: + cache: Final = Cache(type="local", mode=CacheMode.default_off, max_messages=1) + assert cache.should_use_cache(call_type="acompletion") is False + assert cache.should_use_cache(call_type="acompletion", cache={"use-cache": True}) is True + assert ( + cache.should_use_cache( + call_type="acompletion", + cache={"use-cache": True}, + messages=[{"role": "user", "content": "one"}, {"role": "user", "content": "two"}], + ) + is False + )