diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 8bbfed260c7..5f5b8210096 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -353,7 +353,7 @@ class Cache: ) _SEMANTIC_CACHE_END_USER_SCOPE_FIELD: Final = "user_api_key_end_user_id" - def is_semantic_cache(self) -> bool: + def _is_semantic_cache(self) -> bool: return self.type in ( LiteLLMCacheType.REDIS_SEMANTIC, LiteLLMCacheType.QDRANT_SEMANTIC, @@ -395,7 +395,7 @@ class Cache: return preset_cache_key combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params() - is_semantic_cache: Final = self.is_semantic_cache() + is_semantic_cache: Final = self._is_semantic_cache() scope_excluded_params: Final = self._SEMANTIC_CACHE_SCOPE_EXCLUDED_PARAMS if is_semantic_cache else frozenset() for param in kwargs: if param in scope_excluded_params: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 4f3801d6e9b..a4de1be1842 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -443,20 +443,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ return model_names - async def async_should_semantic_cache( - self, - user_api_key_dict: UserAPIKeyAuth, - data: Mapping[str, object], - call_type: str, - ) -> bool: - """Runs once per request the proxy's semantic cache would serve, after the pre-call hooks, - with the authenticated key and the request body. Return True to let the request read from - and write to the semantic cache. Any other return value, or an exception, skips both for that - request. Every callback that overrides this hook must return True for the request to be cached. - Exact-match cache types never call it. - """ - return True - async def async_post_call_response_headers_hook( self, data: dict, diff --git a/litellm/proxy/common_utils/semantic_cache_gate.py b/litellm/proxy/common_utils/semantic_cache_gate.py deleted file mode 100644 index 217fe596e23..00000000000 --- a/litellm/proxy/common_utils/semantic_cache_gate.py +++ /dev/null @@ -1,65 +0,0 @@ -from collections.abc import Mapping, Sequence -from types import MappingProxyType -from typing import Final - -from litellm._logging import verbose_proxy_logger -from litellm.caching.caching import Cache -from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth - -SKIP_CACHE_CONTROLS: Final = MappingProxyType({"no-cache": True, "no-store": True}) -CACHED_CALL_TYPES_BY_ROUTE: Final = MappingProxyType( - { - "aresponses": frozenset({"aresponses", "responses", "acompletion"}), - "anthropic_messages": frozenset({"anthropic_messages", "aanthropic_messages", "acompletion"}), - "agenerate_content": frozenset({"acompletion"}), - "agenerate_content_stream": frozenset({"acompletion"}), - } -) - - -def _cache_serves(cache: Cache, call_type: str) -> bool: - cached_call_types: Final = CACHED_CALL_TYPES_BY_ROUTE.get(call_type, frozenset({call_type})) - return not cached_call_types.isdisjoint(cache.supported_call_types or ()) - - -async def _gate_allows( - gate: CustomLogger, - user_api_key_dict: UserAPIKeyAuth, - data: Mapping[str, object], - call_type: str, -) -> bool: - try: - return await gate.async_should_semantic_cache(user_api_key_dict, data, call_type) is True - except Exception: # noqa: BLE001 # operator hooks can raise anything, and every failure skips the cache - verbose_proxy_logger.exception( - "%s.async_should_semantic_cache raised, so this request skips the semantic cache", type(gate).__name__ - ) - return False - - -async def _every_gate_allows( - gates: Sequence[CustomLogger], - user_api_key_dict: UserAPIKeyAuth, - data: Mapping[str, object], - call_type: str, -) -> bool: - for gate in gates: - if not await _gate_allows(gate, user_api_key_dict, data, call_type): - return False - return True - - -async def gated_cache_controls( - data: Mapping[str, object], - user_api_key_dict: UserAPIKeyAuth, - call_type: str, - gates: Sequence[CustomLogger], - cache: Cache | None, -) -> Mapping[str, object] | None: - if not gates or cache is None or not cache.is_semantic_cache() or not _cache_serves(cache, call_type): - return None - if await _every_gate_allows(gates, user_api_key_dict, data, call_type): - return None - controls: Final = data.get("cache") - return {**(controls if isinstance(controls, Mapping) else {}), **SKIP_CACHE_CONTROLS} diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1966c32fb07..ea1044d4a55 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1496,7 +1496,6 @@ class _CallbackCapabilities: # every string entry in ``litellm.callbacks``. resolved_callbacks: tuple[object, ...] = field(default_factory=tuple) listed_models_filters: tuple[CustomLogger, ...] = field(default_factory=tuple) - semantic_cache_gates: tuple[CustomLogger, ...] = field(default_factory=tuple) def _overrides_hook(callback: CustomLogger, hook_name: str) -> bool: @@ -3276,9 +3275,6 @@ class ProxyLogging: listed_models_filters=tuple( callback for callback in resolved_callbacks if _overrides_hook(callback, "async_filter_listed_models") ), - semantic_cache_gates=tuple( - callback for callback in resolved_callbacks if _overrides_hook(callback, "async_should_semantic_cache") - ), ) # Limit cache to handle test churn without leaking; production # callback lists are stable so this rarely grows past 1 entry. diff --git a/tests/unit/google_genai/test_google_genai_adapter.py b/tests/unit/google_genai/test_google_genai_adapter.py index 5e521506699..dbcc80a5d25 100644 --- a/tests/unit/google_genai/test_google_genai_adapter.py +++ b/tests/unit/google_genai/test_google_genai_adapter.py @@ -2,6 +2,7 @@ """ Test to verify the Google GenAI generate_content adapter functionality """ +import asyncio import json import unittest @@ -1641,3 +1642,40 @@ async def test_generate_content_sends_response_schema_and_tool_parameters_to_the } assert response["candidates"][0]["content"]["parts"] == [{"text": '{"park_name": "EPCOT"}'}] assert "text" not in response + + +@pytest.mark.asyncio +async def test_generate_content_bridge_honors_the_request_cache_controls(respx_mock, monkeypatch): + import httpx + + from litellm.caching.caching import Cache + from litellm.types.caching import LiteLLMCacheType + + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + route = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-capital", + "object": "chat.completion", + "created": 1234567890, + "model": "gpt-4.1", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Paris"}}], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + }, + ) + ) + request = { + "model": "openai/gpt-4.1", + "contents": [{"role": "user", "parts": [{"text": "What is the capital of France?"}]}], + "cache": {"no-cache": True, "no-store": True}, + "api_key": "sk-test", + } + + await agenerate_content(**request) + await asyncio.sleep(0.5) + await agenerate_content(**request) + + assert route.call_count == 2 +