diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 406a4f8c98a..336bf2c9ae5 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -109,6 +109,7 @@ class Cache: qdrant_quantization_config: Optional[str] = None, qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002", qdrant_semantic_cache_vector_size: Optional[int] = None, + qdrant_semantic_cache_embed_api_base: Optional[str] = None, # GCP IAM authentication parameters gcp_service_account: Optional[str] = None, gcp_ssl_ca_certs: Optional[str] = None, @@ -217,6 +218,7 @@ class Cache: quantization_config=qdrant_quantization_config, embedding_model=qdrant_semantic_cache_embedding_model, vector_size=qdrant_semantic_cache_vector_size, + embed_api_base=qdrant_semantic_cache_embed_api_base, ) elif type == LiteLLMCacheType.LOCAL: self.cache = InMemoryCache() diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7cdbd3fc03d..7cc45fbd5b1 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -98,9 +98,9 @@ class LLMCachingHandler: self.request_kwargs = request_kwargs self.original_function = original_function self.start_time = start_time - if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache): + if litellm.cache is not None and isinstance(getattr(litellm.cache, "cache", None), RedisCache): self.dual_cache: Optional[DualCache] = DualCache( - redis_cache=litellm.cache.cache, + redis_cache=getattr(litellm.cache, "cache", None), in_memory_cache=in_memory_cache_obj, ) else: @@ -225,7 +225,7 @@ class LLMCachingHandler: and isinstance(cached_result, list) and litellm.cache is not None and not isinstance( - litellm.cache.cache, S3Cache + getattr(litellm.cache, "cache", None), S3Cache ) # s3 doesn't support bulk writing. Exclude. ): ( @@ -842,7 +842,7 @@ class LLMCachingHandler: isinstance(result, EmbeddingResponse) and litellm.cache is not None and not isinstance( - litellm.cache.cache, S3Cache + getattr(litellm.cache, "cache", None), S3Cache ) # s3 doesn't support bulk writing. Exclude. ): asyncio.create_task( diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 5e3713e5a15..138be85511c 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -32,6 +32,7 @@ class QdrantSemanticCache(BaseCache): embedding_model="text-embedding-ada-002", host_type=None, vector_size=None, + embed_api_base=None, ): import os @@ -54,6 +55,7 @@ class QdrantSemanticCache(BaseCache): raise Exception("similarity_threshold must be provided, passed None") self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model + self.embed_api_base = embed_api_base self.vector_size = ( vector_size if vector_size is not None else QDRANT_VECTOR_SIZE ) @@ -313,12 +315,15 @@ class QdrantSemanticCache(BaseCache): }, ) else: - # convert to embedding - embedding_response = await litellm.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - ) + # convert to embedding, pass api_base if available + embedding_kwargs: dict[str, Any] = { + "model": self.embedding_model, + "input": prompt, + "cache": {"no-store": True, "no-cache": True}, + } + if getattr(self, "embed_api_base", None) is not None: + embedding_kwargs["api_base"] = self.embed_api_base + embedding_response = await litellm.aembedding(**embedding_kwargs) # get the embedding embedding = embedding_response["data"][0]["embedding"] @@ -374,12 +379,15 @@ class QdrantSemanticCache(BaseCache): }, ) else: - # convert to embedding - embedding_response = await litellm.aembedding( - model=self.embedding_model, - input=prompt, - cache={"no-store": True, "no-cache": True}, - ) + # convert to embedding, pass api_base if available + embedding_kwargs: dict[str, Any] = { + "model": self.embedding_model, + "input": prompt, + "cache": {"no-store": True, "no-cache": True}, + } + if getattr(self, "embed_api_base", None) is not None: + embedding_kwargs["api_base"] = self.embed_api_base + embedding_response = await litellm.aembedding(**embedding_kwargs) # get the embedding embedding = embedding_response["data"][0]["embedding"] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 85a12f70f58..6d342d8f03d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2761,10 +2761,10 @@ class ProxyConfig: litellm.cache = Cache(**cache_params) if litellm.cache is not None and isinstance( - litellm.cache.cache, (RedisCache, RedisClusterCache) + getattr(litellm.cache, "cache", None), (RedisCache, RedisClusterCache) ): ## INIT PROXY REDIS USAGE CLIENT ## - redis_usage_cache = litellm.cache.cache + redis_usage_cache = getattr(litellm.cache, "cache", None) spend_counter_cache.redis_cache = redis_usage_cache # Note: PKCE verifier storage uses redis_usage_cache directly (not # user_api_key_cache) to avoid routing all API-key lookups through Redis. diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index fe6830693d6..5107654a5a6 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -9,6 +9,119 @@ sys.path.insert( ) # Adds the parent directory to the system path +def test_proxy_init_cache_does_not_crash_on_non_redis_cache(): + """ + Test that proxy cache initialization does not crash for non-Redis backends. + Verifies that accessing litellm.cache.cache via getattr prevents AttributeError + when the cache type is qdrant-semantic. + """ + import litellm + from litellm.caching.caching import Cache + + with patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") as mock_sync_client, \ + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + # Create a qdrant-semantic cache (has no inner .cache attribute like Redis does) + cache = Cache( + type="qdrant-semantic", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + qdrant_collection_name="test_collection", + similarity_threshold=0.8, + ) + litellm.cache = cache + + # This should not raise AttributeError + from litellm.caching.redis_cache import RedisCache + from litellm.caching.redis_cluster_cache import RedisClusterCache + + inner = getattr(litellm.cache, "cache", None) + result = isinstance(inner, (RedisCache, RedisClusterCache)) + assert result is False + + # Cleanup + litellm.cache = None + + +def test_caching_handler_does_not_crash_on_non_redis_cache(): + """ + Test that CachingHandlerResponse isinstance checks do not crash for non-Redis backends. + Verifies that getattr prevents AttributeError when cache type is qdrant-semantic. + """ + import litellm + from litellm.caching.caching import Cache + from litellm.caching.redis_cache import RedisCache + from litellm.caching.s3_cache import S3Cache + + with patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") as mock_sync_client, \ + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + cache = Cache( + type="qdrant-semantic", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + qdrant_collection_name="test_collection", + similarity_threshold=0.8, + ) + litellm.cache = cache + + # These isinstance checks should not raise AttributeError + inner = getattr(litellm.cache, "cache", None) + assert not isinstance(inner, RedisCache) + assert not isinstance(inner, S3Cache) + + # Cleanup + litellm.cache = None + + +def test_qdrant_semantic_cache_embed_api_base(): + """ + Test that QdrantSemanticCache stores embed_api_base when provided. + Verifies that non-OpenAI embedding models can pass api_base for the fallback path. + """ + with patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") as mock_sync_client, \ + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client"): + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + embedding_model="ollama/nomic-embed-text", + embed_api_base="http://localhost:11434", + ) + + assert qdrant_cache.embed_api_base == "http://localhost:11434" + assert qdrant_cache.embedding_model == "ollama/nomic-embed-text" + + + def test_qdrant_semantic_cache_initialization(monkeypatch): """ Test QDRANT semantic cache initialization with proper parameters.