diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 5e3713e5a15..1e13f1eb8f8 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -17,6 +17,9 @@ import litellm from litellm._logging import print_verbose from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE from litellm.types.utils import EmbeddingResponse +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_str_from_messages, +) from .base_cache import BaseCache @@ -175,10 +178,11 @@ class QdrantSemanticCache(BaseCache): from litellm._uuid import uuid # get the prompt - messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + messages = kwargs.get("messages") or kwargs.get("message") + if not messages: + print_verbose("No messages provided for semantic caching") + return + prompt = get_str_from_messages(messages) # create an embedding for prompt embedding_response = cast( @@ -219,10 +223,11 @@ class QdrantSemanticCache(BaseCache): print_verbose(f"sync qdrant semantic-cache get_cache, kwargs: {kwargs}") # get the messages - messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + messages = kwargs.get("messages") or kwargs.get("message") + if not messages: + print_verbose("No messages provided for semantic lookup") + return + prompt = get_str_from_messages(messages) # convert to embedding embedding_response = cast( @@ -290,10 +295,11 @@ class QdrantSemanticCache(BaseCache): print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}") # get the prompt - messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + messages = kwargs.get("messages") or kwargs.get("message") + if not messages: + print_verbose("No messages provided for semantic caching") + return + prompt = get_str_from_messages(messages) # create an embedding for prompt router_model_names = ( [m["model_name"] for m in llm_model_list] @@ -351,10 +357,11 @@ class QdrantSemanticCache(BaseCache): from litellm.proxy.proxy_server import llm_model_list, llm_router # get the messages - messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + messages = kwargs.get("messages") or kwargs.get("message") + if not messages: + print_verbose("No messages provided for semantic lookup") + return + prompt = get_str_from_messages(messages) router_model_names = ( [m["model_name"] for m in llm_model_list] diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 13dc4b5812c..91fa494d035 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -382,10 +382,108 @@ def test_qdrant_semantic_cache_set_cache(): messages=[{"content": "What is the capital of Italy?"}], ) + # Verify upsert was called + qdrant_cache.sync_client.put.assert_called() + + +def test_qdrant_semantic_cache_set_list_response(): + """ + Test QDRANT semantic cache set method with a list response. + """ + with ( + patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client" + ) as mock_sync_client, + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" + ) as mock_async_client, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + mock_upsert_response = MagicMock() + mock_upsert_response.status_code = 200 + qdrant_cache.sync_client.put = MagicMock(return_value=mock_upsert_response) + + response_to_cache = ["item1", "item2"] + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.1, 0.1]}]} + ): + qdrant_cache.set_cache( + key="test_key", + value=response_to_cache, + messages=[{"content": "What is the list?"}], + ) # Verify upsert was called qdrant_cache.sync_client.put.assert_called() +def test_qdrant_semantic_cache_get_list_response_hit(): + """ + Test QDRANT semantic cache get method when cached response is a list. + """ + with ( + patch( + "litellm.llms.custom_httpx.http_handler._get_httpx_client" + ) as mock_sync_client, + patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" + ) as mock_async_client, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"result": {"exists": True}} + mock_sync_client_instance = MagicMock() + mock_sync_client_instance.get.return_value = mock_response + mock_sync_client.return_value = mock_sync_client_instance + + from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache + + qdrant_cache = QdrantSemanticCache( + collection_name="test_collection", + qdrant_api_base="http://test.qdrant.local", + qdrant_api_key="test_key", + similarity_threshold=0.8, + ) + + mock_search_response = MagicMock() + mock_search_response.status_code = 200 + mock_search_response.json.return_value = { + "result": [ + { + "payload": { + "text": "What is the list?", + "response": '["item1", "item2"]', + }, + "score": 0.9, + } + ] + } + qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response) + + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]} + ): + result = qdrant_cache.get_cache( + key="test_key", messages=[{"content": "What is the list?"}] + ) + assert result == ["item1", "item2"] + + @pytest.mark.asyncio async def test_qdrant_semantic_cache_async_set_cache(): """