mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
fix(caching): handle list-based responses and message key variations in QdrantSemanticCache
When the semantic cache attempted to store or retrieve responses that were structured as lists (e.g., list-based content), the system encountered a `TypeError` during the caching process, resulting in the following error: `LiteLLM: ERROR: caching.py:647 - LiteLLM Cache: Exception add_cache: can only concatenate str (not "list") to str` Additionally, some callers were passing the prompt content using a singular 'message' key instead of the expected plural 'messages' key, leading to further inconsistencies. - Updated `QdrantSemanticCache` to ensure list-type responses are correctly serialized to strings before storage and properly parsed back upon retrieval. - Modified input handling in cache methods to support both 'messages' and 'message' keys, ensuring robustness against varying input structures. - Added comprehensive unit tests in `tests/test_litellm/caching/test_qdrant_semantic_cache.py` to validate list-type response caching and confirm the fix for the reported concatenation error.
This commit is contained in:
Boris Duin
parent
9dcb2bd528
commit
475d29d34a
2 changed files with 121 additions and 16 deletions
|
|
@ -17,6 +17,9 @@ import litellm
|
|||
from litellm._logging import print_verbose
|
||||
from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
)
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
|
|
@ -175,10 +178,11 @@ class QdrantSemanticCache(BaseCache):
|
|||
from litellm._uuid import uuid
|
||||
|
||||
# get the prompt
|
||||
messages = kwargs["messages"]
|
||||
prompt = ""
|
||||
for message in messages:
|
||||
prompt += message["content"]
|
||||
messages = kwargs.get("messages") or kwargs.get("message")
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic caching")
|
||||
return
|
||||
prompt = get_str_from_messages(messages)
|
||||
|
||||
# create an embedding for prompt
|
||||
embedding_response = cast(
|
||||
|
|
@ -219,10 +223,11 @@ class QdrantSemanticCache(BaseCache):
|
|||
print_verbose(f"sync qdrant semantic-cache get_cache, kwargs: {kwargs}")
|
||||
|
||||
# get the messages
|
||||
messages = kwargs["messages"]
|
||||
prompt = ""
|
||||
for message in messages:
|
||||
prompt += message["content"]
|
||||
messages = kwargs.get("messages") or kwargs.get("message")
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic lookup")
|
||||
return
|
||||
prompt = get_str_from_messages(messages)
|
||||
|
||||
# convert to embedding
|
||||
embedding_response = cast(
|
||||
|
|
@ -290,10 +295,11 @@ class QdrantSemanticCache(BaseCache):
|
|||
print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}")
|
||||
|
||||
# get the prompt
|
||||
messages = kwargs["messages"]
|
||||
prompt = ""
|
||||
for message in messages:
|
||||
prompt += message["content"]
|
||||
messages = kwargs.get("messages") or kwargs.get("message")
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic caching")
|
||||
return
|
||||
prompt = get_str_from_messages(messages)
|
||||
# create an embedding for prompt
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
|
|
@ -351,10 +357,11 @@ class QdrantSemanticCache(BaseCache):
|
|||
from litellm.proxy.proxy_server import llm_model_list, llm_router
|
||||
|
||||
# get the messages
|
||||
messages = kwargs["messages"]
|
||||
prompt = ""
|
||||
for message in messages:
|
||||
prompt += message["content"]
|
||||
messages = kwargs.get("messages") or kwargs.get("message")
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic lookup")
|
||||
return
|
||||
prompt = get_str_from_messages(messages)
|
||||
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
|
|
|
|||
|
|
@ -382,10 +382,108 @@ def test_qdrant_semantic_cache_set_cache():
|
|||
messages=[{"content": "What is the capital of Italy?"}],
|
||||
)
|
||||
|
||||
# Verify upsert was called
|
||||
qdrant_cache.sync_client.put.assert_called()
|
||||
|
||||
|
||||
def test_qdrant_semantic_cache_set_list_response():
|
||||
"""
|
||||
Test QDRANT semantic cache set method with a list response.
|
||||
"""
|
||||
with (
|
||||
patch(
|
||||
"litellm.llms.custom_httpx.http_handler._get_httpx_client"
|
||||
) as mock_sync_client,
|
||||
patch(
|
||||
"litellm.llms.custom_httpx.http_handler.get_async_httpx_client"
|
||||
) as mock_async_client,
|
||||
):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"result": {"exists": True}}
|
||||
mock_sync_client_instance = MagicMock()
|
||||
mock_sync_client_instance.get.return_value = mock_response
|
||||
mock_sync_client.return_value = mock_sync_client_instance
|
||||
|
||||
from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache
|
||||
|
||||
qdrant_cache = QdrantSemanticCache(
|
||||
collection_name="test_collection",
|
||||
qdrant_api_base="http://test.qdrant.local",
|
||||
qdrant_api_key="test_key",
|
||||
similarity_threshold=0.8,
|
||||
)
|
||||
|
||||
mock_upsert_response = MagicMock()
|
||||
mock_upsert_response.status_code = 200
|
||||
qdrant_cache.sync_client.put = MagicMock(return_value=mock_upsert_response)
|
||||
|
||||
response_to_cache = ["item1", "item2"]
|
||||
|
||||
with patch(
|
||||
"litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.1, 0.1]}]}
|
||||
):
|
||||
qdrant_cache.set_cache(
|
||||
key="test_key",
|
||||
value=response_to_cache,
|
||||
messages=[{"content": "What is the list?"}],
|
||||
)
|
||||
# Verify upsert was called
|
||||
qdrant_cache.sync_client.put.assert_called()
|
||||
|
||||
|
||||
def test_qdrant_semantic_cache_get_list_response_hit():
|
||||
"""
|
||||
Test QDRANT semantic cache get method when cached response is a list.
|
||||
"""
|
||||
with (
|
||||
patch(
|
||||
"litellm.llms.custom_httpx.http_handler._get_httpx_client"
|
||||
) as mock_sync_client,
|
||||
patch(
|
||||
"litellm.llms.custom_httpx.http_handler.get_async_httpx_client"
|
||||
) as mock_async_client,
|
||||
):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"result": {"exists": True}}
|
||||
mock_sync_client_instance = MagicMock()
|
||||
mock_sync_client_instance.get.return_value = mock_response
|
||||
mock_sync_client.return_value = mock_sync_client_instance
|
||||
|
||||
from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache
|
||||
|
||||
qdrant_cache = QdrantSemanticCache(
|
||||
collection_name="test_collection",
|
||||
qdrant_api_base="http://test.qdrant.local",
|
||||
qdrant_api_key="test_key",
|
||||
similarity_threshold=0.8,
|
||||
)
|
||||
|
||||
mock_search_response = MagicMock()
|
||||
mock_search_response.status_code = 200
|
||||
mock_search_response.json.return_value = {
|
||||
"result": [
|
||||
{
|
||||
"payload": {
|
||||
"text": "What is the list?",
|
||||
"response": '["item1", "item2"]',
|
||||
},
|
||||
"score": 0.9,
|
||||
}
|
||||
]
|
||||
}
|
||||
qdrant_cache.sync_client.post = MagicMock(return_value=mock_search_response)
|
||||
|
||||
with patch(
|
||||
"litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2, 0.3]}]}
|
||||
):
|
||||
result = qdrant_cache.get_cache(
|
||||
key="test_key", messages=[{"content": "What is the list?"}]
|
||||
)
|
||||
assert result == ["item1", "item2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_qdrant_semantic_cache_async_set_cache():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue