From 077ce8b08fd44986b6ab7d5b2f16dd714158b15f Mon Sep 17 00:00:00 2001 From: Boris Duin Date: Fri, 24 Apr 2026 20:02:46 +0000 Subject: [PATCH] fix(caching): handle list-based responses and message key variations in QdrantSemanticCache When the semantic cache attempted to store or retrieve responses that were structured as lists (e.g., list-based content), the system encountered a `TypeError` during the caching process, resulting in the following error: `LiteLLM: ERROR: caching.py:647 - LiteLLM Cache: Exception add_cache: can only concatenate str (not "list") to str` Additionally, some callers were passing the prompt content using a singular 'message' key instead of the expected plural 'messages' key, leading to further inconsistencies. Finally, the use of `str(value)` for serialization generated Python-specific representations (single quotes) incompatible with standard JSON parsing, forcing a reliance on fragile `ast.literal_eval` fallbacks. - Updated `QdrantSemanticCache` to ensure list-type responses are correctly serialized to JSON strings using `json.dumps` before storage, ensuring consistent retrieval. - Modified input handling in cache methods to support both 'messages' and 'message' keys, ensuring robustness against varying input structures. - Added comprehensive unit tests in `tests/test_litellm/caching/test_qdrant_semantic_cache.py` to validate list-type response caching and confirm the fix for the reported concatenation and serialization errors. - Cleaned up test suite by removing unused `mock_async_client` patches in synchronous test cases. --- tests/test_litellm/caching/test_qdrant_semantic_cache.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index 91fa494d035..404d8e834e5 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -394,9 +394,6 @@ def test_qdrant_semantic_cache_set_list_response(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, ): mock_response = MagicMock() mock_response.status_code = 200 @@ -440,9 +437,6 @@ def test_qdrant_semantic_cache_get_list_response_hit(): patch( "litellm.llms.custom_httpx.http_handler._get_httpx_client" ) as mock_sync_client, - patch( - "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" - ) as mock_async_client, ): mock_response = MagicMock() mock_response.status_code = 200