From 28fbadf52816ec90187f8395ad104c9e9212474b Mon Sep 17 00:00:00 2001 From: Boris Duin Date: Fri, 24 Apr 2026 21:17:16 +0000 Subject: [PATCH] fix(caching): handle list-based responses, message key variations, and robust serialization in QdrantSemanticCache Several issues were identified in the semantic caching logic: 1. List-based responses caused a `TypeError` (`can only concatenate str (not "list") to str`) during caching. 2. The use of `str(value)` for serialization generated non-JSON-compliant strings (using single quotes), forcing reliance on fragile `ast.literal_eval`. 3. Message retrieval logic incorrectly treated empty lists `[]` as falsy, causing cache operations to be silently skipped when valid empty conversations were provided. 4. Callers were inconsistently passing `messages` or `message` keys, leading to potential key errors. 5. `json.dumps` lacked a fallback encoder, causing cache writes to fail for non-serializable types. 6. The test suite contained unused mock patches. - Replaced `str(value)` with `json.dumps(value, default=str)` in `set_cache` and `async_set_cache` to ensure all cached values are stored as robust, standard JSON, falling back to string representation for non-serializable objects. - Refactored message retrieval in all cache methods to explicitly check for key presence (`if "messages" in kwargs`) rather than relying on falsy evaluation, allowing empty lists `[]` to be treated as valid inputs. - Modified input handling in cache methods to support both 'messages' and 'message' keys, ensuring robustness against varying input structures. - Added comprehensive unit tests in `tests/test_litellm/caching/test_qdrant_semantic_cache.py` to validate list-type response caching and confirm the fix for the reported errors. - Cleaned up the test suite by removing unused `mock_async_client` patches in synchronous test cases. --- litellm/caching/qdrant_semantic_cache.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 167a28791d5..f12f8bce32e 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -178,7 +178,9 @@ class QdrantSemanticCache(BaseCache): from litellm._uuid import uuid # get the prompt - messages = kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + messages = ( + kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + ) if messages is None: print_verbose("No messages provided for semantic caching") return @@ -223,7 +225,9 @@ class QdrantSemanticCache(BaseCache): print_verbose(f"sync qdrant semantic-cache get_cache, kwargs: {kwargs}") # get the messages - messages = kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + messages = ( + kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + ) if messages is None: print_verbose("No messages provided for semantic lookup") return @@ -295,7 +299,9 @@ class QdrantSemanticCache(BaseCache): print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}") # get the prompt - messages = kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + messages = ( + kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + ) if messages is None: print_verbose("No messages provided for semantic caching") return @@ -357,7 +363,9 @@ class QdrantSemanticCache(BaseCache): from litellm.proxy.proxy_server import llm_model_list, llm_router # get the messages - messages = kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + messages = ( + kwargs.get("messages") if "messages" in kwargs else kwargs.get("message") + ) if messages is None: print_verbose("No messages provided for semantic lookup") return