scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix: support responses input in redis semantic cache

Browse source
This commit is contained in:
Dávid Balatoni 2026-05-19 20:57:12 +02:00
parent d45e9e4d56
commit 05d9657629
No known key found for this signature in database
3 changed files with 262 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/caching/caching.py
View file

@ -511,7 +511,6 @@ class Cache:
try: # never block execution
if self.should_use_cache(**kwargs) is not True:
return
messages = kwargs.get("messages", [])
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
@ -524,11 +523,9 @@ class Cache:
or float("inf")
)
if dynamic_cache_object is not None:
cached_result = dynamic_cache_object.get_cache(
cache_key, messages=messages
)
cached_result = dynamic_cache_object.get_cache(cache_key, **kwargs)
else:
cached_result = self.cache.get_cache(cache_key, messages=messages)
cached_result = self.cache.get_cache(cache_key, **kwargs)
return self._get_cache_logic(
cached_result=cached_result, max_age=max_age
)

105
litellm/caching/redis_semantic_cache.py
View file

@ -213,6 +213,78 @@ class RedisSemanticCache(BaseCache):
ttl = int(ttl)
return ttl
@classmethod
def _get_prompt_from_kwargs(cls, **kwargs) -> Optional[str]:
"""
Extract a semantic-cache prompt from chat or Responses API request kwargs.
"""
messages = kwargs.get("messages")
if messages:
return get_str_from_messages(messages)
if "input" not in kwargs:
return None
prompt_parts: List[str] = []
cls._collect_responses_input_text(kwargs.get("input"), prompt_parts)
prompt = "\n".join(prompt_parts).strip()
return prompt or None
@classmethod
def _collect_responses_input_text(cls, value: Any, prompt_parts: List[str]) -> None:
value = cls._coerce_response_input_value(value)
if value is None:
return
if isinstance(value, str):
stripped_value = value.strip()
if stripped_value:
prompt_parts.append(stripped_value)
return
if isinstance(value, (list, tuple)):
for item in value:
cls._collect_responses_input_text(item, prompt_parts)
return
if isinstance(value, dict):
content = value.get("content")
if content is not None:
cls._collect_responses_input_text(content, prompt_parts)
return
for text_key in ("text", "output", "input_text", "output_text"):
text_value = value.get(text_key)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
return
content = getattr(value, "content", None)
if content is not None:
cls._collect_responses_input_text(content, prompt_parts)
return
for text_key in ("text", "output", "input_text", "output_text"):
text_value = getattr(value, text_key, None)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
@staticmethod
def _coerce_response_input_value(value: Any) -> Any:
model_dump = getattr(value, "model_dump", None)
if callable(model_dump):
return model_dump()
dict_method = getattr(value, "dict", None)
if callable(dict_method):
return dict_method()
return value
def _get_embedding(self, prompt: str) -> List[float]:
"""
Generate an embedding vector for the given prompt using the configured embedding model.
@ -278,13 +350,11 @@ class RedisSemanticCache(BaseCache):
value_str: Optional[str] = None
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic caching")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic caching")
return
prompt = get_str_from_messages(messages)
value_str = str(value)
store_kwargs: Dict[str, Any] = {
@ -315,14 +385,12 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Redis semantic-cache get_cache, kwargs: {kwargs}")
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
prompt = get_str_from_messages(messages)
# Check the cache for semantically similar prompts in this exact
# LiteLLM cache-key scope.
check_kwargs: Dict[str, Any] = {
@ -428,13 +496,11 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Async Redis semantic-cache set_cache, kwargs: {kwargs}")
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic caching")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic caching")
return
prompt = get_str_from_messages(messages)
value_str = str(value)
# Generate embedding for the value (response) to cache
@ -471,15 +537,12 @@ class RedisSemanticCache(BaseCache):
print_verbose(f"Async Redis semantic-cache get_cache, kwargs: {kwargs}")
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
prompt = get_str_from_messages(messages)
# Generate embedding for the prompt
prompt_embedding = await self._get_async_embedding(prompt, **kwargs)

176
tests/test_litellm/caching/test_redis_semantic_cache.py
View file

@ -523,3 +523,179 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter(
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
ttl=60,
)
def test_redis_semantic_cache_set_cache_uses_responses_string_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
redis_semantic_cache.llmcache = MagicMock()
redis_semantic_cache._get_cache_filters = MagicMock(
return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}
)
redis_semantic_cache._get_ttl = MagicMock(return_value=None)
redis_semantic_cache.set_cache(
key="test_key",
value={"content": "Paris"},
input="What is the capital of France?",
)
redis_semantic_cache.llmcache.store.assert_called_once_with(
"What is the capital of France?",
"{'content': 'Paris'}",
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
)
def test_redis_semantic_cache_get_cache_uses_responses_string_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
redis_semantic_cache.similarity_threshold = 0.8
redis_semantic_cache.llmcache = MagicMock()
redis_semantic_cache.llmcache.check = MagicMock(
return_value=[
{
"prompt": "What is the capital of France?",
"response": '{"content": "Paris"}',
"vector_distance": 0.1,
RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key",
}
]
)
with patch.object(
redis_semantic_cache,
"_get_cache_key_filter_expression",
return_value="cache-key-filter",
):
metadata = {}
result = redis_semantic_cache.get_cache(
key="test_key",
input="What is the capital of France?",
metadata=metadata,
)
assert result == {"content": "Paris"}
assert metadata["semantic-similarity"] == pytest.approx(0.9)
redis_semantic_cache.llmcache.check.assert_called_once_with(
prompt="What is the capital of France?",
filter_expression="cache-key-filter",
)
def test_redis_semantic_cache_set_cache_flattens_structured_responses_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
redis_semantic_cache.llmcache = MagicMock()
redis_semantic_cache._get_cache_filters = MagicMock(
return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}
)
redis_semantic_cache._get_ttl = MagicMock(return_value=None)
redis_semantic_cache.set_cache(
key="test_key",
value={"content": "Paris"},
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "What is the capital of France?"},
{"type": "input_text", "text": "Answer briefly."},
{
"type": "input_image",
"image_url": "https://example.com/paris.png",
},
],
}
],
)
redis_semantic_cache.llmcache.store.assert_called_once_with(
"What is the capital of France?\nAnswer briefly.",
"{'content': 'Paris'}",
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
)
@pytest.mark.asyncio
async def test_redis_semantic_cache_async_paths_use_responses_string_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
redis_semantic_cache.similarity_threshold = 0.8
redis_semantic_cache.llmcache = MagicMock()
redis_semantic_cache.llmcache.astore = AsyncMock()
redis_semantic_cache.llmcache.acheck = AsyncMock(
return_value=[
{
"prompt": "What is the capital of France?",
"response": '{"content": "Paris"}',
"vector_distance": 0.1,
RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key",
}
]
)
redis_semantic_cache._get_cache_filters = MagicMock(
return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}
)
redis_semantic_cache._get_ttl = MagicMock(return_value=None)
redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3])
await redis_semantic_cache.async_set_cache(
key="test_key",
value={"content": "Paris"},
input="What is the capital of France?",
)
with patch.object(
redis_semantic_cache,
"_get_cache_key_filter_expression",
return_value="cache-key-filter",
):
metadata = {}
result = await redis_semantic_cache.async_get_cache(
key="test_key",
input="What is the capital of France?",
metadata=metadata,
)
redis_semantic_cache.llmcache.astore.assert_called_once_with(
"What is the capital of France?",
"{'content': 'Paris'}",
vector=[0.1, 0.2, 0.3],
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
)
assert result == {"content": "Paris"}
assert metadata["semantic-similarity"] == pytest.approx(0.9)
redis_semantic_cache.llmcache.acheck.assert_called_once_with(
prompt="What is the capital of France?",
vector=[0.1, 0.2, 0.3],
filter_expression="cache-key-filter",
)
def test_cache_get_cache_passes_responses_input_to_backend_cache():
from litellm.caching.caching import Cache
cache = Cache.__new__(Cache)
cache.cache = MagicMock()
cache.cache.get_cache = MagicMock(return_value=None)
cache.should_use_cache = MagicMock(return_value=True)
cache.get_cache_key = MagicMock(return_value="test_key")
metadata = {}
cache.get_cache(
input="What is the capital of France?",
metadata=metadata,
cache={},
)
cache.cache.get_cache.assert_called_once_with(
"test_key",
input="What is the capital of France?",
metadata=metadata,
cache={},
)