mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-02 02:11:58 +00:00
fix: support Responses input in Redis semantic cache (#29581)
* fix: support responses input in redis semantic cache * test: cover redis semantic prompt extraction * test: handle blank redis semantic text fallbacks * chore: remove async cache dead statement * test: cover redis semantic cache miss paths * fix: filter sensitive cache lookup kwargs * chore: rerun ci after huggingface rate limit
This commit is contained in:
Dávid Balatoni
GitHub
parent
5c915f59a5
commit
4937ade4fd
3 changed files with 586 additions and 25 deletions
|
|
@ -501,6 +501,34 @@ class Cache:
|
|||
return cached_response
|
||||
return cached_result
|
||||
|
||||
@staticmethod
|
||||
def _get_safe_cache_lookup_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]:
|
||||
cache_lookup_kwargs: Dict[str, Any] = {}
|
||||
for prompt_kwarg in ("messages", "input"):
|
||||
if prompt_kwarg in kwargs:
|
||||
cache_lookup_kwargs[prompt_kwarg] = kwargs[prompt_kwarg]
|
||||
|
||||
if isinstance(kwargs.get("metadata"), dict):
|
||||
cache_lookup_kwargs["metadata"] = {}
|
||||
|
||||
return cache_lookup_kwargs
|
||||
|
||||
@staticmethod
|
||||
def _update_metadata_from_cache_lookup_kwargs(
|
||||
original_kwargs: Dict[str, Any], cache_lookup_kwargs: Dict[str, Any]
|
||||
) -> None:
|
||||
original_metadata = original_kwargs.get("metadata")
|
||||
cache_lookup_metadata = cache_lookup_kwargs.get("metadata")
|
||||
if not isinstance(original_metadata, dict) or not isinstance(
|
||||
cache_lookup_metadata, dict
|
||||
):
|
||||
return
|
||||
|
||||
if "semantic-similarity" in cache_lookup_metadata:
|
||||
original_metadata["semantic-similarity"] = cache_lookup_metadata[
|
||||
"semantic-similarity"
|
||||
]
|
||||
|
||||
def get_cache(self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs):
|
||||
"""
|
||||
Retrieves the cached result for the given arguments.
|
||||
|
|
@ -515,7 +543,6 @@ class Cache:
|
|||
try: # never block execution
|
||||
if self.should_use_cache(**kwargs) is not True:
|
||||
return
|
||||
messages = kwargs.get("messages", [])
|
||||
if "cache_key" in kwargs:
|
||||
cache_key = kwargs["cache_key"]
|
||||
else:
|
||||
|
|
@ -527,12 +554,19 @@ class Cache:
|
|||
or cache_control_args.get("s-max-age")
|
||||
or float("inf")
|
||||
)
|
||||
cache_lookup_kwargs = self._get_safe_cache_lookup_kwargs(kwargs)
|
||||
if dynamic_cache_object is not None:
|
||||
cached_result = dynamic_cache_object.get_cache(
|
||||
cache_key, messages=messages
|
||||
cache_key, **cache_lookup_kwargs
|
||||
)
|
||||
else:
|
||||
cached_result = self.cache.get_cache(cache_key, messages=messages)
|
||||
cached_result = self.cache.get_cache(
|
||||
cache_key, **cache_lookup_kwargs
|
||||
)
|
||||
self._update_metadata_from_cache_lookup_kwargs(
|
||||
original_kwargs=kwargs,
|
||||
cache_lookup_kwargs=cache_lookup_kwargs,
|
||||
)
|
||||
return self._get_cache_logic(
|
||||
cached_result=cached_result, max_age=max_age
|
||||
)
|
||||
|
|
@ -553,7 +587,6 @@ class Cache:
|
|||
if self.should_use_cache(**kwargs) is not True:
|
||||
return
|
||||
|
||||
kwargs.get("messages", [])
|
||||
if "cache_key" in kwargs:
|
||||
cache_key = kwargs["cache_key"]
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -213,6 +213,78 @@ class RedisSemanticCache(BaseCache):
|
|||
ttl = int(ttl)
|
||||
return ttl
|
||||
|
||||
@classmethod
|
||||
def _get_prompt_from_kwargs(cls, **kwargs) -> Optional[str]:
|
||||
"""
|
||||
Extract a semantic-cache prompt from chat or Responses API request kwargs.
|
||||
"""
|
||||
messages = kwargs.get("messages")
|
||||
if messages:
|
||||
return get_str_from_messages(messages)
|
||||
|
||||
if "input" not in kwargs:
|
||||
return None
|
||||
|
||||
prompt_parts: List[str] = []
|
||||
cls._collect_responses_input_text(kwargs.get("input"), prompt_parts)
|
||||
prompt = "\n".join(prompt_parts).strip()
|
||||
return prompt or None
|
||||
|
||||
@classmethod
|
||||
def _collect_responses_input_text(cls, value: Any, prompt_parts: List[str]) -> None:
|
||||
value = cls._coerce_response_input_value(value)
|
||||
if value is None:
|
||||
return
|
||||
|
||||
if isinstance(value, str):
|
||||
stripped_value = value.strip()
|
||||
if stripped_value:
|
||||
prompt_parts.append(stripped_value)
|
||||
return
|
||||
|
||||
if isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
cls._collect_responses_input_text(item, prompt_parts)
|
||||
return
|
||||
|
||||
if isinstance(value, dict):
|
||||
content = value.get("content")
|
||||
if content is not None:
|
||||
cls._collect_responses_input_text(content, prompt_parts)
|
||||
return
|
||||
|
||||
for text_key in ("text", "output", "input_text", "output_text"):
|
||||
text_value = value.get(text_key)
|
||||
if isinstance(text_value, str):
|
||||
stripped_text = text_value.strip()
|
||||
if stripped_text:
|
||||
prompt_parts.append(stripped_text)
|
||||
return
|
||||
return
|
||||
|
||||
content = getattr(value, "content", None)
|
||||
if content is not None:
|
||||
cls._collect_responses_input_text(content, prompt_parts)
|
||||
return
|
||||
|
||||
for text_key in ("text", "output", "input_text", "output_text"):
|
||||
text_value = getattr(value, text_key, None)
|
||||
if isinstance(text_value, str):
|
||||
stripped_text = text_value.strip()
|
||||
if stripped_text:
|
||||
prompt_parts.append(stripped_text)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_input_value(value: Any) -> Any:
|
||||
model_dump = getattr(value, "model_dump", None)
|
||||
if callable(model_dump):
|
||||
return model_dump()
|
||||
dict_method = getattr(value, "dict", None)
|
||||
if callable(dict_method):
|
||||
return dict_method()
|
||||
return value
|
||||
|
||||
def _get_embedding(self, prompt: str) -> List[float]:
|
||||
"""
|
||||
Generate an embedding vector for the given prompt using the configured embedding model.
|
||||
|
|
@ -278,13 +350,11 @@ class RedisSemanticCache(BaseCache):
|
|||
|
||||
value_str: Optional[str] = None
|
||||
try:
|
||||
# Extract the prompt from messages
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic caching")
|
||||
prompt = self._get_prompt_from_kwargs(**kwargs)
|
||||
if prompt is None:
|
||||
print_verbose("No prompt provided for semantic caching")
|
||||
return
|
||||
|
||||
prompt = get_str_from_messages(messages)
|
||||
value_str = str(value)
|
||||
|
||||
store_kwargs: Dict[str, Any] = {
|
||||
|
|
@ -315,14 +385,12 @@ class RedisSemanticCache(BaseCache):
|
|||
print_verbose(f"Redis semantic-cache get_cache, kwargs: {kwargs}")
|
||||
|
||||
try:
|
||||
# Extract the prompt from messages
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic cache lookup")
|
||||
prompt = self._get_prompt_from_kwargs(**kwargs)
|
||||
if prompt is None:
|
||||
print_verbose("No prompt provided for semantic cache lookup")
|
||||
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
|
||||
return None
|
||||
|
||||
prompt = get_str_from_messages(messages)
|
||||
# Check the cache for semantically similar prompts in this exact
|
||||
# LiteLLM cache-key scope.
|
||||
check_kwargs: Dict[str, Any] = {
|
||||
|
|
@ -428,13 +496,11 @@ class RedisSemanticCache(BaseCache):
|
|||
print_verbose(f"Async Redis semantic-cache set_cache, kwargs: {kwargs}")
|
||||
|
||||
try:
|
||||
# Extract the prompt from messages
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic caching")
|
||||
prompt = self._get_prompt_from_kwargs(**kwargs)
|
||||
if prompt is None:
|
||||
print_verbose("No prompt provided for semantic caching")
|
||||
return
|
||||
|
||||
prompt = get_str_from_messages(messages)
|
||||
value_str = str(value)
|
||||
|
||||
# Generate embedding for the value (response) to cache
|
||||
|
|
@ -471,15 +537,12 @@ class RedisSemanticCache(BaseCache):
|
|||
print_verbose(f"Async Redis semantic-cache get_cache, kwargs: {kwargs}")
|
||||
|
||||
try:
|
||||
# Extract the prompt from messages
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
print_verbose("No messages provided for semantic cache lookup")
|
||||
prompt = self._get_prompt_from_kwargs(**kwargs)
|
||||
if prompt is None:
|
||||
print_verbose("No prompt provided for semantic cache lookup")
|
||||
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
|
||||
return None
|
||||
|
||||
prompt = get_str_from_messages(messages)
|
||||
|
||||
# Generate embedding for the prompt
|
||||
prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -523,3 +523,468 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter(
|
|||
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
|
||||
ttl=60,
|
||||
)
|
||||
|
||||
|
||||
def test_redis_semantic_cache_set_cache_uses_responses_string_input():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
redis_semantic_cache._get_cache_filters = MagicMock(
|
||||
return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}
|
||||
)
|
||||
redis_semantic_cache._get_ttl = MagicMock(return_value=None)
|
||||
|
||||
redis_semantic_cache.set_cache(
|
||||
key="test_key",
|
||||
value={"content": "Paris"},
|
||||
input="What is the capital of France?",
|
||||
)
|
||||
|
||||
redis_semantic_cache.llmcache.store.assert_called_once_with(
|
||||
"What is the capital of France?",
|
||||
"{'content': 'Paris'}",
|
||||
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
|
||||
)
|
||||
|
||||
|
||||
def test_redis_semantic_cache_get_cache_uses_responses_string_input():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.similarity_threshold = 0.8
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
redis_semantic_cache.llmcache.check = MagicMock(
|
||||
return_value=[
|
||||
{
|
||||
"prompt": "What is the capital of France?",
|
||||
"response": '{"content": "Paris"}',
|
||||
"vector_distance": 0.1,
|
||||
RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
redis_semantic_cache,
|
||||
"_get_cache_key_filter_expression",
|
||||
return_value="cache-key-filter",
|
||||
):
|
||||
metadata = {}
|
||||
result = redis_semantic_cache.get_cache(
|
||||
key="test_key",
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
assert result == {"content": "Paris"}
|
||||
assert metadata["semantic-similarity"] == pytest.approx(0.9)
|
||||
redis_semantic_cache.llmcache.check.assert_called_once_with(
|
||||
prompt="What is the capital of France?",
|
||||
filter_expression="cache-key-filter",
|
||||
)
|
||||
|
||||
|
||||
def test_redis_semantic_cache_set_cache_flattens_structured_responses_input():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
redis_semantic_cache._get_cache_filters = MagicMock(
|
||||
return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}
|
||||
)
|
||||
redis_semantic_cache._get_ttl = MagicMock(return_value=None)
|
||||
|
||||
redis_semantic_cache.set_cache(
|
||||
key="test_key",
|
||||
value={"content": "Paris"},
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "What is the capital of France?"},
|
||||
{"type": "input_text", "text": "Answer briefly."},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/paris.png",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
redis_semantic_cache.llmcache.store.assert_called_once_with(
|
||||
"What is the capital of France?\nAnswer briefly.",
|
||||
"{'content': 'Paris'}",
|
||||
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
|
||||
)
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_prefers_messages():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
prompt = RedisSemanticCache._get_prompt_from_kwargs(
|
||||
messages=[{"content": "message prompt"}],
|
||||
input="responses prompt",
|
||||
)
|
||||
|
||||
assert prompt == "message prompt"
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_handles_model_objects():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
class ModelDumpInput:
|
||||
def model_dump(self):
|
||||
return {"content": [{"text": "model dump prompt"}]}
|
||||
|
||||
class DictInput:
|
||||
def dict(self):
|
||||
return {"content": [{"output_text": "dict prompt"}]}
|
||||
|
||||
prompt = RedisSemanticCache._get_prompt_from_kwargs(
|
||||
input=[
|
||||
ModelDumpInput(),
|
||||
DictInput(),
|
||||
{"content": [{"input_text": "inline prompt"}]},
|
||||
{"content": [{"type": "input_image", "image_url": "https://example.com"}]},
|
||||
]
|
||||
)
|
||||
|
||||
assert prompt == "model dump prompt\ndict prompt\ninline prompt"
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_returns_none_without_text():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
assert RedisSemanticCache._get_prompt_from_kwargs() is None
|
||||
assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None
|
||||
assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None
|
||||
assert (
|
||||
RedisSemanticCache._get_prompt_from_kwargs(
|
||||
input=[{"type": "input_image", "image_url": "https://example.com"}]
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
prompt = RedisSemanticCache._get_prompt_from_kwargs(
|
||||
input={"text": " ", "input_text": "fallback prompt"}
|
||||
)
|
||||
|
||||
assert prompt == "fallback prompt"
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
class ResponseInput:
|
||||
text = " "
|
||||
input_text = "fallback prompt"
|
||||
|
||||
prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput())
|
||||
|
||||
assert prompt == "fallback prompt"
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_handles_object_content():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
class ResponseInput:
|
||||
content = [{"text": "object content prompt"}]
|
||||
|
||||
prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput())
|
||||
|
||||
assert prompt == "object content prompt"
|
||||
|
||||
|
||||
def test_redis_semantic_cache_set_cache_skips_blank_responses_input():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
|
||||
redis_semantic_cache.set_cache(
|
||||
key="test_key",
|
||||
value={"content": "Paris"},
|
||||
input=" ",
|
||||
)
|
||||
|
||||
redis_semantic_cache.llmcache.store.assert_not_called()
|
||||
|
||||
|
||||
def test_redis_semantic_cache_get_cache_sets_similarity_on_blank_responses_input():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
metadata = {}
|
||||
|
||||
result = redis_semantic_cache.get_cache(
|
||||
key="test_key",
|
||||
input=" ",
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert metadata["semantic-similarity"] == 0.0
|
||||
redis_semantic_cache.llmcache.check.assert_not_called()
|
||||
|
||||
|
||||
def test_redis_semantic_cache_get_cache_sets_similarity_when_no_results():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
redis_semantic_cache.llmcache.check = MagicMock(return_value=[])
|
||||
|
||||
with patch.object(
|
||||
redis_semantic_cache,
|
||||
"_get_cache_key_filter_expression",
|
||||
return_value="cache-key-filter",
|
||||
):
|
||||
metadata = {}
|
||||
result = redis_semantic_cache.get_cache(
|
||||
key="test_key",
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert metadata["semantic-similarity"] == 0.0
|
||||
redis_semantic_cache.llmcache.check.assert_called_once_with(
|
||||
prompt="What is the capital of France?",
|
||||
filter_expression="cache-key-filter",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_semantic_cache_async_paths_use_responses_string_input():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.similarity_threshold = 0.8
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
redis_semantic_cache.llmcache.astore = AsyncMock()
|
||||
redis_semantic_cache.llmcache.acheck = AsyncMock(
|
||||
return_value=[
|
||||
{
|
||||
"prompt": "What is the capital of France?",
|
||||
"response": '{"content": "Paris"}',
|
||||
"vector_distance": 0.1,
|
||||
RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key",
|
||||
}
|
||||
]
|
||||
)
|
||||
redis_semantic_cache._get_cache_filters = MagicMock(
|
||||
return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}
|
||||
)
|
||||
redis_semantic_cache._get_ttl = MagicMock(return_value=None)
|
||||
redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3])
|
||||
|
||||
await redis_semantic_cache.async_set_cache(
|
||||
key="test_key",
|
||||
value={"content": "Paris"},
|
||||
input="What is the capital of France?",
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
redis_semantic_cache,
|
||||
"_get_cache_key_filter_expression",
|
||||
return_value="cache-key-filter",
|
||||
):
|
||||
metadata = {}
|
||||
result = await redis_semantic_cache.async_get_cache(
|
||||
key="test_key",
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
redis_semantic_cache.llmcache.astore.assert_called_once_with(
|
||||
"What is the capital of France?",
|
||||
"{'content': 'Paris'}",
|
||||
vector=[0.1, 0.2, 0.3],
|
||||
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
|
||||
)
|
||||
assert result == {"content": "Paris"}
|
||||
assert metadata["semantic-similarity"] == pytest.approx(0.9)
|
||||
redis_semantic_cache.llmcache.acheck.assert_called_once_with(
|
||||
prompt="What is the capital of France?",
|
||||
vector=[0.1, 0.2, 0.3],
|
||||
filter_expression="cache-key-filter",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_semantic_cache_async_paths_set_similarity_on_misses():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
|
||||
redis_semantic_cache.llmcache = MagicMock()
|
||||
redis_semantic_cache.llmcache.astore = AsyncMock()
|
||||
redis_semantic_cache.llmcache.acheck = AsyncMock(return_value=[])
|
||||
redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3])
|
||||
|
||||
await redis_semantic_cache.async_set_cache(
|
||||
key="test_key",
|
||||
value={"content": "Paris"},
|
||||
input=" ",
|
||||
)
|
||||
|
||||
redis_semantic_cache.llmcache.astore.assert_not_called()
|
||||
redis_semantic_cache._get_async_embedding.assert_not_called()
|
||||
|
||||
blank_metadata = {}
|
||||
blank_result = await redis_semantic_cache.async_get_cache(
|
||||
key="test_key",
|
||||
input=" ",
|
||||
metadata=blank_metadata,
|
||||
)
|
||||
|
||||
assert blank_result is None
|
||||
assert blank_metadata["semantic-similarity"] == 0.0
|
||||
redis_semantic_cache.llmcache.acheck.assert_not_called()
|
||||
redis_semantic_cache._get_async_embedding.assert_not_called()
|
||||
|
||||
with patch.object(
|
||||
redis_semantic_cache,
|
||||
"_get_cache_key_filter_expression",
|
||||
return_value="cache-key-filter",
|
||||
):
|
||||
miss_metadata = {}
|
||||
miss_result = await redis_semantic_cache.async_get_cache(
|
||||
key="test_key",
|
||||
input="What is the capital of France?",
|
||||
metadata=miss_metadata,
|
||||
)
|
||||
|
||||
assert miss_result is None
|
||||
assert miss_metadata["semantic-similarity"] == 0.0
|
||||
redis_semantic_cache.llmcache.acheck.assert_called_once_with(
|
||||
prompt="What is the capital of France?",
|
||||
vector=[0.1, 0.2, 0.3],
|
||||
filter_expression="cache-key-filter",
|
||||
)
|
||||
|
||||
|
||||
def test_cache_get_cache_passes_responses_input_to_backend_cache():
|
||||
from litellm.caching.caching import Cache
|
||||
|
||||
cache = Cache.__new__(Cache)
|
||||
cache.cache = MagicMock()
|
||||
cache.cache.get_cache = MagicMock(return_value=None)
|
||||
cache.should_use_cache = MagicMock(return_value=True)
|
||||
cache.get_cache_key = MagicMock(return_value="test_key")
|
||||
|
||||
metadata = {}
|
||||
cache.get_cache(
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
cache={},
|
||||
)
|
||||
|
||||
cache.cache.get_cache.assert_called_once_with(
|
||||
"test_key",
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def test_cache_get_cache_filters_sensitive_kwargs_from_backend_cache():
|
||||
from litellm.caching.caching import Cache
|
||||
|
||||
cache = Cache.__new__(Cache)
|
||||
cache.cache = MagicMock()
|
||||
cache.should_use_cache = MagicMock(return_value=True)
|
||||
cache.get_cache_key = MagicMock(return_value="test_key")
|
||||
cache._get_cache_logic = MagicMock(return_value={"content": "Paris"})
|
||||
|
||||
def _cache_hit(_cache_key, **cache_kwargs):
|
||||
cache_kwargs["metadata"]["semantic-similarity"] = 0.7
|
||||
return {"content": "Paris"}
|
||||
|
||||
cache.cache.get_cache = MagicMock(side_effect=_cache_hit)
|
||||
|
||||
metadata = {"user_api_key": "sk-secret", "trace_id": "trace-id"}
|
||||
result = cache.get_cache(
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
cache={"s-maxage": 10},
|
||||
api_key="sk-secret",
|
||||
headers={"authorization": "Bearer sk-secret"},
|
||||
)
|
||||
|
||||
assert result == {"content": "Paris"}
|
||||
assert metadata == {
|
||||
"user_api_key": "sk-secret",
|
||||
"trace_id": "trace-id",
|
||||
"semantic-similarity": 0.7,
|
||||
}
|
||||
|
||||
forwarded_kwargs = cache.cache.get_cache.call_args.kwargs
|
||||
assert forwarded_kwargs == {
|
||||
"input": "What is the capital of France?",
|
||||
"metadata": {"semantic-similarity": 0.7},
|
||||
}
|
||||
assert forwarded_kwargs["metadata"] is not metadata
|
||||
cache._get_cache_logic.assert_called_once_with(
|
||||
cached_result={"content": "Paris"},
|
||||
max_age=10,
|
||||
)
|
||||
|
||||
|
||||
def test_cache_get_cache_filters_sensitive_kwargs_without_metadata():
|
||||
from litellm.caching.caching import Cache
|
||||
|
||||
cache = Cache.__new__(Cache)
|
||||
cache.cache = MagicMock()
|
||||
cache.cache.get_cache = MagicMock(return_value={"content": "Paris"})
|
||||
cache.should_use_cache = MagicMock(return_value=True)
|
||||
cache.get_cache_key = MagicMock(return_value="test_key")
|
||||
cache._get_cache_logic = MagicMock(return_value={"content": "Paris"})
|
||||
|
||||
result = cache.get_cache(
|
||||
input="What is the capital of France?",
|
||||
cache={"s-maxage": 10},
|
||||
api_key="sk-secret",
|
||||
headers={"authorization": "Bearer sk-secret"},
|
||||
)
|
||||
|
||||
assert result == {"content": "Paris"}
|
||||
cache.cache.get_cache.assert_called_once_with(
|
||||
"test_key",
|
||||
input="What is the capital of France?",
|
||||
)
|
||||
|
||||
|
||||
def test_cache_get_cache_passes_responses_input_to_dynamic_cache():
|
||||
from litellm.caching.caching import Cache
|
||||
|
||||
cache = Cache.__new__(Cache)
|
||||
cache.should_use_cache = MagicMock(return_value=True)
|
||||
cache.get_cache_key = MagicMock(return_value="test_key")
|
||||
cache._get_cache_logic = MagicMock(return_value={"content": "Paris"})
|
||||
dynamic_cache_object = MagicMock()
|
||||
dynamic_cache_object.get_cache = MagicMock(return_value={"content": "Paris"})
|
||||
|
||||
metadata = {}
|
||||
result = cache.get_cache(
|
||||
dynamic_cache_object=dynamic_cache_object,
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
cache={},
|
||||
)
|
||||
|
||||
assert result == {"content": "Paris"}
|
||||
dynamic_cache_object.get_cache.assert_called_once_with(
|
||||
"test_key",
|
||||
input="What is the capital of France?",
|
||||
metadata=metadata,
|
||||
)
|
||||
cache._get_cache_logic.assert_called_once_with(
|
||||
cached_result={"content": "Paris"},
|
||||
max_age=float("inf"),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue