From 397d0b48245c0665979b4d14f955f15df09e9194 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 22 Sep 2026 00:41:24 +0000 Subject: [PATCH] fix(cache): keep native Redis semantic binding and Qdrant batch writes after merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/python-bridge/src/cache/config.rs | 53 +++++++++++++++++-- .../crates/python-bridge/src/cache/native.rs | 22 ++++++-- tests/test_litellm_rust/test_cache.py | 32 +++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs index d161b401928..766d526cf5f 100644 --- a/litellm-rust/crates/python-bridge/src/cache/config.rs +++ b/litellm-rust/crates/python-bridge/src/cache/config.rs @@ -410,7 +410,8 @@ impl NativeCacheConfig { Some("facade and native backend index names must match") } CacheBackendConfig::RedisSemantic(config) - if service.similarity_threshold() != Some(config.similarity_threshold) => + if service.similarity_threshold() + != Some(f64::from(config.similarity_threshold as f32)) => { Some("facade and native backend similarity thresholds must match") } @@ -1177,12 +1178,13 @@ mod tests { use pyo3::{prelude::*, types::PyDict}; use litellm_cache_redis::{RedisNode, RedisTopology}; + use litellm_cache_redis_semantic::RedisSemanticConfig; use super::{ - CacheBackendConfig, CacheConfigProjection, CertificateRequirement, GcsCacheConfig, - NativeCacheConfig, RedisProtocol, UnsupportedCacheConfig, + CacheBackendConfig, CacheConfigProjection, CachePolicy, CertificateRequirement, + GcsCacheConfig, NativeCacheConfig, RedisProtocol, UnsupportedCacheConfig, }; - use crate::cache::native::NativeResponseCache; + use crate::cache::{embedder::PythonEmbedder, native::NativeResponseCache}; fn cluster_facade<'py>(py: Python<'py>, startup_nodes: &str, hook: &str) -> Bound<'py, PyAny> { facade( @@ -1255,6 +1257,49 @@ mod tests { }); } + #[test] + fn redis_semantic_service_mismatch_accepts_backend_precision_threshold() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(_redis_url='redis://127.0.0.1/', _index_name='semantic_idx', similarity_threshold=0.8, embedding_model='text-embedding-3-small', embedding_max_input_tokens=None, embedding_timeout=None)\n\ + facade = SimpleNamespace(type='redis-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let backend = facade.getattr("cache").unwrap(); + let embedder = PythonEmbedder::new(backend.clone().unbind()); + let CacheConfigProjection::Native(config) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("Redis semantic cache should be supported"); + }; + let CacheBackendConfig::RedisSemantic(config) = config.backend else { + panic!("expected Redis semantic configuration"); + }; + let service = NativeResponseCache::redis_semantic( + &config.redis_url, + embedder, + RedisSemanticConfig { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold as f32, + }, + ) + .unwrap(); + let matching_config = NativeCacheConfig { + policy: CachePolicy { + mode: "default-on".into(), + ttl: None, + namespace: None, + supported_call_types: None, + redis_flush_size: None, + semantic_cache_scope: "key".into(), + }, + backend: CacheBackendConfig::RedisSemantic(config), + }; + assert_eq!(matching_config.service_mismatch(&service), None); + }); + } + #[test] fn projects_resolved_redis_tls_configuration() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 7ca62f10ec8..7fef6f55611 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -849,8 +849,13 @@ impl NativeResponseCache { .collect(); cache.async_store_batch(entries, now).await } - Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { - Err(Error::UnsupportedOperation) + Self::RedisSemantic { .. } => Err(Error::UnsupportedOperation), + Self::QdrantSemantic(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::semantic_request(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await } Self::Gcs(cache) => { let entries = entries @@ -922,7 +927,18 @@ impl NativeResponseCache { py, SemanticBody::new(self.clone(), SemanticOperation::StoreBatch(entries.into())), ), - Self::QdrantSemantic(_) => Err(super::cache_error(Error::UnsupportedOperation)), + Self::QdrantSemantic(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { + service + .async_store_batch(entries, super::request::now()) + .await + }, + super::cache_error, + ) + } } } diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py index c4962b533c4..48ca6f5e165 100644 --- a/tests/test_litellm_rust/test_cache.py +++ b/tests/test_litellm_rust/test_cache.py @@ -1830,6 +1830,38 @@ async def test_qdrant_semantic_async_parity( assert python_value["response"] == {"id": "native"} +async def test_qdrant_semantic_async_store_batch_shares_entries( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + entries: Final = [ + qdrant_request("batch-one", [{"role": "user", "content": "first batch prompt"}]), + qdrant_request("batch-two", [{"role": "user", "content": "second batch prompt"}]), + ] + await binding.async_store_batch(entries, [{"id": "one"}, {"id": "two"}]) + + assert binding.lookup(entries[0]) == {"id": "one"} + assert binding.lookup(entries[1]) == {"id": "two"} + assert ( + (await facade.cache.async_get_cache("batch-one", messages=entries[0]["messages"]))["response"] + == {"id": "one"} + ) + assert ( + (await facade.cache.async_get_cache("batch-two", messages=entries[1]["messages"]))["response"] + == {"id": "two"} + ) + + async def test_qdrant_semantic_malformed_entries_and_unsupported_operations( qdrant_url: str, fake_embedding_endpoint: str ) -> None: