feat(python-bridge): serve redis-semantic caches natively

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm-rust/Cargo.lock

@@ -2683,6 +2683,7 @@ dependencies = [
  "litellm-cache",
  "litellm-cache-memory",
  "litellm-cache-redis",
+ "litellm-cache-redis-semantic",
  "litellm-cache-response",
  "litellm-callbacks-legacy-python",
  "litellm-core",

litellm-rust/crates/python-bridge/Cargo.toml

@@ -23,6 +23,7 @@ bytes.workspace = true
 litellm-cache.workspace = true
 litellm-cache-memory.workspace = true
 litellm-cache-redis.workspace = true
+litellm-cache-redis-semantic.workspace = true
 litellm-cache-response.workspace = true
 serde.workspace = true
 litellm-auth.workspace = true

litellm-rust/crates/python-bridge/src/cache/config.rs

@@ -73,9 +73,23 @@ pub(super) struct RedisCacheConfig {
     pub(super) connection: RedisConnectionConfig,
 }
 
+#[allow(
+    dead_code,
+    reason = "embedding settings are projected so drift falls back to Python"
+)]
+pub(super) struct RedisSemanticCacheConfig {
+    pub(super) redis_url: String,
+    pub(super) index_name: String,
+    pub(super) similarity_threshold: f64,
+    pub(super) embedding_model: String,
+    pub(super) embedding_max_input_tokens: Option<u64>,
+    pub(super) embedding_timeout: Option<f64>,
+}
+
 pub(super) enum CacheBackendConfig {
     Memory(MemoryCacheConfig),
     Redis(Box<RedisCacheConfig>),
+    RedisSemantic(Box<RedisSemanticCacheConfig>),
 }
 
 #[allow(dead_code, reason = "consumed by the cache activation follow-up")]
@@ -142,9 +156,14 @@ impl NativeCacheConfig {
                 }))),
                 Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
             },
+            Some(CacheType::RedisSemantic) => project_redis_semantic(&backend).map(|backend| {
+                CacheConfigProjection::Native(Box::new(Self {
+                    policy,
+                    backend: CacheBackendConfig::RedisSemantic(Box::new(backend)),
+                }))
+            }),
             Some(
-                CacheType::RedisSemantic
-                | CacheType::ValkeySemantic
+                CacheType::ValkeySemantic
                 | CacheType::S3
                 | CacheType::Disk
                 | CacheType::QdrantSemantic
@@ -159,10 +178,11 @@ impl NativeCacheConfig {
 
     pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
         if service.default_ttl()
-            != Some(match &self.backend {
-                CacheBackendConfig::Memory(config) => config.default_ttl,
-                CacheBackendConfig::Redis(config) => config.default_ttl,
-            })
+            != match &self.backend {
+                CacheBackendConfig::Memory(config) => Some(config.default_ttl),
+                CacheBackendConfig::Redis(config) => Some(config.default_ttl),
+                CacheBackendConfig::RedisSemantic(_) => None,
+            }
         {
             return Some("facade and native backend default TTLs must match");
         }
@@ -185,10 +205,45 @@ impl NativeCacheConfig {
             CacheBackendConfig::Redis(config) => (service.namespace()
                 != config.namespace.as_deref())
             .then_some("facade and native backend namespaces must match"),
+            CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => {
+                Some("facade and native backend types must match")
+            }
+            CacheBackendConfig::RedisSemantic(config)
+                if service.index_name() != Some(config.index_name.as_str()) =>
+            {
+                Some("facade and native backend index names must match")
+            }
+            CacheBackendConfig::RedisSemantic(config)
+                if service.similarity_threshold() != Some(config.similarity_threshold as f32) =>
+            {
+                Some("facade and native backend similarity thresholds must match")
+            }
+            CacheBackendConfig::RedisSemantic(_) => None,
         }
     }
 }
 
+#[inline(never)]
+pub(super) fn project_redis_semantic(
+    backend: &Bound<'_, PyAny>,
+) -> PyResult<RedisSemanticCacheConfig> {
+    Ok(RedisSemanticCacheConfig {
+        redis_url: backend.getattr("_redis_url")?.extract::<String>()?,
+        index_name: backend
+            .getattr("_index_name")?
+            .extract::<Option<String>>()?
+            .unwrap_or_else(|| "litellm_semantic_cache_index".into()),
+        similarity_threshold: backend.getattr("similarity_threshold")?.extract::<f64>()?,
+        embedding_model: backend.getattr("embedding_model")?.extract::<String>()?,
+        embedding_max_input_tokens: backend
+            .getattr("embedding_max_input_tokens")?
+            .extract::<Option<u64>>()?,
+        embedding_timeout: backend
+            .getattr("embedding_timeout")?
+            .extract::<Option<f64>>()?,
+    })
+}
+
 #[inline(never)]
 fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
     let max_size_kib = backend.getattr("max_size_per_item")?.extract::<usize>()?;

litellm-rust/crates/python-bridge/src/cache/embedder.rs

@@ -0,0 +1,85 @@
+use std::future::Future;
+
+use litellm_cache::Error;
+use litellm_cache_redis_semantic::Embedder;
+use litellm_host_python::to_py;
+use pyo3::{PyTraverseError, PyVisit, prelude::*, types::PyDict};
+use serde_json::{Map, Value};
+
+pub(super) struct PythonEmbedder(Py<PyAny>);
+
+impl PythonEmbedder {
+    pub(super) fn new(object: Py<PyAny>) -> Self {
+        Self(object)
+    }
+
+    pub(super) fn object(&self) -> &Py<PyAny> {
+        &self.0
+    }
+
+    pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        visit.call(&self.0)
+    }
+
+    fn metadata_kwargs<'py>(
+        py: Python<'py>,
+        metadata: &Map<String, Value>,
+    ) -> PyResult<Bound<'py, PyDict>> {
+        let kwargs = PyDict::new(py);
+        if metadata.is_empty() {
+            kwargs.set_item("metadata", py.None())?;
+        } else {
+            kwargs.set_item("metadata", to_py(py, metadata)?)?;
+        }
+        Ok(kwargs)
+    }
+
+    fn extract(vector: Bound<'_, PyAny>) -> PyResult<Vec<f32>> {
+        Ok(vector
+            .extract::<Vec<f64>>()?
+            .into_iter()
+            .map(|value| value as f32)
+            .collect())
+    }
+}
+
+impl Embedder for PythonEmbedder {
+    fn embed(&self, prompt: &str, metadata: &Map<String, Value>) -> Result<Vec<f32>, Error> {
+        Python::attach(|py| {
+            let kwargs = Self::metadata_kwargs(py, metadata)?;
+            Self::extract(self.0.bind(py).call_method(
+                "_get_embedding",
+                (prompt,),
+                Some(&kwargs),
+            )?)
+        })
+        .map_err(|_| Error::Unavailable)
+    }
+
+    fn async_embed(
+        &self,
+        prompt: &str,
+        metadata: &Map<String, Value>,
+    ) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
+        let coroutine = Python::attach(|py| {
+            let kwargs = Self::metadata_kwargs(py, metadata)?;
+            self.0
+                .bind(py)
+                .call_method("_get_async_embedding", (prompt,), Some(&kwargs))
+                .map(Bound::unbind)
+        })
+        .map_err(|_| Error::Unavailable);
+        async move {
+            let coroutine = coroutine?;
+            let awaited = Python::attach(|py| {
+                pyo3_async_runtimes::tokio::into_future(coroutine.into_bound(py))
+            })
+            .map_err(|_| Error::Unavailable)?
+            .await
+            .map_err(|_| Error::Unavailable)?;
+            let vector = Python::attach(|py| awaited.extract::<Vec<f64>>(py))
+                .map_err(|_| Error::Unavailable)?;
+            Ok(vector.into_iter().map(|value| value as f32).collect())
+        }
+    }
+}

litellm-rust/crates/python-bridge/src/cache/facade.rs

@@ -192,6 +192,11 @@ impl FacadeGuard {
         let (module, name, cache_kind) = match kind {
             "memory" => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
             "redis" => ("litellm.caching.redis_cache", "RedisCache", "redis"),
+            "redis_semantic" => (
+                "litellm.caching.redis_semantic_cache",
+                "RedisSemanticCache",
+                "redis-semantic",
+            ),
             _ => unreachable!(),
         };
         let backend = facade.getattr("cache")?;
@@ -211,6 +216,15 @@ impl FacadeGuard {
         if let Some(message) = config.service_mismatch(service) {
             return Err(PyTypeError::new_err(message));
         }
+        if kind == "redis_semantic"
+            && service
+                .embedder_object()
+                .is_none_or(|embedder| !backend.is(embedder.bind(py)))
+        {
+            return Err(PyTypeError::new_err(
+                "facade backend must be the native embedder",
+            ));
+        }
         Ok(Self {
             outer: ObjectGuard::capture(
                 py,
@@ -235,6 +249,13 @@ impl FacadeGuard {
                     "max_size_per_item",
                     "redis_kwargs",
                     "redis_flush_size",
+                    "similarity_threshold",
+                    "distance_threshold",
+                    "embedding_model",
+                    "embedding_max_input_tokens",
+                    "embedding_timeout",
+                    "_index_name",
+                    "_redis_url",
                 ],
             )?,
             redis_pool: (kind == "redis")

litellm-rust/crates/python-bridge/src/cache/handle.rs

@@ -1,7 +1,15 @@
+use litellm_cache_redis_semantic::RedisSemanticConfig;
 use litellm_host_python::release_gil;
-use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
+use pyo3::{
+    PyTraverseError, PyVisit,
+    exceptions::{PyRuntimeError, PyTypeError},
+    prelude::*,
+};
 
-use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
+use super::{
+    cache_error, config::project_redis_semantic, embedder::PythonEmbedder, facade::FacadeGuard,
+    native::NativeResponseCache, request::duration,
+};
 
 #[pyclass(frozen, name = "_CacheTestHandle")]
 pub(crate) struct CacheTestHandle {
@@ -51,6 +59,36 @@ impl CacheTestHandle {
         })
     }
 
+    #[staticmethod]
+    fn redis_semantic(py: Python<'_>, backend: Bound<'_, PyAny>) -> PyResult<Self> {
+        let class = py
+            .import("litellm.caching.redis_semantic_cache")?
+            .getattr("RedisSemanticCache")?;
+        if !backend.get_type().is(&class) {
+            return Err(PyTypeError::new_err(
+                "native redis-semantic handles require the built-in RedisSemanticCache",
+            ));
+        }
+        let config = project_redis_semantic(&backend)?;
+        let embedder = PythonEmbedder::new(backend.unbind());
+        let service = release_gil(py, move || {
+            NativeResponseCache::redis_semantic(
+                &config.redis_url,
+                embedder,
+                RedisSemanticConfig {
+                    index_name: config.index_name,
+                    similarity_threshold: config.similarity_threshold as f32,
+                },
+            )
+        })
+        .map_err(cache_error)?;
+        Ok(Self {
+            service,
+            guard: None,
+            pid: std::process::id(),
+        })
+    }
+
     #[getter]
     fn backend(&self) -> &'static str {
         self.service.kind()
@@ -76,6 +114,7 @@ impl CacheTestHandle {
     }
 
     fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
+        self.service.traverse(&visit)?;
         if let Some(guard) = &self.guard {
             guard.traverse(visit)?;
         }

litellm-rust/crates/python-bridge/src/cache/mod.rs

@@ -1,6 +1,7 @@
 mod binding;
 mod callback;
 mod config;
+mod embedder;
 mod facade;
 mod future;
 mod handle;
@@ -10,7 +11,7 @@ mod resolver;
 
 use litellm_cache::Error;
 use pyo3::{
-    exceptions::{PyRuntimeError, PyValueError},
+    exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
     prelude::*,
 };
 
@@ -21,6 +22,7 @@ pub(crate) use self::{
 fn cache_error(error: Error) -> PyErr {
     match error {
         Error::InvalidEntry => PyValueError::new_err(error.to_string()),
+        Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()),
         _ => PyRuntimeError::new_err(error.to_string()),
     }
 }

litellm-rust/crates/python-bridge/src/cache/native.rs

@@ -1,13 +1,17 @@
 use std::{sync::Arc, time::Duration};
 
-use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
+use litellm_cache::{CacheCodec, CacheConnectionResult, Error, ExactCacheContext};
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_redis::RedisCache;
+use litellm_cache_redis_semantic::{RedisSemanticCache, RedisSemanticConfig};
 use litellm_cache_response::{
     CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer,
 };
+use pyo3::{Py, PyAny, PyTraverseError, PyVisit};
 use serde_json::Value;
 
+use super::{embedder::PythonEmbedder, request::CacheRequest};
+
 #[derive(Clone)]
 pub(super) enum NativeResponseCache {
     Memory(Arc<ResponseCache<InMemoryCache<CacheEntry>>>),
@@ -15,6 +19,7 @@ pub(super) enum NativeResponseCache {
         cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
         buffer: Option<Arc<WriteBuffer>>,
     },
+    RedisSemantic(Arc<ResponseCache<RedisSemanticCache<PythonEmbedder>>>),
 }
 
 impl NativeResponseCache {
@@ -43,6 +48,17 @@ impl NativeResponseCache {
             buffer: None,
         })
     }
+
+    pub fn redis_semantic(
+        url: &str,
+        embedder: PythonEmbedder,
+        config: RedisSemanticConfig,
+    ) -> Result<Self, Error> {
+        let backend = RedisSemanticCache::new(url, embedder, config)?;
+        Ok(Self::RedisSemantic(Arc::new(ResponseCache::new(Arc::new(
+            backend,
+        )))))
+    }
 }
 
 impl NativeResponseCache {
@@ -50,6 +66,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(_) => "memory",
             Self::Redis { .. } => "redis",
+            Self::RedisSemantic(_) => "redis_semantic",
         }
     }
 
@@ -57,12 +74,13 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.default_ttl(),
             Self::Redis { cache, .. } => cache.default_ttl(),
+            Self::RedisSemantic(cache) => cache.default_ttl(),
         }
     }
 
     pub fn namespace(&self) -> Option<&str> {
         match self {
-            Self::Memory(_) => None,
+            Self::Memory(_) | Self::RedisSemantic(_) => None,
             Self::Redis { cache, .. } => cache.backend().namespace(),
         }
     }
@@ -70,110 +88,189 @@ impl NativeResponseCache {
     pub fn capacity(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::RedisSemantic(_) => None,
         }
     }
 
     pub fn max_entry_bytes(&self) -> Option<usize> {
         match self {
             Self::Memory(cache) => cache.backend().max_entry_bytes(),
-            Self::Redis { .. } => None,
+            Self::Redis { .. } | Self::RedisSemantic(_) => None,
         }
     }
 
+    pub fn index_name(&self) -> Option<&str> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().index_name()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
+    pub fn similarity_threshold(&self) -> Option<f32> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().similarity_threshold()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
+    pub fn embedder_object(&self) -> Option<&Py<PyAny>> {
+        match self {
+            Self::RedisSemantic(cache) => Some(cache.backend().embedder().object()),
+            Self::Memory(_) | Self::Redis { .. } => None,
+        }
+    }
+
+    pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
+        if let Self::RedisSemantic(cache) = self {
+            cache.backend().embedder().traverse(visit)?;
+        }
+        Ok(())
+    }
+
     pub fn with_redis_flush_size(self, flush_size: Option<usize>) -> Self {
         match self {
             Self::Redis { cache, .. } => Self::Redis {
                 cache,
                 buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))),
             },
-            memory => memory,
+            other => other,
         }
     }
 
-    pub fn lookup(
-        &self,
-        request: &ResponseCacheRequest,
-        now: Duration,
-    ) -> Result<Option<Value>, Error> {
+    fn exact_requests(requests: &[CacheRequest]) -> Vec<ResponseCacheRequest<ExactCacheContext>> {
+        requests.iter().map(CacheRequest::exact).collect()
+    }
+
+    pub fn lookup(&self, request: &CacheRequest, now: Duration) -> Result<Option<Value>, Error> {
         match self {
-            Self::Memory(cache) => cache.lookup(request, now),
-            Self::Redis { cache, .. } => cache.lookup(request, now),
+            Self::Memory(cache) => cache.lookup(&request.exact(), now),
+            Self::Redis { cache, .. } => cache.lookup(&request.exact(), now),
+            Self::RedisSemantic(cache) => cache.lookup(&request.semantic(), now),
         }
     }
 
     pub fn store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &CacheRequest,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.store(request, response, now),
-            Self::Redis { cache, .. } => cache.store(request, response, now),
+            Self::Memory(cache) => cache.store(&request.exact(), response, now),
+            Self::Redis { cache, .. } => cache.store(&request.exact(), response, now),
+            Self::RedisSemantic(cache) => cache.store(&request.semantic(), response, now),
         }
     }
 
     pub fn lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[CacheRequest],
         now: Duration,
     ) -> Result<PartialHits, Error> {
         match self {
-            Self::Memory(cache) => cache.lookup_batch(requests, now),
-            Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
+            Self::Memory(cache) => cache.lookup_batch(&Self::exact_requests(requests), now),
+            Self::Redis { cache, .. } => cache.lookup_batch(&Self::exact_requests(requests), now),
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 
     pub async fn async_lookup(
         &self,
-        request: &ResponseCacheRequest,
+        request: &CacheRequest,
         now: Duration,
     ) -> Result<Option<Value>, Error> {
         match self {
-            Self::Memory(cache) => cache.async_lookup(request, now).await,
-            Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
+            Self::Memory(cache) => cache.async_lookup(&request.exact(), now).await,
+            Self::Redis { cache, .. } => cache.async_lookup(&request.exact(), now).await,
+            Self::RedisSemantic(cache) => cache.async_lookup(&request.semantic(), now).await,
         }
     }
 
     pub async fn async_store(
         &self,
-        request: &ResponseCacheRequest,
+        request: &CacheRequest,
         response: Value,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.async_store(request, response, now).await,
+            Self::Memory(cache) => cache.async_store(&request.exact(), response, now).await,
             Self::Redis {
                 cache,
                 buffer: None,
-            } => cache.async_store(request, response, now).await,
+            } => cache.async_store(&request.exact(), response, now).await,
             Self::Redis {
                 cache,
                 buffer: Some(buffer),
-            } => buffer.async_store(cache, request, response, now).await,
+            } => {
+                buffer
+                    .async_store(cache, &request.exact(), response, now)
+                    .await
+            }
+            Self::RedisSemantic(cache) => {
+                cache.async_store(&request.semantic(), response, now).await
+            }
         }
     }
 
     pub async fn async_lookup_batch(
         &self,
-        requests: &[ResponseCacheRequest],
+        requests: &[CacheRequest],
         now: Duration,
     ) -> Result<PartialHits, Error> {
         match self {
-            Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
-            Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
+            Self::Memory(cache) => {
+                cache
+                    .async_lookup_batch(&Self::exact_requests(requests), now)
+                    .await
+            }
+            Self::Redis { cache, .. } => {
+                cache
+                    .async_lookup_batch(&Self::exact_requests(requests), now)
+                    .await
+            }
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 
     pub async fn async_store_batch(
         &self,
-        entries: Vec<(ResponseCacheRequest, Value)>,
+        entries: Vec<(CacheRequest, Value)>,
         now: Duration,
     ) -> Result<(), Error> {
         match self {
-            Self::Memory(cache) => cache.async_store_batch(entries, now).await,
-            Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
+            Self::Memory(cache) => {
+                cache
+                    .async_store_batch(
+                        entries
+                            .into_iter()
+                            .map(|(request, value)| (request.exact(), value))
+                            .collect(),
+                        now,
+                    )
+                    .await
+            }
+            Self::Redis { cache, .. } => {
+                cache
+                    .async_store_batch(
+                        entries
+                            .into_iter()
+                            .map(|(request, value)| (request.exact(), value))
+                            .collect(),
+                        now,
+                    )
+                    .await
+            }
+            Self::RedisSemantic(cache) => {
+                cache
+                    .async_store_batch(
+                        entries
+                            .into_iter()
+                            .map(|(request, value)| (request.semantic(), value))
+                            .collect(),
+                        now,
+                    )
+                    .await
+            }
         }
     }
 
@@ -186,6 +283,7 @@ impl NativeResponseCache {
                 }
                 cache.async_flush().await
             }
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 
@@ -193,6 +291,7 @@ impl NativeResponseCache {
         match self {
             Self::Memory(cache) => cache.test_connection().await,
             Self::Redis { cache, .. } => cache.test_connection().await,
+            Self::RedisSemantic(_) => Err(Error::UnsupportedOperation),
         }
     }
 }

litellm-rust/crates/python-bridge/src/cache/request.rs

@@ -1,9 +1,11 @@
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
+use litellm_cache::{ExactCacheContext, SemanticCacheContext};
 use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest};
 use litellm_host_python::from_py;
 use pyo3::{exceptions::PyValueError, prelude::*};
 use serde::Deserialize;
+use serde_json::{Map, Value};
 
 #[derive(Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -12,24 +14,68 @@ struct RequestInput {
     controls: Option<CacheControls>,
     ttl_seconds: Option<f64>,
     max_age_seconds: Option<f64>,
+    input: Option<Value>,
+    messages: Option<Vec<Value>>,
+    metadata: Option<Map<String, Value>>,
+    scope: Option<String>,
 }
 
-pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<ResponseCacheRequest> {
+pub(super) struct CacheRequest {
+    key: CacheKeyInput,
+    controls: CacheControls,
+    ttl: Option<Duration>,
+    max_age: Option<Duration>,
+    input: Option<Value>,
+    messages: Vec<Value>,
+    metadata: Map<String, Value>,
+    scope: Option<String>,
+}
+
+impl CacheRequest {
+    pub(super) fn exact(&self) -> ResponseCacheRequest<ExactCacheContext> {
+        let mut request = ResponseCacheRequest::new(self.key.clone());
+        request.controls = self.controls;
+        request.context.ttl = self.ttl;
+        request.max_age = self.max_age;
+        request
+    }
+
+    pub(super) fn semantic(&self) -> ResponseCacheRequest<SemanticCacheContext> {
+        ResponseCacheRequest {
+            key: self.key.clone(),
+            controls: self.controls,
+            context: SemanticCacheContext {
+                input: self.input.clone(),
+                messages: self.messages.clone(),
+                metadata: self.metadata.clone(),
+                scope: self.scope.clone(),
+                ttl: self.ttl,
+            },
+            max_age: self.max_age,
+        }
+    }
+}
+
+pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<CacheRequest> {
     let input: RequestInput = from_py(value)?;
     request_input(input)
 }
 
-fn request_input(input: RequestInput) -> PyResult<ResponseCacheRequest> {
-    let mut request = ResponseCacheRequest::new(input.key);
-    if let Some(controls) = input.controls {
-        request.controls = controls;
-    }
-    request.context.ttl = input.ttl_seconds.map(duration).transpose()?;
-    request.max_age = input.max_age_seconds.map(duration).transpose()?;
-    Ok(request)
+fn request_input(input: RequestInput) -> PyResult<CacheRequest> {
+    let defaults = ResponseCacheRequest::<ExactCacheContext>::new(input.key.clone());
+    Ok(CacheRequest {
+        key: input.key,
+        controls: input.controls.unwrap_or(defaults.controls),
+        ttl: input.ttl_seconds.map(duration).transpose()?,
+        max_age: input.max_age_seconds.map(duration).transpose()?,
+        input: input.input,
+        messages: input.messages.unwrap_or_default(),
+        metadata: input.metadata.unwrap_or_default(),
+        scope: input.scope,
+    })
 }
 
-pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<ResponseCacheRequest>> {
+pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<CacheRequest>> {
     from_py::<Vec<RequestInput>>(value)?
         .into_iter()
         .map(request_input)