From 8ae1f763394bcd5a74cf3bf76ccd3756399d1958 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 16 Sep 2026 22:36:56 +0000 Subject: [PATCH] feat(rust): scaffold redis cache crate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 55 ++++++++ litellm-rust/Cargo.toml | 1 + litellm-rust/crates/cache-redis/Cargo.toml | 11 ++ litellm-rust/crates/cache-redis/src/cache.rs | 127 ++++++++++++++++++ litellm-rust/crates/cache-redis/src/lib.rs | 3 + .../crates/cache-redis/tests/cache.rs | 6 + 6 files changed, 203 insertions(+) create mode 100644 litellm-rust/crates/cache-redis/Cargo.toml create mode 100644 litellm-rust/crates/cache-redis/src/cache.rs create mode 100644 litellm-rust/crates/cache-redis/src/lib.rs create mode 100644 litellm-rust/crates/cache-redis/tests/cache.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 7397742369b..d5b94e261e6 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -70,6 +70,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + [[package]] name = "async-compression" version = "0.4.46" @@ -1915,6 +1921,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-cache-redis" +version = "0.1.0" +dependencies = [ + "litellm-cache", + "redis", + "serde_json", +] + [[package]] name = "litellm-core" version = "0.1.0" @@ -2140,6 +2155,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2656,6 +2681,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redis" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +dependencies = [ + "arcstr", + "combine", + "itoa", + "num-bigint", + "percent-encoding", + "ryu", + "sha1_smol", + "socket2 0.6.5", + "url", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3096,6 +3139,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -4182,6 +4231,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "yoke" version = "0.8.3" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 879090870d8..eb3413eb89a 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -23,6 +23,7 @@ pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" +redis = "1.7.0" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "http2", "stream"] } rstest = "0.26.1" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } diff --git a/litellm-rust/crates/cache-redis/Cargo.toml b/litellm-rust/crates/cache-redis/Cargo.toml new file mode 100644 index 00000000000..2db2ae33840 --- /dev/null +++ b/litellm-rust/crates/cache-redis/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "litellm-cache-redis" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +redis.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs new file mode 100644 index 00000000000..963783d31a6 --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/cache.rs @@ -0,0 +1,127 @@ +use std::sync::Mutex; +use std::time::Duration; + +use litellm_cache::{ + BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs, + Error, +}; +use redis::Commands; + +const DEFAULT_TTL: Duration = Duration::from_secs(600); + +pub struct RedisCache { + connection: Mutex, + default_ttl: Duration, +} + +impl RedisCache { + pub fn new(url: &str, default_ttl: Option) -> Result { + let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?; + let connection = client.get_connection().map_err(|_| Error::Unavailable)?; + Ok(Self { + connection: Mutex::new(connection), + default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), + }) + } + + fn connection(&self) -> Result, Error> { + self.connection.lock().map_err(|_| Error::Unavailable) + } + + fn encode(value: &CacheEntry) -> Result, Error> { + serde_json::to_vec(value).map_err(|_| Error::InvalidEntry) + } + + fn decode(value: Vec) -> Result { + serde_json::from_slice(&value).map_err(|_| Error::InvalidEntry) + } + + fn ttl_seconds(ttl: Duration) -> u64 { + ttl.as_secs().max(1) + } +} + +impl BaseCache for RedisCache { + type Value = CacheEntry; + + fn default_ttl(&self) -> Duration { + self.default_ttl + } + + fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> { + let payload = Self::encode(&value)?; + let ttl = Self::ttl_seconds(self.get_ttl(&kwargs)); + self.connection()? + .set_ex::<_, _, ()>(key, payload, ttl) + .map_err(|_| Error::Unavailable) + } + + fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result, Error> { + self.connection()? + .get::<_, Option>>(key) + .map_err(|_| Error::Unavailable)? + .map(Self::decode) + .transpose() + } + + fn delete_cache(&self, key: &str) -> Result<(), Error> { + self.connection()? + .del::<_, ()>(key) + .map_err(|_| Error::Unavailable) + } + + fn flush_cache(&self) -> Result<(), Error> { + self.connection()? + .flushdb::<()>() + .map_err(|_| Error::Unavailable) + } + + fn disconnect(&self) -> CacheFuture<'_, ()> { + Box::pin(async { Ok(()) }) + } + + fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> { + Box::pin(async { + let mut connection = self.connection()?; + redis::cmd("PING") + .query::(&mut *connection) + .map_err(|_| Error::Unavailable)?; + Ok(CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: "Redis cache connection test successful".into(), + error: None, + }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::RedisCache; + use litellm_cache::CacheEntry; + use serde_json::json; + use std::time::Duration; + + #[test] + fn cache_entries_round_trip_through_json() { + let entry = CacheEntry { + timestamp: 123.0, + response: json!({"choices": [{"text": "cached"}]}), + }; + + let encoded = RedisCache::encode(&entry).unwrap(); + assert_eq!(RedisCache::decode(encoded).unwrap(), entry); + } + + #[test] + fn invalid_json_is_rejected() { + assert!(RedisCache::decode(b"not json".to_vec()).is_err()); + } + + #[test] + fn ttl_seconds_keeps_redis_expiration_positive() { + assert_eq!(RedisCache::ttl_seconds(Duration::ZERO), 1); + assert_eq!(RedisCache::ttl_seconds(Duration::from_millis(1500)), 1); + assert_eq!(RedisCache::ttl_seconds(Duration::from_secs(15)), 15); + } +} diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs new file mode 100644 index 00000000000..37b35c5ea4a --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/lib.rs @@ -0,0 +1,3 @@ +mod cache; + +pub use cache::RedisCache; diff --git a/litellm-rust/crates/cache-redis/tests/cache.rs b/litellm-rust/crates/cache-redis/tests/cache.rs new file mode 100644 index 00000000000..76f73145da8 --- /dev/null +++ b/litellm-rust/crates/cache-redis/tests/cache.rs @@ -0,0 +1,6 @@ +use litellm_cache_redis::RedisCache; + +#[test] +fn constructor_rejects_invalid_urls() { + assert!(RedisCache::new("not a redis url", None).is_err()); +}