wip

.github/workflows/test-rust.yml

@@ -95,7 +95,7 @@ jobs:
 
       - run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
-      - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
+      - run: cargo clippy -p litellm-core --all-targets --no-default-features --locked -- -D warnings
 
   rust-test:
     runs-on: ubuntu-latest
@@ -129,7 +129,7 @@ jobs:
       - run: cargo test --workspace --locked
         working-directory: litellm-rust
 
-      - run: cargo test -p litellm-core --features bedrock-auth --locked
+      - run: cargo test -p litellm-core --no-default-features --locked
         working-directory: litellm-rust
 
       - run: uv build --wheel --out-dir dist

litellm-rust/Cargo.lock

@@ -1917,6 +1917,27 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "litellm-cache"
+version = "0.1.0"
+dependencies = [
+ "rstest",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "thiserror 2.0.19",
+]
+
+[[package]]
+name = "litellm-cache-memory"
+version = "0.1.0"
+dependencies = [
+ "litellm-cache",
+ "rstest",
+ "serde_json",
+ "tokio",
+]
+
 [[package]]
 name = "litellm-config"
 version = "0.1.0"
@@ -1964,6 +1985,7 @@ version = "0.1.0"
 dependencies = [
  "criterion",
  "futures-util",
+ "litellm-auth",
  "litellm-core",
  "litellm-python-interop",
  "litellm-token-counter",

litellm-rust/Cargo.toml

@@ -1,17 +1,6 @@
 [workspace]
-members = [
-    "crates/auth",
-    "crates/auth-aws",
-    "crates/auth-azure",
-    "crates/auth-gcp",
-    "crates/core",
-    "crates/token-counter",
-    "crates/config",
-    "crates/ai-gateway",
-    "crates/python-interop",
-    "crates/python-bridge",
-]
 resolver = "2"
+members = ["crates/*"]
 
 [workspace.package]
 edition = "2024"
@@ -26,6 +15,8 @@ litellm-auth = { path = "crates/auth" }
 litellm-auth-aws = { path = "crates/auth-aws" }
 litellm-auth-azure = { path = "crates/auth-azure" }
 litellm-auth-gcp = { path = "crates/auth-gcp" }
+litellm-cache = { path = "crates/cache" }
+litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-token-counter = { path = "crates/token-counter" }
 litellm-python-interop = { path = "crates/python-interop" }
 pyo3 = "0.29.2"

litellm-rust/crates/cache-memory/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "litellm-cache-memory"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-cache.workspace = true
+serde_json.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tokio.workspace = true

litellm-rust/crates/cache-memory/src/cache.rs

@@ -0,0 +1,254 @@
+use std::cmp::Reverse;
+use std::collections::{BinaryHeap, HashMap};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use litellm_cache::{
+    BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs,
+    Error,
+};
+
+const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200;
+const DEFAULT_TTL: Duration = Duration::from_secs(600);
+
+type ValueMeasure<V> = Arc<dyn Fn(&V) -> Result<usize, Error> + Send + Sync>;
+type ValueValidator<V> = Arc<dyn Fn(&V) -> Result<(), Error> + Send + Sync>;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CacheWrite {
+    Stored,
+    Disabled,
+    TooLarge,
+}
+
+struct CacheState<V> {
+    values: HashMap<String, V>,
+    expirations: HashMap<String, Duration>,
+    expiration_heap: BinaryHeap<Reverse<(Duration, String)>>,
+}
+
+pub struct InMemoryCache<V: Clone> {
+    state: Mutex<CacheState<V>>,
+    max_size_in_memory: usize,
+    default_ttl: Duration,
+    max_entry_bytes: Option<usize>,
+    measure_value: Option<ValueMeasure<V>>,
+    validate_value: Option<ValueValidator<V>>,
+    now: Arc<dyn Fn() -> Duration + Send + Sync>,
+}
+
+impl<V: Clone> Default for InMemoryCache<V> {
+    fn default() -> Self {
+        Self::new(None, None)
+    }
+}
+
+impl<V: Clone> InMemoryCache<V> {
+    pub fn new(max_size_in_memory: Option<usize>, default_ttl: Option<Duration>) -> Self {
+        Self::with_clock(max_size_in_memory, default_ttl, || {
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or_default()
+        })
+    }
+
+    pub fn with_clock(
+        max_size_in_memory: Option<usize>,
+        default_ttl: Option<Duration>,
+        now: impl Fn() -> Duration + Send + Sync + 'static,
+    ) -> Self {
+        Self::with_clock_and_size_measurement(max_size_in_memory, default_ttl, None, None, now)
+    }
+
+    pub fn with_clock_and_size_measurement(
+        max_size_in_memory: Option<usize>,
+        default_ttl: Option<Duration>,
+        max_entry_bytes: Option<usize>,
+        measure_value: Option<ValueMeasure<V>>,
+        now: impl Fn() -> Duration + Send + Sync + 'static,
+    ) -> Self {
+        Self {
+            state: Mutex::new(CacheState {
+                values: HashMap::new(),
+                expirations: HashMap::new(),
+                expiration_heap: BinaryHeap::new(),
+            }),
+            max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY),
+            default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
+            max_entry_bytes,
+            measure_value,
+            validate_value: None,
+            now: Arc::new(now),
+        }
+    }
+
+    pub fn set_cache(
+        &self,
+        key: impl Into<String>,
+        value: V,
+        ttl: Option<Duration>,
+    ) -> Result<CacheWrite, Error> {
+        if self.max_size_in_memory == 0 {
+            return Ok(CacheWrite::Disabled);
+        }
+        if let Some(validate) = &self.validate_value {
+            validate(&value)?;
+        }
+        if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value)
+            && measure(&value)? > limit
+        {
+            return Ok(CacheWrite::TooLarge);
+        }
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::evict(&mut state, self.max_size_in_memory, now);
+        let key = key.into();
+        state.values.insert(key.clone(), value);
+        let expiration = state.expirations.get(&key).copied();
+        if expiration.is_none_or(|expiration| expiration < now) {
+            let expiration = now + ttl.unwrap_or(self.default_ttl);
+            state.expirations.insert(key.clone(), expiration);
+            state.expiration_heap.push(Reverse((expiration, key)));
+        }
+        Ok(CacheWrite::Stored)
+    }
+
+    pub fn get_cache(&self, key: &str) -> Result<Option<V>, Error> {
+        let now = (self.now)();
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        if state
+            .expirations
+            .get(key)
+            .is_some_and(|expiration| *expiration < now)
+        {
+            Self::remove(&mut state, key);
+        }
+        Ok(state.values.get(key).cloned())
+    }
+
+    pub fn expires_at(&self, key: &str) -> Result<Option<Duration>, Error> {
+        Ok(self
+            .state
+            .lock()
+            .map_err(|_| Error::Unavailable)?
+            .expirations
+            .get(key)
+            .copied())
+    }
+
+    pub fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        Self::remove(&mut state, key);
+        Ok(())
+    }
+
+    pub fn flush_cache(&self) -> Result<(), Error> {
+        let mut state = self.state.lock().map_err(|_| Error::Unavailable)?;
+        state.values.clear();
+        state.expirations.clear();
+        state.expiration_heap.clear();
+        Ok(())
+    }
+
+    fn evict(state: &mut CacheState<V>, capacity: usize, now: Duration) {
+        while let Some(Reverse((expiration, key))) = state.expiration_heap.peek().cloned() {
+            if state.expirations.get(&key).copied() != Some(expiration) {
+                state.expiration_heap.pop();
+            } else if expiration < now {
+                state.expiration_heap.pop();
+                Self::remove(state, &key);
+            } else {
+                break;
+            }
+        }
+        while state.values.len() >= capacity {
+            let Some(Reverse((expiration, key))) = state.expiration_heap.pop() else {
+                break;
+            };
+            if state.expirations.get(&key).copied() == Some(expiration) {
+                Self::remove(state, &key);
+            }
+        }
+    }
+
+    fn remove(state: &mut CacheState<V>, key: &str) {
+        state.values.remove(key);
+        state.expirations.remove(key);
+    }
+}
+
+impl InMemoryCache<CacheEntry> {
+    pub fn response_cache(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self {
+        Self::response_cache_with_clock(capacity, ttl, max_entry_bytes, || {
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or_default()
+        })
+    }
+
+    pub fn response_cache_with_clock(
+        capacity: usize,
+        ttl: Duration,
+        max_entry_bytes: usize,
+        now: impl Fn() -> Duration + Send + Sync + 'static,
+    ) -> Self {
+        let mut cache = Self::with_clock_and_size_measurement(
+            Some(capacity),
+            Some(ttl),
+            Some(max_entry_bytes),
+            Some(Arc::new(|entry: &CacheEntry| {
+                serde_json::to_vec(entry)
+                    .map(|bytes| bytes.len())
+                    .map_err(|_| Error::InvalidEntry)
+            })),
+            now,
+        );
+        cache.validate_value = Some(Arc::new(|entry: &CacheEntry| {
+            entry
+                .timestamp
+                .is_finite()
+                .then_some(())
+                .ok_or(Error::InvalidEntry)
+        }));
+        cache
+    }
+}
+
+impl BaseCache for InMemoryCache<CacheEntry> {
+    type Value = CacheEntry;
+
+    fn default_ttl(&self) -> Duration {
+        self.default_ttl
+    }
+
+    fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> {
+        let ttl = self.get_ttl(&kwargs);
+        self.set_cache(key, value, Some(ttl)).map(|_| ())
+    }
+
+    fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result<Option<Self::Value>, Error> {
+        self.get_cache(key)
+    }
+
+    fn delete_cache(&self, key: &str) -> Result<(), Error> {
+        self.delete_cache(key)
+    }
+
+    fn flush_cache(&self) -> Result<(), Error> {
+        self.flush_cache()
+    }
+
+    fn disconnect(&self) -> CacheFuture<'_, ()> {
+        Box::pin(async { Ok(()) })
+    }
+
+    fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> {
+        Box::pin(async {
+            Ok(CacheConnectionResult {
+                status: CacheConnectionStatus::Success,
+                message: "In-memory cache connection test successful".into(),
+                error: None,
+            })
+        })
+    }
+}

litellm-rust/crates/cache-memory/src/lib.rs

@@ -0,0 +1,3 @@
+mod cache;
+
+pub use cache::{CacheWrite, InMemoryCache};

litellm-rust/crates/cache-memory/tests/cache.rs

@@ -0,0 +1,140 @@
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Duration;
+
+use litellm_cache::{BaseCache, CacheConnectionStatus, CacheEntry, Error};
+use litellm_cache_memory::{CacheWrite, InMemoryCache};
+use rstest::{fixture, rstest};
+
+#[fixture]
+fn clock() -> Arc<AtomicU64> {
+    Arc::new(AtomicU64::new(100))
+}
+
+fn cache(clock: Arc<AtomicU64>, capacity: usize) -> InMemoryCache<String> {
+    InMemoryCache::with_clock(Some(capacity), Some(Duration::from_secs(60)), move || {
+        Duration::from_secs(clock.load(Ordering::SeqCst))
+    })
+}
+
+#[rstest]
+fn default_explicit_and_override_ttls_follow_python_rules(clock: Arc<AtomicU64>) {
+    let cache = cache(clock.clone(), 4);
+    cache.set_cache("key", "first".into(), None).unwrap();
+    assert_eq!(
+        cache.expires_at("key").unwrap(),
+        Some(Duration::from_secs(160))
+    );
+    cache
+        .set_cache("key", "second".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+    assert_eq!(
+        cache.expires_at("key").unwrap(),
+        Some(Duration::from_secs(160))
+    );
+    clock.store(160, Ordering::SeqCst);
+    assert_eq!(cache.get_cache("key").unwrap(), Some("second".into()));
+    clock.store(161, Ordering::SeqCst);
+    assert_eq!(cache.get_cache("key").unwrap(), None);
+    cache
+        .set_cache("key", "third".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+    assert_eq!(
+        cache.expires_at("key").unwrap(),
+        Some(Duration::from_secs(171))
+    );
+}
+
+#[rstest]
+fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Arc<AtomicU64>) {
+    let cache = cache(clock, 2);
+    cache
+        .set_cache("early", "a".into(), Some(Duration::from_secs(10)))
+        .unwrap();
+    cache
+        .set_cache("late", "b".into(), Some(Duration::from_secs(20)))
+        .unwrap();
+    cache.delete_cache("early").unwrap();
+    cache
+        .set_cache("new", "c".into(), Some(Duration::from_secs(30)))
+        .unwrap();
+    assert_eq!(cache.get_cache("late").unwrap(), Some("b".into()));
+    cache
+        .set_cache("last", "d".into(), Some(Duration::from_secs(40)))
+        .unwrap();
+    assert_eq!(cache.get_cache("late").unwrap(), None);
+}
+
+#[test]
+fn disabled_size_limited_and_synchronized_response_writes_are_observable() {
+    let disabled = InMemoryCache::<CacheEntry>::response_cache(0, Duration::from_secs(60), 80);
+    assert_eq!(
+        disabled
+            .set_cache(
+                "a",
+                CacheEntry {
+                    timestamp: 1.0,
+                    response: serde_json::json!("x")
+                },
+                None
+            )
+            .unwrap(),
+        CacheWrite::Disabled
+    );
+    let cache = InMemoryCache::<CacheEntry>::response_cache(2, Duration::from_secs(60), 80);
+    assert_eq!(
+        cache
+            .set_cache(
+                "large",
+                CacheEntry {
+                    timestamp: 1.0,
+                    response: serde_json::json!("x".repeat(100))
+                },
+                None
+            )
+            .unwrap(),
+        CacheWrite::TooLarge
+    );
+    cache
+        .set_cache(
+            "small",
+            CacheEntry {
+                timestamp: 1.0,
+                response: serde_json::json!("ok"),
+            },
+            None,
+        )
+        .unwrap();
+    assert!(cache.get_cache("small").unwrap().is_some());
+    assert_eq!(
+        cache
+            .set_cache(
+                "invalid",
+                CacheEntry {
+                    timestamp: f64::NAN,
+                    response: serde_json::json!("bad"),
+                },
+                None,
+            )
+            .unwrap_err(),
+        Error::InvalidEntry
+    );
+    cache.delete_cache("small").unwrap();
+    cache.flush_cache().unwrap();
+}
+
+#[tokio::test]
+async fn connection_test_matches_python_result_contract() {
+    let cache = InMemoryCache::<CacheEntry>::default();
+    let result = BaseCache::test_connection(&cache).await.unwrap();
+    assert_eq!(result.status, CacheConnectionStatus::Success);
+    assert_eq!(result.message, "In-memory cache connection test successful");
+    assert_eq!(result.error, None);
+    assert_eq!(
+        serde_json::to_value(result).unwrap(),
+        serde_json::json!({
+            "status": "success",
+            "message": "In-memory cache connection test successful"
+        })
+    );
+}

litellm-rust/crates/cache/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "litellm-cache"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true

litellm-rust/crates/cache/src/base_cache.rs

@@ -0,0 +1,98 @@
+use std::future::Future;
+use std::pin::Pin;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+
+use crate::Error;
+
+pub type CacheFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
+
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct CacheKwargs {
+    pub ttl: Option<Duration>,
+    pub extras: Map<String, Value>,
+}
+
+#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum CacheConnectionStatus {
+    Success,
+    Failed,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+pub struct CacheConnectionResult {
+    pub status: CacheConnectionStatus,
+    pub message: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+}
+
+pub trait BaseCache: Send + Sync {
+    type Value: Clone + Send + Sync + 'static;
+
+    fn default_ttl(&self) -> Duration {
+        Duration::from_secs(60)
+    }
+
+    fn get_ttl(&self, kwargs: &CacheKwargs) -> Duration {
+        kwargs.ttl.unwrap_or_else(|| self.default_ttl())
+    }
+
+    fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error>;
+
+    fn get_cache(&self, key: &str, kwargs: &CacheKwargs) -> Result<Option<Self::Value>, Error>;
+
+    fn async_set_cache<'a>(
+        &'a self,
+        key: &'a str,
+        value: Self::Value,
+        kwargs: CacheKwargs,
+    ) -> CacheFuture<'a, ()> {
+        Box::pin(async move { self.set_cache(key, value, kwargs) })
+    }
+
+    fn async_get_cache<'a>(
+        &'a self,
+        key: &'a str,
+        kwargs: &'a CacheKwargs,
+    ) -> CacheFuture<'a, Option<Self::Value>> {
+        Box::pin(async move { self.get_cache(key, kwargs) })
+    }
+
+    fn async_set_cache_pipeline<'a>(
+        &'a self,
+        cache_list: Vec<(String, Self::Value)>,
+        kwargs: CacheKwargs,
+    ) -> CacheFuture<'a, ()> {
+        Box::pin(async move {
+            for (key, value) in cache_list {
+                self.set_cache(&key, value, kwargs.clone())?;
+            }
+            Ok(())
+        })
+    }
+
+    fn batch_cache_write<'a>(
+        &'a self,
+        key: &'a str,
+        value: Self::Value,
+        kwargs: CacheKwargs,
+    ) -> CacheFuture<'a, ()> {
+        self.async_set_cache(key, value, kwargs)
+    }
+
+    fn delete_cache(&self, key: &str) -> Result<(), Error>;
+
+    fn async_delete_cache<'a>(&'a self, key: &'a str) -> CacheFuture<'a, ()> {
+        Box::pin(async move { self.delete_cache(key) })
+    }
+
+    fn flush_cache(&self) -> Result<(), Error>;
+
+    fn disconnect(&self) -> CacheFuture<'_, ()>;
+
+    fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult>;
+}

litellm-rust/crates/cache/src/caching.rs

@@ -0,0 +1,166 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+
+use crate::{BaseCache, CacheKwargs, Error};
+
+pub use crate::BaseCache as Cache;
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
+pub enum CacheMode {
+    #[default]
+    #[serde(rename = "default_on")]
+    DefaultOn,
+    #[serde(rename = "default_off")]
+    DefaultOff,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct CacheKeyField {
+    pub name: String,
+    pub value: Option<String>,
+    pub api_parameter: bool,
+    pub internal_parameter: bool,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct CacheKeyInput {
+    pub fields: Vec<CacheKeyField>,
+    pub preset: Option<String>,
+    pub namespace: Option<String>,
+    pub include_provider_parameters: bool,
+}
+
+#[derive(Default)]
+pub struct CacheKeyContext {
+    pub model_group: Option<String>,
+    pub caching_groups: Vec<(Vec<String>, String)>,
+    pub file_checksum: Option<String>,
+    pub file_object_name: Option<String>,
+    pub metadata_file_name: Option<String>,
+    pub parameters_file_name: Option<String>,
+}
+
+impl CacheKeyContext {
+    pub fn apply(self, input: &mut CacheKeyInput) {
+        let group = self.model_group.as_ref().and_then(|model| {
+            self.caching_groups
+                .iter()
+                .find(|(models, _)| models.contains(model))
+        });
+        for field in &mut input.fields {
+            match field.name.as_str() {
+                "model" => {
+                    field.value = group
+                        .map(|(_, formatted)| formatted.clone())
+                        .or_else(|| self.model_group.clone())
+                        .or_else(|| field.value.take())
+                }
+                "file" => {
+                    field.value = self
+                        .file_checksum
+                        .clone()
+                        .or_else(|| self.file_object_name.clone())
+                        .or_else(|| self.metadata_file_name.clone())
+                        .or_else(|| self.parameters_file_name.clone())
+                }
+                _ => {}
+            }
+        }
+    }
+}
+
+pub fn get_cache_key(input: &CacheKeyInput) -> String {
+    cache_key(input)
+}
+
+pub fn cache_key(input: &CacheKeyInput) -> String {
+    if let Some(preset) = &input.preset {
+        return preset.clone();
+    }
+    let mut digest = Sha256::new();
+    for field in &input.fields {
+        if (field.api_parameter || (input.include_provider_parameters && !field.internal_parameter))
+            && let Some(value) = &field.value
+        {
+            digest.update(field.name.as_bytes());
+            digest.update(b": ");
+            digest.update(value.as_bytes());
+        }
+    }
+    let hash = format!("{:x}", digest.finalize());
+    input
+        .namespace
+        .as_deref()
+        .filter(|namespace| !namespace.is_empty())
+        .map_or(hash.clone(), |namespace| format!("{namespace}:{hash}"))
+}
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)]
+pub struct CacheControls {
+    pub supported_call_type: bool,
+    pub configured: bool,
+    pub native_backend: bool,
+    pub default_on: bool,
+    pub caching: Option<bool>,
+    pub no_cache: bool,
+    pub no_store: bool,
+    #[serde(default)]
+    pub use_cache: bool,
+}
+
+impl CacheControls {
+    pub fn reads(self) -> bool {
+        self.supported_call_type
+            && self.configured
+            && self.caching.unwrap_or(true)
+            && !self.no_cache
+            && (self.default_on || self.use_cache)
+    }
+
+    pub fn writes(self) -> bool {
+        self.supported_call_type
+            && self.configured
+            && !self.no_store
+            && (self.default_on || self.use_cache)
+    }
+}
+
+pub fn should_use_cache(controls: CacheControls) -> bool {
+    controls.reads() || controls.writes()
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct CacheEntry {
+    pub timestamp: f64,
+    pub response: Value,
+}
+
+impl CacheEntry {
+    pub fn fresh(&self, now: Duration, max_age: Option<Duration>) -> bool {
+        self.timestamp.is_finite()
+            && max_age.is_none_or(|age| now.as_secs_f64() - self.timestamp <= age.as_secs_f64())
+    }
+}
+
+pub fn get_cache(
+    cache: &dyn BaseCache<Value = CacheEntry>,
+    key: &str,
+    kwargs: &CacheKwargs,
+) -> Result<Option<CacheEntry>, Error> {
+    cache.get_cache(key, kwargs)
+}
+
+pub fn set_cache(
+    cache: &dyn BaseCache<Value = CacheEntry>,
+    key: &str,
+    entry: CacheEntry,
+    kwargs: CacheKwargs,
+) -> Result<(), Error> {
+    cache.set_cache(key, entry, kwargs)
+}
+
+pub type CacheBackend = Arc<dyn BaseCache<Value = CacheEntry>>;

litellm-rust/crates/cache/src/error.rs

@@ -0,0 +1,7 @@
+#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum Error {
+    #[error("cache is unavailable")]
+    Unavailable,
+    #[error("invalid cache entry")]
+    InvalidEntry,
+}

litellm-rust/crates/cache/src/lib.rs

@@ -0,0 +1,12 @@
+mod base_cache;
+mod caching;
+mod error;
+
+pub use base_cache::{
+    BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheFuture, CacheKwargs,
+};
+pub use caching::{
+    Cache, CacheBackend, CacheControls, CacheEntry, CacheKeyContext, CacheKeyField, CacheKeyInput,
+    CacheMode, cache_key, get_cache, get_cache_key, set_cache, should_use_cache,
+};
+pub use error::Error;

litellm-rust/crates/cache/tests/caching.rs

@@ -0,0 +1,139 @@
+use litellm_cache::{
+    BaseCache, CacheConnectionResult, CacheControls, CacheEntry, CacheFuture, CacheKeyContext,
+    CacheKeyField, CacheKeyInput, CacheKwargs, Error, cache_key, get_cache_key,
+};
+use sha2::{Digest, Sha256};
+use std::time::Duration;
+
+struct TestCache {
+    default_ttl: Duration,
+}
+
+impl BaseCache for TestCache {
+    type Value = CacheEntry;
+
+    fn default_ttl(&self) -> Duration {
+        self.default_ttl
+    }
+
+    fn set_cache(&self, _: &str, _: Self::Value, _: CacheKwargs) -> Result<(), Error> {
+        Ok(())
+    }
+
+    fn get_cache(&self, _: &str, _: &CacheKwargs) -> Result<Option<Self::Value>, Error> {
+        Ok(None)
+    }
+
+    fn delete_cache(&self, _: &str) -> Result<(), Error> {
+        Ok(())
+    }
+
+    fn flush_cache(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
+    fn disconnect(&self) -> CacheFuture<'_, ()> {
+        Box::pin(async { Ok(()) })
+    }
+
+    fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> {
+        unreachable!()
+    }
+}
+
+#[test]
+fn ttl_uses_default_and_allows_per_call_override() {
+    let cache = TestCache {
+        default_ttl: Duration::from_secs(60),
+    };
+    assert_eq!(
+        cache.get_ttl(&CacheKwargs::default()),
+        Duration::from_secs(60)
+    );
+    assert_eq!(
+        cache.get_ttl(&CacheKwargs {
+            ttl: Some(Duration::from_secs(5)),
+            ..Default::default()
+        }),
+        Duration::from_secs(5)
+    );
+}
+
+#[test]
+fn keys_match_python_order_groups_files_presets_and_namespaces() {
+    let mut input = CacheKeyInput {
+        fields: vec![
+            CacheKeyField {
+                name: "model".into(),
+                value: Some("deployment".into()),
+                api_parameter: true,
+                internal_parameter: false,
+            },
+            CacheKeyField {
+                name: "file".into(),
+                value: None,
+                api_parameter: true,
+                internal_parameter: false,
+            },
+        ],
+        namespace: Some("team".into()),
+        ..Default::default()
+    };
+    CacheKeyContext {
+        model_group: Some("group".into()),
+        caching_groups: vec![(vec!["group".into()], "['group']".into())],
+        file_checksum: Some("checksum".into()),
+        ..Default::default()
+    }
+    .apply(&mut input);
+    assert_eq!(
+        cache_key(&input),
+        format!(
+            "team:{:x}",
+            Sha256::digest(b"model: ['group']file: checksum")
+        )
+    );
+    input.preset = Some("preset".into());
+    assert_eq!(get_cache_key(&input), "preset");
+}
+
+#[test]
+fn cache_controls_honor_default_modes_and_directives() {
+    let enabled = CacheControls {
+        supported_call_type: true,
+        configured: true,
+        default_on: true,
+        ..Default::default()
+    };
+    assert!(enabled.reads());
+    assert!(enabled.writes());
+    assert!(
+        !CacheControls {
+            default_on: false,
+            ..enabled
+        }
+        .reads()
+    );
+    assert!(
+        CacheControls {
+            default_on: false,
+            use_cache: true,
+            ..enabled
+        }
+        .reads()
+    );
+    assert!(
+        !CacheControls {
+            no_cache: true,
+            ..enabled
+        }
+        .reads()
+    );
+    assert!(
+        !CacheControls {
+            no_store: true,
+            ..enabled
+        }
+        .writes()
+    );
+}

litellm-rust/crates/core/Cargo.toml

@@ -12,7 +12,7 @@ futures-util.workspace = true
 base64.workspace = true
 data-url = "0.3.2"
 litellm-auth.workspace = true
-litellm-auth-aws = { workspace = true, optional = true }
+litellm-auth-aws.workspace = true
 litellm-auth-azure.workspace = true
 litellm-auth-gcp.workspace = true
 moka.workspace = true
@@ -33,7 +33,6 @@ url.workspace = true
 veil.workspace = true
 [features]
 default = []
-bedrock-auth = ["dep:litellm-auth-aws"]
 observability = ["dep:tracing-subscriber"]
 
 [dev-dependencies]

litellm-rust/crates/core/src/audio_transcription/handler.rs

@@ -41,7 +41,6 @@ pub async fn execute_audio_transcription_provider_call(
         .into_json())
 }
 
-#[cfg(feature = "bedrock-auth")]
 async fn signed_headers(
     request: &ProviderAudioTranscriptionRequest,
     body: &[u8],
@@ -73,18 +72,3 @@ async fn signed_headers(
     )?;
     Ok(unsigned.into_iter().chain(signature).collect())
 }
-
-#[cfg(not(feature = "bedrock-auth"))]
-async fn signed_headers(
-    request: &ProviderAudioTranscriptionRequest,
-    _body: &[u8],
-) -> Result<Vec<(String, String)>, Error> {
-    use crate::audio_transcription::transformation::AudioTranscriptionAuth;
-
-    match request.auth {
-        AudioTranscriptionAuth::AwsSigV4 { .. } => Err(Error::Unsupported(
-            "AWS SigV4 requires the bedrock-auth feature",
-        )),
-        AudioTranscriptionAuth::Bearer => Ok(request.upstream_headers.clone()),
-    }
-}

litellm-rust/crates/core/src/audio_transcription/prepare.rs

@@ -1,6 +1,5 @@
 use crate::error::Error;
 use crate::http_utils::{has_header, string_headers};
-#[cfg(feature = "bedrock-auth")]
 use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG;
 use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider};
 
@@ -8,12 +7,10 @@ use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderCo
 use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
 
 fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProviderConfig> {
-    #[cfg(feature = "bedrock-auth")]
-    if provider == "bedrock" {
-        return Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG);
+    match provider {
+        "bedrock" => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
+        _ => None,
     }
-    let _ = provider;
-    None
 }
 
 pub fn prepare_audio_transcription_provider_call(
@@ -65,7 +62,6 @@ pub fn prepare_audio_transcription_provider_call(
         body: transformed.body,
         upstream_headers: headers,
         auth,
-        #[cfg(feature = "bedrock-auth")]
         optional_params: request.optional_params,
         timeout: request.timeout,
     })

litellm-rust/crates/core/src/audio_transcription/types.rs

@@ -26,7 +26,6 @@ pub struct ProviderAudioTranscriptionRequest {
     pub(super) body: Value,
     pub(super) upstream_headers: Vec<(String, String)>,
     pub(super) auth: AudioTranscriptionAuth,
-    #[cfg(feature = "bedrock-auth")]
     pub(super) optional_params: OpaqueParams,
     pub(super) timeout: Option<Duration>,
 }

litellm-rust/crates/core/src/auth/mod.rs

@@ -1 +0,0 @@
-pub use litellm_auth::*;

litellm-rust/crates/core/src/caching/in_memory_cache.rs

@@ -1,258 +0,0 @@
-use std::cmp::Reverse;
-use std::collections::{BinaryHeap, HashMap};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-
-const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200;
-const DEFAULT_TTL: Duration = Duration::from_secs(600);
-
-pub struct InMemoryCache<V: Clone> {
-    pub cache_dict: HashMap<String, V>,
-    pub ttl_dict: HashMap<String, Duration>,
-    pub expiration_heap: BinaryHeap<Reverse<(Duration, String)>>,
-    pub max_size_in_memory: usize,
-    pub default_ttl: Duration,
-    now: Box<dyn Fn() -> Duration + Send + Sync>,
-}
-
-impl<V: Clone> Default for InMemoryCache<V> {
-    fn default() -> Self {
-        Self::new(None, None)
-    }
-}
-
-impl<V: Clone> InMemoryCache<V> {
-    pub fn new(max_size_in_memory: Option<usize>, default_ttl: Option<Duration>) -> Self {
-        Self::with_clock(max_size_in_memory, default_ttl, || {
-            SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .unwrap_or_default()
-        })
-    }
-
-    pub fn with_clock(
-        max_size_in_memory: Option<usize>,
-        default_ttl: Option<Duration>,
-        now: impl Fn() -> Duration + Send + Sync + 'static,
-    ) -> Self {
-        Self {
-            cache_dict: HashMap::new(),
-            ttl_dict: HashMap::new(),
-            expiration_heap: BinaryHeap::new(),
-            max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY),
-            default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
-            now: Box::new(now),
-        }
-    }
-
-    pub fn evict_cache(&mut self) {
-        if self.max_size_in_memory == 0 {
-            return;
-        }
-
-        let current_time = (self.now)();
-        while let Some(Reverse((expiration_time, key))) = self.expiration_heap.peek().cloned() {
-            if self.ttl_dict.get(&key).copied() != Some(expiration_time) {
-                self.expiration_heap.pop();
-            } else if expiration_time <= current_time {
-                self.expiration_heap.pop();
-                self.remove_key(&key);
-            } else {
-                break;
-            }
-        }
-
-        while self.cache_dict.len() >= self.max_size_in_memory {
-            let Some(Reverse((expiration_time, key))) = self.expiration_heap.pop() else {
-                break;
-            };
-            if self.ttl_dict.get(&key).copied() == Some(expiration_time) {
-                self.remove_key(&key);
-            }
-        }
-    }
-
-    pub fn allow_ttl_override(&self, key: &str) -> bool {
-        match self.ttl_dict.get(key).copied() {
-            None => true,
-            Some(expiration_time) => expiration_time < (self.now)(),
-        }
-    }
-
-    pub fn set_cache(&mut self, key: impl Into<String>, value: V, ttl: Option<Duration>) {
-        if self.max_size_in_memory == 0 {
-            return;
-        }
-
-        self.evict_cache();
-        let key = key.into();
-        self.cache_dict.insert(key.clone(), value);
-        if self.allow_ttl_override(&key) {
-            let expiration_time = (self.now)() + ttl.unwrap_or(self.default_ttl);
-            self.ttl_dict.insert(key.clone(), expiration_time);
-            self.expiration_heap.push(Reverse((expiration_time, key)));
-        }
-    }
-
-    // Generic values intentionally omit Python's per-item size check.
-    pub fn get_cache(&mut self, key: &str) -> Option<V> {
-        if self.cache_dict.contains_key(key) {
-            if self.is_key_expired(key) {
-                self.remove_key(key);
-                return None;
-            }
-            return self.cache_dict.get(key).cloned();
-        }
-        None
-    }
-
-    pub fn get_ttl(&self, key: &str) -> Option<Duration> {
-        self.ttl_dict.get(key).copied()
-    }
-
-    pub fn delete_cache(&mut self, key: &str) {
-        self.remove_key(key);
-    }
-
-    pub fn flush_cache(&mut self) {
-        self.cache_dict.clear();
-        self.ttl_dict.clear();
-        self.expiration_heap.clear();
-    }
-
-    fn is_key_expired(&self, key: &str) -> bool {
-        self.ttl_dict
-            .get(key)
-            .is_some_and(|expiration_time| *expiration_time < (self.now)())
-    }
-
-    fn remove_key(&mut self, key: &str) {
-        self.cache_dict.remove(key);
-        self.ttl_dict.remove(key);
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::{
-        Arc,
-        atomic::{AtomicU64, Ordering},
-    };
-
-    use super::InMemoryCache;
-    use std::time::Duration;
-
-    fn cache(now: Arc<AtomicU64>, max_size: usize, default_ttl: Duration) -> InMemoryCache<String> {
-        InMemoryCache::with_clock(Some(max_size), Some(default_ttl), move || {
-            Duration::from_secs(now.load(Ordering::Relaxed))
-        })
-    }
-
-    #[test]
-    fn ttl_expiry_is_deterministic() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now.clone(), 10, Duration::from_secs(60));
-        cache.set_cache("key", "value".to_string(), None);
-        assert_eq!(cache.get_cache("key"), Some("value".to_string()));
-        now.store(161, Ordering::Relaxed);
-        assert_eq!(cache.get_cache("key"), None);
-        assert_eq!(cache.get_ttl("key"), None);
-    }
-
-    #[test]
-    fn default_and_per_set_ttl_are_applied() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now.clone(), 10, Duration::from_secs(60));
-        cache.set_cache("default", "value".to_string(), None);
-        cache.set_cache("custom", "value".to_string(), Some(Duration::from_secs(20)));
-        assert_eq!(cache.get_ttl("default"), Some(Duration::from_secs(160)));
-        assert_eq!(cache.get_ttl("custom"), Some(Duration::from_secs(120)));
-    }
-
-    #[test]
-    fn unexpired_entries_do_not_allow_ttl_override() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now.clone(), 10, Duration::from_secs(60));
-        cache.set_cache("key", "first".to_string(), Some(Duration::from_secs(20)));
-        cache.set_cache("key", "second".to_string(), Some(Duration::from_secs(80)));
-        assert_eq!(cache.get_cache("key"), Some("second".to_string()));
-        assert_eq!(cache.get_ttl("key"), Some(Duration::from_secs(120)));
-        now.store(121, Ordering::Relaxed);
-        cache.set_cache("key", "third".to_string(), Some(Duration::from_secs(80)));
-        assert_eq!(cache.get_ttl("key"), Some(Duration::from_secs(201)));
-    }
-
-    #[test]
-    fn max_size_evicts_earliest_expiration() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now, 2, Duration::from_secs(60));
-        cache.set_cache("early", "value".to_string(), Some(Duration::from_secs(10)));
-        cache.set_cache("late", "value".to_string(), Some(Duration::from_secs(20)));
-        cache.set_cache("new", "value".to_string(), Some(Duration::from_secs(30)));
-        assert_eq!(cache.get_cache("early"), None);
-        assert!(cache.get_cache("late").is_some());
-        assert!(cache.get_cache("new").is_some());
-    }
-
-    #[test]
-    fn expired_entries_are_evicted_before_live_entries() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now.clone(), 3, Duration::from_secs(60));
-        cache.set_cache(
-            "expired-one",
-            "value".to_string(),
-            Some(Duration::from_secs(10)),
-        );
-        cache.set_cache(
-            "expired-two",
-            "value".to_string(),
-            Some(Duration::from_secs(20)),
-        );
-        cache.set_cache("live", "value".to_string(), Some(Duration::from_secs(100)));
-        now.store(121, Ordering::Relaxed);
-        cache.set_cache("new", "value".to_string(), Some(Duration::from_secs(100)));
-        assert_eq!(cache.get_cache("expired-one"), None);
-        assert_eq!(cache.get_cache("expired-two"), None);
-        assert!(cache.get_cache("live").is_some());
-        assert!(cache.get_cache("new").is_some());
-    }
-
-    #[test]
-    fn stale_heap_entries_are_skipped() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now, 1, Duration::from_secs(60));
-        cache.set_cache(
-            "removed",
-            "value".to_string(),
-            Some(Duration::from_secs(10)),
-        );
-        cache.delete_cache("removed");
-        cache.set_cache("kept", "value".to_string(), Some(Duration::from_secs(20)));
-        cache.set_cache("new", "value".to_string(), Some(Duration::from_secs(30)));
-        assert_eq!(cache.get_cache("removed"), None);
-        assert_eq!(cache.get_cache("kept"), None);
-        assert!(cache.get_cache("new").is_some());
-    }
-
-    #[test]
-    fn delete_and_flush_remove_values_and_ttls() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now, 10, Duration::from_secs(60));
-        cache.set_cache("one", "value".to_string(), None);
-        cache.set_cache("two", "value".to_string(), None);
-        cache.delete_cache("one");
-        assert_eq!(cache.get_cache("one"), None);
-        cache.flush_cache();
-        assert!(cache.cache_dict.is_empty());
-        assert!(cache.ttl_dict.is_empty());
-        assert!(cache.expiration_heap.is_empty());
-    }
-
-    #[test]
-    fn zero_max_size_does_not_cache() {
-        let now = Arc::new(AtomicU64::new(100));
-        let mut cache = cache(now, 0, Duration::from_secs(60));
-        cache.set_cache("key", "value".to_string(), None);
-        assert_eq!(cache.get_cache("key"), None);
-        assert!(cache.cache_dict.is_empty());
-    }
-}

litellm-rust/crates/core/src/caching/mod.rs

@@ -1 +0,0 @@
-pub mod in_memory_cache;

litellm-rust/crates/core/src/chat_completions/common_utils.rs

@@ -12,7 +12,6 @@ pub(super) fn chat_completions_provider_config(
 ) -> Option<&'static dyn ChatCompletionsProviderConfig> {
     match provider {
         "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG),
-        #[cfg(feature = "bedrock-auth")]
         "bedrock" => Some(
             &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
         ),

litellm-rust/crates/core/src/chat_completions/handler.rs

@@ -79,7 +79,6 @@ pub(super) fn as_response_error(err: Error) -> Error {
     }
 }
 
-#[cfg(feature = "bedrock-auth")]
 pub(super) async fn signed_headers(
     request: &ProviderChatCompletionsRequest,
     body: &[u8],
@@ -135,16 +134,3 @@ pub(super) async fn signed_headers(
     // that would collide, so no name appears twice.
     Ok(unsigned.into_iter().chain(signature).collect())
 }
-
-#[cfg(not(feature = "bedrock-auth"))]
-pub(super) async fn signed_headers(
-    request: &ProviderChatCompletionsRequest,
-    _body: &[u8],
-) -> Result<Vec<(String, String)>, Error> {
-    match &request.auth {
-        ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported(
-            "AWS SigV4 requires the bedrock-auth feature",
-        )),
-        _ => Ok(request.upstream_headers.clone()),
-    }
-}

litellm-rust/crates/core/src/chat_completions/tests.rs

@@ -270,7 +270,6 @@ fn rejects_non_string_extra_headers() {
     );
 }
 
-#[cfg(feature = "bedrock-auth")]
 #[test]
 fn prepares_a_bedrock_call_without_resolving_credentials() {
     let mut call = request(
@@ -302,7 +301,6 @@ fn prepares_a_bedrock_call_without_resolving_credentials() {
     assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
 }
 
-#[cfg(feature = "bedrock-auth")]
 #[tokio::test]
 async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
     // Python signs only the AWS header set and reattaches the rest, so a header
@@ -351,7 +349,6 @@ async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
     );
 }
 
-#[cfg(feature = "bedrock-auth")]
 #[tokio::test]
 async fn a_forwarded_header_the_signer_computes_declines_to_python() {
     // Reattaching the caller's copy next to the computed one puts the name on
@@ -386,7 +383,6 @@ async fn a_forwarded_header_the_signer_computes_declines_to_python() {
     }
 }
 
-#[cfg(feature = "bedrock-auth")]
 #[test]
 fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
     // `get_request_headers` assigns `headers["Authorization"]` unconditionally
@@ -453,7 +449,6 @@ fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
     );
 }
 
-#[cfg(feature = "bedrock-auth")]
 #[test]
 fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
     // The configured bearer identity has its own account and quota boundary,

litellm-rust/crates/core/src/chat_completions/types.rs

@@ -41,7 +41,6 @@ pub(super) struct ProviderChatCompletionsRequest {
     pub(super) body: Value,
     pub(super) upstream_headers: Vec<(String, String)>,
     pub(super) auth: ChatCompletionsAuth,
-    #[cfg_attr(not(feature = "bedrock-auth"), allow(dead_code))]
     pub(super) optional_params: OpaqueParams,
     pub(super) timeout: Option<Duration>,
 }

litellm-rust/crates/core/src/lib.rs

@@ -1,6 +1,4 @@
 pub mod audio_transcription;
-pub mod auth;
-pub mod caching;
 pub mod call_lifecycle;
 pub mod chat_completions;
 pub mod constants;
@@ -15,5 +13,5 @@ pub mod providers;
 pub mod responses;
 mod url_utils;
 
-pub use auth::Error as AuthError;
 pub use error::Error;
+pub use litellm_auth::Error as AuthError;

litellm-rust/crates/core/src/llms/azure_ai/ocr/common_utils.rs

@@ -1,9 +1,9 @@
 use std::sync::OnceLock;
 
 use crate::Error;
-use crate::auth::{InputSource, Sourced};
 use crate::ocr::error::OcrError;
 use crate::ocr::types::OcrConnection;
+use litellm_auth::{InputSource, Sourced};
 use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
 
 pub(super) async fn resolve_entra(

litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs

@@ -614,7 +614,6 @@ mod polling {
 }
 
 use crate::Error;
-use crate::auth::{InputSource, Sourced};
 use crate::constants::{AZURE_DI_API_VERSION, AZURE_DI_SUBSCRIPTION_HEADER};
 use crate::llms::base_llm::ocr::transformation::BaseOcrConfig;
 use crate::ocr::OcrClient;
@@ -622,6 +621,7 @@ use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError};
 use crate::ocr::prepare::{credential_env, transform_request_body};
 use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrResponseFormat};
 use crate::url_utils::ApiUrl;
+use litellm_auth::{InputSource, Sourced};
 use litellm_auth_azure::AzureAuthInputs;
 
 const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";

litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs

@@ -1,5 +1,4 @@
 use crate::Error;
-use crate::auth::{InputSource, Sourced};
 use crate::constants::AZURE_AI_OCR_PATH;
 use crate::llms::base_llm::ocr::transformation::BaseOcrConfig;
 use crate::llms::mistral::ocr::MistralOcrResponse;
@@ -10,6 +9,7 @@ use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError};
 use crate::ocr::prepare::{credential_env, transform_request_body};
 use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection};
 use crate::url_utils::ApiUrl;
+use litellm_auth::{InputSource, Sourced};
 use litellm_auth_azure::AzureAuthInputs;
 
 const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY";

litellm-rust/crates/core/src/llms/vertex_ai/ocr/common_utils.rs

@@ -1,7 +1,7 @@
 use crate::Error;
-use crate::auth::InputSource;
 use crate::ocr::error::OcrError;
 use crate::ocr::types::OcrConnection;
+use litellm_auth::InputSource;
 
 pub(super) fn validate_destination(connection: &OcrConnection) -> Result<(), OcrError> {
     if connection.api_base.is_some() && connection.api_base_source == InputSource::Request {

litellm-rust/crates/core/src/ocr/lifecycle.rs

@@ -12,11 +12,11 @@ use super::hooks::{
 use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient};
 use crate::AuthError;
 use crate::Error;
-use crate::auth::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
 use crate::call_lifecycle::host::{
     HostCall, HostCallFuture, HostCallStep, HostFailure, HostLifecycle, HostPhase,
 };
 use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming};
+use litellm_auth::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
 
 pub type NativeResult<T> = Result<NativeOutcome<T>, Error>;
 

litellm-rust/crates/core/src/ocr/types.rs

@@ -8,9 +8,9 @@ use serde_json::{Map, Value};
 use super::hooks::{NoopOcrHooks, OcrHooks};
 use super::provider_config::{OcrConfigKind, resolve_provider_config};
 use crate::Error;
-use crate::auth::{InputSource, TokenProviderHandle};
 use crate::constants::OCR_HTTP_TIMEOUT_SECS;
 use crate::params::OpaqueParams;
+use litellm_auth::{InputSource, TokenProviderHandle};
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(tag = "type")]

litellm-rust/crates/core/src/ocr/wire.rs

@@ -5,8 +5,8 @@ use std::time::Duration;
 
 use super::types::{LiteLLMOcrRequest, OcrConnection, OcrDocument};
 use crate::Error;
-use crate::auth::InputSource;
 use crate::params::OpaqueParams;
+use litellm_auth::InputSource;
 use serde::{
     Deserialize,
     de::{DeserializeOwned, IntoDeserializer},

litellm-rust/crates/core/src/providers/bedrock/mod.rs

@@ -2,7 +2,6 @@
 //! with Python's `BaseAWSLLM`; the broader core purity guidance is reconciled
 //! separately.
 
-#[cfg(feature = "bedrock-auth")]
 pub mod audio_transcription;
 pub mod aws_base;
 pub mod chat_completions;

litellm-rust/crates/core/src/providers/mod.rs

@@ -1,6 +1,5 @@
 pub mod anthropic;
 pub mod azure_ai;
-#[cfg(feature = "bedrock-auth")]
 pub mod bedrock;
 pub mod custom_llm_provider;
 pub(crate) mod model;

litellm-rust/crates/python-bridge/Cargo.toml

@@ -17,7 +17,9 @@ panic-test = []
 
 [dependencies]
 futures-util.workspace = true
-litellm-core = { workspace = true, features = ["bedrock-auth"] }
+tracing = { workspace = true, optional = true }
+litellm-core.workspace = true
+litellm-auth.workspace = true
 litellm-token-counter.workspace = true
 litellm-python-interop.workspace = true
 pyo3.workspace = true

litellm-rust/crates/python-bridge/src/auth.rs

@@ -1,4 +1,4 @@
-use litellm_core::auth::{ResolvedCredential, SecretValue};
+use litellm_auth::{ResolvedCredential, SecretValue};
 use pyo3::exceptions::{PyException, PyRuntimeError, PyTypeError};
 use pyo3::gc::{PyTraverseError, PyVisit};
 use pyo3::prelude::*;

litellm-rust/crates/python-bridge/src/lifecycle/preparation.rs

@@ -1,4 +1,4 @@
-use litellm_core::auth::{credential_default_fields, credential_index};
+use litellm_auth::{credential_default_fields, credential_index};
 use pyo3::prelude::*;
 use pyo3::types::{PyDict, PyList};
 

litellm-rust/crates/python-bridge/src/marshal.rs

@@ -6,7 +6,7 @@ use pyo3::prelude::*;
 use pyo3::types::PyDict;
 use serde_json::{Map, Value};
 
-use litellm_core::auth::InputSource;
+use litellm_auth::InputSource;
 use litellm_python_interop::from_py_preserving_errors as from_py;
 
 pub(crate) struct RouteOptions {

litellm-rust/crates/python-bridge/src/routes/ocr/lifecycle.rs

@@ -1,7 +1,7 @@
 use pyo3::prelude::*;
 use pyo3::types::{PyDict, PyTuple};
 
-use litellm_core::auth::ResolvedCredential;
+use litellm_auth::ResolvedCredential;
 use litellm_core::ocr::hooks::{OcrDuringCallRequest, OcrPostCallRequest, OcrPreCallRequest};
 use litellm_core::ocr::{OcrAdmission, OcrCall, OcrClient, OcrHostOperation, OcrHostResult};
 use litellm_python_interop::{