scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-25 01:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 131

Merge remote-tracking branch 'origin/main' into litellm_logging_worker_flush_loop_change

Browse source
This commit is contained in:
mateo-berri 2026-09-21 16:02:04 -07:00
commit 37f1670a1e
105 changed files with 11100 additions and 3802 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.circleci/config.yml
View file

@ -3050,28 +3050,29 @@ jobs:
- run:
name: Run Docker container with bad DATABASE_URL
command: |
set +e
docker run --name my-app \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
myapp:latest \
--port 4000 > docker_output.log 2>&1 || true
--port 4000 > docker_output.log 2>&1
echo "$?" > docker_exit_code
set -e
- run:
name: Display Docker logs
command: cat docker_output.log
- run:
name: Check for expected error
name: Proxy must refuse to serve on an unreachable database
command: |
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
(grep -q "Database setup failed after multiple retries" docker_output.log || \
grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
echo "Expected error found. Test passed."
else
echo "Expected error not found. Test failed."
cat docker_output.log
exit 1
fi
fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; }
exit_code="$(cat docker_exit_code)"
[ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database"
grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server"
! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state"
! docker exec my-app true 2>/dev/null || fail "container is still running"
echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
provider_replay_harness:
docker:

4
.github/workflows/test-linting.yml vendored
View file

@ -130,6 +130,10 @@ jobs:
echo "File content around line 43:"
head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- name: Check MCP operation boundary
if: steps.changes.outputs.decision != 'skip'
run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
- name: Run Ruff linting
if: steps.changes.outputs.decision != 'skip'
run: |

1
Makefile
View file

@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
# Linting targets
lint-ruff: $(LINT_DEP_INSTALL)
$(UV_RUN) python scripts/check_mcp_operation_boundary.py
cd litellm && $(UV_RUN) ruff check . && cd ..
$(UV_RUN) ruff check --config ruff-tests.toml tests

11
docker/README.md
View file

@ -2,6 +2,17 @@
This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
> **Just want to run LiteLLM?** This guide builds from source. To run the published
> image instead, use `docker-compose.quickstart.yml` in this directory — the
> two-service stack (gateway + Postgres) that the
> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents:
>
> ```bash
> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
> docker compose -f docker-compose.quickstart.yml up -d
> ```
## Prerequisites
- Docker

41
docker/docker-compose.quickstart.yml Normal file
View file

@ -0,0 +1,41 @@
# LiteLLM quickstart stack: the gateway plus a Postgres database that stores
# models, virtual keys, and spend logs. Used by
# https://docs.litellm.ai/docs/proxy/docker_quick_start
#
# curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
# printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
# docker compose -f docker-compose.quickstart.yml up -d
#
# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY
# makes credentials already stored in the database unreadable. For anything
# beyond local evaluation, pin the image to a specific release tag.
services:
litellm:
image: docker.litellm.ai/berriai/litellm:main-stable
ports:
- "4000:4000"
environment:
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file}
LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file}
DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
STORE_MODEL_IN_DB: "True"
depends_on:
db:
condition: service_healthy
db:
image: postgres:16
environment:
POSTGRES_USER: litellm
POSTGRES_PASSWORD: litellm
POSTGRES_DB: litellm
healthcheck:
test: ["CMD-SHELL", "pg_isready -U litellm"]
interval: 5s
timeout: 5s
retries: 10
volumes:
- postgres_data:/var/lib/postgresql/data
volumes:
postgres_data:

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN;
ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3);

2
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -246,6 +246,8 @@ model LiteLLM_UserTable {
organization_id String?
object_permission_id String?
password String?
password_reset_required Boolean?
last_breach_check_at DateTime?
teams String[] @default([])
user_role String?
max_budget Float?

144
litellm-rust/Cargo.lock generated
View file

@ -1377,6 +1377,18 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fancy-regex"
version = "0.17.0"
@ -1428,6 +1440,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@ -1892,11 +1910,32 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
dependencies = [
"foldhash",
]
[[package]]
name = "hashlink"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
dependencies = [
"hashbrown 0.17.1",
]
[[package]]
name = "heck"
@ -2277,6 +2316,12 @@ version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iter-read"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071ed4cc1afd86650602c7b11aa2e1ce30762a1c27193201cb5cee9c6ebb1294"
[[package]]
name = "itertools"
version = "0.13.0"
@ -2435,6 +2480,17 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libsqlite3-sys"
version = "0.38.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@ -2540,6 +2596,36 @@ dependencies = [
"url",
]
[[package]]
name = "litellm-cache-disk"
version = "0.1.0"
dependencies = [
"litellm-cache",
"py_literal",
"rand 0.8.7",
"rstest",
"rusqlite",
"serde-pickle",
"serde_json",
"tempfile",
"tokio",
]
[[package]]
name = "litellm-cache-gcs"
version = "0.1.0"
dependencies = [
"futures-util",
"litellm-auth-gcp",
"litellm-auth-types",
"litellm-cache",
"percent-encoding",
"reqwest 0.12.28",
"serde_json",
"tokio",
"wiremock",
]
[[package]]
name = "litellm-cache-memory"
version = "0.1.0"
@ -2743,6 +2829,8 @@ dependencies = [
"litellm-auth-gcp",
"litellm-cache",
"litellm-cache-azure-blob",
"litellm-cache-disk",
"litellm-cache-gcs",
"litellm-cache-memory",
"litellm-cache-redis",
"litellm-cache-response",
@ -4033,6 +4121,16 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rsqlite-vfs"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
dependencies = [
"hashbrown 0.16.1",
"thiserror 2.0.19",
]
[[package]]
name = "rstest"
version = "0.26.1"
@ -4073,6 +4171,21 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "rusqlite"
version = "0.40.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
dependencies = [
"bitflags 2.13.1",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
"sqlite-wasm-rs",
]
[[package]]
name = "rustc-hash"
version = "2.1.3"
@ -4330,6 +4443,19 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-pickle"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b641fdc8bcf2781ee78b30c599700d64ad4f412976143e4c5d0b9df906bb4843"
dependencies = [
"byteorder",
"iter-read",
"num-bigint 0.4.8",
"num-traits",
"serde",
]
[[package]]
name = "serde_core"
version = "1.0.229"
@ -4557,6 +4683,18 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "sqlite-wasm-rs"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
dependencies = [
"cc",
"js-sys",
"rsqlite-vfs",
"wasm-bindgen",
]
[[package]]
name = "sse-stream"
version = "0.2.6"
@ -5303,6 +5441,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "veil"
version = "0.3.0"

3
litellm-rust/Cargo.toml
View file

@ -32,6 +32,8 @@ litellm-cache = { path = "crates/cache" }
litellm-cache-azure-blob = { path = "crates/cache-azure-blob" }
litellm-cache-memory = { path = "crates/cache-memory" }
litellm-cache-redis = { path = "crates/cache-redis" }
litellm-cache-gcs = { path = "crates/cache-gcs" }
litellm-cache-disk = { path = "crates/cache-disk" }
litellm-cache-response = { path = "crates/cache-response" }
litellm-token-counter = { path = "crates/token-counter" }
litellm-token-counter-fast = { path = "crates/token-counter-fast" }
@ -69,6 +71,7 @@ base64 = "0.22"
moka = { version = "0.12.16", features = ["future"] }
strum = { version = "0.28.0", features = ["derive"] }
url = "2.5.8"
percent-encoding = "2.3"
webpki-roots = "1"
time = { version = "0.3.53", features = ["parsing"] }
criterion = "0.8.2"

8
litellm-rust/crates/auth-gcp/src/lib.rs
View file

@ -128,6 +128,14 @@ impl VertexAuth {
}
}
pub async fn access_token(
&self,
config: &VertexConfig,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> Result<String, Error> {
self.load_provider(config, env_lookup).await?.token().await
}
pub async fn validate_environment(
&self,
headers: Vec<(String, String)>,

19
litellm-rust/crates/cache-disk/Cargo.toml Normal file
View file

@ -0,0 +1,19 @@
[package]
name = "litellm-cache-disk"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-cache.workspace = true
py_literal = "0.4.0"
rand.workspace = true
rusqlite = { version = "0.40", features = ["bundled"] }
serde-pickle = "1.2"
serde_json.workspace = true
tokio.workspace = true
[dev-dependencies]
rstest.workspace = true
tempfile = "3.27.0"

10
litellm-rust/crates/cache-disk/src/adapter.rs Normal file
View file

@ -0,0 +1,10 @@
use litellm_cache::Error;
use crate::StoredValue;
pub trait ValueAdapter: Send + Sync + 'static {
fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error>;
fn write(&self, payload: Vec<u8>) -> StoredValue;
fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error>;
fn counter_value(&self, value: f64) -> StoredValue;
}

301
litellm-rust/crates/cache-disk/src/cache.rs Normal file
View file

@ -0,0 +1,301 @@
use std::{
path::Path,
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use litellm_cache::{
BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus,
CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache,
};
use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter};
pub struct DiskCache<S, D = DiskcacheSqliteStore, A = PythonDiskCacheAdapter> {
store: Arc<D>,
adapter: Arc<A>,
codec: S,
}
impl<S: CacheCodec> DiskCache<S> {
pub fn open(directory: impl AsRef<Path>, codec: S) -> Result<Self, Error> {
Ok(Self {
store: Arc::new(DiskcacheSqliteStore::open(directory)?),
adapter: Arc::new(PythonDiskCacheAdapter),
codec,
})
}
}
impl<S: CacheCodec, D: DiskStore> DiskCache<S, D, PythonDiskCacheAdapter> {
pub fn with_store(store: D, codec: S) -> Self {
Self {
store: Arc::new(store),
adapter: Arc::new(PythonDiskCacheAdapter),
codec,
}
}
}
impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DiskCache<S, D, A> {
pub fn with_adapter(store: D, adapter: A, codec: S) -> Self {
Self {
store: Arc::new(store),
adapter: Arc::new(adapter),
codec,
}
}
pub fn directory(&self) -> &Path {
self.store.directory()
}
fn decode_stored(&self, value: StoredValue) -> Result<Option<S::Value>, Error> {
let Some(bytes) = self.adapter.read(value)? else {
return Ok(None);
};
self.codec.decode(&bytes).map(Some)
}
async fn run_blocking<T, F>(store: Arc<D>, operation: F) -> Result<T, Error>
where
T: Send + 'static,
F: FnOnce(&D) -> Result<T, Error> + Send + 'static,
{
tokio::task::spawn_blocking(move || operation(&store))
.await
.map_err(|_| Error::Unavailable)?
}
}
impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BaseCache for DiskCache<S, D, A> {
type Value = S::Value;
type Context = ExactCacheContext;
fn get_ttl(&self, context: &Self::Context) -> Option<Duration> {
context.ttl
}
fn set_cache(
&self,
key: &str,
value: Self::Value,
context: &Self::Context,
) -> Result<(), Error> {
let value = self.adapter.write(self.codec.encode(&value)?);
let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
self.store.set(key, value, expire_time, unix_now())
}
fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
self.store
.get(key, unix_now())?
.map(|value| self.decode_stored(value))
.transpose()
.map(|value| value.flatten())
}
async fn async_set_cache(
&self,
key: &str,
value: Self::Value,
context: ExactCacheContext,
) -> Result<(), Error> {
let value = self.adapter.write(self.codec.encode(&value)?);
let ttl = context.ttl;
let key = key.to_string();
Self::run_blocking(Arc::clone(&self.store), move |store| {
let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64());
store.set(&key, value, expire_time, unix_now())
})
.await
}
async fn async_get_cache(
&self,
key: &str,
_: &ExactCacheContext,
) -> Result<Option<Self::Value>, Error> {
let key = key.to_string();
let value = Self::run_blocking(Arc::clone(&self.store), move |store| {
store.get(&key, unix_now())
})
.await?;
value
.map(|value| self.decode_stored(value))
.transpose()
.map(|value| value.flatten())
}
async fn async_set_cache_pipeline(
&self,
entries: Vec<(String, Self::Value)>,
context: ExactCacheContext,
) -> Result<(), Error> {
let entries = entries
.into_iter()
.map(|(key, value)| {
self.codec
.encode(&value)
.map(|value| (key, self.adapter.write(value)))
})
.collect::<Result<Vec<_>, _>>()?;
let expire_after = context.ttl;
Self::run_blocking(Arc::clone(&self.store), move |store| {
for (key, value) in entries {
let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64());
store.set(&key, value, expire_time, unix_now())?;
}
Ok(())
})
.await
}
async fn disconnect(&self) -> Result<(), Error> {
Ok(())
}
async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
let result = Self::run_blocking(Arc::clone(&self.store), |store| {
store.probe().map(|_| CacheConnectionResult {
status: CacheConnectionStatus::Success,
message: "Disk cache connection test successful".into(),
error: None,
})
})
.await;
Ok(match result {
Ok(result) => result,
Err(error) => CacheConnectionResult {
status: CacheConnectionStatus::Failed,
message: format!("Disk cache connection failed: {error}"),
error: Some(error.to_string()),
},
})
}
}
impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> BatchCache for DiskCache<S, D, A> {
fn batch_get_cache(
&self,
keys: &[String],
context: &ExactCacheContext,
) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
keys.iter()
.map(|key| match self.get_cache(key, context) {
Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
Ok(None) => Ok(BatchEntry::Miss),
Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
Err(error) => Err(error),
})
.collect()
}
async fn async_batch_get_cache(
&self,
keys: Vec<String>,
_: ExactCacheContext,
) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
let values = Self::run_blocking(Arc::clone(&self.store), move |store| {
keys.into_iter()
.map(|key| store.get(&key, unix_now()).map(|value| (key, value)))
.collect::<Result<Vec<_>, _>>()
})
.await?;
values
.into_iter()
.map(|(_, value)| match value {
None => Ok(BatchEntry::Miss),
Some(value) => match self.decode_stored(value) {
Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
Ok(None) => Ok(BatchEntry::Miss),
Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
Err(error) => Err(error),
},
})
.collect()
}
}
impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> DeleteCache for DiskCache<S, D, A> {
fn delete_cache(&self, key: &str) -> Result<(), Error> {
self.store.pop(key, unix_now()).map(|_| ())
}
async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
let key = key.to_string();
Self::run_blocking(Arc::clone(&self.store), move |store| {
store.pop(&key, unix_now()).map(|_| ())
})
.await
}
}
impl<S: CacheCodec, D: DiskStore, A: ValueAdapter> FlushCache for DiskCache<S, D, A> {
fn flush_cache(&self) -> Result<(), Error> {
self.store.clear()
}
async fn async_flush_cache(&self) -> Result<(), Error> {
Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await
}
}
impl<S: CacheCodec<Value = f64>, D: DiskStore, A: ValueAdapter> CounterCache
for DiskCache<S, D, A>
{
fn increment_cache(
&self,
key: &str,
amount: f64,
context: ExactCacheContext,
) -> Result<f64, Error> {
increment(
self.adapter.as_ref(),
self.store.as_ref(),
key,
amount,
context.ttl,
)
}
async fn async_increment(
&self,
key: &str,
amount: f64,
context: ExactCacheContext,
) -> Result<f64, Error> {
let key = key.to_string();
let adapter = Arc::clone(&self.adapter);
Self::run_blocking(Arc::clone(&self.store), move |store| {
increment(adapter.as_ref(), store, &key, amount, context.ttl)
})
.await
}
}
fn increment<A: ValueAdapter, D: DiskStore>(
adapter: &A,
store: &D,
key: &str,
amount: f64,
ttl: Option<Duration>,
) -> Result<f64, Error> {
let mut result = None;
let mut apply = |current: Option<StoredValue>| {
let initial = adapter.counter_seed(current)?;
let value = initial + amount;
let stored = adapter.counter_value(value);
result = Some(value);
Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64())))
};
store.update(key, unix_now(), &mut apply)?;
result.ok_or(Error::InvalidEntry)
}
fn unix_now() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64()
}

11
litellm-rust/crates/cache-disk/src/lib.rs Normal file
View file

@ -0,0 +1,11 @@
mod adapter;
mod cache;
mod python;
mod sqlite;
mod store;
pub use adapter::ValueAdapter;
pub use cache::DiskCache;
pub use python::PythonDiskCacheAdapter;
pub use sqlite::DiskcacheSqliteStore;
pub use store::{DiskStore, StoredValue};

77
litellm-rust/crates/cache-disk/src/python/mod.rs Normal file
View file

@ -0,0 +1,77 @@
mod value;
use litellm_cache::Error;
use py_literal::Value;
use crate::{StoredValue, ValueAdapter};
#[derive(Clone, Copy, Debug, Default)]
pub struct PythonDiskCacheAdapter;
impl PythonDiskCacheAdapter {
fn python_get_cache(value: StoredValue) -> Result<Option<Value>, Error> {
let value = match value {
StoredValue::Bytes(value) => Value::Bytes(value),
StoredValue::Text(value) => Value::String(value),
StoredValue::Integer(value) => Value::Integer(value.into()),
StoredValue::Float(value) => Value::Float(value),
StoredValue::Pickle(value) => value::from_pickle(&value)?,
};
if !value::is_truthy(&value) {
return Ok(None);
}
match value {
Value::String(text) => Ok(Some(
value::from_json_text(&text).unwrap_or(Value::String(text)),
)),
Value::Bytes(bytes) => match std::str::from_utf8(&bytes) {
Ok(text) => Ok(Some(
value::from_json_text(text).unwrap_or(Value::Bytes(bytes)),
)),
Err(_) => Ok(Some(Value::Bytes(bytes))),
},
value => Ok(Some(value)),
}
}
}
impl ValueAdapter for PythonDiskCacheAdapter {
fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, Error> {
match value {
StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())),
StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)),
value => {
let Some(value) = Self::python_get_cache(value)? else {
return Ok(None);
};
value::to_json(&value).map(Some)
}
}
}
fn write(&self, payload: Vec<u8>) -> StoredValue {
StoredValue::Bytes(payload)
}
fn counter_seed(&self, value: Option<StoredValue>) -> Result<f64, Error> {
let Some(value) = value else {
return Ok(0.0);
};
let Some(value) = Self::python_get_cache(value)? else {
return Ok(0.0);
};
Ok(if value::is_int(&value) {
value::to_f64(&value).unwrap_or(0.0)
} else {
0.0
})
}
fn counter_value(&self, value: f64) -> StoredValue {
if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 {
StoredValue::Integer(value as i64)
} else {
StoredValue::Float(value)
}
}
}

173
litellm-rust/crates/cache-disk/src/python/value.rs Normal file
View file

@ -0,0 +1,173 @@
use litellm_cache::Error;
use py_literal::Value;
use serde_json::{Map, Number};
pub(crate) fn from_pickle(bytes: &[u8]) -> Result<Value, Error> {
let value = serde_pickle::value_from_slice(bytes, Default::default())
.map_err(|_| Error::InvalidEntry)?;
from_pickle_value(value)
}
fn from_pickle_value(value: serde_pickle::Value) -> Result<Value, Error> {
match value {
serde_pickle::Value::None => Ok(Value::None),
serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)),
serde_pickle::Value::I64(value) => integer(value.to_string()),
serde_pickle::Value::Int(value) => integer(value.to_string()),
serde_pickle::Value::F64(value) => Ok(Value::Float(value)),
serde_pickle::Value::String(value) => Ok(Value::String(value)),
serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)),
serde_pickle::Value::List(values) => values
.into_iter()
.map(from_pickle_value)
.collect::<Result<Vec<_>, _>>()
.map(Value::List),
serde_pickle::Value::Tuple(values) => values
.into_iter()
.map(from_pickle_value)
.collect::<Result<Vec<_>, _>>()
.map(Value::Tuple),
serde_pickle::Value::Set(values) => values
.into_iter()
.map(from_pickle_hashable)
.collect::<Result<Vec<_>, _>>()
.map(Value::Set),
serde_pickle::Value::FrozenSet(values) => values
.into_iter()
.map(from_pickle_hashable)
.collect::<Result<Vec<_>, _>>()
.map(Value::Set),
serde_pickle::Value::Dict(values) => values
.into_iter()
.map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?)))
.collect::<Result<Vec<_>, Error>>()
.map(Value::Dict),
}
}
fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result<Value, Error> {
Ok(match value {
serde_pickle::HashableValue::None => Value::None,
serde_pickle::HashableValue::Bool(value) => Value::Boolean(value),
serde_pickle::HashableValue::I64(value) => integer(value.to_string())?,
serde_pickle::HashableValue::Int(value) => integer(value.to_string())?,
serde_pickle::HashableValue::F64(value) => Value::Float(value),
serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value),
serde_pickle::HashableValue::String(value) => Value::String(value),
serde_pickle::HashableValue::Tuple(values) => Value::Tuple(
values
.into_iter()
.map(from_pickle_hashable)
.collect::<Result<Vec<_>, _>>()?,
),
serde_pickle::HashableValue::FrozenSet(values) => Value::Set(
values
.into_iter()
.map(from_pickle_hashable)
.collect::<Result<Vec<_>, _>>()?,
),
})
}
fn integer(value: String) -> Result<Value, Error> {
value.parse().map_err(|_| Error::InvalidEntry)
}
pub(crate) fn from_json(value: serde_json::Value) -> Value {
match value {
serde_json::Value::Null => Value::None,
serde_json::Value::Bool(value) => Value::Boolean(value),
serde_json::Value::Number(value) => {
if value.is_i64() || value.is_u64() {
integer(value.to_string())
.unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN)))
} else {
Value::Float(value.as_f64().unwrap_or(f64::NAN))
}
}
serde_json::Value::String(value) => Value::String(value),
serde_json::Value::Array(values) => {
Value::List(values.into_iter().map(from_json).collect())
}
serde_json::Value::Object(values) => Value::Dict(
values
.into_iter()
.map(|(key, value)| (Value::String(key), from_json(value)))
.collect(),
),
}
}
pub(crate) fn from_json_text(value: &str) -> Result<Value, Error> {
serde_json::from_str(value)
.map(from_json)
.map_err(|_| Error::InvalidEntry)
}
pub(crate) fn is_truthy(value: &Value) -> bool {
match value {
Value::None => false,
Value::Boolean(value) => *value,
Value::Integer(value) => value.to_string() != "0",
Value::Float(value) => *value != 0.0,
Value::Complex(value) => value.re != 0.0 || value.im != 0.0,
Value::String(value) => !value.is_empty(),
Value::Bytes(value) => !value.is_empty(),
Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(),
Value::Dict(value) => !value.is_empty(),
}
}
pub(crate) fn is_int(value: &Value) -> bool {
matches!(value, Value::Integer(_) | Value::Boolean(_))
}
pub(crate) fn to_f64(value: &Value) -> Option<f64> {
match value {
Value::Integer(value) => value.to_string().parse().ok(),
Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }),
_ => None,
}
}
pub(crate) fn to_json(value: &Value) -> Result<Vec<u8>, Error> {
serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry)
}
fn to_json_value(value: &Value) -> Result<serde_json::Value, Error> {
Ok(match value {
Value::None => serde_json::Value::Null,
Value::Boolean(value) => serde_json::Value::Bool(*value),
Value::Integer(value) => serde_json::Value::Number(
value
.to_string()
.parse::<Number>()
.map_err(|_| Error::InvalidEntry)?,
),
Value::Float(value) => {
serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?)
}
Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry),
Value::String(value) => serde_json::Value::String(value.clone()),
Value::Tuple(values) | Value::List(values) | Value::Set(values) => {
serde_json::Value::Array(
values
.iter()
.map(to_json_value)
.collect::<Result<Vec<_>, _>>()?,
)
}
Value::Dict(values) => {
let values = values
.iter()
.map(|(key, value)| {
let Value::String(key) = key else {
return Err(Error::InvalidEntry);
};
Ok((key.clone(), to_json_value(value)?))
})
.collect::<Result<Map<String, serde_json::Value>, _>>()?;
serde_json::Value::Object(values)
}
})
}

817
litellm-rust/crates/cache-disk/src/sqlite.rs Normal file
View file

@ -0,0 +1,817 @@
use std::{
collections::HashMap,
fs::{self, OpenOptions},
io::Write,
path::{Path, PathBuf},
sync::Mutex,
};
use litellm_cache::Error;
use rand::RngCore;
use rusqlite::{Connection, OptionalExtension, params, types::Value};
use crate::{DiskStore, StoredValue};
const MODE_RAW: i64 = 1;
const MODE_BINARY: i64 = 2;
const MODE_TEXT: i64 = 3;
const MODE_PICKLE: i64 = 4;
const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15);
const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30);
const DEFAULT_CULL_LIMIT: i64 = 10;
pub struct DiskcacheSqliteStore {
directory: PathBuf,
connection: Mutex<Connection>,
min_file_size: usize,
eviction_policy: String,
size_limit: i64,
cull_limit: i64,
statistics: bool,
}
struct StoredColumns {
size: i64,
mode: i64,
filename: Option<String>,
value: Option<Value>,
}
struct Row {
rowid: i64,
mode: i64,
filename: Option<String>,
value: Value,
}
impl DiskcacheSqliteStore {
pub fn open(directory: impl AsRef<Path>) -> Result<Self, Error> {
let directory = directory.as_ref().to_path_buf();
fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?;
let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?;
let database = directory.join("cache.db");
let connection = Connection::open(database).map_err(|_| Error::Unavailable)?;
connection
.busy_timeout(std::time::Duration::from_secs(60))
.map_err(|_| Error::Unavailable)?;
let mut settings = read_settings(&connection)?;
for (key, value) in default_settings() {
settings.entry(key).or_insert(value);
}
for (key, value) in settings
.iter()
.filter(|(key, _)| key.starts_with("sqlite_"))
{
apply_pragma(&connection, key, value)?;
}
connection
.execute_batch(
"CREATE TABLE IF NOT EXISTS Settings (
key TEXT NOT NULL UNIQUE,
value
)",
)
.map_err(|_| Error::Unavailable)?;
for (key, value) in &settings {
if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") {
connection
.execute(
"INSERT OR REPLACE INTO Settings VALUES (?, ?)",
params![key, value],
)
.map_err(|_| Error::Unavailable)?;
}
}
for (key, value) in [
("count", Value::Integer(0)),
("size", Value::Integer(0)),
("hits", Value::Integer(0)),
("misses", Value::Integer(0)),
] {
connection
.execute(
"INSERT OR IGNORE INTO Settings VALUES (?, ?)",
params![key, value],
)
.map_err(|_| Error::Unavailable)?;
}
connection
.execute_batch(
"CREATE TABLE IF NOT EXISTS Cache (
rowid INTEGER PRIMARY KEY,
key BLOB,
raw INTEGER,
store_time REAL,
expire_time REAL,
access_time REAL,
access_count INTEGER DEFAULT 0,
tag BLOB,
size INTEGER DEFAULT 0,
mode INTEGER DEFAULT 0,
filename TEXT,
value BLOB
);
CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw);
CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);",
)
.map_err(|_| Error::Unavailable)?;
let eviction_policy = setting_string(&settings, "eviction_policy")
.unwrap_or_else(|| "least-recently-stored".to_string());
match eviction_policy.as_str() {
"none" => {}
"least-recently-stored" => {
connection
.execute_batch(
"CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)",
)
.map_err(|_| Error::Unavailable)?;
}
"least-recently-used" => {
connection
.execute_batch(
"CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)",
)
.map_err(|_| Error::Unavailable)?;
}
"least-frequently-used" => {
connection
.execute_batch(
"CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)",
)
.map_err(|_| Error::Unavailable)?;
}
_ => return Err(Error::Unavailable),
}
connection
.execute_batch(
"CREATE TRIGGER IF NOT EXISTS Settings_count_insert
AFTER INSERT ON Cache FOR EACH ROW BEGIN
UPDATE Settings SET value = value + 1
WHERE key = \"count\"; END;
CREATE TRIGGER IF NOT EXISTS Settings_count_delete
AFTER DELETE ON Cache FOR EACH ROW BEGIN
UPDATE Settings SET value = value - 1
WHERE key = \"count\"; END;
CREATE TRIGGER IF NOT EXISTS Settings_size_insert
AFTER INSERT ON Cache FOR EACH ROW BEGIN
UPDATE Settings SET value = value + NEW.size
WHERE key = \"size\"; END;
CREATE TRIGGER IF NOT EXISTS Settings_size_update
AFTER UPDATE ON Cache FOR EACH ROW BEGIN
UPDATE Settings
SET value = value + NEW.size - OLD.size
WHERE key = \"size\"; END;
CREATE TRIGGER IF NOT EXISTS Settings_size_delete
AFTER DELETE ON Cache FOR EACH ROW BEGIN
UPDATE Settings SET value = value - OLD.size
WHERE key = \"size\"; END;",
)
.map_err(|_| Error::Unavailable)?;
let min_file_size = setting_i64(&settings, "disk_min_file_size")
.unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE)
.try_into()
.map_err(|_| Error::Unavailable)?;
let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT);
let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT);
let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0;
Ok(Self {
directory,
connection: Mutex::new(connection),
min_file_size,
eviction_policy,
size_limit,
cull_limit,
statistics,
})
}
fn set_locked(
&self,
connection: &Connection,
key: &str,
columns: StoredColumns,
expire_time: Option<f64>,
now: f64,
) -> Result<Vec<String>, Error> {
let mut cleanup = Vec::new();
if let Some(old_filename) = connection
.query_row(
"SELECT filename FROM Cache WHERE key = ? AND raw = 1",
params![key],
|row| row.get::<_, Option<String>>(0),
)
.optional()
.map_err(|_| Error::Unavailable)?
.flatten()
{
cleanup.push(old_filename);
}
let (size, mode, filename, value) =
(columns.size, columns.mode, columns.filename, columns.value);
let rowid = connection
.query_row(
"SELECT rowid FROM Cache WHERE key = ? AND raw = 1",
params![key],
|row| row.get::<_, i64>(0),
)
.optional()
.map_err(|_| Error::Unavailable)?;
if let Some(rowid) = rowid {
connection
.execute(
"UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?,
access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ?
WHERE rowid = ?",
params![now, expire_time, now, size, mode, filename, value, rowid],
)
.map_err(|_| Error::Unavailable)?;
} else {
connection
.execute(
"INSERT INTO Cache(
key, raw, store_time, expire_time, access_time, access_count,
tag, size, mode, filename, value
) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)",
params![key, now, expire_time, now, size, mode, filename, value],
)
.map_err(|_| Error::Unavailable)?;
}
cleanup.extend(self.cull(connection, now)?);
Ok(cleanup)
}
fn cull(&self, connection: &Connection, now: f64) -> Result<Vec<String>, Error> {
if self.cull_limit <= 0 {
return Ok(Vec::new());
}
let mut cleanup = Vec::new();
let expired = connection
.prepare(
"SELECT rowid, filename FROM Cache
WHERE expire_time IS NOT NULL AND expire_time < ?
ORDER BY expire_time LIMIT ?",
)
.map_err(|_| Error::Unavailable)?
.query_map(params![now, self.cull_limit], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
})
.map_err(|_| Error::Unavailable)?
.collect::<Result<Vec<_>, _>>()
.map_err(|_| Error::Unavailable)?;
for (_, filename) in &expired {
if let Some(filename) = filename {
cleanup.push(filename.clone());
}
}
for (rowid, _) in &expired {
connection
.execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
.map_err(|_| Error::Unavailable)?;
}
let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit);
if remaining <= 0 || self.volume(connection)? < self.size_limit {
return Ok(cleanup);
}
let order = match self.eviction_policy.as_str() {
"none" => return Ok(cleanup),
"least-recently-stored" => "store_time",
"least-recently-used" => "access_time",
"least-frequently-used" => "access_count",
_ => return Err(Error::Unavailable),
};
let rows = connection
.prepare(&format!(
"SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?"
))
.map_err(|_| Error::Unavailable)?
.query_map(params![remaining], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
})
.map_err(|_| Error::Unavailable)?
.collect::<Result<Vec<_>, _>>()
.map_err(|_| Error::Unavailable)?;
for (_, filename) in &rows {
if let Some(filename) = filename {
cleanup.push(filename.clone());
}
}
for (rowid, _) in rows {
connection
.execute("DELETE FROM Cache WHERE rowid = ?", params![rowid])
.map_err(|_| Error::Unavailable)?;
}
Ok(cleanup)
}
fn volume(&self, connection: &Connection) -> Result<i64, Error> {
let page_count: i64 = connection
.query_row("PRAGMA page_count", [], |row| row.get(0))
.map_err(|_| Error::Unavailable)?;
let page_size: i64 = connection
.query_row("PRAGMA page_size", [], |row| row.get(0))
.map_err(|_| Error::Unavailable)?;
let size: i64 = connection
.query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| {
row.get(0)
})
.map_err(|_| Error::Unavailable)?;
Ok(page_count.saturating_mul(page_size).saturating_add(size))
}
}
impl DiskStore for DiskcacheSqliteStore {
fn directory(&self) -> &Path {
&self.directory
}
fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache
WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)";
let row = connection
.query_row(select, params![key, now], row_from_query)
.optional()
.map_err(|_| Error::Unavailable)?;
if !self.statistics && !has_get_update(&self.eviction_policy) {
return row
.map(|row| fetch_row(&self.directory, row))
.transpose()
.map(|value| value.flatten());
}
transactional(&connection, |connection| {
let row = connection
.query_row(select, params![key, now], row_from_query)
.optional()
.map_err(|_| Error::Unavailable)?;
let Some(row) = row else {
if self.statistics {
connection
.execute(
"UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
[],
)
.map_err(|_| Error::Unavailable)?;
}
return Ok(None);
};
let rowid = row.rowid;
let value = fetch_row(&self.directory, row);
let hit = value.as_ref().is_ok_and(Option::is_some);
if hit && self.statistics {
connection
.execute(
"UPDATE Settings SET value = value + 1 WHERE key = 'hits'",
[],
)
.map_err(|_| Error::Unavailable)?;
} else if !hit && self.statistics {
connection
.execute(
"UPDATE Settings SET value = value + 1 WHERE key = 'misses'",
[],
)
.map_err(|_| Error::Unavailable)?;
}
if has_get_update(&self.eviction_policy) && hit {
let update = match self.eviction_policy.as_str() {
"least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?",
"least-frequently-used" => {
"UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?"
}
_ => return Err(Error::Unavailable),
};
if self.eviction_policy == "least-recently-used" {
connection
.execute(update, params![now, rowid])
.map_err(|_| Error::Unavailable)?;
} else {
connection
.execute(update, params![rowid])
.map_err(|_| Error::Unavailable)?;
}
}
value
})
}
fn set(
&self,
key: &str,
value: StoredValue,
expire_time: Option<f64>,
now: f64,
) -> Result<(), Error> {
let columns = store_value(&self.directory, self.min_file_size, value)?;
let new_filename = columns.filename.clone();
let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
let result = transactional(&connection, |connection| {
self.set_locked(connection, key, columns, expire_time, now)
});
match result {
Ok(cleanup) => {
cleanup_files(&self.directory, cleanup);
Ok(())
}
Err(error) => {
if let Some(filename) = new_filename {
remove_file(&self.directory, &filename);
}
Err(error)
}
}
}
fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error> {
let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
let selected = transactional(&connection, |connection| {
let row = connection
.query_row(
"SELECT rowid, expire_time, mode, filename, value FROM Cache
WHERE key = ? AND raw = 1
AND (expire_time IS NULL OR expire_time > ?)",
params![key, now],
row_from_query,
)
.optional()
.map_err(|_| Error::Unavailable)?;
let Some(row) = row else {
return Ok(None);
};
connection
.execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid])
.map_err(|_| Error::Unavailable)?;
Ok(Some(row))
})?;
let Some(row) = selected else {
return Ok(None);
};
let filename = row.filename.clone();
let result = fetch_row(&self.directory, row)?;
if let Some(filename) = filename {
remove_file(&self.directory, &filename);
}
Ok(result)
}
fn clear(&self) -> Result<(), Error> {
let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
let mut last_rowid = 0_i64;
loop {
let batch = transactional(&connection, |connection| {
let rows = connection
.prepare(
"SELECT rowid, filename FROM Cache
WHERE rowid > ? ORDER BY rowid LIMIT 100",
)
.map_err(|_| Error::Unavailable)?
.query_map(params![last_rowid], |row| {
Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?))
})
.map_err(|_| Error::Unavailable)?
.collect::<Result<Vec<_>, _>>()
.map_err(|_| Error::Unavailable)?;
if rows.is_empty() {
return Ok(rows);
}
let ids = rows
.iter()
.map(|(rowid, _)| rowid.to_string())
.collect::<Vec<_>>()
.join(",");
connection
.execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), [])
.map_err(|_| Error::Unavailable)?;
Ok(rows)
})?;
if batch.is_empty() {
return Ok(());
}
last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid);
cleanup_files(
&self.directory,
batch
.into_iter()
.filter_map(|(_, filename)| filename)
.collect(),
);
}
}
fn update(
&self,
key: &str,
now: f64,
apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
) -> Result<(), Error> {
let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
let mut created_filename = None;
let result = transactional(&connection, |connection| {
let current = connection
.query_row(
"SELECT rowid, expire_time, mode, filename, value FROM Cache
WHERE key = ? AND raw = 1
AND (expire_time IS NULL OR expire_time > ?)",
params![key, now],
row_from_query,
)
.optional()
.map_err(|_| Error::Unavailable)?
.map(|row| fetch_row(&self.directory, row))
.transpose()?
.flatten();
let (value, expire_time) = apply(current)?;
let columns = store_value(&self.directory, self.min_file_size, value)?;
created_filename = columns.filename.clone();
let cleanup = self.set_locked(connection, key, columns, expire_time, now)?;
Ok(cleanup)
});
match result {
Ok(cleanup) => {
cleanup_files(&self.directory, cleanup);
Ok(())
}
Err(error) => {
if let Some(filename) = created_filename {
remove_file(&self.directory, &filename);
}
Err(error)
}
}
}
fn probe(&self) -> Result<(), Error> {
let connection = self.connection.lock().map_err(|_| Error::Unavailable)?;
connection
.query_row(
"SELECT value FROM Settings WHERE key = 'count'",
[],
|row| row.get::<_, i64>(0),
)
.map(|_| ())
.map_err(|_| Error::Unavailable)
}
}
fn default_settings() -> HashMap<String, Value> {
HashMap::from([
("statistics".to_string(), Value::Integer(0)),
("tag_index".to_string(), Value::Integer(0)),
(
"eviction_policy".to_string(),
Value::Text("least-recently-stored".to_string()),
),
("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)),
("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)),
("sqlite_auto_vacuum".to_string(), Value::Integer(1)),
("sqlite_cache_size".to_string(), Value::Integer(8192)),
(
"sqlite_journal_mode".to_string(),
Value::Text("wal".to_string()),
),
(
"sqlite_mmap_size".to_string(),
Value::Integer(2_i64.pow(26)),
),
("sqlite_synchronous".to_string(), Value::Integer(1)),
(
"disk_min_file_size".to_string(),
Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE),
),
("disk_pickle_protocol".to_string(), Value::Integer(5)),
])
}
fn read_settings(connection: &Connection) -> Result<HashMap<String, Value>, Error> {
let mut statement = match connection.prepare("SELECT key, value FROM Settings") {
Ok(statement) => statement,
Err(_) => return Ok(HashMap::new()),
};
statement
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
.map_err(|_| Error::Unavailable)?
.collect::<Result<HashMap<_, _>, _>>()
.map_err(|_| Error::Unavailable)
}
fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> {
let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?;
match value {
Value::Integer(value) => connection
.pragma_update(None, pragma, value)
.map_err(|_| Error::Unavailable),
Value::Text(value) => connection
.pragma_update(None, pragma, value)
.map_err(|_| Error::Unavailable),
_ => Err(Error::Unavailable),
}
}
fn setting_i64(settings: &HashMap<String, Value>, key: &str) -> Option<i64> {
match settings.get(key) {
Some(Value::Integer(value)) => Some(*value),
_ => None,
}
}
fn setting_string(settings: &HashMap<String, Value>, key: &str) -> Option<String> {
match settings.get(key) {
Some(Value::Text(value)) => Some(value.clone()),
_ => None,
}
}
fn has_get_update(policy: &str) -> bool {
matches!(policy, "least-recently-used" | "least-frequently-used")
}
fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result<Row> {
Ok(Row {
rowid: row.get(0)?,
mode: row.get(2)?,
filename: row.get(3)?,
value: row.get(4)?,
})
}
fn fetch_row(directory: &Path, row: Row) -> Result<Option<StoredValue>, Error> {
match row.mode {
MODE_RAW => match row.value {
Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))),
Value::Text(value) => Ok(Some(StoredValue::Text(value))),
Value::Integer(value) => Ok(Some(StoredValue::Integer(value))),
Value::Real(value) => Ok(Some(StoredValue::Float(value))),
Value::Null => Err(Error::InvalidEntry),
},
MODE_BINARY | MODE_PICKLE => {
let bytes = match row.value {
Value::Blob(value) => value,
Value::Null => {
let Some(value) = read_file(directory, row.filename.as_deref())? else {
return Ok(None);
};
value
}
_ => return Err(Error::InvalidEntry),
};
Ok(Some(if row.mode == MODE_BINARY {
StoredValue::Bytes(bytes)
} else {
StoredValue::Pickle(bytes)
}))
}
MODE_TEXT => {
let bytes = match row.value {
Value::Null => {
let Some(value) = read_file(directory, row.filename.as_deref())? else {
return Ok(None);
};
value
}
Value::Blob(value) => value,
Value::Text(value) => value.into_bytes(),
_ => return Err(Error::InvalidEntry),
};
Ok(Some(StoredValue::Text(
String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?,
)))
}
_ => Err(Error::InvalidEntry),
}
}
fn read_file(directory: &Path, filename: Option<&str>) -> Result<Option<Vec<u8>>, Error> {
let Some(filename) = filename else {
return Err(Error::InvalidEntry);
};
match fs::read(directory.join(filename)) {
Ok(value) => Ok(Some(value)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(_) => Err(Error::Unavailable),
}
}
fn store_value(
directory: &Path,
min_file_size: usize,
value: StoredValue,
) -> Result<StoredColumns, Error> {
match value {
StoredValue::Integer(value) => Ok(StoredColumns {
size: 0,
mode: MODE_RAW,
filename: None,
value: Some(Value::Integer(value)),
}),
StoredValue::Float(value) => Ok(StoredColumns {
size: 0,
mode: MODE_RAW,
filename: None,
value: Some(Value::Real(value)),
}),
StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns {
size: 0,
mode: MODE_RAW,
filename: None,
value: Some(Value::Text(value)),
}),
StoredValue::Text(value) => {
let bytes = value.into_bytes();
let filename = write_file(directory, &bytes)?;
Ok(StoredColumns {
size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?,
mode: MODE_TEXT,
filename: Some(filename),
value: None,
})
}
StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns {
size: 0,
mode: MODE_RAW,
filename: None,
value: Some(Value::Blob(value)),
}),
StoredValue::Bytes(value) => {
let filename = write_file(directory, &value)?;
Ok(StoredColumns {
size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
mode: MODE_BINARY,
filename: Some(filename),
value: None,
})
}
StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns {
size: 0,
mode: MODE_PICKLE,
filename: None,
value: Some(Value::Blob(value)),
}),
StoredValue::Pickle(value) => {
let filename = write_file(directory, &value)?;
Ok(StoredColumns {
size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?,
mode: MODE_PICKLE,
filename: Some(filename),
value: None,
})
}
}
}
fn write_file(directory: &Path, bytes: &[u8]) -> Result<String, Error> {
let mut random = [0_u8; 16];
rand::rngs::OsRng.fill_bytes(&mut random);
let hex = random
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>();
let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]);
let path = directory.join(&filename);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?;
}
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|_| Error::Unavailable)?;
file.write_all(bytes).map_err(|_| Error::Unavailable)?;
Ok(filename)
}
fn cleanup_files(directory: &Path, filenames: Vec<String>) {
for filename in filenames {
remove_file(directory, &filename);
}
}
fn remove_file(directory: &Path, filename: &str) {
let path = directory.join(filename);
let _ = fs::remove_file(&path);
}
fn transactional<T>(
connection: &Connection,
operation: impl FnOnce(&Connection) -> Result<T, Error>,
) -> Result<T, Error> {
connection
.execute_batch("BEGIN IMMEDIATE")
.map_err(|_| Error::Unavailable)?;
match operation(connection) {
Ok(value) => {
connection
.execute_batch("COMMIT")
.map_err(|_| Error::Unavailable)?;
Ok(value)
}
Err(error) => {
let _ = connection.execute_batch("ROLLBACK");
Err(error)
}
}
}

33
litellm-rust/crates/cache-disk/src/store.rs Normal file
View file

@ -0,0 +1,33 @@
use std::path::Path;
use litellm_cache::Error;
#[derive(Clone, Debug, PartialEq)]
pub enum StoredValue {
Bytes(Vec<u8>),
Text(String),
Integer(i64),
Float(f64),
Pickle(Vec<u8>),
}
pub trait DiskStore: Send + Sync + 'static {
fn directory(&self) -> &Path;
fn get(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
fn set(
&self,
key: &str,
value: StoredValue,
expire_time: Option<f64>,
now: f64,
) -> Result<(), Error>;
fn pop(&self, key: &str, now: f64) -> Result<Option<StoredValue>, Error>;
fn clear(&self) -> Result<(), Error>;
fn update(
&self,
key: &str,
now: f64,
apply: &mut dyn FnMut(Option<StoredValue>) -> Result<(StoredValue, Option<f64>), Error>,
) -> Result<(), Error>;
fn probe(&self) -> Result<(), Error>;
}

431
litellm-rust/crates/cache-disk/tests/cache.rs Normal file
View file

@ -0,0 +1,431 @@
use std::{
fs,
path::{Path, PathBuf},
sync::Arc,
thread,
time::Duration,
};
use litellm_cache::{
BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext,
FlushCache, JsonCodec,
};
use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter};
use rstest::{fixture, rstest};
use rusqlite::Connection;
use serde_json::{Value, json};
use tempfile::TempDir;
struct Sandbox {
directory: TempDir,
}
#[fixture]
fn sandbox() -> Sandbox {
Sandbox {
directory: tempfile::tempdir().unwrap(),
}
}
impl Sandbox {
fn store(&self) -> DiskcacheSqliteStore {
DiskcacheSqliteStore::open(self.directory.path()).unwrap()
}
fn cache<V>(&self) -> DiskCache<JsonCodec<V>>
where
JsonCodec<V>: CacheCodec,
{
DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap()
}
fn db(&self) -> Connection {
Connection::open(self.directory.path().join("cache.db")).unwrap()
}
fn value_files(&self) -> Vec<PathBuf> {
fn visit(directory: &Path, files: &mut Vec<PathBuf>) {
for entry in fs::read_dir(directory).unwrap() {
let path = entry.unwrap().path();
if path.is_dir() {
visit(&path, files);
} else if path.extension().is_some_and(|extension| extension == "val") {
files.push(path);
}
}
}
let mut files = Vec::new();
visit(self.directory.path(), &mut files);
files
}
}
#[rstest]
fn relative_store_directory_is_absolutized(sandbox: Sandbox) {
let relative = PathBuf::from(format!(
".litellm-cache-disk-{}",
sandbox
.directory
.path()
.file_name()
.unwrap()
.to_string_lossy()
));
let store = DiskcacheSqliteStore::open(&relative).unwrap();
assert!(store.directory().is_absolute());
assert!(store.directory().ends_with(&relative));
let directory = store.directory().to_path_buf();
drop(store);
fs::remove_dir_all(directory).unwrap();
}
#[derive(Clone, Copy, Debug, Default)]
struct TextAdapter;
impl ValueAdapter for TextAdapter {
fn read(&self, value: StoredValue) -> Result<Option<Vec<u8>>, litellm_cache::Error> {
match value {
StoredValue::Text(value) => Ok(Some(value.into_bytes())),
_ => Ok(None),
}
}
fn write(&self, payload: Vec<u8>) -> StoredValue {
StoredValue::Text(String::from_utf8(payload).unwrap())
}
fn counter_seed(&self, _: Option<StoredValue>) -> Result<f64, litellm_cache::Error> {
Ok(0.0)
}
fn counter_value(&self, value: f64) -> StoredValue {
if value.fract() == 0.0 {
StoredValue::Integer(value as i64)
} else {
StoredValue::Float(value)
}
}
}
#[rstest]
fn roundtrip_persists_and_reopens(sandbox: Sandbox) {
let context = ExactCacheContext::default();
let opened = sandbox.cache::<Value>();
opened
.set_cache("key", json!({"answer": 42}), &context)
.unwrap();
assert_eq!(
opened.get_cache("key", &context).unwrap(),
Some(json!({"answer": 42}))
);
drop(opened);
let reopened = sandbox.cache::<Value>();
assert_eq!(
reopened.get_cache("key", &context).unwrap(),
Some(json!({"answer": 42}))
);
}
#[rstest]
fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) {
let store = sandbox.store();
store
.set(
"expired",
StoredValue::Bytes(b"old".to_vec()),
Some(10.0),
0.0,
)
.unwrap();
assert_eq!(store.get("expired", 10.0).unwrap(), None);
store
.set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0)
.unwrap();
assert_eq!(
sandbox
.db()
.query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0))
.unwrap(),
1
);
assert_eq!(
sandbox
.db()
.query_row(
"SELECT value FROM Settings WHERE key = 'count'",
[],
|row| row.get::<_, i64>(0)
)
.unwrap(),
1
);
}
#[rstest]
fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) {
let store = sandbox.store();
store
.set(
"hit",
StoredValue::Bytes(br#"{"ok":true}"#.to_vec()),
None,
0.0,
)
.unwrap();
store
.set(
"invalid",
StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
None,
0.0,
)
.unwrap();
let entries = sandbox
.cache::<Value>()
.batch_get_cache(
&["hit".into(), "missing".into(), "invalid".into()],
&ExactCacheContext::default(),
)
.unwrap();
assert_eq!(
entries,
vec![
BatchEntry::Hit(json!({"ok": true})),
BatchEntry::Miss,
BatchEntry::Invalid
]
);
}
#[rstest]
#[case(StoredValue::Bytes(Vec::new()))]
#[case(StoredValue::Text(String::new()))]
#[case(StoredValue::Integer(0))]
#[case(StoredValue::Float(0.0))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))]
#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))]
fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) {
sandbox.store().set("key", value, None, 0.0).unwrap();
assert_eq!(
sandbox
.cache::<Value>()
.get_cache("key", &ExactCacheContext::default())
.unwrap(),
None
);
}
#[rstest]
#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")]
#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")]
#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")]
#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")]
#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")]
#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")]
#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")]
#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")]
#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")]
#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")]
fn counters_follow_python_initialization(
sandbox: Sandbox,
#[case] initial: Option<StoredValue>,
#[case] amount: f64,
#[case] expected: f64,
#[case] sqlite_type: &str,
) {
if let Some(initial) = initial {
sandbox.store().set("counter", initial, None, 0.0).unwrap();
}
let cache = sandbox.cache::<f64>();
assert_eq!(
cache
.increment_cache("counter", amount, ExactCacheContext::default())
.unwrap(),
expected
);
assert_eq!(
sandbox
.db()
.query_row(
"SELECT typeof(value) FROM Cache WHERE key = 'counter'",
[],
|row| row.get::<_, String>(0)
)
.unwrap(),
sqlite_type
);
}
#[rstest]
fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) {
let cache = Arc::new(sandbox.cache::<f64>());
let workers = (0..8)
.map(|_| {
let cache = Arc::clone(&cache);
thread::spawn(move || {
for _ in 0..25 {
cache
.increment_cache("counter", 1.0, ExactCacheContext::default())
.unwrap();
}
})
})
.collect::<Vec<_>>();
for worker in workers {
worker.join().unwrap();
}
assert_eq!(
cache
.increment_cache("counter", 0.0, ExactCacheContext::default())
.unwrap(),
200.0
);
}
#[rstest]
fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) {
let cache = sandbox.cache::<f64>();
assert_eq!(
cache
.increment_cache("counter", 3.5, ExactCacheContext::default())
.unwrap(),
3.5
);
assert_eq!(
cache
.increment_cache("counter", 1.0, ExactCacheContext::default())
.unwrap(),
1.0
);
}
#[rstest]
fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) {
let cache = sandbox.cache::<f64>();
cache
.increment_cache(
"counter",
1.0,
ExactCacheContext {
ttl: Some(Duration::from_secs(60)),
},
)
.unwrap();
assert!(
sandbox
.db()
.query_row(
"SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
[],
|row| row.get::<_, bool>(0)
)
.unwrap()
);
cache
.increment_cache("counter", 1.0, ExactCacheContext::default())
.unwrap();
assert!(
!sandbox
.db()
.query_row(
"SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'",
[],
|row| row.get::<_, bool>(0)
)
.unwrap()
);
}
#[rstest]
fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) {
let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::<Value>::new());
cache
.set_cache("key", json!({"answer": 42}), &ExactCacheContext::default())
.unwrap();
assert!(matches!(
sandbox.store().get("key", 0.0).unwrap(),
Some(StoredValue::Text(_))
));
assert_eq!(
cache
.get_cache("key", &ExactCacheContext::default())
.unwrap(),
Some(json!({"answer": 42}))
);
}
#[rstest]
fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) {
let large = vec![b'x'; 32 * 1024];
sandbox
.store()
.set("large", StoredValue::Bytes(large.clone()), None, 0.0)
.unwrap();
assert_eq!(sandbox.value_files().len(), 1);
sandbox
.store()
.set(
"large",
StoredValue::Bytes(vec![b'y'; 32 * 1024]),
None,
0.0,
)
.unwrap();
assert_eq!(sandbox.value_files().len(), 1);
sandbox.store().pop("large", 0.0).unwrap();
assert!(sandbox.value_files().is_empty());
sandbox
.store()
.set("a", StoredValue::Bytes(large.clone()), None, 0.0)
.unwrap();
sandbox
.store()
.set("b", StoredValue::Bytes(large), None, 0.0)
.unwrap();
sandbox.store().clear().unwrap();
assert!(sandbox.value_files().is_empty());
}
#[rstest]
#[tokio::test]
async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) {
let cache = sandbox.cache::<Value>();
let context = ExactCacheContext {
ttl: Some(Duration::from_secs(60)),
};
cache
.async_set_cache("a", json!(1), context.clone())
.await
.unwrap();
cache
.async_set_cache_pipeline(
vec![("b".into(), json!(2)), ("c".into(), json!(3))],
context.clone(),
)
.await
.unwrap();
assert_eq!(
cache.async_get_cache("a", &context).await.unwrap(),
Some(json!(1))
);
assert_eq!(
cache
.async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone())
.await
.unwrap(),
vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss]
);
cache.async_delete_cache("a").await.unwrap();
cache.async_flush_cache().await.unwrap();
assert_eq!(
cache.test_connection().await.unwrap().status,
litellm_cache::CacheConnectionStatus::Success
);
}

113
litellm-rust/crates/cache-disk/tests/python_compat.rs Normal file
View file

@ -0,0 +1,113 @@
use litellm_cache::Error;
use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter};
use rstest::rstest;
enum ReadExpectation {
Bytes(&'static [u8]),
Miss,
Invalid,
}
#[rstest]
#[case::pickled_dictionary_with_string_keys(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]),
ReadExpectation::Bytes(br#"{"a":1}"#)
)]
#[case::pickled_list_of_integers(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]),
ReadExpectation::Bytes(br#"[1,2]"#)
)]
#[case::pickled_tuple_of_integers(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]),
ReadExpectation::Bytes(br#"[1,2]"#)
)]
#[case::pickled_set_of_integers(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]),
ReadExpectation::Bytes(br#"[1,2]"#)
)]
#[case::pickled_response_envelope(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]),
ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#)
)]
#[case::non_json_text(
StoredValue::Text("not json".into()),
ReadExpectation::Bytes(b"not json")
)]
#[case::json_text(
StoredValue::Text("{\"a\": 1}".into()),
ReadExpectation::Bytes(br#"{"a": 1}"#)
)]
#[case::non_utf8_bytes(
StoredValue::Bytes(vec![0xff, 0xfe]),
ReadExpectation::Bytes(&[0xff, 0xfe])
)]
#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))]
#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))]
#[case::pickled_true(
StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]),
ReadExpectation::Bytes(b"true")
)]
#[case::pickled_negative_integer(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]),
ReadExpectation::Bytes(b"-3")
)]
#[case::pickled_bytes(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]),
ReadExpectation::Invalid
)]
#[case::pickled_dictionary_with_integer_key(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]),
ReadExpectation::Invalid
)]
#[case::pickled_complex(
StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]),
ReadExpectation::Invalid
)]
#[case::truncated_pickle(
StoredValue::Pickle(vec![0x80, 0x05, 0x2e]),
ReadExpectation::Invalid
)]
#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)]
#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)]
#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)]
#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)]
#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)]
#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)]
fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) {
let result = PythonDiskCacheAdapter.read(row);
match expected {
ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected),
ReadExpectation::Miss => assert_eq!(result.unwrap(), None),
ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))),
}
}
#[rstest]
#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)]
#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)]
#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)]
#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)]
#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)]
#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)]
#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)]
#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)]
#[case::missing(None, 0.0)]
#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)]
fn python_counter_seed_cases(#[case] row: Option<StoredValue>, #[case] expected: f64) {
assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected);
}
#[rstest]
#[case::integer_three(3.0, StoredValue::Integer(3))]
#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))]
#[case::negative_zero(-0.0, StoredValue::Integer(0))]
#[case::large_float(1e300, StoredValue::Float(1e300))]
fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) {
assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected);
}

20
litellm-rust/crates/cache-gcs/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
[package]
name = "litellm-cache-gcs"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
futures-util.workspace = true
litellm-auth-gcp.workspace = true
litellm-auth-types.workspace = true
litellm-cache.workspace = true
percent-encoding.workspace = true
reqwest.workspace = true
tokio.workspace = true
[dev-dependencies]
serde_json.workspace = true
tokio.workspace = true
wiremock = "0.6.5"

260
litellm-rust/crates/cache-gcs/src/cache.rs Normal file
View file

@ -0,0 +1,260 @@
use std::{future::Future, sync::Arc, time::Duration};
use futures_util::future::try_join_all;
use litellm_cache::{
BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext,
FlushCache,
};
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
use reqwest::Client;
use crate::{GcpTokenSource, TokenSource};
pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
pub fn key_prefix(gcs_path: Option<&str>) -> String {
match gcs_path {
Some(path) if !path.is_empty() => format!("{}/", path.trim_end_matches('/')),
_ => String::new(),
}
}
#[derive(Clone, Debug)]
pub struct GcsConfig {
pub bucket_name: String,
pub gcs_path: Option<String>,
pub path_service_account: Option<String>,
pub endpoint: String,
}
impl GcsConfig {
pub fn new(bucket_name: impl Into<String>) -> Self {
Self {
bucket_name: bucket_name.into(),
gcs_path: None,
path_service_account: None,
endpoint: DEFAULT_ENDPOINT.to_string(),
}
}
}
pub struct GcsCache<S: CacheCodec> {
config: GcsConfig,
key_prefix: String,
client: Client,
token: Arc<dyn TokenSource>,
codec: S,
}
impl<S: CacheCodec> GcsCache<S> {
pub fn new(config: GcsConfig, codec: S) -> Result<Self, Error> {
let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
Self::with_token_source(config, codec, token)
}
pub fn with_token_source(
config: GcsConfig,
codec: S,
token: Arc<dyn TokenSource>,
) -> Result<Self, Error> {
let client = Client::builder().build().map_err(|_| Error::Unavailable)?;
let key_prefix = key_prefix(config.gcs_path.as_deref());
Ok(Self {
config,
key_prefix,
client,
token,
codec,
})
}
pub fn bucket_name(&self) -> &str {
&self.config.bucket_name
}
pub fn key_prefix(&self) -> &str {
&self.key_prefix
}
pub fn path_service_account(&self) -> Option<&str> {
self.config.path_service_account.as_deref()
}
pub fn object_name(&self, key: &str) -> String {
format!("{}{}", self.key_prefix, key)
}
fn encoded_object_name(&self, key: &str) -> String {
percent_encode(self.object_name(key).as_bytes(), OBJECT_NAME_ENCODE_SET).to_string()
}
fn endpoint(&self, path: &str) -> String {
format!("{}{}", self.config.endpoint.trim_end_matches('/'), path)
}
async fn async_set(&self, key: &str, value: S::Value) -> Result<(), Error> {
let token = self.token.bearer_token().await?;
let payload = self.codec.encode(&value)?;
let url = self.endpoint(&format!(
"/upload/storage/v1/b/{}/o?uploadType=media&name={}",
self.config.bucket_name,
self.encoded_object_name(key)
));
let response = self
.client
.post(url)
.bearer_auth(token)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(payload)
.send()
.await
.map_err(|_| Error::Unavailable)?;
if !response.status().is_success() {
return Err(Error::Unavailable);
}
Ok(())
}
async fn async_get(&self, key: &str) -> Result<Option<S::Value>, Error> {
let token = self.token.bearer_token().await?;
let url = self.endpoint(&format!(
"/storage/v1/b/{}/o/{}?alt=media",
self.config.bucket_name,
self.encoded_object_name(key)
));
let response = self
.client
.get(url)
.bearer_auth(token)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.send()
.await
.map_err(|_| Error::Unavailable)?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
if !response.status().is_success() {
return Err(Error::Unavailable);
}
let body = response.bytes().await.map_err(|_| Error::Unavailable)?;
self.codec
.decode(&body)
.map(Some)
.map_err(|_| Error::InvalidEntry)
}
fn run_sync<T, F>(future: F) -> Result<T, Error>
where
F: Future<Output = Result<T, Error>> + Send,
T: Send,
{
let run = || {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|_| Error::Unavailable)
.and_then(|runtime| runtime.block_on(future))
};
if let Ok(handle) = tokio::runtime::Handle::try_current() {
if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread {
return tokio::task::block_in_place(run);
}
return std::thread::scope(|scope| {
scope
.spawn(run)
.join()
.map_err(|_| Error::Unavailable)
.and_then(|result| result)
});
}
run()
}
}
impl<S: CacheCodec> BaseCache for GcsCache<S> {
type Value = S::Value;
type Context = ExactCacheContext;
fn get_ttl(&self, _: &Self::Context) -> Option<Duration> {
None
}
fn set_cache(&self, key: &str, value: Self::Value, _: &Self::Context) -> Result<(), Error> {
Self::run_sync(self.async_set(key, value))
}
fn get_cache(&self, key: &str, _: &Self::Context) -> Result<Option<Self::Value>, Error> {
Self::run_sync(self.async_get(key))
}
async fn async_set_cache(
&self,
key: &str,
value: Self::Value,
_: Self::Context,
) -> Result<(), Error> {
self.async_set(key, value).await
}
async fn async_get_cache(
&self,
key: &str,
_: &Self::Context,
) -> Result<Option<Self::Value>, Error> {
self.async_get(key).await
}
async fn async_set_cache_pipeline(
&self,
entries: Vec<(String, Self::Value)>,
context: Self::Context,
) -> Result<(), Error> {
try_join_all(entries.into_iter().map(|(key, value)| {
let context = context.clone();
async move { self.async_set_cache(&key, value, context).await }
}))
.await
.map(|_| ())
}
async fn disconnect(&self) -> Result<(), Error> {
Ok(())
}
async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
Err(Error::UnsupportedOperation)
}
}
impl<S: CacheCodec> BatchCache for GcsCache<S> {
async fn async_batch_get_cache(
&self,
keys: Vec<String>,
context: Self::Context,
) -> Result<Vec<BatchEntry<Self::Value>>, Error> {
try_join_all(keys.into_iter().map(|key| {
let context = context.clone();
async move {
match self.async_get_cache(&key, &context).await {
Ok(Some(value)) => Ok(BatchEntry::Hit(value)),
Ok(None) => Ok(BatchEntry::Miss),
Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid),
Err(error) => Err(error),
}
}
}))
.await
}
}
impl<S: CacheCodec> FlushCache for GcsCache<S> {
fn flush_cache(&self) -> Result<(), Error> {
Ok(())
}
}

5
litellm-rust/crates/cache-gcs/src/lib.rs Normal file
View file

@ -0,0 +1,5 @@
mod cache;
mod token;
pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix};
pub use token::{GcpTokenSource, StaticTokenSource, TokenSource};

44
litellm-rust/crates/cache-gcs/src/token.rs Normal file
View file

@ -0,0 +1,44 @@
use std::{future::Future, pin::Pin};
use litellm_auth_gcp::{VertexAuth, VertexConfig};
use litellm_auth_types::{InputSource, SecretValue, Sourced};
use litellm_cache::Error;
pub trait TokenSource: Send + Sync + 'static {
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
}
pub struct GcpTokenSource {
auth: VertexAuth,
config: VertexConfig,
}
impl GcpTokenSource {
pub fn new(path_service_account: Option<String>) -> Self {
let credentials = path_service_account
.map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
Self {
auth: VertexAuth::default(),
config: VertexConfig::new(credentials, None, None),
}
}
}
impl TokenSource for GcpTokenSource {
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
Box::pin(async move {
self.auth
.access_token(&self.config, &|name| std::env::var(name).ok())
.await
.map_err(|_| Error::Unavailable)
})
}
}
pub struct StaticTokenSource(pub String);
impl TokenSource for StaticTokenSource {
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
Box::pin(async move { Ok(self.0.clone()) })
}
}

324
litellm-rust/crates/cache-gcs/tests/cache.rs Normal file
View file

@ -0,0 +1,324 @@
use std::{sync::Arc, time::Duration};
use litellm_cache::{
BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache,
JsonCodec,
};
use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix};
use serde_json::json;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{body_bytes, header, method, path, query_param},
};
fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig {
GcsConfig {
bucket_name: "bucket".into(),
gcs_path: gcs_path.map(str::to_string),
path_service_account: None,
endpoint: server.uri(),
}
}
fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache<JsonCodec<serde_json::Value>> {
GcsCache::with_token_source(
config(server, gcs_path),
JsonCodec::new(),
Arc::new(StaticTokenSource("tok".into())),
)
.unwrap()
}
#[tokio::test]
async fn set_writes_encoded_object_and_headers() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/upload/storage/v1/b/bucket/o"))
.and(query_param("uploadType", "media"))
.and(header("authorization", "Bearer tok"))
.and(header("content-type", "application/json"))
.and(body_bytes(br#"{"value":"entry"}"#))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&server)
.await;
cache(&server, Some("cache/"))
.set_cache(
"team:a b/c",
json!({"value": "entry"}),
&ExactCacheContext::default(),
)
.unwrap();
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert_eq!(
requests[0].url.query(),
Some("uploadType=media&name=cache%2Fteam%3Aa%20b%2Fc")
);
}
#[tokio::test]
async fn get_maps_statuses_and_decode_failures() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/hit"))
.and(query_param("alt", "media"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/missing"))
.respond_with(ResponseTemplate::new(404))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/server-error"))
.respond_with(ResponseTemplate::new(500))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/invalid"))
.respond_with(ResponseTemplate::new(200).set_body_string("not json"))
.mount(&server)
.await;
let cache = cache(&server, None);
assert_eq!(
cache
.get_cache("hit", &ExactCacheContext::default())
.unwrap(),
Some(json!({"value": "entry"}))
);
assert_eq!(
cache
.get_cache("missing", &ExactCacheContext::default())
.unwrap(),
None
);
assert_eq!(
cache
.get_cache("server-error", &ExactCacheContext::default())
.unwrap_err(),
Error::Unavailable
);
assert_eq!(
cache
.get_cache("invalid", &ExactCacheContext::default())
.unwrap_err(),
Error::InvalidEntry
);
}
#[test]
fn key_prefix_normalizes_paths() {
assert_eq!(key_prefix(None), "");
assert_eq!(key_prefix(Some("a/b/")), "a/b/");
assert_eq!(key_prefix(Some("a/b")), "a/b/");
assert_eq!(key_prefix(Some("")), "");
}
#[tokio::test]
async fn object_names_use_python_quote_encoding() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/upload/storage/v1/b/bucket/o"))
.and(query_param("uploadType", "media"))
.respond_with(ResponseTemplate::new(200))
.expect(2)
.mount(&server)
.await;
let cache = cache(&server, Some("p/"));
cache
.set_cache(
"a~b-c_d.e/f g%h",
json!({"value": "punctuation"}),
&ExactCacheContext::default(),
)
.unwrap();
cache
.set_cache(
"ключ",
json!({"value": "utf8"}),
&ExactCacheContext::default(),
)
.unwrap();
let requests = server.received_requests().await.unwrap();
let queries: Vec<_> = requests
.iter()
.filter_map(|request| request.url.query())
.collect();
assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h"));
assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87"));
}
#[tokio::test]
async fn ignores_ttl_and_writes_pipeline_concurrently() {
let server = MockServer::start().await;
for key in ["one", "two", "three"] {
Mock::given(method("POST"))
.and(path("/upload/storage/v1/b/bucket/o"))
.and(query_param("uploadType", "media"))
.and(query_param("name", key))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&server)
.await;
}
let cache = cache(&server, None);
assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None);
assert_eq!(
cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))),
None
);
cache
.async_set_cache_pipeline(
vec![
("one".into(), json!({"key": "one"})),
("two".into(), json!({"key": "two"})),
("three".into(), json!({"key": "three"})),
],
ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))),
)
.await
.unwrap();
}
#[tokio::test]
async fn async_batch_get_preserves_hits_misses_and_invalid_entries() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/hit"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/missing"))
.respond_with(ResponseTemplate::new(404))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/invalid"))
.respond_with(ResponseTemplate::new(200).set_body_string("not json"))
.mount(&server)
.await;
assert_eq!(
cache(&server, None)
.async_batch_get_cache(
vec!["hit".into(), "missing".into(), "invalid".into()],
ExactCacheContext::default(),
)
.await
.unwrap(),
vec![
BatchEntry::Hit(json!({"value": "entry"})),
BatchEntry::Miss,
BatchEntry::Invalid,
]
);
}
#[tokio::test]
async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() {
let server = MockServer::start().await;
let cache = cache(&server, None);
assert_eq!(cache.flush_cache(), Ok(()));
assert_eq!(cache.disconnect().await, Ok(()));
assert_eq!(
cache.test_connection().await,
Err(Error::UnsupportedOperation)
);
}
#[test]
fn sync_operations_work_without_an_active_runtime() {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let server = runtime.block_on(MockServer::start());
runtime.block_on(
Mock::given(method("POST"))
.and(path("/upload/storage/v1/b/bucket/o"))
.respond_with(ResponseTemplate::new(200))
.mount(&server),
);
runtime.block_on(
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/key"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
.mount(&server),
);
let cache = cache(&server, None);
cache
.set_cache(
"key",
json!({"value": "entry"}),
&ExactCacheContext::default(),
)
.unwrap();
assert_eq!(
cache
.get_cache("key", &ExactCacheContext::default())
.unwrap(),
Some(json!({"value": "entry"}))
);
}
#[tokio::test(flavor = "multi_thread")]
async fn sync_operations_work_inside_a_multi_thread_runtime() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/upload/storage/v1/b/bucket/o"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/storage/v1/b/bucket/o/key"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})))
.mount(&server)
.await;
let cache = cache(&server, None);
cache
.set_cache(
"key",
json!({"value": "entry"}),
&ExactCacheContext::default(),
)
.unwrap();
assert_eq!(
cache
.get_cache("key", &ExactCacheContext::default())
.unwrap(),
Some(json!({"value": "entry"}))
);
}
struct FailingTokenSource;
impl TokenSource for FailingTokenSource {
fn bearer_token(
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<String, Error>> + Send + '_>>
{
Box::pin(async { Err(Error::Unavailable) })
}
}
#[tokio::test]
async fn token_source_failure_skips_http() {
let server = MockServer::start().await;
let cache = GcsCache::with_token_source(
config(&server, None),
JsonCodec::<serde_json::Value>::new(),
Arc::new(FailingTokenSource),
)
.unwrap();
assert_eq!(
cache
.get_cache("key", &ExactCacheContext::default())
.unwrap_err(),
Error::Unavailable
);
assert_eq!(server.received_requests().await.unwrap().len(), 0);
}

2
litellm-rust/crates/cache/src/error.rs vendored
View file

@ -6,4 +6,6 @@ pub enum Error {
InvalidEntry,
#[error("flushing Redis requires an explicit namespace")]
UnscopedFlush,
#[error("operation is not supported by this cache")]
UnsupportedOperation,
}

2
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -24,6 +24,8 @@ litellm-cache.workspace = true
litellm-cache-azure-blob.workspace = true
litellm-cache-memory.workspace = true
litellm-cache-redis.workspace = true
litellm-cache-gcs.workspace = true
litellm-cache-disk.workspace = true
litellm-cache-response.workspace = true
serde.workspace = true
litellm-auth.workspace = true

257
litellm-rust/crates/python-bridge/src/cache/config.rs vendored
View file

@ -1,4 +1,4 @@
use std::time::Duration;
use std::{path::PathBuf, time::Duration};
use litellm_cache::CacheType;
use litellm_cache_redis::{RedisNode, RedisTopology};
@ -26,6 +26,10 @@ pub(super) struct MemoryCacheConfig {
pub(super) max_entry_bytes: usize,
}
pub(super) struct DiskCacheConfig {
pub(super) directory: PathBuf,
}
#[derive(Debug, PartialEq)]
pub(super) enum RedisProtocol {
Resp2,
@ -75,6 +79,13 @@ pub(super) struct RedisCacheConfig {
pub(super) connection: RedisConnectionConfig,
}
#[derive(Debug, PartialEq)]
pub(super) struct GcsCacheConfig {
pub(super) bucket_name: String,
pub(super) key_prefix: String,
pub(super) path_service_account: Option<String>,
}
pub(super) struct AzureBlobCacheConfig {
pub(super) account_url: String,
pub(super) container: String,
@ -94,6 +105,8 @@ const REDIS_PY_DEFAULT_MAX_CONNECTIONS: usize = 1 << 31;
pub(super) enum CacheBackendConfig {
Memory(MemoryCacheConfig),
Redis(Box<RedisCacheConfig>),
Gcs(GcsCacheConfig),
Disk(DiskCacheConfig),
AzureBlob(AzureBlobCacheConfig),
}
@ -109,6 +122,8 @@ pub(super) enum UnsupportedCacheConfig {
RedisCredentials,
RedisConnection,
RedisOption,
GcsBucket,
DiskStore,
}
impl UnsupportedCacheConfig {
@ -119,6 +134,8 @@ impl UnsupportedCacheConfig {
Self::RedisCredentials => "native Redis credentials require Python",
Self::RedisConnection => "native Redis connection type is not implemented",
Self::RedisOption => "native Redis configuration requires Python",
Self::GcsBucket => "native GCS cache requires a configured bucket name",
Self::DiskStore => "native disk cache requires the built-in diskcache store",
}
}
}
@ -161,6 +178,20 @@ impl NativeCacheConfig {
}))),
Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
},
Some(CacheType::Gcs) => match project_gcs(&backend)? {
Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
policy,
backend: CacheBackendConfig::Gcs(backend),
}))),
Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
},
Some(CacheType::Disk) => match project_disk(&backend)? {
Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self {
policy,
backend: CacheBackendConfig::Disk(backend),
}))),
Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)),
},
Some(CacheType::AzureBlob) => project_azure_blob(&backend).map(|backend| {
CacheConfigProjection::Native(Box::new(Self {
policy,
@ -171,9 +202,7 @@ impl NativeCacheConfig {
CacheType::RedisSemantic
| CacheType::ValkeySemantic
| CacheType::S3
| CacheType::Disk
| CacheType::QdrantSemantic
| CacheType::Gcs,
| CacheType::QdrantSemantic,
)
| None => Ok(CacheConfigProjection::Unsupported(
UnsupportedCacheConfig::Backend,
@ -185,7 +214,9 @@ impl NativeCacheConfig {
let default_ttl = match &self.backend {
CacheBackendConfig::Memory(config) => Some(config.default_ttl),
CacheBackendConfig::Redis(config) => Some(config.default_ttl),
CacheBackendConfig::AzureBlob(_) => None,
CacheBackendConfig::Disk(_)
| CacheBackendConfig::AzureBlob(_)
| CacheBackendConfig::Gcs(_) => None,
};
if service.default_ttl() != default_ttl {
return Some("facade and native backend default TTLs must match");
@ -212,6 +243,42 @@ impl NativeCacheConfig {
CacheBackendConfig::Redis(config) => (service.namespace()
!= config.namespace.as_deref())
.then_some("facade and native backend namespaces must match"),
CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::Gcs(config)
if service
.gcs_backend()
.is_none_or(|backend| backend.bucket_name() != config.bucket_name) =>
{
Some("facade and native backend buckets must match")
}
CacheBackendConfig::Gcs(config)
if service
.gcs_backend()
.is_none_or(|backend| backend.key_prefix() != config.key_prefix) =>
{
Some("facade and native backend key prefixes must match")
}
CacheBackendConfig::Gcs(config)
if service.gcs_backend().is_none_or(|backend| {
backend.path_service_account() != config.path_service_account.as_deref()
}) =>
{
Some("facade and native backend credentials must match")
}
CacheBackendConfig::Gcs(_) => None,
CacheBackendConfig::Disk(_) if service.kind() != "disk" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::Disk(config) => {
let Some(directory) = service.directory() else {
return Some("facade and native backend types must match");
};
let native = std::fs::canonicalize(directory).ok();
let facade = std::fs::canonicalize(&config.directory).ok();
(native != facade).then_some("facade and native backend directories must match")
}
CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() {
None => Some("facade and native backend types must match"),
Some((account_url, container))
@ -252,6 +319,38 @@ fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult<MemoryCacheConfig> {
})
}
#[inline(never)]
fn project_gcs(
backend: &Bound<'_, PyAny>,
) -> PyResult<Result<GcsCacheConfig, UnsupportedCacheConfig>> {
let bucket_name = match backend.getattr("bucket_name")?.extract::<Option<String>>() {
Ok(Some(bucket_name)) if !bucket_name.is_empty() => bucket_name,
_ => return Ok(Err(UnsupportedCacheConfig::GcsBucket)),
};
Ok(Ok(GcsCacheConfig {
bucket_name,
key_prefix: backend.getattr("key_prefix")?.extract::<String>()?,
path_service_account: backend
.getattr("path_service_account")?
.extract::<Option<String>>()?,
}))
}
#[inline(never)]
fn project_disk(
backend: &Bound<'_, PyAny>,
) -> PyResult<Result<DiskCacheConfig, UnsupportedCacheConfig>> {
let store = backend.getattr("disk_cache")?;
if !instance_class_is(&store, "diskcache.core", "Cache")?
|| !instance_class_is(&store.getattr("_disk")?, "diskcache.core", "Disk")?
{
return Ok(Err(UnsupportedCacheConfig::DiskStore));
}
Ok(Ok(DiskCacheConfig {
directory: PathBuf::from(store.getattr("directory")?.extract::<String>()?),
}))
}
#[inline(never)]
fn project_redis(
backend: &Bound<'_, PyAny>,
@ -639,8 +738,8 @@ mod tests {
use litellm_cache_redis::{RedisNode, RedisTopology};
use super::{
CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig,
RedisProtocol,
CacheBackendConfig, CacheConfigProjection, CachePolicy, CertificateRequirement,
DiskCacheConfig, GcsCacheConfig, NativeCacheConfig, RedisProtocol, UnsupportedCacheConfig,
};
use crate::cache::native::NativeResponseCache;
@ -715,6 +814,71 @@ mod tests {
});
}
#[test]
fn projects_gcs_configuration() {
Python::initialize();
Python::attach(|py| {
let facade = facade(
py,
"backend = SimpleNamespace(bucket_name='bucket', key_prefix='cache/', path_service_account='credentials.json')\n\
facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
);
let CacheConfigProjection::Native(config) =
NativeCacheConfig::project(&facade).unwrap()
else {
panic!("GCS cache should be supported");
};
let CacheBackendConfig::Gcs(gcs) = config.backend else {
panic!("expected GCS configuration");
};
assert_eq!(
gcs,
GcsCacheConfig {
bucket_name: "bucket".into(),
key_prefix: "cache/".into(),
path_service_account: Some("credentials.json".into()),
}
);
let matching = NativeResponseCache::gcs(
litellm_cache_gcs::GcsConfig {
bucket_name: "bucket".into(),
gcs_path: Some("cache/".into()),
path_service_account: Some("credentials.json".into()),
endpoint: litellm_cache_gcs::DEFAULT_ENDPOINT.into(),
},
Some("token".into()),
)
.unwrap();
let matching_config = NativeCacheConfig {
policy: config.policy,
backend: CacheBackendConfig::Gcs(gcs),
};
assert_eq!(matching_config.service_mismatch(&matching), None);
});
}
#[test]
fn rejects_gcs_without_a_bucket_name() {
Python::initialize();
Python::attach(|py| {
let facade = facade(
py,
"backend = SimpleNamespace(bucket_name=None, key_prefix='', path_service_account=None)\n\
facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)",
);
let CacheConfigProjection::Unsupported(reason) =
NativeCacheConfig::project(&facade).unwrap()
else {
panic!("GCS cache without a bucket should be unsupported");
};
assert!(matches!(&reason, UnsupportedCacheConfig::GcsBucket));
assert_eq!(
reason.message(),
"native GCS cache requires a configured bucket name"
);
});
}
#[test]
fn projects_resolved_redis_tls_configuration() {
Python::initialize();
@ -775,6 +939,85 @@ mod tests {
assert_eq!(reason.message(), "native Redis credentials require Python");
});
}
#[test]
fn projects_builtin_disk_configuration_and_rejects_custom_stores() {
Python::initialize();
Python::attach(|py| {
let root =
std::env::temp_dir().join(format!("litellm-disk-config-{}", std::process::id()));
let directory = root.to_string_lossy();
let disk_facade = facade(
py,
&format!(
"Cache = type('Cache', (), {{'__module__': 'diskcache.core'}})\n\
Disk = type('Disk', (), {{'__module__': 'diskcache.core'}})\n\
store = Cache()\n\
store._disk = Disk()\n\
store.directory = {directory:?}\n\
backend = SimpleNamespace(disk_cache=store)\n\
facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)"
),
);
let CacheConfigProjection::Native(config) =
NativeCacheConfig::project(&disk_facade).unwrap()
else {
panic!("disk cache should be supported");
};
let CacheBackendConfig::Disk(disk) = config.backend else {
panic!("expected disk configuration");
};
assert_eq!(disk.directory, root);
let matching = NativeResponseCache::disk(&directory).unwrap();
assert_eq!(
(NativeCacheConfig {
policy: config.policy,
backend: CacheBackendConfig::Disk(disk),
})
.service_mismatch(&matching),
None
);
let other = NativeResponseCache::disk(&root.join("other").to_string_lossy()).unwrap();
let mismatch = NativeCacheConfig {
policy: CachePolicy {
mode: "default-on".into(),
ttl: None,
namespace: None,
supported_call_types: None,
redis_flush_size: None,
semantic_cache_scope: "key".into(),
},
backend: CacheBackendConfig::Disk(DiskCacheConfig {
directory: root.clone(),
}),
};
assert_eq!(
mismatch.service_mismatch(&other),
Some("facade and native backend directories must match")
);
let custom = facade(
py,
&format!(
"CustomCache = type('CustomCache', (), {{'__module__': 'mypkg'}})\n\
CustomDisk = type('CustomDisk', (), {{'__module__': 'mypkg'}})\n\
store = CustomCache()\n\
store._disk = CustomDisk()\n\
store.directory = {directory:?}\n\
backend = SimpleNamespace(disk_cache=store)\n\
facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)"
),
);
let CacheConfigProjection::Unsupported(reason) =
NativeCacheConfig::project(&custom).unwrap()
else {
panic!("custom disk store must stay on Python");
};
assert_eq!(
reason.message(),
"native disk cache requires the built-in diskcache store"
);
});
}
#[test]
fn projects_cluster_startup_nodes_as_redis_topology() {

43
litellm-rust/crates/python-bridge/src/cache/facade.rs vendored
View file

@ -34,6 +34,11 @@ struct RedisPoolGuard {
attributes: RedisPoolAttributes,
}
struct DiskStoreGuard {
reference: Py<PyAny>,
directory: String,
}
struct AzureBlobClientGuard {
sync_client: Py<PyAny>,
async_client: Py<PyAny>,
@ -46,7 +51,6 @@ enum ConnectionGuard {
RedisPool(RedisPoolGuard),
AzureBlob(AzureBlobClientGuard),
}
struct RedisPoolAttributes {
pool: &'static str,
connection_class: &'static str,
@ -68,6 +72,7 @@ const CLUSTER_POOL: RedisPoolAttributes = RedisPoolAttributes {
pub(super) struct FacadeGuard {
outer: ObjectGuard,
backend: ObjectGuard,
disk_store: Option<DiskStoreGuard>,
connection: ConnectionGuard,
}
@ -218,6 +223,26 @@ impl RedisPoolGuard {
}
}
impl DiskStoreGuard {
fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
let store = backend.getattr("disk_cache")?;
Ok(Self {
reference: store.clone().unbind(),
directory: store.getattr("directory")?.extract()?,
})
}
fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
let store = backend.getattr("disk_cache")?;
Ok(self.reference.bind(py).is(&store)
&& self.directory == store.getattr("directory")?.extract::<String>()?)
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.reference)
}
}
impl AzureBlobClientGuard {
fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
let sync_client = backend.getattr("container_client")?;
@ -295,6 +320,8 @@ impl FacadeGuard {
"RedisClusterCache",
"redis",
),
("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"),
("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"),
("azure-blob", _) => (
"litellm.caching.azure_blob_cache",
"AzureBlobCache",
@ -343,8 +370,14 @@ impl FacadeGuard {
"max_size_per_item",
"redis_kwargs",
"redis_flush_size",
"bucket_name",
"key_prefix",
"path_service_account",
],
)?,
disk_store: (kind == "disk")
.then(|| DiskStoreGuard::capture(&backend))
.transpose()?,
connection: ConnectionGuard::capture(kind, cluster, &backend)?,
})
}
@ -357,12 +390,20 @@ impl FacadeGuard {
if !self.backend.matches(py, &backend)? {
return Ok(false);
}
if let Some(guard) = &self.disk_store
&& !guard.matches(py, &backend)?
{
return Ok(false);
}
self.connection.matches(py, &backend)
}
pub(super) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
self.outer.traverse(&visit)?;
self.backend.traverse(&visit)?;
if let Some(guard) = &self.disk_store {
guard.traverse(&visit)?;
}
self.connection.traverse(&visit)
}
}

39
litellm-rust/crates/python-bridge/src/cache/handle.rs vendored
View file

@ -2,6 +2,8 @@ use litellm_cache_redis::{RedisNode, RedisTopology};
use litellm_host_python::{release_gil, run_sync_value};
use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration};
#[pyclass(frozen, name = "_CacheTestHandle")]
@ -64,6 +66,43 @@ impl CacheTestHandle {
})
}
#[staticmethod]
#[pyo3(signature = (bucket_name, *, gcs_path=None, path_service_account=None, endpoint=None, token=None))]
fn gcs(
py: Python<'_>,
bucket_name: String,
gcs_path: Option<String>,
path_service_account: Option<String>,
endpoint: Option<String>,
token: Option<String>,
) -> PyResult<Self> {
let config = GcsConfig {
bucket_name,
gcs_path,
path_service_account,
endpoint: endpoint.unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()),
};
let service = release_gil(py, move || NativeResponseCache::gcs(config, token))
.map_err(cache_error)?;
Ok(Self {
service,
guard: None,
pid: std::process::id(),
})
}
#[staticmethod]
#[pyo3(signature = (directory))]
fn disk(py: Python<'_>, directory: String) -> PyResult<Self> {
let service =
release_gil(py, move || NativeResponseCache::disk(&directory)).map_err(cache_error)?;
Ok(Self {
service,
guard: None,
pid: std::process::id(),
})
}
#[staticmethod]
#[pyo3(signature = (account_url, container))]
fn azure_blob(py: Python<'_>, account_url: String, container: String) -> PyResult<Self> {

3
litellm-rust/crates/python-bridge/src/cache/mod.rs vendored
View file

@ -10,7 +10,7 @@ mod resolver;
use litellm_cache::Error;
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError},
prelude::*,
};
@ -21,6 +21,7 @@ pub(crate) use self::{
fn cache_error(error: Error) -> PyErr {
match error {
Error::InvalidEntry => PyValueError::new_err(error.to_string()),
Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()),
_ => PyRuntimeError::new_err(error.to_string()),
}
}

69
litellm-rust/crates/python-bridge/src/cache/native.rs vendored
View file

@ -1,7 +1,9 @@
use std::{sync::Arc, time::Duration};
use std::{path::Path, sync::Arc, time::Duration};
use litellm_cache::{CacheCodec, CacheConnectionResult, Error};
use litellm_cache_azure_blob::AzureBlobCache;
use litellm_cache_disk::DiskCache;
use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource};
use litellm_cache_memory::InMemoryCache;
use litellm_cache_redis::{RedisCache, RedisTopology};
use litellm_cache_response::{
@ -16,6 +18,8 @@ pub(super) enum NativeResponseCache {
cache: Arc<ResponseCache<RedisCache<ResponseCacheCodec>>>,
buffer: Option<Arc<WriteBuffer>>,
},
Gcs(Arc<ResponseCache<GcsCache<ResponseCacheCodec>>>),
Disk(Arc<ResponseCache<DiskCache<ResponseCacheCodec>>>),
AzureBlob(Arc<ResponseCache<AzureBlobCache<ResponseCacheCodec>>>),
}
@ -47,6 +51,22 @@ impl NativeResponseCache {
buffer: None,
})
}
pub fn disk(directory: &str) -> Result<Self, Error> {
let cache = DiskCache::open(directory, ResponseCacheCodec)?;
Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache)))))
}
pub fn gcs(config: GcsConfig, token: Option<String>) -> Result<Self, Error> {
let backend = match token {
Some(token) => GcsCache::with_token_source(
config,
ResponseCacheCodec,
Arc::new(StaticTokenSource(token)),
)?,
None => GcsCache::new(config, ResponseCacheCodec)?,
};
Ok(Self::Gcs(Arc::new(ResponseCache::new(Arc::new(backend)))))
}
pub async fn azure_blob(account_url: &str, container: &str) -> Result<Self, Error> {
let backend = AzureBlobCache::connect(
@ -67,7 +87,7 @@ impl NativeResponseCache {
cache.backend().account_url(),
cache.backend().container_name(),
)),
Self::Memory(_) | Self::Redis { .. } => None,
Self::Memory(_) | Self::Redis { .. } | Self::Disk(_) | Self::Gcs(_) => None,
}
}
}
@ -77,6 +97,8 @@ impl NativeResponseCache {
match self {
Self::Memory(_) => "memory",
Self::Redis { .. } => "redis",
Self::Gcs(_) => "gcs",
Self::Disk(_) => "disk",
Self::AzureBlob(_) => "azure-blob",
}
}
@ -85,20 +107,23 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.default_ttl(),
Self::Redis { cache, .. } => cache.default_ttl(),
Self::Gcs(cache) => cache.default_ttl(),
Self::Disk(cache) => cache.default_ttl(),
Self::AzureBlob(cache) => cache.default_ttl(),
}
}
pub fn namespace(&self) -> Option<&str> {
match self {
Self::Memory(_) | Self::AzureBlob(_) => None,
Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) => None,
Self::Redis { cache, .. } => cache.backend().namespace(),
Self::Gcs(_) => None,
}
}
pub fn topology(&self) -> Option<&RedisTopology> {
match self {
Self::Memory(_) | Self::AzureBlob(_) => None,
Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) | Self::Gcs(_) => None,
Self::Redis { cache, .. } => Some(cache.backend().topology()),
}
}
@ -106,14 +131,14 @@ impl NativeResponseCache {
pub fn capacity(&self) -> Option<usize> {
match self {
Self::Memory(cache) => Some(cache.backend().max_size_in_memory()),
Self::Redis { .. } | Self::AzureBlob(_) => None,
Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) | Self::Gcs(_) => None,
}
}
pub fn max_entry_bytes(&self) -> Option<usize> {
match self {
Self::Memory(cache) => cache.backend().max_entry_bytes(),
Self::Redis { .. } | Self::AzureBlob(_) => None,
Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) | Self::Gcs(_) => None,
}
}
@ -127,6 +152,13 @@ impl NativeResponseCache {
}
}
pub fn directory(&self) -> Option<&Path> {
match self {
Self::Disk(cache) => Some(cache.backend().directory()),
Self::Memory(_) | Self::Redis { .. } | Self::AzureBlob(_) | Self::Gcs(_) => None,
}
}
pub fn lookup(
&self,
request: &ResponseCacheRequest,
@ -135,6 +167,8 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.lookup(request, now),
Self::Redis { cache, .. } => cache.lookup(request, now),
Self::Gcs(cache) => cache.lookup(request, now),
Self::Disk(cache) => cache.lookup(request, now),
Self::AzureBlob(cache) => cache.lookup(request, now),
}
}
@ -148,6 +182,8 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.store(request, response, now),
Self::Redis { cache, .. } => cache.store(request, response, now),
Self::Gcs(cache) => cache.store(request, response, now),
Self::Disk(cache) => cache.store(request, response, now),
Self::AzureBlob(cache) => cache.store(request, response, now),
}
}
@ -160,6 +196,8 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.lookup_batch(requests, now),
Self::Redis { cache, .. } => cache.lookup_batch(requests, now),
Self::Gcs(cache) => cache.lookup_batch(requests, now),
Self::Disk(cache) => cache.lookup_batch(requests, now),
Self::AzureBlob(cache) => cache.lookup_batch(requests, now),
}
}
@ -172,6 +210,8 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.async_lookup(request, now).await,
Self::Redis { cache, .. } => cache.async_lookup(request, now).await,
Self::Gcs(cache) => cache.async_lookup(request, now).await,
Self::Disk(cache) => cache.async_lookup(request, now).await,
Self::AzureBlob(cache) => cache.async_lookup(request, now).await,
}
}
@ -192,6 +232,8 @@ impl NativeResponseCache {
cache,
buffer: Some(buffer),
} => buffer.async_store(cache, request, response, now).await,
Self::Gcs(cache) => cache.async_store(request, response, now).await,
Self::Disk(cache) => cache.async_store(request, response, now).await,
Self::AzureBlob(cache) => cache.async_store(request, response, now).await,
}
}
@ -204,6 +246,8 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.async_lookup_batch(requests, now).await,
Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await,
Self::Gcs(cache) => cache.async_lookup_batch(requests, now).await,
Self::Disk(cache) => cache.async_lookup_batch(requests, now).await,
Self::AzureBlob(cache) => cache.async_lookup_batch(requests, now).await,
}
}
@ -216,6 +260,8 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.async_store_batch(entries, now).await,
Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await,
Self::Gcs(cache) => cache.async_store_batch(entries, now).await,
Self::Disk(cache) => cache.async_store_batch(entries, now).await,
Self::AzureBlob(cache) => cache.async_store_batch(entries, now).await,
}
}
@ -229,6 +275,8 @@ impl NativeResponseCache {
}
cache.async_flush().await
}
Self::Gcs(cache) => cache.async_flush().await,
Self::Disk(cache) => cache.async_flush().await,
Self::AzureBlob(cache) => cache.async_flush().await,
}
}
@ -237,7 +285,16 @@ impl NativeResponseCache {
match self {
Self::Memory(cache) => cache.test_connection().await,
Self::Redis { cache, .. } => cache.test_connection().await,
Self::Gcs(cache) => cache.test_connection().await,
Self::Disk(cache) => cache.test_connection().await,
Self::AzureBlob(cache) => cache.test_connection().await,
}
}
pub fn gcs_backend(&self) -> Option<&GcsCache<ResponseCacheCodec>> {
match self {
Self::Gcs(cache) => Some(cache.backend()),
_ => None,
}
}
}

3
litellm/constants.py
View file

@ -2121,3 +2121,6 @@ BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit"
# Shared read-only empty mapping, for defaulting optional Mapping parameters without
# constructing a fresh mutable dict at each call site.
EMPTY_MAPPING: Final = MappingProxyType({})
# API endpoint for breached password k-anonymity search
HIBP_RANGE_API_BASE: Final = "https://api.pwnedpasswords.com/range"

2
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -238,7 +238,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "message": "content"}
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
)
_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}

1
litellm/llms/xai/chat/transformation.py
View file

@ -118,6 +118,7 @@ class XAIChatConfig(OpenAIGPTConfig):
base_openai_params: Final = [
"logit_bias",
"logprobs",
"max_completion_tokens",
"max_tokens",
"n",
"parallel_tool_calls",

26
litellm/model_prices_and_context_window_backup.json
View file

@ -43037,21 +43037,21 @@
"supports_web_search": false
},
"openrouter/deepseek/deepseek-v4-pro": {
"input_cost_per_token": 8.95578e-07,
"input_cost_per_token": 8.92272e-07,
"input_cost_per_token_cache_hit": 4.4e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.791156e-06,
"output_cost_per_token": 1.784544e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 7.46315e-08,
"cache_read_input_token_cost": 7.4356e-08,
"supports_audio_input": false,
"supports_pdf_input": false,
"supports_vision": false,
@ -68212,13 +68212,13 @@
"supports_web_search": false
},
"openrouter/z-ai/glm-5.3-flash": {
"input_cost_per_token": 7.5e-08,
"output_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 5e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1310720,
"max_output_tokens": 102400,
"max_tokens": 102400,
"max_output_tokens": 943718,
"max_tokens": 943718,
"mode": "chat",
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
@ -73252,14 +73252,14 @@
"supports_web_search": true
},
"openrouter/~z-ai/glm-flash-latest": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 7.5e-08,
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1310720,
"max_output_tokens": 102400,
"max_tokens": 102400,
"max_output_tokens": 943718,
"max_tokens": 943718,
"mode": "chat",
"output_cost_per_token": 2.5e-07,
"output_cost_per_token": 5e-07,
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,

2
litellm/models/user.py
View file

@ -24,6 +24,8 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase):
organization_id: str | None = None
object_permission_id: str | None = None
password: str | None = Field(default=None, exclude=True)
password_reset_required: bool | None = None
last_breach_check_at: datetime | None = None
teams: list[str] = []
user_role: str | None = None
max_budget: float | None = None

95
litellm/proxy/_experimental/mcp_server/contracts.py Normal file
View file

@ -0,0 +1,95 @@
from collections.abc import Mapping
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import Final, Protocol
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
def copy_caller(auth: UserAPIKeyAuth | None) -> UserAPIKeyAuth | None:
if auth is None:
return None
span: Final = auth.parent_otel_span
return deepcopy(auth, {id(span): span} if span is not None else None) # mutable-ok: deepcopy mutates its memo
@dataclass(frozen=True, slots=True)
class OperationContext:
_caller: UserAPIKeyAuth | None = field(repr=False)
mcp_auth_header: str | None = field(default=None, repr=False)
mcp_servers: tuple[str, ...] | None = None
mcp_server_auth_headers: Mapping[str, Mapping[str, str]] | None = field(default=None, repr=False)
oauth2_headers: Mapping[str, str] | None = field(default=None, repr=False)
raw_headers: Mapping[str, str] | None = field(default=None, repr=False)
client_ip: str | None = None
mcp_proxy_mode: bool = False
def __post_init__(self) -> None:
object.__setattr__(self, "_caller", copy_caller(self._caller))
object.__setattr__(self, "mcp_servers", tuple(self.mcp_servers) if self.mcp_servers is not None else None)
object.__setattr__(
self,
"oauth2_headers",
MappingProxyType(dict(self.oauth2_headers)) if self.oauth2_headers is not None else None,
)
object.__setattr__(
self, "raw_headers", MappingProxyType(dict(self.raw_headers)) if self.raw_headers is not None else None
)
object.__setattr__(
self,
"mcp_server_auth_headers",
MappingProxyType(
{key: MappingProxyType(dict(value)) for key, value in self.mcp_server_auth_headers.items()}
)
if self.mcp_server_auth_headers is not None
else None,
)
@property
def user_api_key_auth(self) -> UserAPIKeyAuth | None:
return copy_caller(self._caller)
def legacy_auth(
self,
) -> tuple[
UserAPIKeyAuth | None,
str | None,
list[str] | None, # mutable-ok: detached legacy server-list payload
dict[str, dict[str, str]] | None, # mutable-ok: legacy auth dispatch requires concrete dict headers
dict[str, str] | None, # mutable-ok: detached legacy header payload
dict[str, str] | None, # mutable-ok: detached legacy header payload
str | None,
]:
return (
self.user_api_key_auth,
self.mcp_auth_header,
list(self.mcp_servers) if self.mcp_servers is not None else None, # mutable-ok: legacy policy list input
{
key: dict(value) for key, value in self.mcp_server_auth_headers.items()
} # mutable-ok: legacy auth dispatch checks concrete dict headers
if self.mcp_server_auth_headers is not None
else None,
dict(self.oauth2_headers)
if self.oauth2_headers is not None
else None, # mutable-ok: legacy OAuth header input
dict(self.raw_headers) if self.raw_headers is not None else None, # mutable-ok: legacy request header input
self.client_ip,
)
class ProgressCallback(Protocol):
async def __call__(self, progress: float, total: float | None, /) -> None: ...
@dataclass(frozen=True, slots=True)
class AuthorizedToolCall:
name: str
arguments: Mapping[str, object]
allowed_mcp_servers: tuple[MCPServer, ...]
start_time: datetime
host_progress_callback: ProgressCallback | None
guardrail_context: Mapping[str, object] | None
logging_data: Mapping[str, object]

83
litellm/proxy/_experimental/mcp_server/legacy_callbacks.py Normal file
View file

@ -0,0 +1,83 @@
from collections.abc import Mapping
from typing import Final, Protocol
from mcp.client.session import ClientRequestContext
from mcp.types import (
CreateMessageRequestParams,
CreateMessageResult,
CreateMessageResultWithTools,
ElicitRequestParams,
ElicitResult,
ErrorData,
)
from litellm.proxy._experimental.mcp_server.contracts import OperationContext
from litellm.proxy._types import UserAPIKeyAuth
class SamplingCallback(Protocol):
async def __call__(
self, context: ClientRequestContext, params: CreateMessageRequestParams, /
) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData: ...
class ElicitationCallback(Protocol):
async def __call__(self, context: object, params: ElicitRequestParams, /) -> ElicitResult | ErrorData: ...
def create_sampling_callback(
user_api_key_auth: UserAPIKeyAuth | None = None,
raw_headers: Mapping[str, str] | None = None,
client_ip: str | None = None,
operation_context: OperationContext | None = None,
) -> SamplingCallback:
from litellm.proxy._experimental.mcp_server.server import get_active_auth_context
auth: Final = get_active_auth_context() if operation_context is None and user_api_key_auth is None else None
captured: Final = (
operation_context
if operation_context is not None
else OperationContext(
_caller=user_api_key_auth if user_api_key_auth is not None else (auth.user_api_key_auth if auth else None),
raw_headers=raw_headers if raw_headers is not None else (auth.raw_headers if auth else None),
client_ip=client_ip if client_ip is not None else (auth.client_ip if auth else None),
)
)
async def callback(
context: ClientRequestContext, params: CreateMessageRequestParams
) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData:
import litellm
from litellm.proxy._experimental.mcp_server.sampling_handler import handle_sampling_create_message
return await handle_sampling_create_message(
context=context,
params=params,
default_model=getattr(litellm, "default_mcp_sampling_model", None),
user_api_key_auth=captured.user_api_key_auth,
raw_headers=dict(captured.raw_headers)
if captured.raw_headers is not None
else None, # mutable-ok: handler consumes an owned request header dict
client_ip=captured.client_ip,
)
return callback
def create_elicitation_callback() -> ElicitationCallback:
from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session
downstream_session: Final = get_active_mcp_session()
downstream_capabilities: Final = getattr(downstream_session, "capabilities", None)
async def callback(context: object, params: ElicitRequestParams) -> ElicitResult | ErrorData:
from litellm.proxy._experimental.mcp_server.elicitation_handler import handle_elicitation_request
return await handle_elicitation_request(
context=context,
params=params,
downstream_session=downstream_session,
downstream_capabilities=downstream_capabilities,
)
return callback

129
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -73,6 +73,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPServerAccess,
_is_mcp_admitted_user_subject,
)
from litellm.proxy._experimental.mcp_server.contracts import OperationContext
from litellm.proxy._experimental.mcp_server.elicitation_handler import (
MCP_ELICITATION_AVAILABLE,
)
@ -195,9 +196,6 @@ from litellm.types.mcp_server.mcp_server_manager import (
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
from mcp.client.session import ClientRequestContext
from mcp.types import CreateMessageRequestParams
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.mcp_server.mcp_toolset import MCPToolset
@ -1218,7 +1216,7 @@ async def _resolve_byok_mcp_auth_header(
if not mcp_server.is_byok:
return mcp_auth_header
from litellm.proxy._experimental.mcp_server.server import (
from litellm.proxy._experimental.mcp_server.operations import (
_check_byok_credential,
_get_byok_credential,
)
@ -1577,77 +1575,25 @@ def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None:
mcp_info["mcp_server_cost_info"] = normalized
def _create_sampling_callback(user_api_key_auth: UserAPIKeyAuth | None = None):
"""
Create a sampling callback for MCP ClientSession.
Returns a callable that handles sampling/createMessage requests from
upstream MCP servers by routing them through litellm.acompletion().
"""
def _create_sampling_callback(
user_api_key_auth: UserAPIKeyAuth | None = None,
raw_headers: Mapping[str, str] | None = None,
client_ip: str | None = None,
operation_context: OperationContext | None = None,
):
if not MCP_SAMPLING_AVAILABLE:
return None
from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_sampling_callback
async def _sampling_callback(
context: "ClientRequestContext",
params: "CreateMessageRequestParams",
):
import litellm
from litellm.proxy._experimental.mcp_server.sampling_handler import (
handle_sampling_create_message,
)
from litellm.proxy._experimental.mcp_server.server import (
get_active_auth_context,
)
auth_context: Final = get_active_auth_context()
resolved_auth: Final = user_api_key_auth or (auth_context.user_api_key_auth if auth_context else None)
# Forward original HTTP headers and client IP so that
# header-dependent guardrails, tag-based routing, trace
# correlation, and forward_llm_provider_auth_headers work
# correctly for sampling sub-calls.
_raw_headers: Final = getattr(auth_context, "raw_headers", None)
_client_ip: Final = getattr(auth_context, "client_ip", None)
return await handle_sampling_create_message(
context=context,
params=params,
default_model=getattr(litellm, "default_mcp_sampling_model", None),
user_api_key_auth=resolved_auth,
raw_headers=_raw_headers,
client_ip=_client_ip,
)
return _sampling_callback
return create_sampling_callback(user_api_key_auth, raw_headers, client_ip, operation_context)
def _create_elicitation_callback():
"""
Create an elicitation callback for MCP ClientSession.
Returns a callable that handles elicitation/create requests from
upstream MCP servers. In gateway mode, this relays to the downstream
client; in tool bridge mode, it returns a decline response.
"""
if not MCP_ELICITATION_AVAILABLE:
return None
from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_elicitation_callback
async def _elicitation_callback(context, params):
from litellm.proxy._experimental.mcp_server.elicitation_handler import (
handle_elicitation_request,
)
from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session
# In Gateway mode, we relay the elicitation request to the downstream client
# that triggered the current operation.
downstream_session: Final = get_active_mcp_session()
downstream_capabilities = getattr(downstream_session, "capabilities", None) if downstream_session else None
return await handle_elicitation_request(
context=context,
params=params,
downstream_session=downstream_session,
downstream_capabilities=downstream_capabilities,
)
return _elicitation_callback
return create_elicitation_callback()
def _record_mcp_guardrail_evaluations(
@ -3386,17 +3332,13 @@ class MCPServerManager:
listable but uninvokable.
Empty inside a toolset scope: toolset_mcp_route / dynamic_mcp_route set
``_mcp_active_toolset_id`` before calling the handler, pinning the request to the toolset's
the caller's server-only ``mcp_toolset_id`` before calling the handler, pinning the request to the toolset's
own servers (checking op.mcp_toolsets==[] instead would false-positive on DB-default rows
where Postgres initialises the column to ARRAY[]::TEXT[]).
``allow_all_server_ids`` / ``submitted_server_ids`` are injectable so the server union,
which precomputes both for its fallback path, does not compute them twice."""
from litellm.proxy._experimental.mcp_server.mcp_context import ( # noqa: PLC0415
_mcp_active_toolset_id,
)
if _mcp_active_toolset_id.get() is not None:
if user_api_key_auth is not None and user_api_key_auth.mcp_toolset_id is not None:
return set()
if allow_all_server_ids is None:
allow_all_server_ids = self.get_allow_all_keys_server_ids()
@ -4164,6 +4106,8 @@ class MCPServerManager:
subject_token: str | None = None,
user_api_key_auth: UserAPIKeyAuth | None = None,
cred_provider: UpstreamCredentialProvider | None = None,
raw_headers: Mapping[str, str] | None = None,
client_ip: str | None = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
@ -4212,7 +4156,13 @@ class MCPServerManager:
# Create sampling and elicitation callbacks for this client
sampling_cb = (
_create_sampling_callback(user_api_key_auth=user_api_key_auth) if resolved_server.allow_sampling else None
_create_sampling_callback(
operation_context=OperationContext(
_caller=user_api_key_auth, raw_headers=raw_headers, client_ip=client_ip
)
)
if resolved_server.allow_sampling
else None
)
elicitation_cb: Final = _create_elicitation_callback() if resolved_server.allow_elicitation else None
@ -4357,6 +4307,7 @@ class MCPServerManager:
raw_headers: dict[str, str] | None = None,
user_api_key_auth: UserAPIKeyAuth | None = None,
oauth2_headers: dict[str, str] | None = None,
client_ip: str | None = None,
) -> list[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@ -4446,6 +4397,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
## HANDLE OPENAPI TOOLS
@ -4556,6 +4509,7 @@ class MCPServerManager:
extra_headers: dict[str, str] | None = None,
add_prefix: bool = True,
raw_headers: dict[str, str] | None = None,
client_ip: str | None = None,
) -> list[Prompt]:
try:
headers: Final = (
@ -4576,6 +4530,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
credential_fingerprint: Final = await client.discovery_auth_fingerprint()
key: Final = self._discovery_key(
@ -4599,6 +4555,7 @@ class MCPServerManager:
extra_headers: dict[str, str] | None = None,
add_prefix: bool = True,
raw_headers: dict[str, str] | None = None,
client_ip: str | None = None,
) -> list[Resource]:
try:
headers: Final = (
@ -4619,6 +4576,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
credential_fingerprint: Final = await client.discovery_auth_fingerprint()
key: Final = self._discovery_key(
@ -4642,6 +4601,7 @@ class MCPServerManager:
extra_headers: dict[str, str] | None = None,
add_prefix: bool = True,
raw_headers: dict[str, str] | None = None,
client_ip: str | None = None,
) -> list[ResourceTemplate]:
try:
headers: Final = (
@ -4662,6 +4622,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
credential_fingerprint: Final = await client.discovery_auth_fingerprint()
key: Final = self._discovery_key(
@ -4685,6 +4647,7 @@ class MCPServerManager:
mcp_auth_header: str | dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
raw_headers: dict[str, str] | None = None,
client_ip: str | None = None,
) -> ReadResourceResult:
"""Read resource contents from a specific MCP server."""
@ -4705,6 +4668,9 @@ class MCPServerManager:
extra_headers=extra_headers,
stdio_env=stdio_env,
subject_token=subject_token,
raw_headers=raw_headers,
client_ip=client_ip,
user_api_key_auth=user_api_key_auth,
)
return await client.read_resource(url)
@ -4718,6 +4684,7 @@ class MCPServerManager:
mcp_auth_header: str | dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
raw_headers: dict[str, str] | None = None,
client_ip: str | None = None,
) -> GetPromptResult:
"""Fetch a specific prompt definition from a single MCP server."""
@ -4738,6 +4705,9 @@ class MCPServerManager:
extra_headers=extra_headers,
stdio_env=stdio_env,
subject_token=subject_token,
raw_headers=raw_headers,
client_ip=client_ip,
user_api_key_auth=user_api_key_auth,
)
get_prompt_request_params: Final = GetPromptRequestParams(
@ -5818,6 +5788,8 @@ class MCPServerManager:
stdio_env: dict[str, str] | None,
subject_token: str | None,
user_api_key_auth: UserAPIKeyAuth | None,
raw_headers: Mapping[str, str] | None = None,
client_ip: str | None = None,
) -> CallToolResult:
"""Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry.
@ -5843,6 +5815,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
return await retry_client.call_tool(call_tool_params, host_progress_callback=host_progress_callback)
@ -5860,6 +5834,7 @@ class MCPServerManager:
host_progress_callback: Callable | None = None,
hook_extra_headers: dict[str, str] | None = None,
user_api_key_auth: UserAPIKeyAuth | None = None,
client_ip: str | None = None,
) -> CallToolResult:
"""
Call a regular MCP tool using the MCP client.
@ -6004,6 +5979,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
call_tool_params: Final = MCPCallToolRequestParams(
@ -6027,6 +6004,8 @@ class MCPServerManager:
stdio_env=stdio_env,
subject_token=subject_token,
user_api_key_auth=user_api_key_auth,
raw_headers=raw_headers,
client_ip=client_ip,
)
tool_call_coro = _obo_call_tool_limited()
@ -6202,7 +6181,7 @@ class MCPServerManager:
return oauth2_headers
try:
from litellm.proxy._experimental.mcp_server.server import ( # noqa: PLC0415
from litellm.proxy._experimental.mcp_server.operations import ( # noqa: PLC0415
_get_user_oauth_extra_headers_from_db,
)
@ -6308,6 +6287,7 @@ class MCPServerManager:
host_progress_callback: Callable | None = None,
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
guardrail_context: Mapping[str, object] | None = None,
client_ip: str | None = None,
) -> CallToolResult:
"""
Call a tool with the given name and arguments
@ -6434,6 +6414,7 @@ class MCPServerManager:
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
client_ip=client_ip,
proxy_logging_obj=proxy_logging_obj,
host_progress_callback=host_progress_callback,
hook_extra_headers=hook_result.get("extra_headers"),

3102
litellm/proxy/_experimental/mcp_server/operations.py Normal file
View file

File diff suppressed because it is too large Load diff

13
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -203,17 +203,19 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.oauth_utils import (
get_request_base_url,
)
from litellm.proxy._experimental.mcp_server.server import (
from litellm.proxy._experimental.mcp_server.operations import (
ListMCPToolsRestAPIResponseObject,
MCPInfo,
MCPServer,
_aggregate_server_key, # pyright: ignore[reportPrivateUsage] # same per-server key as the tools/list _meta outcomes
_apply_toolset_scope,
_aggregate_server_key,
_fire_mcp_tool_call_logging,
execute_mcp_tool,
filter_tools_by_allowed_tools,
filter_tools_by_key_team_permissions,
fire_mcp_tool_call_failure_logging,
)
from litellm.proxy._experimental.mcp_server.server import (
_apply_toolset_scope,
reject_disallowed_mcp_client,
)
@ -670,6 +672,7 @@ if MCP_AVAILABLE:
user_api_key_auth: UserAPIKeyAuth | None = None,
extra_headers: dict[str, str] | None = None,
apply_tool_filters: bool = True,
client_ip: str | None = None,
):
"""Helper function to get tools for a single server.
@ -684,6 +687,7 @@ if MCP_AVAILABLE:
extra_headers=extra_headers,
add_prefix=False,
raw_headers=raw_headers,
client_ip=client_ip,
user_api_key_auth=user_api_key_auth,
)
@ -797,6 +801,7 @@ if MCP_AVAILABLE:
user_api_key_dict,
extra_headers=user_oauth_extra_headers,
apply_tool_filters=apply_tool_filters,
client_ip=rest_client_ip,
)
except MCPUpstreamAuthError:
# Surface the upstream 401/403 to the caller so it can emit the
@ -1016,6 +1021,7 @@ if MCP_AVAILABLE:
user_api_key_dict,
extra_headers=user_oauth_extra_headers,
apply_tool_filters=apply_tool_filters,
client_ip=_rest_client_ip,
)
except Exception as e:
verbose_logger.warning(
@ -1193,6 +1199,7 @@ if MCP_AVAILABLE:
mcp_server_auth_headers=data.get("mcp_server_auth_headers"),
oauth2_headers=user_oauth_extra_headers or data.get("oauth2_headers"),
raw_headers=data.get("raw_headers"),
client_ip=IPAddressUtils.get_mcp_client_ip(request),
litellm_logging_obj=data.get("litellm_logging_obj"),
guardrail_context=MCPRequestContext.resolve_guardrail_context(data),
requested_server_id=canonical_server_id,

3062
litellm/proxy/_experimental/mcp_server/server.py
View file

File diff suppressed because it is too large Load diff

11
litellm/proxy/_experimental/mcp_server/tool_search.py
View file

@ -463,8 +463,8 @@ async def handle_mcp_tool_search(
oauth2_headers: dict[str, str] | None = None,
raw_headers: dict[str, str] | None = None,
) -> CallToolResult:
from litellm.proxy._experimental.mcp_server.server import (
_list_mcp_tools, # pyright: ignore[reportPrivateUsage] # shared catalog owner
from litellm.proxy._experimental.mcp_server.operations import (
_list_mcp_tools,
)
from litellm.proxy.proxy_server import llm_router, proxy_logging_obj
@ -519,8 +519,8 @@ async def handle_mcp_proxy_tool(
from jsonschema import validate
from litellm.proxy import proxy_server
from litellm.proxy._experimental.mcp_server.server import ( # pyright: ignore[reportPrivateUsage] # shared catalog owner
_list_mcp_tools, # pyright: ignore[reportPrivateUsage] # shared catalog owner
from litellm.proxy._experimental.mcp_server.operations import (
_list_mcp_tools,
)
listing: Final = await _list_mcp_tools(
@ -607,7 +607,7 @@ async def handle_mcp_tool_call(
requested_server_id: str | None = None,
guardrail_context: Mapping[str, object] | None = None,
) -> CallToolResult:
from litellm.proxy._experimental.mcp_server.server import (
from litellm.proxy._experimental.mcp_server.operations import (
_get_allowed_mcp_servers,
execute_mcp_tool,
raise_denied_scoped_mcp_access,
@ -643,6 +643,7 @@ async def handle_mcp_tool_call(
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
client_ip=client_ip,
litellm_logging_obj=litellm_logging_obj,
requested_server_id=requested_server_id,
guardrail_context=guardrail_context,

33
litellm/proxy/_types.py
View file

@ -905,6 +905,7 @@ class LiteLLMRoutes(enum.Enum):
"/claude_code_gateway/v1/traces",
"/user/list", # org admins checked in endpoint; non-admins get 403
"/management/v1/users/bulk_delete", # proxy admins delete anyone, org admins only their orgs' users; others 403
"/user/password/change", # endpoint only ever writes the caller's own row
"/model/{model_id}/update",
"/prompt/list",
"/prompt/info",
@ -1865,6 +1866,17 @@ class NewUserRequest(GenerateRequestBase):
send_invite_email: bool | None = None
sso_user_id: str | None = None
organizations: list[str] | None = None
password: str | None = None
@field_validator("password")
@classmethod
def password_not_supported(cls, value: str | None) -> str | None:
if value is not None:
raise ValueError(
"password cannot be set via /user/new. Users set their own password through an "
"invitation link (POST /invitation/new)."
)
return value
class NewUserResponse(GenerateKeyResponse):
@ -1887,7 +1899,8 @@ class NewUserResponse(GenerateKeyResponse):
class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest
password: str | None = None
# repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model
password: str | None = Field(default=None, repr=False)
spend: float | None = None
metadata: dict | None = None
user_alias: str | None = None
@ -1917,6 +1930,16 @@ class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail):
return values
class ChangePasswordRequest(LiteLLMPydanticObjectBase):
current_password: str = Field(repr=False)
new_password: str = Field(repr=False)
class ChangePasswordResponse(LiteLLMPydanticObjectBase):
user_id: str
message: str
class DeleteUserRequest(LiteLLMPydanticObjectBase):
user_ids: list[str] # required
@ -3239,6 +3262,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
# above; a forged value could at most narrow, but the stripping keeps the field's provenance
# single-owner so its meaning stays trustworthy.
mcp_session_resource_server_id: str | None = Field(default=None, exclude=True)
mcp_toolset_id: str | None = Field(default=None, exclude=True)
via_virtual_key: bool = Field(
default=False,
exclude=True,
@ -3280,6 +3304,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
values.pop("mcp_admitted_user_subject", None)
values.pop("mcp_source_team_rpm_limits", None)
values.pop("mcp_session_resource_server_id", None)
values.pop("mcp_toolset_id", None)
values.pop("via_virtual_key", None)
if values.get("api_key") is not None:
values.update({"token": cls._safe_hash_litellm_api_key(values.get("api_key"))})
@ -3938,6 +3963,12 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
)
class HTTPExceptionErrorDetail(TypedDict):
"""The `{"error": <message>}` shape most proxy endpoints raise as `HTTPException.detail`."""
error: ReadOnly[str]
class SpendLogsRouterMetadata(TypedDict):
"""
Router provenance stamped on spend logs for deployments flagged with

89
litellm/proxy/auth/login_utils.py
View file

@ -10,14 +10,16 @@ import secrets
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from types import MappingProxyType
from typing import Final, Literal, cast
from typing import TYPE_CHECKING, Final, Literal, cast
import jwt
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import LITELLM_PROXY_ADMIN_NAME, LITELLM_UI_SESSION_DURATION
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import (
LiteLLM_UserTable,
LitellmUserRoles,
@ -28,6 +30,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured
from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle
from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
from litellm.proxy.management_endpoints.key_management_endpoints import (
generate_key_helper_fn,
@ -50,6 +53,57 @@ INVALID_UI_CREDENTIALS_MESSAGE: Final = (
)
INVALID_USER_PASSWORD_MESSAGE: Final = "Invalid credentials used to access UI. Check the password set for your user"
if TYPE_CHECKING:
from prisma import types as prisma_types
BREACH_RECHECK_INTERVAL: Final = timedelta(hours=24)
PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change",)
PASSWORD_SESSION_METADATA: Final = MappingProxyType({"login_method": "username_password"})
def _breach_recheck_due(last_breach_check_at: datetime | None) -> bool:
if last_breach_check_at is None:
return True
last_checked_utc: Final = (
last_breach_check_at
if last_breach_check_at.tzinfo is not None
else last_breach_check_at.replace(tzinfo=timezone.utc)
)
return datetime.now(timezone.utc) - last_checked_utc >= BREACH_RECHECK_INTERVAL
async def screen_login_password_for_breach(
user_id: str,
password: str,
last_breach_check_at: datetime | None,
general_settings: Mapping[str, object],
prisma_client: PrismaClient,
client: AsyncHTTPHandler | None = None,
) -> bool:
"""Screens a successfully verified login password against HIBP, stamps
``password_reset_required`` when breached, and returns whether a breach was
found so the login it runs in can restrict the session it is about to mint.
Fails open (HIBP or DB trouble never fails the login) and rechecks a given
user at most once per ``BREACH_RECHECK_INTERVAL``."""
if not is_breach_check_enabled(general_settings):
return False
if not _breach_recheck_due(last_breach_check_at):
return False
breached: Final = await is_password_breached(password, general_settings, client)
checked_at: Final = datetime.now(timezone.utc)
breached_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {
"last_breach_check_at": checked_at,
"password_reset_required": True,
}
recheck_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {"last_breach_check_at": checked_at}
update_data: Final = breached_update if breached else recheck_update
find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id}
try:
await UserRepository(prisma_client).table.update(where=find_user, data=update_data)
except Exception as e: # noqa: BLE001 # a failed stamp must never surface into the login
verbose_proxy_logger.warning("Login-time breach screening could not update user %s: %s", user_id, e)
return breached
async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None:
"""Rehash legacy password (SHA256) to scrypt on successful login."""
@ -137,6 +191,7 @@ class LoginResult:
user_email: str | None
user_role: str
login_method: Literal["sso", "username_password"]
password_reset_required: bool
def __init__(
self,
@ -145,12 +200,14 @@ class LoginResult:
user_email: str | None,
user_role: str,
login_method: Literal["sso", "username_password"] = "username_password",
password_reset_required: bool = False,
):
self.user_id = user_id
self.key = key
self.user_email = user_email
self.user_role = user_role
self.login_method = login_method
self.password_reset_required = password_reset_required
async def authenticate_user(
@ -356,20 +413,28 @@ async def _sign_in(
if verify_password(password, _password):
await _rehash_password_if_needed(_user_row.user_id, password, _password)
breached_now: Final = prisma_client is not None and await screen_login_password_for_breach(
user_id=_user_row.user_id,
password=password,
last_breach_check_at=getattr(_user_row, "last_breach_check_at", None),
general_settings=general_settings,
prisma_client=prisma_client,
)
password_reset_required: Final = breached_now or getattr(_user_row, "password_reset_required", None) is True
if os.getenv("DATABASE_URL") is not None:
response = await generate_key_helper_fn(
llm_router=None,
request_type="key",
**{
"user_role": user_role,
"duration": LITELLM_UI_SESSION_DURATION,
"key_max_budget": litellm.max_ui_session_budget,
"models": [],
"aliases": {},
"config": {},
"spend": 0,
"user_id": user_id,
"team_id": "litellm-dashboard",
user_role=user_role,
duration=LITELLM_UI_SESSION_DURATION,
key_max_budget=litellm.max_ui_session_budget,
spend=0,
user_id=user_id,
team_id="litellm-dashboard",
allowed_routes=list(PASSWORD_RESET_ALLOWED_ROUTES) if password_reset_required else None,
metadata={
**PASSWORD_SESSION_METADATA,
**({"password_reset_required": True} if password_reset_required else {}),
},
)
else:
@ -390,6 +455,7 @@ async def _sign_in(
user_email=user_email,
user_role=cast(str, user_role),
login_method="username_password",
password_reset_required=password_reset_required,
)
else:
await attempt.failed()
@ -460,4 +526,5 @@ def create_ui_token_object(
auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
server_root_path=get_server_root_path(),
password_reset_required=login_result.password_reset_required,
)

128
litellm/proxy/auth/password_policy.py
View file

@ -4,13 +4,28 @@ Applied at every path that persists a new or changed password for a DB-backed
user (``/user/update``, ``/user/bulk_update``, and the invitation onboarding
claim flow), so the strength bar is configured in one place instead of
per-endpoint.
Also screens new passwords against known data breaches via the
haveibeenpwned.com (HIBP) k-anonymity range API: only the first 5 characters
of the password's SHA-1 hash ever leave the proxy, and the check fails open
(allows the password) when HIBP is unreachable.
"""
from collections.abc import Mapping
import asyncio
import hashlib
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm._version import version
from litellm.constants import HIBP_RANGE_API_BASE
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.custom_http import httpxSpecialProvider
HIBP_TIMEOUT_SECONDS: Final = 5.0
DEFAULT_MIN_LENGTH: Final = 12
MIN_ALLOWED_LENGTH: Final = 8
@ -90,3 +105,114 @@ def validate_password_policy(password: str, general_settings: Mapping[str, objec
param="password",
code=400,
)
def _hibp_client() -> AsyncHTTPHandler:
return get_async_httpx_client(
llm_provider=httpxSpecialProvider.PasswordBreachCheck,
params={"timeout": HIBP_TIMEOUT_SECONDS}, # mutable-ok: callee takes a bare dict (PEP 589)
)
def _is_suffix_in_range_response(response_body: str, hash_suffix: str) -> bool:
for line in response_body.upper().splitlines():
entry_suffix, _, count = line.strip().partition(":")
if entry_suffix == hash_suffix:
return int(count.strip() or "0") > 0
return False
async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool:
# usedforsecurity=False: SHA-1 is only a lookup key into the HIBP dataset, so no security property rests on it
sha1_hex: Final = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
headers: Final = { # mutable-ok: callee takes a bare dict (PEP 589)
"Add-Padding": "true",
"User-Agent": f"litellm-proxy/{version}",
}
try:
response: Final = await client.get(
f"{HIBP_RANGE_API_BASE}/{sha1_hex[:5]}",
headers=headers,
)
response.raise_for_status()
breached: Final = _is_suffix_in_range_response(response.text, sha1_hex[5:])
except Exception as e: # noqa: BLE001 # fail-open: any HIBP failure skips the check, never breaks the caller
verbose_proxy_logger.warning("Breached-password check skipped, HIBP lookup failed: %s", e)
return False
return breached
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
return general_settings.get("password_policy_check_breached_passwords", True) is not False
async def is_password_breached(
password: str,
general_settings: Mapping[str, object],
client: AsyncHTTPHandler | None = None,
) -> bool:
"""False when the check is disabled, the password is absent from the HIBP
corpus, or HIBP is unreachable (fail open)."""
if not is_breach_check_enabled(general_settings):
return False
return await _is_password_breached(password, client if client is not None else _hibp_client())
def breached_password_error() -> ProxyException:
return ProxyException(
message=(
"This password appears in known data breaches and cannot be used. Please choose a different password."
),
type=ProxyErrorTypes.validation_error,
param="password",
code=400,
)
async def validate_password_not_breached(
password: str,
general_settings: Mapping[str, object],
client: AsyncHTTPHandler | None = None,
) -> None:
"""Raise ``ProxyException`` (400) if ``password`` appears in a known data breach.
Fails open: an unreachable or misbehaving HIBP allows the password."""
if not await is_password_breached(password, general_settings, client):
return
raise breached_password_error()
def _strength_verdict(password: str, general_settings: Mapping[str, object]) -> ProxyException | None:
try:
validate_password_policy(password, general_settings)
except ProxyException as e:
return e
return None
async def validate_passwords_bulk(
passwords: Sequence[str],
general_settings: Mapping[str, object],
client: AsyncHTTPHandler | None = None,
) -> Mapping[str, ProxyException | None]:
"""Per-unique-password policy verdicts for a batch: the ProxyException to
surface, or None when the password is acceptable.
Deduplicates first, then issues every needed HIBP lookup concurrently, so a
batch caller pays one HIBP timeout window in the worst case instead of one
per password (each lookup still fails open independently)."""
unique_passwords: Final = tuple(dict.fromkeys(passwords))
strength_verdicts: Final[Mapping[str, ProxyException | None]] = MappingProxyType(
{password: _strength_verdict(password, general_settings) for password in unique_passwords}
)
to_screen: Final = tuple(password for password in unique_passwords if strength_verdicts[password] is None)
breached_flags: Final = await asyncio.gather(
*(is_password_breached(password, general_settings, client) for password in to_screen)
)
breached_passwords: Final = frozenset(password for password, breached in zip(to_screen, breached_flags) if breached)
return MappingProxyType(
{
password: breached_password_error() if password in breached_passwords else strength_verdicts[password]
for password in unique_passwords
}
)

27
litellm/proxy/auth/route_checks.py
View file

@ -194,6 +194,16 @@ class RouteChecks:
if denied_auth_enforced_pass_through_route:
raise RouteChecks._auth_pass_through_denied_exception(route=route)
if valid_token.metadata.get("password_reset_required") is True:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=(
"This account's password must be changed before the session can be used: "
"it was either found in a known data breach or set by an admin. "
"Change it via POST /user/password/change (UI: /ui/change-password), then log in again."
),
)
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"Virtual key is not allowed to call this route. Only allowed to call routes: {valid_token.allowed_routes}. Tried to call route: {route}",
@ -812,7 +822,8 @@ class RouteChecks:
in the codebase is automatically readable by Admin Viewer
without needing to remember to add it to an allowlist.
3. Unsafe HTTP method (POST/PUT/PATCH/DELETE):
- Allow `/user/update` only when restricted to user_email/password.
- Allow `/user/update` only when restricted to user_email.
- Allow `/user/password/change` (endpoint only writes the caller's own row).
- Block all explicit writes in `_ADMIN_VIEWER_BLOCKED_WRITE_ROUTES`.
- Otherwise allow only if the route is in admin_viewer_routes /
global_spend_tracking_routes (legacy explicit-allow set).
@ -832,10 +843,10 @@ class RouteChecks:
if request_data is not None and isinstance(request_data, dict):
_params_updated: Final = request_data.keys()
for param in _params_updated:
if param not in ["user_email", "password"]:
if param != "user_email":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated",
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email can be updated",
)
elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or (
route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES)
@ -854,21 +865,25 @@ class RouteChecks:
return
# ── Unsafe HTTP method: explicit checks ──────────────────────────
# Allow `/user/update` for self-service email / password change.
# Allow `/user/update` for self-service email change.
if route == "/user/update":
if request_data is not None and isinstance(request_data, dict):
for param in request_data:
if param not in ["user_email", "password"]:
if param != "user_email":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=(
f"user not allowed to access this route, role= {_user_role}. "
f"Trying to access: {route} and updating invalid param: {param}. "
"only user_email and password can be updated"
"only user_email can be updated"
),
)
return
# Self-service password change; the endpoint only writes the caller's own row.
if route == "/user/password/change":
return
# Hard-block known write routes regardless of HTTP method (defensive
# — these are POSTs in practice, but pinning them here protects
# against future GET-shaped writes).

62
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -28,6 +28,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import *
from litellm.proxy.auth.auth_checks import (
delete_cache_key_objects,
@ -35,7 +36,11 @@ from litellm.proxy.auth.auth_checks import (
get_team_object,
get_user_object,
)
from litellm.proxy.auth.password_policy import validate_password_policy
from litellm.proxy.auth.password_policy import (
validate_password_not_breached,
validate_password_policy,
validate_passwords_bulk,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.common_utils.user_api_key_cache import (
@ -173,11 +178,23 @@ def _team_membership_table(
return team_membership_table
def _hash_password_in_dict(data: dict, general_settings: Mapping[str, object]) -> None:
"""Validate and hash password field in-place if present."""
async def _hash_password_in_dict(
data: dict, general_settings: Mapping[str, object], password_prevalidated: bool = False
) -> None:
"""Validate and hash password field in-place if present.
``password_prevalidated`` skips the policy checks for callers that already
validated the password (the bulk path screens its whole batch upfront).
An admin-set password is known to whoever set it, so the user is also
flagged for a forced password change at next login."""
if "password" in data and data["password"] is not None:
validate_password_policy(data["password"], general_settings)
if not password_prevalidated:
validate_password_policy(data["password"], general_settings)
await validate_password_not_breached(data["password"], general_settings)
data["password"] = hash_password(data["password"])
data["password_reset_required"] = True
data["last_breach_check_at"] = None
def _strip_password_from_response(response) -> None:
@ -505,6 +522,7 @@ async def new_user(
- prompts: Optional[List[str]] - List of allowed prompts for the user. If specified, the user will only be able to use these specific prompts.
- organizations: List[str] - List of organization id's the user is a member of
- budget_limits: Optional[list] - List of concurrent budget windows for the user. Each window specifies a budget_limit, time_period, and optional budget_duration. Example - [{"budget_limit": 10.0, "time_period": "1d"}, {"budget_limit": 50.0, "time_period": "7d"}].
- password: Optional[str] - Not supported; any value is rejected with a 422. Users set their own password through an invitation link (POST /invitation/new).
Returns:
- key: (str) The generated api key for the user
- expires: (datetime) Datetime object for when key expires.
@ -524,7 +542,7 @@ async def new_user(
```
"""
try:
from litellm.proxy.proxy_server import _license_check, general_settings, prisma_client
from litellm.proxy.proxy_server import _license_check, prisma_client
if prisma_client is None:
raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value)
@ -572,7 +590,7 @@ async def new_user(
# generate_key_helper_fn only forwards object_permission_id, so without this the entitlement
# the caller sent would be dropped on the floor.
data_json = await _set_object_permission(data_json=data_json, prisma_client=prisma_client)
_hash_password_in_dict(data_json, general_settings)
data_json.pop("password", None)
teams = data.teams
if teams is None:
teams = check_if_default_team_set()
@ -1438,6 +1456,7 @@ async def _update_single_user_helper(
user_request: UpdateUserRequest,
user_api_key_dict: UserAPIKeyAuth,
litellm_changed_by: str | None = None,
password_prevalidated: bool = False,
) -> dict[str, Any]:
"""
Helper function to update a single user.
@ -1460,7 +1479,7 @@ async def _update_single_user_helper(
data_json: Final[dict] = user_request.model_dump(exclude_unset=True)
non_default_values = _update_internal_user_params(data_json=data_json, data=user_request)
_hash_password_in_dict(non_default_values, general_settings)
await _hash_password_in_dict(non_default_values, general_settings, password_prevalidated=password_prevalidated)
existing_user_row: BaseModel | None = None
if user_request.user_id:
@ -1641,7 +1660,7 @@ async def user_update(
Parameters:
- user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated.
- user_email: Optional[str] - Specify a user email.
- password: Optional[str] - Specify a user password.
- password: Optional[str] - Set the user's password (admin only). Must satisfy the configured password policy. The user is required to change it at their next login. Users change their own password with POST /user/password/change.
- user_alias: Optional[str] - A descriptive name for you to know who this user id refers to.
- teams: Optional[list] - specify a list of team id's a user belongs to.
- send_invite_email: Optional[bool] - Specify if an invite email should be sent.
@ -1709,19 +1728,38 @@ async def bulk_update_processed_users(
users_to_update: list[UpdateUserRequest],
user_api_key_dict: UserAPIKeyAuth,
litellm_changed_by: str | None = None,
hibp_client: AsyncHTTPHandler | None = None,
) -> BulkUpdateUserResponse:
from litellm.proxy.proxy_server import general_settings
results: Final[list[UserUpdateResult]] = []
successful_updates = 0
failed_updates = 0
# Screen the batch's passwords upfront and concurrently: done per-user
# inside the loop below, each HIBP lookup would be awaited serially and a
# degraded-slow HIBP could stretch a full batch to minutes, timing out the
# request after some updates already persisted.
password_verdicts: Final = await validate_passwords_bulk(
tuple(u.password for u in users_to_update if u.password is not None),
general_settings,
client=hibp_client,
)
# Process each user update independently
try:
for user_request in users_to_update:
try:
if (
user_request.password is not None
and (password_error := password_verdicts.get(user_request.password)) is not None
):
raise password_error
response = await _update_single_user_helper(
user_request=user_request,
user_api_key_dict=user_api_key_dict,
litellm_changed_by=litellm_changed_by,
password_prevalidated=True,
)
# Record success
results.append(
@ -1859,6 +1897,14 @@ async def bulk_user_update(
status_code=403,
detail="Only proxy admins can update all users at once.",
)
if data.user_updates.password is not None:
bulk_password_error: Final[HTTPExceptionErrorDetail] = {
"error": (
"Setting one password for all users is not supported. "
"Use per-user updates via the 'users' list instead."
)
}
raise HTTPException(status_code=400, detail=bulk_password_error)
# Optimized path for updating all users directly in database
all_users_in_db: Final = await _user_table(prisma_client).find_many(order={"created_at": "desc"})

154
litellm/proxy/management_endpoints/password_endpoints.py Normal file
View file

@ -0,0 +1,154 @@
"""
Self-service password management.
/user/password/change
Deliberately NOT wrapped in `management_endpoint_wrapper`: the wrapper emits
request kwargs to OTEL spans, which would log plaintext passwords. The audit
signal is emitted by hand below, with field names only, never values.
"""
from typing import TYPE_CHECKING, Annotated, Final
from fastapi import APIRouter, Depends, HTTPException
from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import (
UI_TEAM_ID,
ChangePasswordRequest,
ChangePasswordResponse,
CommonProxyErrors,
HTTPExceptionErrorDetail,
LitellmTableNames,
UserAPIKeyAuth,
)
from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
from litellm.proxy.utils import hash_password, verify_password
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.user_repository import UserRepository
if TYPE_CHECKING:
from prisma import models as prisma_models
from prisma import types as prisma_types
from litellm.proxy.utils import PrismaClient
router: Final = APIRouter()
_PASSWORD_CHANGED_AUDIT_VALUES: Final = '{"fields_changed": ["password"]}'
_KEY_METADATA: Final = TypeAdapter(dict[str, object])
def _error_detail(message: str) -> HTTPExceptionErrorDetail:
detail: Final[HTTPExceptionErrorDetail] = {"error": message}
return detail
def _is_password_login_session(user_api_key_dict: UserAPIKeyAuth) -> bool:
if user_api_key_dict.team_id != UI_TEAM_ID:
return False
key_metadata: Final = _KEY_METADATA.validate_python(user_api_key_dict.metadata)
return all(key_metadata.get(k) == v for k, v in PASSWORD_SESSION_METADATA.items())
def _user_table(
prisma_client: "PrismaClient | None",
) -> "TableActions[prisma_models.LiteLLM_UserTable]":
user_table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = UserRepository(prisma_client).table
return user_table
@router.post(
"/user/password/change",
tags=("Internal User management",),
dependencies=(Depends(user_api_key_auth),),
)
async def change_password(
data: ChangePasswordRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ChangePasswordResponse:
"""
Change the calling user's own password.
Only callable with the dashboard session issued by a username/password
login; SSO sessions and virtual keys are rejected with 403. Requires the
current password. The new password must differ from the
current one and satisfy the configured password policy
(`general_settings.password_policy_*`: minimum length, character classes,
and, when enabled, breached-password screening via haveibeenpwned.com).
A successful change lifts any pending forced password reset
(`password_reset_required`) on the account.
Parameters:
- current_password: str - The user's current password.
- new_password: str - The password to change to.
"""
from litellm.proxy.proxy_server import general_settings, litellm_proxy_admin_name, prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail=_error_detail(CommonProxyErrors.db_not_connected_error.value),
)
if not _is_password_login_session(user_api_key_dict):
raise HTTPException(
status_code=403,
detail=_error_detail(
"Passwords can only be changed from a dashboard session created by logging in with a password."
),
)
user_id: Final = user_api_key_dict.user_id
if user_id is None:
raise HTTPException(
status_code=400,
detail=_error_detail("No user is associated with this session, so there is no password to change."),
)
find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id}
user_row: Final = await _user_table(prisma_client).find_first(where=find_user)
stored_password: Final = user_row.password if user_row is not None else None
if stored_password is None:
raise HTTPException(
status_code=400,
detail=_error_detail(
"This account has no password set, so there is no password to change. "
"Passwords are set through an invitation link (POST /invitation/new)."
),
)
if not verify_password(data.current_password, stored_password):
raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect."))
if data.new_password == data.current_password:
raise HTTPException(
status_code=400,
detail=_error_detail("New password must be different from the current password."),
)
validate_password_policy(data.new_password, general_settings)
await validate_password_not_breached(data.new_password, general_settings)
password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {
"password": hash_password(data.new_password),
"password_reset_required": False,
"last_breach_check_at": None,
}
await _user_table(prisma_client).update(where=find_user, data=password_update)
verbose_proxy_logger.info("Password changed via /user/password/change for user_id=%s", user_id)
await create_object_audit_log(
object_id=user_id,
action="updated",
litellm_changed_by=None,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
table_name=LitellmTableNames.USER_TABLE_NAME,
after_value=_PASSWORD_CHANGED_AUDIT_VALUES,
)
return ChangePasswordResponse(user_id=user_id, message="Password updated successfully.")

1
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -3665,6 +3665,7 @@ class SSOAuthenticationHandler:
auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
server_root_path=get_server_root_path(),
password_reset_required=False,
)
from litellm.proxy.auth.login_utils import encode_ui_session_jwt

16
litellm/proxy/proxy_server.py
View file

@ -359,7 +359,7 @@ from litellm.proxy.auth.model_checks import (
get_mcp_server_ids,
get_team_models,
)
from litellm.proxy.auth.password_policy import validate_password_policy
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
from litellm.proxy.auth.user_api_key_auth import (
_fetch_global_spend_with_event_coordination,
user_api_key_auth,
@ -601,6 +601,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
from litellm.proxy.management_endpoints.organization_endpoints import (
router as organization_router,
)
from litellm.proxy.management_endpoints.password_endpoints import (
router as password_management_router,
)
from litellm.proxy.management_endpoints.prompt_caching_requests import (
router as prompt_caching_requests_router,
)
@ -16647,6 +16650,7 @@ async def onboarding(invite_link: str, request: Request):
auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
server_root_path=get_server_root_path(),
password_reset_required=False,
)
jwt_token: Final = jwt.encode(
cast(dict, returned_ui_token_object),
@ -16757,6 +16761,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str:
auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
server_root_path=get_server_root_path(),
password_reset_required=False,
)
assert master_key is not None
return jwt.encode(
@ -16827,6 +16832,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
)
validate_password_policy(data.password, general_settings)
await validate_password_not_breached(data.password, general_settings)
hashed_pw: Final = hash_password(data.password)
current_time = litellm.utils.get_utc_datetime()
async with prisma_client.db.tx() as tx:
@ -16846,7 +16852,12 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
### UPDATE USER OBJECT ###
user_obj: Final[_UserTableRow | None] = await tx.litellm_usertable.update(
where={"user_id": invite_obj.user_id}, data={"password": hashed_pw}
where={"user_id": invite_obj.user_id},
data={
"password": hashed_pw,
"password_reset_required": False,
"last_breach_check_at": None,
},
)
if user_obj is None:
@ -19284,6 +19295,7 @@ app.include_router(pass_through_router)
app.include_router(health_router)
app.include_router(key_management_router)
app.include_router(internal_user_router)
app.include_router(password_management_router)
app.include_router(team_router)
app.include_router(ui_sso_router)
app.include_router(organization_router)

2
litellm/proxy/schema.prisma
View file

@ -246,6 +246,8 @@ model LiteLLM_UserTable {
organization_id String?
object_permission_id String?
password String?
password_reset_required Boolean?
last_breach_check_at DateTime?
teams String[] @default([])
user_role String?
max_budget Float?

1
litellm/types/llms/custom_http.py
View file

@ -31,6 +31,7 @@ class httpxSpecialProvider(str, Enum):
UI = "ui"
Sandbox = "sandbox"
ModelCostMap = "model_cost_map"
PasswordBreachCheck = "password_breach_check"
VerifyTypes = str | bool | ssl.SSLContext

3
litellm/types/proxy/ui_sso.py
View file

@ -1,6 +1,6 @@
from typing import Literal
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
class ReturnedUITokenObject(TypedDict):
@ -17,6 +17,7 @@ class ReturnedUITokenObject(TypedDict):
auth_header_name: str
disabled_non_admin_personal_key_creation: bool
server_root_path: str # e.g. `/litellm`
password_reset_required: ReadOnly[bool]
class ParsedOpenIDResult(TypedDict, total=False):

26
model_prices_and_context_window.json
View file

@ -43037,21 +43037,21 @@
"supports_web_search": false
},
"openrouter/deepseek/deepseek-v4-pro": {
"input_cost_per_token": 8.95578e-07,
"input_cost_per_token": 8.92272e-07,
"input_cost_per_token_cache_hit": 4.4e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"max_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.791156e-06,
"output_cost_per_token": 1.784544e-06,
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 7.46315e-08,
"cache_read_input_token_cost": 7.4356e-08,
"supports_audio_input": false,
"supports_pdf_input": false,
"supports_vision": false,
@ -68212,13 +68212,13 @@
"supports_web_search": false
},
"openrouter/z-ai/glm-5.3-flash": {
"input_cost_per_token": 7.5e-08,
"output_cost_per_token": 2.5e-07,
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 5e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 1310720,
"max_output_tokens": 102400,
"max_tokens": 102400,
"max_output_tokens": 943718,
"max_tokens": 943718,
"mode": "chat",
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
@ -73252,14 +73252,14 @@
"supports_web_search": true
},
"openrouter/~z-ai/glm-flash-latest": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 7.5e-08,
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1310720,
"max_output_tokens": 102400,
"max_tokens": 102400,
"max_output_tokens": 943718,
"max_tokens": 943718,
"mode": "chat",
"output_cost_per_token": 2.5e-07,
"output_cost_per_token": 5e-07,
"source": "https://openrouter.ai/api/v1/models",
"supports_audio_input": false,
"supports_function_calling": true,

2
schema.prisma
View file

@ -246,6 +246,8 @@ model LiteLLM_UserTable {
organization_id String?
object_permission_id String?
password String?
password_reset_required Boolean?
last_breach_check_at DateTime?
teams String[] @default([])
user_role String?
max_budget Float?

65
scripts/check_mcp_operation_boundary.py Normal file
View file

@ -0,0 +1,65 @@
import ast
import sys
from pathlib import Path
from typing import Final
PACKAGE: Final = Path("litellm/proxy/_experimental/mcp_server")
LEGACY_ADAPTERS: Final = frozenset({"server.py", "legacy_callbacks.py", "mcp_context.py", "mcp_debug.py"})
CONFINED_NAMES: Final = frozenset(
{
"auth_context_var",
"active_mcp_session_var",
"active_mcp_request_ctx_var",
"get_active_auth_context",
"get_active_mcp_session",
"get_active_mcp_request_ctx",
"get_or_extract_auth_context",
"_session_obj_auth_storage",
"WeakKeyDictionary",
"_mcp_active_toolset_id",
"_mcp_gateway_initialize_instructions",
"_mcp_gateway_server_name",
"_mcp_proxy_mode",
}
)
def is_confined(name: str) -> bool:
return name in CONFINED_NAMES or name.startswith("_stateful_session_")
def violations(path: Path, source: str) -> tuple[str, ...]:
if path.name in LEGACY_ADAPTERS:
return ()
tree: Final = ast.parse(source, filename=str(path))
return tuple(
f"{path}:{node.lineno}: MCP request/session state belongs in a legacy adapter"
for node in ast.walk(tree)
if (
isinstance(node, ast.ImportFrom)
and (
(node.module or "").endswith(".mcp_context")
or any(is_confined(alias.name) for alias in node.names)
or (path.name in {"operations.py", "contracts.py"} and (node.module or "").endswith(".server"))
)
or isinstance(node, ast.Name)
and is_confined(node.id)
or isinstance(node, ast.Attribute)
and is_confined(node.attr)
)
)
def main() -> int:
findings: Final = tuple(
finding for path in sorted(PACKAGE.rglob("*.py")) for finding in violations(path, path.read_text())
)
if findings:
print("\n".join(findings), file=sys.stderr)
return 1
print("MCP operation boundary: passed")
return 0
if __name__ == "__main__":
sys.exit(main())

3
scripts/pre_commit_lint.sh
View file

@ -102,6 +102,9 @@ ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|s
ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$'
litellm_py_files=$(scope_match "$litellm_py_pattern")
if [ -n "$(scope_match '^(litellm/proxy/_experimental/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then
uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1
fi
e2e_py_files=$(scope_match "$e2e_py_pattern")
test_tree_files=$(scope_match "$test_tree_pattern")
# ruff format (and CI's format step) skip enterprise; the rest of make lint covers it.

1
terraform/provider/tools/endpointaudit/coverage_allowlist.txt
View file

@ -86,6 +86,7 @@ POST /team/key/bulk_update
POST /team/permissions_bulk_update
POST /team/{team_id}/disable_logging
POST /user/bulk_update
POST /user/password/change
# Alternate method or path for functionality the provider already manages elsewhere
GET /credentials/by_model/{model_id}

6
tests/mcp_tests/test_mcp_logging.py
View file

@ -142,7 +142,7 @@ async def test_mcp_cost_tracking():
local_mcp_server_manager,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
local_mcp_server_manager,
),
):
@ -293,7 +293,7 @@ async def test_mcp_cost_tracking_per_tool():
local_mcp_server_manager,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
local_mcp_server_manager,
),
):
@ -451,7 +451,7 @@ async def test_mcp_tool_call_hook():
local_mcp_server_manager,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
local_mcp_server_manager,
),
):

24
tests/mcp_tests/test_mcp_server.py
View file

@ -922,7 +922,7 @@ async def test_get_tools_from_mcp_servers():
)
with patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
mock_manager,
):
# Test with specific servers
@ -950,6 +950,7 @@ async def test_get_tools_from_mcp_servers():
extra_headers=None,
add_prefix=False,
raw_headers=None,
client_ip=None,
user_api_key_auth=None,
oauth2_headers=None,
):
@ -966,7 +967,7 @@ async def test_get_tools_from_mcp_servers():
)
with patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
mock_manager_2,
):
result = await _get_tools_from_mcp_servers(
@ -998,7 +999,7 @@ async def test_get_tools_from_mcp_servers():
)
with patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
mock_manager,
):
with patch(
@ -1981,6 +1982,7 @@ async def test_get_tools_for_single_server():
extra_headers=None,
add_prefix=False,
raw_headers=None,
client_ip=None,
user_api_key_auth=None,
)
@ -2076,7 +2078,7 @@ async def test_rest_listing_hides_key_grants_dispatch_would_refuse():
with patch(
"litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager"
) as mock_manager, patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
) as mock_server_manager, patch.object(
MCPRequestHandler,
"get_allowed_tools_for_server",
@ -2473,7 +2475,7 @@ async def test_filter_tools_by_allowed_tools_integration():
# Mock the global MCP server manager
with patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
) as mock_manager:
# Mock manager methods
mock_manager.get_allowed_mcp_servers = AsyncMock(
@ -2588,7 +2590,7 @@ async def test_filter_tools_by_disallowed_tools_integration():
# Mock the global MCP server manager
with patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
) as mock_manager:
# Mock manager methods
mock_manager.get_allowed_mcp_servers = AsyncMock(
@ -2689,7 +2691,7 @@ async def test_filter_tools_no_restrictions_integration():
# Mock the global MCP server manager
with patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager"
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager"
) as mock_manager:
# Mock manager methods
mock_manager.get_allowed_mcp_servers = AsyncMock(
@ -2970,10 +2972,10 @@ async def test_call_mcp_tool_uses_manager_permission_lookup():
return_value=mock_server,
) as mock_get_server,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry"
"litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry"
) as mock_tool_registry,
patch(
"litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool",
new_callable=AsyncMock,
) as mock_handle_managed,
patch(
@ -3046,10 +3048,10 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission
return_value=mock_server,
) as mock_get_server,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry"
"litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry"
) as mock_tool_registry,
patch(
"litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool",
new_callable=AsyncMock,
) as mock_handle_managed,
patch(

13
tests/otel_tests/test_e2e_budgeting.py
View file

@ -5,6 +5,7 @@ import uuid
from typing import Any, Optional
import aiohttp
import openai
import pytest
from httpx import AsyncClient
@ -23,7 +24,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k
call_count += 1
await asyncio.sleep(0.1) # allow spend tracking to catch up
pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls")
except Exception as e:
except openai.APIStatusError as e:
print("vars: ", vars(e))
print("e.body: ", e.body)
@ -32,8 +33,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k
# Check error structure and values that should be consistent
assert (
error_dict["code"] == "429"
), f"Expected error code 429, got: {error_dict['code']}"
error_dict["code"] == "422"
), f"Expected error code 422, got: {error_dict['code']}"
assert (
error_dict["type"] == "budget_exceeded"
), f"Expected error type budget_exceeded, got: {error_dict['type']}"
@ -506,9 +507,9 @@ async def make_calls_until_team_budget_exceeded_cli_sso(
call_count += 1
await asyncio.sleep(0.1)
pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls")
except Exception as e:
except openai.APIStatusError as e:
error_dict = e.body
assert error_dict["code"] == "429"
assert error_dict["code"] == "422"
assert error_dict["type"] == "budget_exceeded"
message = error_dict["message"]
assert "Budget has been exceeded!" in message
@ -556,7 +557,7 @@ async def test_team_budget_enforcement_cli_sso_token():
1. Create team with a tiny max_budget and a user on that team
2. Obtain a CLI SSO JWT (HTTP poll flow when Redis is shared, else mint)
3. Make chat completion calls until the team budget is exceeded
4. Verify HTTP 429 budget_exceeded names the team
4. Verify HTTP 422 budget_exceeded names the team
"""
user_id = f"cli-budget-user-{uuid.uuid4().hex[:8]}"
user_email = f"{user_id}@example.com"

50
tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
View file

@ -2297,6 +2297,56 @@ class TestStructuredMessagesWriteBack:
}
assert result["input"][3] == {"role": "user", "content": "What is the codename?"}
@pytest.mark.asyncio
async def test_codex_custom_tool_items_survive_tool_output_compression(self):
handler = OpenAIResponsesHandler()
additional_tools_item = {
"type": "additional_tools",
"tools": [{"type": "custom", "name": "exec", "description": "Run a JavaScript snippet"}],
}
reasoning_item = {
"id": "rs_456",
"type": "reasoning",
"summary": [],
"encrypted_content": "gAAAAA-signed-reasoning",
}
custom_tool_call_item = {
"id": "ctc_456",
"type": "custom_tool_call",
"call_id": "call_exec",
"name": "exec",
"input": 'const r = await tools.exec_command({"cmd": "cat memo.txt"});\ntext(r.output);',
"status": "completed",
}
data = {
"model": "gpt-5.6",
"input": [
additional_tools_item,
{"role": "user", "content": "What is the codename?"},
reasoning_item,
custom_tool_call_item,
{
"type": "custom_tool_call_output",
"call_id": "call_exec",
"output": [
{"type": "input_text", "text": "Script completed\nOutput:\n"},
{"type": "input_text", "text": "memo " * 400},
],
},
],
}
result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail())
assert result["input"][0] is additional_tools_item
assert result["input"][1] == {"role": "user", "content": "What is the codename?"}
assert result["input"][2] is reasoning_item
assert result["input"][3] is custom_tool_call_item
assert result["input"][4]["type"] == "custom_tool_call_output"
assert result["input"][4]["call_id"] == "call_exec"
assert COMPRESSED_MARKER in str(result["input"][4]["output"])
assert len(result["input"]) == 5
@pytest.mark.asyncio
async def test_web_search_call_item_preserved_verbatim(self):
handler = OpenAIResponsesHandler()

10
tests/test_litellm/llms/xai/test_xai_chat_transformation.py
View file

@ -90,6 +90,16 @@ class TestXAIReasoningTokenFolding:
assert response.usage.total_tokens == 999
def test_max_completion_tokens_is_accepted_and_mapped_to_max_tokens() -> None:
optional_params = litellm.get_optional_params(
model="grok-4.20",
custom_llm_provider="xai",
max_completion_tokens=64,
)
assert optional_params["max_tokens"] == 64, optional_params
assert "max_completion_tokens" not in optional_params, optional_params
class TestXAIParallelToolCalls:
"""Test suite for XAI parallel tool calls functionality."""

19
tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py
View file

@ -1,3 +1,4 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
"""
Unit tests for the BYOK OAuth 2.1 authorization server endpoints.
@ -592,7 +593,7 @@ async def test_check_byok_credential_missing_credential(monkeypatch):
monkeypatch.delenv("PROXY_BASE_URL", raising=False)
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
server_module.byok_credential_cache.flush_cache()
mcp_operations.byok_credential_cache.flush_cache()
mock_prisma = MagicMock()
with (
@ -628,13 +629,13 @@ async def test_execute_byok_tool_missing_credential_advertises_api_key_flow(monk
from litellm.types.mcp_server.mcp_server_manager import MCPServer
monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy")
mcp_module.byok_credential_cache.flush_cache()
mcp_operations.byok_credential_cache.flush_cache()
server = MCPServer(server_id="byok-discovery", name="byok-discovery", transport=MCPTransport.http, is_byok=True)
prisma = MagicMock()
prisma.db.litellm_mcpusercredentials.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(HTTPException) as exc_info:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="list_regions",
arguments={},
allowed_mcp_servers=[server],
@ -687,7 +688,7 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same
server = MCPServer(server_id="byok-revoke", name="byok-server", transport=MCPTransport.http, is_byok=True)
user_auth = UserAPIKeyAuth(user_id="mallory", api_key="sk-test")
server_module.byok_credential_cache.flush_cache()
mcp_operations.byok_credential_cache.flush_cache()
db_lookup = AsyncMock(side_effect=["sk-before-revoke", None])
publish = AsyncMock()
@ -699,13 +700,13 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same
"litellm.proxy.proxy_server.prisma_client", MagicMock()
),
patch.object( # test-quality-ok: the redis publisher is module-level; asserting the broadcast without a redis
server_module, "publish_auth_cache_invalidation", new=publish
mcp_operations, "publish_auth_cache_invalidation", new=publish
),
):
assert await server_module._get_byok_credential(server, user_auth) == "sk-before-revoke"
assert await server_module._get_byok_credential(server, user_auth) == "sk-before-revoke"
await server_module._invalidate_byok_cred_cache("mallory", "byok-revoke")
assert await server_module._get_byok_credential(server, user_auth) is None
assert await mcp_operations._get_byok_credential(server, user_auth) == "sk-before-revoke"
assert await mcp_operations._get_byok_credential(server, user_auth) == "sk-before-revoke"
await mcp_operations._invalidate_byok_cred_cache("mallory", "byok-revoke")
assert await mcp_operations._get_byok_credential(server, user_auth) is None
assert db_lookup.await_count == 2
publish.assert_awaited_once_with(cache_key=byok_credential_cache_key("mallory", "byok-revoke"))

60
tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py Normal file
View file

@ -0,0 +1,60 @@
from dataclasses import FrozenInstanceError
import pytest
from litellm.proxy._experimental.mcp_server.operations import prepare_context
from litellm.proxy._types import UserAPIKeyAuth
def test_operation_context_isolates_nested_headers_and_caller_permissions():
caller = UserAPIKeyAuth(user_id="alpha", models=["allowed"])
caller.mcp_admitted_user_subject = True
caller.mcp_session_resource_server_id = "alpha-server"
caller.mcp_toolset_id = "toolset-alpha"
caller.mcp_source_team_rpm_limits = {"team": {"alpha-server": 2}}
headers = {"x-caller": "alpha"}
server_headers = {"alpha-server": {"authorization": "alpha-token"}}
context = prepare_context(caller, raw_headers=headers, mcp_server_auth_headers=server_headers)
caller.models.append("forbidden")
caller.mcp_source_team_rpm_limits["team"]["alpha-server"] = 999
headers["x-caller"] = "bravo"
server_headers["alpha-server"]["authorization"] = "bravo-token"
captured = context.user_api_key_auth
assert captured is not None
assert captured.models == ["allowed"]
assert captured.mcp_admitted_user_subject is True
assert captured.mcp_session_resource_server_id == "alpha-server"
assert captured.mcp_toolset_id == "toolset-alpha"
assert captured.mcp_source_team_rpm_limits == {"team": {"alpha-server": 2}}
captured.models.append("also-forbidden")
assert context.user_api_key_auth.models == ["allowed"]
assert context.raw_headers == {"x-caller": "alpha"}
assert context.mcp_server_auth_headers == {"alpha-server": {"authorization": "alpha-token"}}
with pytest.raises(TypeError):
context.raw_headers["x-caller"] = "changed"
with pytest.raises(TypeError):
context.mcp_server_auth_headers["alpha-server"]["authorization"] = "changed"
with pytest.raises(FrozenInstanceError):
context.client_ip = "untrusted"
def test_operation_context_preserves_missing_and_empty_inputs():
missing = prepare_context()
empty = prepare_context(mcp_servers=[], raw_headers={}, oauth2_headers={}, mcp_server_auth_headers={})
assert missing.user_api_key_auth is None
assert missing.mcp_servers is None
assert missing.raw_headers is None
assert missing.oauth2_headers is None
assert missing.mcp_server_auth_headers is None
assert empty.mcp_servers == ()
assert empty.raw_headers == {}
assert empty.oauth2_headers == {}
assert empty.mcp_server_auth_headers == {}
def test_toolset_request_marker_cannot_be_supplied_by_caller_or_serialized():
auth = UserAPIKeyAuth.model_validate({"user_id": "alpha", "mcp_toolset_id": "forged"})
assert auth.mcp_toolset_id is None
auth.mcp_toolset_id = "server-resolved"
assert "mcp_toolset_id" not in auth.model_dump()

5
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py
View file

@ -1,5 +1,6 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
"""Tests for guardrail-block recording in
``litellm.proxy._experimental.mcp_server.server.call_mcp_tool``.
``litellm.proxy._experimental.mcp_server.operations.call_mcp_tool``.
A pre-call MCP guardrail block *raises* into ``call_mcp_tool``'s
``except Exception``. The failure spend-log row that the Guardrails Monitor's
@ -70,7 +71,7 @@ async def _call_block(logging_obj, order: list, *, user_api_key_auth=mock.sentin
with mock.patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}):
with contextlib.suppress(HTTPException):
await server.call_mcp_tool.__wrapped__(
await mcp_operations.call_mcp_tool.__wrapped__(
name="t",
arguments=None,
user_api_key_auth=user_api_key_auth,

6
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py
View file

@ -1229,7 +1229,7 @@ class TestResolveByokMcpAuthHeader:
user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard")
with patch(
"litellm.proxy._experimental.mcp_server.server._get_byok_credential",
"litellm.proxy._experimental.mcp_server.operations._get_byok_credential",
new=AsyncMock(return_value="stored-cred"),
):
result = await _resolve_byok_mcp_auth_header(server, user_auth, None)
@ -1249,7 +1249,7 @@ class TestResolveByokMcpAuthHeader:
user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard")
with patch(
"litellm.proxy._experimental.mcp_server.server._get_byok_credential",
"litellm.proxy._experimental.mcp_server.operations._get_byok_credential",
new=AsyncMock(return_value=None),
):
with pytest.raises(HTTPException) as exc_info:
@ -1272,7 +1272,7 @@ class TestResolveByokMcpAuthHeader:
check_mock = AsyncMock(return_value=None)
with patch(
"litellm.proxy._experimental.mcp_server.server._check_byok_credential",
"litellm.proxy._experimental.mcp_server.operations._check_byok_credential",
new=check_mock,
):
result = await _resolve_byok_mcp_auth_header(server, user_auth, "caller-header")

61
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
View file

@ -1,3 +1,4 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
"""Unit tests for MCP OAuth passthrough tool-fetch behavior."""
import logging
@ -339,16 +340,16 @@ async def test_aggregate_list_tools_absorbs_one_unauthenticated_server():
raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
return [good_tool]
with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object(
mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object(
mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
), patch.object(
mcp_server, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools)
mcp_operations, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools)
), patch.object(
mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
):
listing = await mcp_server._get_tools_from_mcp_servers(
listing = await mcp_operations._get_tools_from_mcp_servers(
user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
mcp_auth_header=None,
mcp_servers=None,
@ -382,14 +383,14 @@ async def test_single_server_route_also_absorbs_upstream_auth_error():
# /<server>/mcp sets the path-derived single-server scope; absorption must hold even then.
token = _mcp_gateway_server_name.set("delegate_docs")
try:
with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
), patch.object(
mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
):
listing = await mcp_server._get_tools_from_mcp_servers(
listing = await mcp_operations._get_tools_from_mcp_servers(
user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
mcp_auth_header=None,
mcp_servers=["delegate_docs"],
@ -419,15 +420,15 @@ async def test_aggregate_with_single_accessible_server_still_absorbs():
async def fake_get_tools(server, **kwargs):
raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name)
with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object(
mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={})
), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object(
mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None)
), patch.object(
mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools)
):
# Aggregate route: no explicit server filter, even though only one server is accessible.
listing = await mcp_server._get_tools_from_mcp_servers(
listing = await mcp_operations._get_tools_from_mcp_servers(
user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"),
mcp_auth_header=None,
mcp_servers=None,
@ -475,3 +476,25 @@ async def test_client_creation_failure_logs_sanitized_exchange(monkeypatch, capl
await manager._get_tools_from_server(server)
assert "POST https://upstream/ -> HTTP 500" in caplog.text
assert "missing_scope" in caplog.text and "query-secret" not in caplog.text
@pytest.mark.parametrize(
"oauth_headers,server_headers,authorized",
[
({"Authorization": "Bearer upstream"}, None, True),
({"AUTHORIZATION": "Bearer upstream"}, None, True),
({"x-unrelated": "present"}, None, False),
(None, {"catalog": {"Authorization": "Bearer scoped"}}, True),
(None, {"other-server": {"Authorization": "Bearer unrelated"}}, False),
(None, {"catalog": {"x-unrelated": "present"}}, False),
(None, {"catalog": "Bearer legacy"}, True),
(None, {"catalog": " "}, False),
],
)
def test_passthrough_admission_recognizes_only_matching_authorization(oauth_headers, server_headers, authorized):
from litellm.proxy._experimental.mcp_server.operations import _client_has_passthrough_authorization
from litellm.types.mcp import MCPTransport
from litellm.types.mcp_server.mcp_server_manager import MCPServer
server = MCPServer(server_id="catalog", name="catalog", alias="catalog", transport=MCPTransport.http)
assert _client_has_passthrough_authorization(server, oauth_headers, server_headers) is authorized

8
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py
View file

@ -1,3 +1,4 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
import json
from datetime import datetime
@ -27,8 +28,8 @@ def proxy_mode():
@pytest.mark.asyncio
@pytest.mark.usefixtures("proxy_mode")
async def test_proxy_call_rejects_non_proxy_tool_names() -> None:
result = await server._dispatch_virtual_mcp_tool(
name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None
result = await mcp_operations._dispatch_virtual_mcp_tool(
name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None, mcp_proxy_mode=True
)
assert result is not None
@ -105,12 +106,13 @@ async def test_proxy_scope_exception_emits_failure_log(monkeypatch: pytest.Monke
arguments = {"tool_id": "denied-scope", "arguments": {}}
with pytest.raises(HTTPException) as denied:
await server._dispatch_virtual_mcp_tool(
await mcp_operations._dispatch_virtual_mcp_tool(
name="call_tool",
arguments=arguments,
user_api_key_auth=auth,
client_ip=None,
mcp_servers=["ungranted"],
mcp_proxy_mode=True,
raw_headers={"authorization": "Bearer raw-scope-secret", "x-litellm-call-id": "scope-denial"},
)

385
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
View file

File diff suppressed because it is too large Load diff

179
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
View file

@ -73,6 +73,135 @@ from litellm.proxy.utils import ProxyLogging
from litellm.types.guardrails import GuardrailEventHooks
@pytest.mark.asyncio
async def test_manager_sampling_preserves_explicit_headers_without_ambient_context():
from litellm.proxy._experimental.mcp_server import server as legacy_server
caller = UserAPIKeyAuth(user_id="sampling-caller")
upstream = MCPServer(
server_id="sampling-context",
name="sampling_context",
url="https://example.invalid/mcp",
transport=MCPTransport.http,
allow_sampling=True,
)
sampling = AsyncMock()
client = MagicMock()
client.call_tool = AsyncMock(return_value=CallToolResult(content=[]))
assert legacy_server.get_active_auth_context() is None
with (
patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient", return_value=client) as factory,
patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
):
await MCPServerManager()._call_regular_mcp_tool(
mcp_server=upstream,
original_tool_name="probe",
arguments={},
tasks=[],
mcp_auth_header=None,
mcp_server_auth_headers=None,
oauth2_headers=None,
raw_headers={"x-test-caller": "sampling-caller"},
proxy_logging_obj=None,
user_api_key_auth=caller,
)
callback = factory.call_args.kwargs["sampling_callback"]
await callback(None, None)
assert sampling.await_args.kwargs["user_api_key_auth"].user_id == "sampling-caller"
assert sampling.await_args.kwargs["raw_headers"] == {"x-test-caller": "sampling-caller"}
@pytest.mark.asyncio
async def test_sampling_callback_keeps_creation_context_after_caller_switch():
from mcp.server.auth.middleware.auth_context import auth_context_var
from litellm.proxy._experimental.mcp_server import server as legacy_server
from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback
token = auth_context_var.set(None)
recorder = AsyncMock()
try:
original = UserAPIKeyAuth(user_id="alpha", models=["alpha-model"])
original.mcp_admitted_user_subject = True
headers = {"x-caller": "alpha"}
legacy_server.set_auth_context(original, raw_headers=headers, client_ip="192.0.2.1")
callback = _create_sampling_callback()
original.models.append("bravo-model")
headers["x-caller"] = "bravo"
legacy_server.set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"})
with patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", recorder):
await callback(None, None)
observed = recorder.await_args.kwargs
assert observed["user_api_key_auth"].user_id == "alpha"
assert observed["user_api_key_auth"].models == ["alpha-model"]
assert observed["user_api_key_auth"].mcp_admitted_user_subject is True
assert observed["raw_headers"] == {"x-caller": "alpha"}
assert observed["client_ip"] == "192.0.2.1"
finally:
auth_context_var.reset(token)
@pytest.mark.asyncio
async def test_elicitation_callback_keeps_initiating_session():
from litellm.proxy._experimental.mcp_server import server as legacy_server
from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_elicitation_callback
initiating = MagicMock()
replacement = MagicMock()
recorder = AsyncMock()
token = legacy_server.active_mcp_session_var.set(initiating)
try:
callback = _create_elicitation_callback()
legacy_server.active_mcp_session_var.set(replacement)
with patch("litellm.proxy._experimental.mcp_server.elicitation_handler.handle_elicitation_request", recorder):
await callback(None, None)
assert recorder.await_args.kwargs["downstream_session"] is initiating
assert recorder.await_args.kwargs["downstream_capabilities"] is initiating.capabilities
finally:
legacy_server.active_mcp_session_var.reset(token)
@pytest.mark.asyncio
async def test_sampling_callbacks_isolate_callers_and_cancellation():
from mcp.types import ErrorData
from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback
started = asyncio.Event()
cancelled = asyncio.Event()
observed = {}
async def record_sampling(*, user_api_key_auth, raw_headers, **kwargs):
label = user_api_key_auth.user_id
if label == "cancelled":
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
raise
await asyncio.sleep(0)
observed[label] = raw_headers["x-caller"]
return ErrorData(code=-1, message=label)
callbacks = tuple(
_create_sampling_callback(UserAPIKeyAuth(user_id=label), raw_headers={"x-caller": label})
for label in ("alpha", "bravo", "cancelled")
)
with patch(
"litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", record_sampling
):
tasks = tuple(asyncio.create_task(callback(None, None)) for callback in callbacks)
await asyncio.wait_for(started.wait(), timeout=2)
tasks[2].cancel()
results = await asyncio.gather(*tasks, return_exceptions=True)
assert observed == {"alpha": "alpha", "bravo": "bravo"}
assert [result.message for result in results[:2]] == ["alpha", "bravo"]
assert isinstance(results[2], asyncio.CancelledError)
assert cancelled.is_set()
def _reload_mcp_manager_module():
utils_module = sys.modules["litellm.proxy._experimental.mcp_server.utils"]
manager_module = sys.modules["litellm.proxy._experimental.mcp_server.mcp_server_manager"]
@ -84,6 +213,9 @@ def _reload_mcp_manager_module():
server_module = sys.modules.get("litellm.proxy._experimental.mcp_server.server")
if server_module is not None and hasattr(server_module, "global_mcp_server_manager"):
server_module.global_mcp_server_manager = reloaded.global_mcp_server_manager
operations_module = sys.modules.get("litellm.proxy._experimental.mcp_server.operations")
if operations_module is not None:
operations_module.global_mcp_server_manager = reloaded.global_mcp_server_manager
return reloaded
@ -3923,6 +4055,7 @@ class TestMCPServerManager:
result = await manager.get_resource_templates_from_server(
server=server,
user_api_key_auth=None,
raw_headers=None,
mcp_auth_header="auth",
extra_headers=None,
add_prefix=False,
@ -3935,6 +4068,8 @@ class TestMCPServerManager:
stdio_env=None,
subject_token=None,
user_api_key_auth=None,
raw_headers=None,
client_ip=None,
)
mock_client.list_resource_templates.assert_awaited_once()
assert result == expected_templates
@ -5849,7 +5984,7 @@ class TestMCPServerManager:
stored = {"Authorization": "Bearer stored-user-token"}
with patch(
"litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
"litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
new=AsyncMock(return_value=stored),
) as mock_lookup:
result = await manager._resolve_oauth2_headers_for_tool_call(
@ -5876,7 +6011,7 @@ class TestMCPServerManager:
user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice")
with patch(
"litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
"litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
new=AsyncMock(return_value={"Authorization": "Bearer should-not-be-used"}),
) as mock_lookup:
result = await manager._resolve_oauth2_headers_for_tool_call(
@ -5902,7 +6037,7 @@ class TestMCPServerManager:
user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice")
with patch(
"litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
"litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
new=AsyncMock(side_effect=RuntimeError("redis down")),
):
result = await manager._resolve_oauth2_headers_for_tool_call(
@ -6058,7 +6193,7 @@ class TestMCPServerManager:
user_auth = UserAPIKeyAuth(api_key="sk-test")
with patch(
"litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
"litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
new=AsyncMock(return_value={"Authorization": "Bearer x"}),
) as mock_lookup:
result = await manager._resolve_oauth2_headers_for_tool_call(
@ -6862,7 +6997,8 @@ class TestMCPServerManager:
}
user_api_key_auth = UserAPIKeyAuth(api_key="sk-test", user_id="user-123")
token = _mcp_active_toolset_id.set("toolset-abc")
user_api_key_auth.mcp_toolset_id = "toolset-abc"
token = _mcp_active_toolset_id.set("unrelated-ambient-toolset")
try:
with (
patch.object(proxy_server_module, "user_api_key_cache", cache),
@ -14332,3 +14468,36 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon
assert guardrail_started.is_set() is selected
assert result.is_error is False
assert result.content[0].text == "executed"
@pytest.mark.asyncio
@pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)])
async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory):
from mcp.server.auth.middleware.auth_context import auth_context_var
from litellm.proxy._experimental.mcp_server import server as legacy_server
from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback
upstream = MCPServer(server_id="explicit-empty", name="explicit_empty", url="https://example.invalid/mcp", transport=MCPTransport.http, allow_sampling=True)
token = auth_context_var.set(None)
sampling = AsyncMock()
try:
legacy_server.set_auth_context(UserAPIKeyAuth(user_id="unrelated"), raw_headers={"authorization": "unrelated-credential"}, client_ip="192.0.2.99")
with (
patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as factory,
patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
):
if legacy_factory:
callback = _create_sampling_callback(user_api_key_auth=UserAPIKeyAuth(user_id="explicit"))
else:
await MCPServerManager()._create_mcp_client(upstream, user_api_key_auth=UserAPIKeyAuth(user_id="explicit") if with_caller else None)
callback = factory.call_args.kwargs["sampling_callback"]
await callback(None, None)
captured = sampling.await_args.kwargs
if with_caller:
assert captured["user_api_key_auth"].user_id == "explicit"
else:
assert captured["user_api_key_auth"] is None
assert captured["raw_headers"] is None
assert captured["client_ip"] is None
finally:
auth_context_var.reset(token)

40
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py
View file

@ -639,12 +639,12 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401():
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
new_callable=AsyncMock,
return_value=False,
) as mock_has_token,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=oauth_server,
),
patch.object(
@ -727,12 +727,12 @@ async def test_admitted_subject_missing_stored_token_challenged_with_resource_me
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
new_callable=AsyncMock,
return_value=False,
) as mock_has_token,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=oauth_server,
),
patch.object(
@ -833,11 +833,11 @@ async def test_client_credentials_server_is_not_preemptively_challenged(m2m_fiel
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
new_callable=AsyncMock,
) as mock_has_token,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=m2m_server,
),
patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request,
@ -929,16 +929,16 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db",
"litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db",
new_callable=AsyncMock,
return_value=None,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=delegated_server,
),
patch( # test-quality-ok: registry is empty in unit tests; key owns the delegated server
"litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
"litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=[delegated_server],
),
@ -1022,12 +1022,12 @@ async def test_per_user_oauth_with_stored_token_skips_preemptive_401():
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
new_callable=AsyncMock,
return_value=True,
) as mock_has_token,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=oauth_server,
),
patch.object(
@ -1126,11 +1126,11 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token",
new_callable=AsyncMock,
) as mock_has_token,
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=delegated_server,
),
patch.object(
@ -1218,7 +1218,7 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns
return_value=False,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=obo_server,
),
patch.object(
@ -1317,7 +1317,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_g
True,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=od_server,
),
patch.object(
@ -1391,7 +1391,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk
new_callable=AsyncMock,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=od_server,
),
patch.object(
@ -1453,7 +1453,7 @@ async def _run_passthrough_connect(
new_callable=AsyncMock,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=server,
),
patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request,
@ -1574,7 +1574,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface
return_value=probe_client,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=tp_server,
),
patch.object(
@ -1642,7 +1642,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges
return_value=probe_client,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=bridge_server,
),
patch.object(
@ -1720,7 +1720,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_prob
return_value=probe_client,
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name",
return_value=tp_server,
),
patch.object(

40
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
View file

@ -1,3 +1,4 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
"""
Tests for MCP tool search feature.
@ -572,7 +573,7 @@ class TestCallToolRestApiVirtualTools:
mock_tool.input_schema = {"type": "object", "properties": {}}
with patch(
"litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
"litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
new_callable=AsyncMock,
return_value=AggregateToolListing(tools=[mock_tool], outcomes={}),
):
@ -616,12 +617,12 @@ class TestCallToolRestApiVirtualTools:
with (
patch(
"litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
"litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=[MagicMock()],
),
patch(
"litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
new_callable=AsyncMock,
return_value=fake_result,
) as mock_execute,
@ -669,12 +670,12 @@ class TestCallToolRestApiVirtualTools:
return_value="203.0.113.7",
),
patch(
"litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
"litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=[MagicMock()],
) as mock_allowed,
patch(
"litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
new_callable=AsyncMock,
return_value=fake_result,
),
@ -699,7 +700,7 @@ class TestCallToolRestApiVirtualTools:
return_value="203.0.113.7",
),
patch(
"litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
"litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
new_callable=AsyncMock,
return_value=AggregateToolListing(tools=[], outcomes={}),
) as mock_list,
@ -832,7 +833,7 @@ class TestCallToolRestApiVirtualTools:
"litellm.proxy.proxy_server.proxy_logging_obj", key_limits
),
patch( # test-quality-ok: the authorized catalog is the seam every virtual tool shares; the ranking under test stays real
"litellm.proxy._experimental.mcp_server.server._list_mcp_tools",
"litellm.proxy._experimental.mcp_server.operations._list_mcp_tools",
new_callable=AsyncMock,
return_value=AggregateToolListing(tools=list(CATALOG), outcomes={}),
) as mock_list,
@ -939,7 +940,7 @@ class TestDispatchVirtualMcpTool:
new_callable=AsyncMock,
return_value="SEARCH_RESULT",
) as mock_search:
result = await srv._dispatch_virtual_mcp_tool(
result = await mcp_operations._dispatch_virtual_mcp_tool(
name=MCP_TOOL_SEARCH_TOOL_NAME,
arguments={"query": "q", "top_k": 3},
user_api_key_auth=uak,
@ -961,7 +962,7 @@ class TestDispatchVirtualMcpTool:
new_callable=AsyncMock,
return_value="AGENT_RESULT",
) as mock_agent_search:
result = await srv._dispatch_virtual_mcp_tool(
result = await mcp_operations._dispatch_virtual_mcp_tool(
name=AGENT_SEARCH_TOOL_NAME,
arguments={"query": "translate a document", "top_k": "2"},
user_api_key_auth=uak,
@ -996,7 +997,7 @@ class TestDispatchVirtualMcpTool:
new_callable=AsyncMock,
return_value="CALL_RESULT",
) as mock_call:
result = await srv._dispatch_virtual_mcp_tool(
result = await mcp_operations._dispatch_virtual_mcp_tool(
name=MCP_TOOL_CALL_TOOL_NAME,
arguments={"tool_name": "math-add", "arguments": {"a": 1, "b": 2}},
user_api_key_auth=uak,
@ -1027,8 +1028,7 @@ class TestDispatchVirtualMcpTool:
sentinel_logging_obj = object()
with (
patch.object(
srv,
"_build_virtual_call_logging_obj",
mcp_operations, "_build_virtual_call_logging_obj",
new_callable=AsyncMock,
return_value=sentinel_logging_obj,
) as mock_build,
@ -1038,7 +1038,7 @@ class TestDispatchVirtualMcpTool:
return_value="CALL_RESULT",
) as mock_call,
):
await srv._dispatch_virtual_mcp_tool(
await mcp_operations._dispatch_virtual_mcp_tool(
name=MCP_TOOL_CALL_TOOL_NAME,
arguments={"tool_name": "math-add", "arguments": {"a": 1}},
user_api_key_auth=uak,
@ -1060,7 +1060,7 @@ class TestDispatchVirtualMcpTool:
new_callable=AsyncMock,
return_value="SEARCH_RESULT",
) as mock_search:
await srv._dispatch_virtual_mcp_tool(
await mcp_operations._dispatch_virtual_mcp_tool(
name=MCP_TOOL_SEARCH_TOOL_NAME,
arguments={"query": "issue", "top_k": "not-a-number"},
user_api_key_auth=uak,
@ -1083,12 +1083,12 @@ class TestDispatchVirtualMcpTool:
fake = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False)
with (
patch(
"litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
"litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=[MagicMock()],
) as mock_allowed,
patch(
"litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
new_callable=AsyncMock,
return_value=fake,
) as mock_exec,
@ -1130,12 +1130,12 @@ class TestDispatchVirtualMcpTool:
uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True))
with (
patch(
"litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
"litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=[],
),
patch(
"litellm.proxy._experimental.mcp_server.server.execute_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool",
new_callable=AsyncMock,
) as mock_exec,
):
@ -1217,7 +1217,7 @@ class TestMcpServerToolCallErrorHandling:
return_value=(uak, None, None, None, None, None, None),
),
patch(
"litellm.proxy._experimental.mcp_server.server._dispatch_virtual_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._dispatch_virtual_mcp_tool",
new_callable=AsyncMock,
side_effect=HTTPException(status_code=403, detail="User not allowed to call this tool"),
),
@ -1254,7 +1254,7 @@ async def test_handle_mcp_tool_call_scoped_denial_names_the_binding_agent() -> N
]
with patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam
"litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers",
"litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers",
new=AsyncMock(side_effect=resolve),
):
with pytest.raises(HTTPException) as exc_info:

18
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py
View file

@ -58,6 +58,22 @@ class TestApplyToolsetScope:
assert set(op.mcp_servers or []) == {"server-a", "server-b"}
assert op.mcp_tool_permissions == toolset_perms
from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager
from litellm.proxy._experimental.mcp_server.operations import prepare_context
manager = MCPServerManager()
unscoped_open = await manager.operator_open_server_ids(
auth, allow_all_server_ids=["operator-open-outside-toolset"], submitted_server_ids=[]
)
scoped_open = await manager.operator_open_server_ids(
prepare_context(result).user_api_key_auth,
allow_all_server_ids=["operator-open-outside-toolset"],
submitted_server_ids=[],
)
assert unscoped_open == {"operator-open-outside-toolset"}
assert scoped_open == set()
assert auth.mcp_toolset_id is None
@pytest.mark.asyncio
async def test_admin_creates_object_permission_when_none(self):
"""Admin key with object_permission=None can access any toolset."""
@ -564,7 +580,7 @@ class TestMCPActiveToolsetContextVar:
MagicMock(get_mcp_client_ip=MagicMock(return_value="127.0.0.1")),
),
patch(
"litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
"litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager",
MagicMock(get_mcp_server_by_name=MagicMock(return_value=None)),
),
patch(

75
tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
View file

@ -1,3 +1,4 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
"""
VERIA-7 regression: OpenAPI-backed (local-registry) MCP tools must run
through `pre_call_tool_check` before dispatch, the same as managed
@ -49,22 +50,22 @@ async def test_openapi_local_tool_runs_pre_call_tool_check():
with (
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"_get_mcp_server_from_tool_name",
return_value=fake_server,
),
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"pre_call_tool_check",
new=pre_call,
),
patch.object(
mcp_module.global_mcp_tool_registry,
mcp_operations.global_mcp_tool_registry,
"get_tool",
return_value=fake_tool,
),
patch(
"litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
new=handle_local,
),
patch(
@ -72,7 +73,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check():
return_value=True,
),
):
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="list_pets",
arguments={"limit": 10},
allowed_mcp_servers=[fake_server],
@ -92,7 +93,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check():
assert pre_call_kwargs["guardrail_context"] == {"metadata": {"guardrails": ("block-all",)}}
assert pre_call_kwargs["name"] == "list_pets"
assert pre_call_kwargs["server"] is fake_server
assert pre_call_kwargs["user_api_key_auth"] is user
assert pre_call_kwargs["user_api_key_auth"] == user
# `proxy_logging_obj` must be sourced from the canonical proxy_server
# module (same as the managed path) — passing None would crash the
# downstream `_create_mcp_request_object_from_kwargs` call with
@ -134,22 +135,22 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises():
with (
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"_get_mcp_server_from_tool_name",
return_value=fake_server,
),
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"pre_call_tool_check",
new=pre_call,
),
patch.object(
mcp_module.global_mcp_tool_registry,
mcp_operations.global_mcp_tool_registry,
"get_tool",
return_value=fake_tool,
),
patch(
"litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
new=handle_local,
),
patch(
@ -158,7 +159,7 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises():
),
):
with pytest.raises(HTTPException) as exc:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="delete_pet",
arguments={},
allowed_mcp_servers=[fake_server],
@ -195,24 +196,24 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
# `_get_mcp_server_from_tool_name` returns None — no server context.
with (
patch.object(mcp_module, "_resolve_openapi_tool_auth", new=resolve_auth),
patch.object(mcp_operations, "_resolve_openapi_tool_auth", new=resolve_auth),
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"_get_mcp_server_from_tool_name",
return_value=None,
),
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"pre_call_tool_check",
new=pre_call,
),
patch.object(
mcp_module.global_mcp_tool_registry,
mcp_operations.global_mcp_tool_registry,
"get_tool",
return_value=fake_tool,
),
patch(
"litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
new=handle_local,
),
patch(
@ -221,7 +222,7 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable():
),
):
with pytest.raises(HTTPException) as exc:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="list_pets",
arguments={},
allowed_mcp_servers=[],
@ -280,27 +281,27 @@ async def test_openapi_local_tool_injects_resolved_oauth_token():
with (
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"_get_mcp_server_from_tool_name",
return_value=oauth_server,
),
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"pre_call_tool_check",
new=AsyncMock(return_value={}),
),
patch.object(
mcp_module.global_mcp_tool_registry,
mcp_operations.global_mcp_tool_registry,
"get_tool",
return_value=fake_tool,
),
patch.object(
mcp_module.global_mcp_server_manager._cred_provider,
mcp_operations.global_mcp_server_manager._cred_provider,
"resolve_credentials",
new=AsyncMock(return_value=Ok(StaticHeaderAuth("Bearer stored-user-token"))),
),
patch(
"litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
new=handle_local,
),
patch(
@ -308,7 +309,7 @@ async def test_openapi_local_tool_injects_resolved_oauth_token():
return_value=True,
),
):
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="get_values",
arguments={},
allowed_mcp_servers=[oauth_server],
@ -417,7 +418,7 @@ async def test_legacy_local_tool_fallback_refuses_unentitled_caller(legacy_local
)
with pytest.raises(HTTPException) as exc:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}",
arguments={},
allowed_mcp_servers=[server],
@ -451,7 +452,7 @@ async def test_legacy_local_tool_fallback_still_dispatches_entitled_caller(
server, executed = legacy_local_tool
user = _caller_entitled_to([LEGACY_TOOL])
result = await mcp_module.execute_mcp_tool(
result = await mcp_operations.execute_mcp_tool(
name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}",
arguments={},
allowed_mcp_servers=[server],
@ -481,7 +482,7 @@ async def test_legacy_local_tool_fallback_fails_closed_on_empty_prefix(
_server, executed = legacy_local_tool
with pytest.raises(HTTPException) as exc:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name=f"-{LEGACY_TOOL}",
arguments={},
allowed_mcp_servers=[],
@ -523,7 +524,7 @@ async def test_legacy_local_tool_fallback_fails_closed_when_prefix_names_no_serv
return_value=True,
):
with pytest.raises(HTTPException) as exc:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}",
arguments={},
allowed_mcp_servers=[other_server],
@ -546,7 +547,7 @@ async def test_unknown_tool_name_still_reports_not_found():
from litellm.proxy._experimental.mcp_server import server as mcp_module
with pytest.raises(HTTPException) as exc:
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="tool_no_registry_knows",
arguments={},
allowed_mcp_servers=[],
@ -610,7 +611,7 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
captured["injected"] = _request_auth_header.get()
return []
manager = mcp_module.global_mcp_server_manager
manager = mcp_operations.global_mcp_server_manager
with (
patch.object(manager, "resolve_openapi_upstream_auth", new=fake_resolver),
patch.object(manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
@ -620,9 +621,9 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
fake_tool.name = "list_reports"
with (
patch.object(manager, "_get_mcp_server_from_tool_name", return_value=server),
patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
patch(
"litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool",
"litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool",
new=capture_local,
),
patch(
@ -630,7 +631,7 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
return_value=True,
),
):
await mcp_module.execute_mcp_tool(
await mcp_operations.execute_mcp_tool(
name="list_reports",
arguments={},
allowed_mcp_servers=[server],
@ -702,11 +703,11 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st
user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
with (
patch.object(mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server),
patch.object(mcp_module.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
patch.object(mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server),
patch.object(mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
patch.object(
mcp_module.global_mcp_server_manager,
mcp_operations.global_mcp_server_manager,
"resolve_openapi_upstream_auth",
new=AsyncMock(return_value=(None, None)),
),
@ -715,7 +716,7 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st
return_value=True,
),
):
call = mcp_module.execute_mcp_tool(
call = mcp_operations.execute_mcp_tool(
name="list_reports",
arguments={},
allowed_mcp_servers=[server],

365
tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py Normal file
View file

@ -0,0 +1,365 @@
import asyncio
from unittest.mock import AsyncMock, patch
import pytest
from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult
from litellm.proxy._experimental.mcp_server.operations import GatewayOperations, prepare_context
from litellm.proxy._types import UserAPIKeyAuth
@pytest.mark.asyncio
async def test_oauth_prefetch_failure_does_not_log_caller_or_exception_text(caplog):
from litellm.proxy._experimental.mcp_server.operations import _prefetch_oauth_creds_for_user
user_id = "caller\nFORGED-USER-LINE"
fetch = AsyncMock(side_effect=RuntimeError("database\nFORGED-ERROR-LINE"))
database = object()
with (
patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=database),
patch("litellm.proxy._experimental.mcp_server.db.list_user_oauth_credentials", fetch),
caplog.at_level("WARNING", logger="LiteLLM"),
):
result = await _prefetch_oauth_creds_for_user(UserAPIKeyAuth(user_id=user_id))
assert result == {}
fetch.assert_awaited_once_with(database, user_id)
warnings = [record.getMessage() for record in caplog.records if "prefetch" in record.getMessage()]
assert len(warnings) == 1
assert "failed" in warnings[0]
assert "\n" not in warnings[0]
assert "FORGED" not in warnings[0]
@pytest.mark.asyncio
async def test_dispatch_uses_explicit_context_when_ambient_caller_differs():
from mcp.server.auth.middleware.auth_context import auth_context_var
from litellm.proxy._experimental.mcp_server.server import set_auth_context
context = prepare_context(
UserAPIKeyAuth(user_id="alpha"),
raw_headers={"x-caller": "alpha"},
mcp_servers=["alpha-server"],
client_ip="192.0.2.1",
)
token = auth_context_var.set(None)
handler = AsyncMock(return_value=GetPromptResult(messages=[]))
try:
set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"})
with patch("litellm.proxy._experimental.mcp_server.operations.mcp_get_prompt", handler):
result = await GatewayOperations().execute(
GetPromptRequest(params=GetPromptRequestParams(name="alpha-prompt")), context
)
assert result.messages == []
assert handler.await_args.kwargs["name"] == "alpha-prompt"
assert handler.await_args.kwargs["user_api_key_auth"].user_id == "alpha"
assert handler.await_args.kwargs["raw_headers"] == {"x-caller": "alpha"}
assert handler.await_args.kwargs["mcp_servers"] == ["alpha-server"]
assert handler.await_args.kwargs["client_ip"] == "192.0.2.1"
finally:
auth_context_var.reset(token)
@pytest.mark.asyncio
async def test_legacy_adapter_cleans_context_after_cancelled_operation():
from types import SimpleNamespace
from litellm.proxy._experimental.mcp_server import server
from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
previous_session = server.active_mcp_session_var.get()
previous_request = active_mcp_request_ctx_var.get()
request = SimpleNamespace(session=object())
auth = (None, None, None, None, None, None, None)
async def cancelled_operation():
async with server._legacy_operation_context(request, trace=False):
assert server.active_mcp_session_var.get() is request.session
assert active_mcp_request_ctx_var.get() is request
raise asyncio.CancelledError
with patch(
"litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", AsyncMock(return_value=auth)
):
with pytest.raises(asyncio.CancelledError):
await cancelled_operation()
assert server.active_mcp_session_var.get() is previous_session
assert active_mcp_request_ctx_var.get() is previous_request
@pytest.mark.asyncio
async def test_legacy_adapter_cleans_context_when_trace_setup_fails():
from types import SimpleNamespace
from litellm.proxy._experimental.mcp_server import server
from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var
previous_session = server.active_mcp_session_var.get()
previous_request = active_mcp_request_ctx_var.get()
request = SimpleNamespace(session=object())
async def enter_operation():
async with server._legacy_operation_context(request, trace=True):
pytest.fail("Trace setup failure must prevent dispatch")
with patch.object(server, "_otel_set_mcp_transport_span", side_effect=RuntimeError("trace failure")):
with pytest.raises(RuntimeError, match="trace failure"):
await enter_operation()
assert server.active_mcp_session_var.get() is previous_session
assert active_mcp_request_ctx_var.get() is previous_request
@pytest.mark.asyncio
async def test_prompt_sampling_receives_explicit_operation_caller_headers_and_ip():
from unittest.mock import MagicMock
from litellm.proxy._experimental.mcp_server import operations
from litellm.types.mcp import MCPTransport
from litellm.types.mcp_server.mcp_server_manager import MCPServer
upstream = MCPServer(
server_id="explicit-prompt",
name="explicit_prompt",
url="https://example.invalid/mcp",
transport=MCPTransport.http,
allow_sampling=True,
)
context = prepare_context(
UserAPIKeyAuth(user_id="prompt-caller"),
raw_headers={"x-caller": "prompt-caller"},
client_ip="192.0.2.41",
)
client = MagicMock()
client.get_prompt = AsyncMock(return_value=GetPromptResult(messages=[]))
sampling = AsyncMock()
with (
patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[upstream])),
patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient", return_value=client) as factory,
patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling),
):
result = await GatewayOperations().execute(
GetPromptRequest(params=GetPromptRequestParams(name="explicit_prompt-prompt")), context
)
assert result.messages == []
await factory.call_args.kwargs["sampling_callback"](None, None)
captured = sampling.await_args.kwargs
assert captured["user_api_key_auth"] is not None
assert captured["user_api_key_auth"].user_id == "prompt-caller"
assert captured["raw_headers"] == {"x-caller": "prompt-caller"}
assert captured["client_ip"] == "192.0.2.41"
def _catalog_case(method):
from mcp import types
cases = {
"prompts/list": (
types.ListPromptsRequest(),
"list_prompts",
"get_prompts_from_server",
[types.Prompt(name="catalog-prompt")],
"prompts",
),
"prompts/get": (
types.GetPromptRequest(
params=types.GetPromptRequestParams(name="catalog-prompt", arguments={"topic": "test"})
),
"get_prompt",
"get_prompt_from_server",
types.GetPromptResult(messages=[]),
None,
),
"resources/list": (
types.ListResourcesRequest(),
"list_resources",
"get_resources_from_server",
[types.Resource(name="document", uri="https://example.com/document")],
"resources",
),
"resources/templates/list": (
types.ListResourceTemplatesRequest(),
"list_resource_templates",
"get_resource_templates_from_server",
[types.ResourceTemplate(name="document", uri_template="https://example.com/{name}")],
"resource_templates",
),
"resources/read": (
types.ReadResourceRequest(params=types.ReadResourceRequestParams(uri="https://example.com/document")),
"read_resource",
"read_resource_from_server",
types.ReadResourceResult(
contents=[types.TextResourceContents(uri="https://example.com/document", text="document body")]
),
None,
),
}
return cases[method]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method", ["prompts/list", "prompts/get", "resources/list", "resources/templates/list", "resources/read"]
)
@pytest.mark.parametrize("state", ["success", "denied", "upstream_failure", "scope_failure"])
async def test_native_catalog_operations_preserve_context_results_and_failure_policy(method, state):
from types import SimpleNamespace
from fastapi import HTTPException
from mcp.server.context import ServerRequestContext
from mcp.types import PaginatedRequestParams
from litellm.proxy._experimental.mcp_server import operations, server
from litellm.types.mcp import MCPTransport
from litellm.types.mcp_server.mcp_server_manager import MCPServer
operation, handler_name, manager_method, payload, collection = _catalog_case(method)
caller = UserAPIKeyAuth(user_id="catalog-caller")
headers = {"x-caller": "catalog-caller"}
upstream_server = MCPServer(server_id="catalog", name="catalog", transport=MCPTransport.http)
allowed = AsyncMock(
return_value=[] if state == "denied" else [upstream_server],
side_effect=HTTPException(status_code=403, detail="scope denied") if state == "scope_failure" else None,
)
upstream = AsyncMock(
return_value=payload, side_effect=RuntimeError("upstream unavailable") if state == "upstream_failure" else None
)
ctx = ServerRequestContext(
session=SimpleNamespace(), lifespan_context={}, protocol_version="2025-06-18", method=method
)
auth = (caller, None, ["catalog"], None, None, headers, "192.0.2.41")
with (
patch.object(server, "get_or_extract_auth_context", AsyncMock(return_value=auth)),
patch.object(operations, "_get_allowed_mcp_servers", allowed),
patch.object(operations.global_mcp_server_manager, manager_method, upstream),
):
if collection is None and state != "success":
expected_error = RuntimeError if state == "upstream_failure" else HTTPException
with pytest.raises(expected_error):
await getattr(server, handler_name)(ctx, operation.params)
else:
result = await getattr(server, handler_name)(ctx, operation.params or PaginatedRequestParams())
if collection:
assert getattr(result, collection) == (payload if state == "success" else [])
else:
assert result == payload
assert allowed.await_args.kwargs == {
"user_api_key_auth": caller,
"mcp_servers": ["catalog"],
"client_ip": "192.0.2.41",
}
if state in ("denied", "scope_failure"):
upstream.assert_not_awaited()
else:
upstream.assert_awaited_once()
forwarded = upstream.await_args.kwargs
assert forwarded["user_api_key_auth"] == caller
assert forwarded["raw_headers"] == headers
assert forwarded["client_ip"] == "192.0.2.41"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method", ["prompts/list", "prompts/get", "resources/list", "resources/templates/list", "resources/read"]
)
async def test_explicit_proxy_context_rejects_catalog_operations_before_upstream_access(method):
from mcp.shared.exceptions import MCPError
from mcp.types import METHOD_NOT_FOUND
from litellm.proxy._experimental.mcp_server import operations
operation, _, manager_method, _, _ = _catalog_case(method)
upstream = AsyncMock()
with patch.object(operations.global_mcp_server_manager, manager_method, upstream):
with pytest.raises(MCPError) as rejected:
await GatewayOperations().execute(operation, prepare_context(mcp_proxy_mode=True))
assert rejected.value.error.code == METHOD_NOT_FOUND
assert rejected.value.error.message == "Operation unavailable on /mcp/proxy"
upstream.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["missing_env", "pii", "guardrail", "unexpected"])
async def test_tool_operation_preserves_failure_messages_and_request_trace(failure):
from mcp.types import CallToolRequest, CallToolRequestParams
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.proxy._experimental.mcp_server import operations
from litellm.proxy._experimental.mcp_server.utils import MCPMissingUserEnvVarsError
failures = {
"missing_env": (
MCPMissingUserEnvVarsError(
server_id="server", server_name="server", missing=["TOKEN"], setup_url="https://example.com/setup"
),
"https://example.com/setup",
),
"pii": (
BlockedPiiEntityError(entity_type="EMAIL_ADDRESS", guardrail_name="test"),
"Blocked PII entity detected",
),
"guardrail": (GuardrailRaisedException(message="request denied"), "Guardrail violation"),
"unexpected": (RuntimeError("upstream unavailable"), "Error: upstream unavailable"),
}
error, expected = failures[failure]
dispatch = AsyncMock(side_effect=error)
context = prepare_context(
raw_headers={"x-litellm-trace-id": "operation-trace", "authorization": "private-test-header"}
)
with patch.object(operations, "call_mcp_tool", dispatch):
result = await GatewayOperations().execute(
CallToolRequest(params=CallToolRequestParams(name="catalog-tool", arguments={})), context
)
assert result.is_error is True
assert expected in result.content[0].text
assert "private-test-header" not in result.content[0].text
dispatch.assert_awaited_once()
assert dispatch.await_args.kwargs["litellm_trace_id"] == "operation-trace"
assert dispatch.await_args.kwargs["litellm_session_id"] == "operation-trace"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method,helper",
[
("prompts/list", "_list_mcp_prompts"),
("resources/list", "_list_mcp_resources"),
("resources/templates/list", "_list_mcp_resource_templates"),
],
)
async def test_catalog_operation_preserves_empty_result_for_malformed_upstream_items(method, helper):
from litellm.proxy._experimental.mcp_server import operations
operation, _, _, _, collection = _catalog_case(method)
with patch.object(operations, helper, AsyncMock(return_value=[{"unexpected": "item"}])):
result = await GatewayOperations().execute(operation, prepare_context())
assert getattr(result, collection) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("catalog_unavailable", [False, True])
async def test_tool_listing_returns_empty_result_without_dispatch_for_unavailable_catalog(catalog_unavailable):
from mcp.types import ListToolsRequest
from litellm.proxy._experimental.mcp_server import operations
allowed = AsyncMock(
return_value=[], side_effect=RuntimeError("catalog unavailable") if catalog_unavailable else None
)
upstream = AsyncMock()
with (
patch.object(operations, "_get_allowed_mcp_servers", allowed),
patch.object(operations.global_mcp_server_manager, "_get_tools_from_server", upstream),
):
result = await GatewayOperations().execute(ListToolsRequest(), prepare_context())
assert result.tools == []
allowed.assert_awaited_once()
upstream.assert_not_awaited()
@pytest.mark.asyncio
async def test_explicit_proxy_context_lists_builtin_tools_and_blocks_direct_tool_dispatch():
from mcp.types import CallToolRequest, CallToolRequestParams, ListToolsRequest
from litellm.proxy._experimental.mcp_server import operations
context = prepare_context(mcp_proxy_mode=True)
allowed = AsyncMock()
with patch.object(operations, "_get_allowed_mcp_servers", allowed):
listing = await GatewayOperations().execute(ListToolsRequest(), context)
denied = await GatewayOperations().execute(
CallToolRequest(params=CallToolRequestParams(name="catalog-tool", arguments={})), context
)
assert {tool.name for tool in listing.tools} == {"search_tools", "get_tool_schema", "call_tool"}
assert denied.is_error is True
assert "unavailable on /mcp/proxy" in denied.content[0].text
allowed.assert_not_awaited()

13
tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py
View file

@ -1,3 +1,4 @@
from litellm.proxy._experimental.mcp_server import operations as mcp_operations
import asyncio
import inspect
import json
@ -1253,6 +1254,7 @@ class TestListToolsRestAPI:
user_api_key_auth=None,
extra_headers=None,
apply_tool_filters=True,
client_ip=None,
):
captured["called"] = True
captured["server"] = server
@ -1338,6 +1340,7 @@ class TestListToolsRestAPI:
user_api_key_auth=None,
extra_headers=None,
apply_tool_filters=True,
client_ip=None,
):
captured["user_api_key_auth"] = user_api_key_auth
return ["tool-1"]
@ -1891,6 +1894,7 @@ class TestListToolsRestAPI:
user_api_key_auth=None,
extra_headers=None,
apply_tool_filters=True,
client_ip=None,
):
captured["called"] = True
captured["server_arg"] = server
@ -2027,6 +2031,7 @@ class TestListToolsRestAPI:
user_api_key_auth=None,
extra_headers=None,
apply_tool_filters=True,
client_ip=None,
):
captured["called"] = True
captured["server_arg"] = server
@ -2112,6 +2117,7 @@ class TestListToolsRestAPI:
user_api_key_auth=None,
extra_headers=None,
apply_tool_filters=True,
client_ip=None,
):
return ["scoped-tool"]
@ -2319,6 +2325,7 @@ class TestListToolsRestAPI:
user_api_key_auth=None,
extra_headers=None,
apply_tool_filters=True,
client_ip=None,
):
captured["server"] = server
captured["auth_header"] = server_auth_header
@ -3145,10 +3152,10 @@ async def test_request_selected_tool_specific_guardrail_applies_to_virtual_execu
monkeypatch.setattr(litellm, "callbacks", [guardrail])
monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry)
monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager)
monkeypatch.setattr(server, "global_mcp_tool_registry", registry)
monkeypatch.setattr(server, "global_mcp_server_manager", manager)
monkeypatch.setattr(mcp_operations, "global_mcp_tool_registry", registry)
monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager)
monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager)
monkeypatch.setattr(server, "_get_allowed_mcp_servers", AsyncMock(return_value=[managed_server]))
monkeypatch.setattr(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[managed_server]))
monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", passthrough_request_data)
monkeypatch.setattr(proxy_server, "proxy_config", {})

281
tests/test_litellm/proxy/auth/test_login_utils.py
View file

@ -5,12 +5,15 @@ This module tests the refactored login logic that was moved from proxy_server.py
to login_utils.py for better reusability.
"""
import hashlib
import os
from collections.abc import Mapping
from contextlib import ExitStack
from typing import TYPE_CHECKING, Final
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
if TYPE_CHECKING:
@ -34,6 +37,7 @@ def _unlimited_throttle():
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import (
LiteLLM_UserTable,
LitellmUserRoles,
@ -46,8 +50,13 @@ from litellm.proxy.auth.login_utils import (
authenticate_user,
get_ui_credentials,
is_env_credential_login_enabled,
screen_login_password_for_breach,
)
# Successful DB-user logins schedule the background HIBP screen; disable it so
# no test ever does live network I/O to haveibeenpwned.com from CI.
_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False}
def test_get_ui_credentials_prefers_explicit_password():
"""The configured UI password should be returned when available."""
@ -326,6 +335,7 @@ async def test_authenticate_user_email_case_insensitive_login():
master_key=master_key,
prisma_client=mock_prisma_client,
throttle=_unlimited_throttle(),
general_settings=_POLICY_NO_BREACH_CHECK,
)
result_lower = await authenticate_user(
username=stored_email,
@ -333,6 +343,7 @@ async def test_authenticate_user_email_case_insensitive_login():
master_key=master_key,
prisma_client=mock_prisma_client,
throttle=_unlimited_throttle(),
general_settings=_POLICY_NO_BREACH_CHECK,
)
assert result_mixed.user_id == result_lower.user_id == "test-user-123"
@ -576,6 +587,7 @@ async def test_authenticate_user_database_login_with_non_ascii_password():
master_key=master_key,
prisma_client=mock_prisma_client,
throttle=_unlimited_throttle(),
general_settings=_POLICY_NO_BREACH_CHECK,
)
assert isinstance(result, LoginResult)
@ -721,7 +733,12 @@ async def _db_login(throttle, username: str, password: str, *, correct: bool):
),
):
return await authenticate_user(
username=username, password=password, master_key="sk-master", prisma_client=MagicMock(), throttle=throttle
username=username,
password=password,
master_key="sk-master",
prisma_client=MagicMock(),
throttle=throttle,
general_settings=_POLICY_NO_BREACH_CHECK,
)
@ -2064,3 +2081,265 @@ class TestIsEnvCredentialLoginEnabled:
with ExitStack() as stack:
_patch_sso_configured(stack, configured=False)
assert is_env_credential_login_enabled({"disable_password_login_when_sso_enabled": True}) is True
def _db_user_row(*, password: str, password_reset_required: bool | None = None, last_breach_check_at=None):
hashed = hash_token(token=password)
row = MagicMock()
row.user_id = "reset-user-1"
row.user_email = "reset@example.com"
row.password = hashed
row.user_role = LitellmUserRoles.INTERNAL_USER
row.password_reset_required = password_reset_required
row.last_breach_check_at = last_breach_check_at
return row
def _prisma_with_user(row) -> MagicMock:
mock_prisma_client = MagicMock()
mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=row)
mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value=row)
return mock_prisma_client
_DB_LOGIN_ENV = {
"DATABASE_URL": "postgresql://test:test@localhost/test",
"UI_USERNAME": "admin",
"UI_PASSWORD": "admin-password",
}
class TestPasswordResetRequiredSessionMinting:
"""A user flagged `password_reset_required` must receive a UI session key
restricted to the change-password endpoint (server-side enforcement, so a
script driving the management API with the session key is blocked too);
an unflagged user must keep getting an unrestricted key."""
async def _login(self, mock_prisma_client) -> tuple[LoginResult, dict]:
with patch.dict(os.environ, _DB_LOGIN_ENV):
with patch( # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs
"litellm.proxy.auth.login_utils.generate_key_helper_fn",
new_callable=AsyncMock,
return_value={"token": "session-token"},
) as mock_generate_key:
result = await authenticate_user(
username="reset@example.com",
password="Str0ng!Passw0rd",
master_key="sk-1234",
prisma_client=mock_prisma_client,
throttle=_unlimited_throttle(),
general_settings=_POLICY_NO_BREACH_CHECK,
)
return result, mock_generate_key.call_args.kwargs
@pytest.mark.asyncio
async def test_flagged_user_gets_key_restricted_to_change_password(self):
row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=True)
result, key_kwargs = await self._login(_prisma_with_user(row))
assert key_kwargs["allowed_routes"] == ["/user/password/change"]
assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True}
assert result.password_reset_required is True
@pytest.mark.asyncio
async def test_unflagged_user_gets_unrestricted_key(self):
row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None)
result, key_kwargs = await self._login(_prisma_with_user(row))
assert key_kwargs["allowed_routes"] is None
assert key_kwargs["metadata"] == {"login_method": "username_password"}
assert result.password_reset_required is False
async def _login_with_screen_result(self, mock_prisma_client, breached: bool) -> tuple[LoginResult, dict, dict]:
with patch.dict(os.environ, _DB_LOGIN_ENV):
with patch( # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs
"litellm.proxy.auth.login_utils.generate_key_helper_fn",
new_callable=AsyncMock,
return_value={"token": "session-token"},
) as mock_generate_key:
with (
patch( # test-quality-ok: authenticate_user has no HIBP client seam; the screen itself is tested against MockTransport below
"litellm.proxy.auth.login_utils.screen_login_password_for_breach",
new_callable=AsyncMock,
return_value=breached,
) as mock_screen
):
result = await authenticate_user(
username="reset@example.com",
password="Str0ng!Passw0rd",
master_key="sk-1234",
prisma_client=mock_prisma_client,
throttle=_unlimited_throttle(),
general_settings=_POLICY_NO_BREACH_CHECK,
)
return result, mock_generate_key.call_args.kwargs, mock_screen.call_args.kwargs
@pytest.mark.asyncio
async def test_login_screens_with_row_state_before_minting(self):
"""The login must hand the screen the row's recheck timestamp, or the
24h throttle can never work."""
checked_at = datetime.now(timezone.utc) - timedelta(hours=1)
row = _db_user_row(password="Str0ng!Passw0rd", last_breach_check_at=checked_at)
mock_prisma_client = _prisma_with_user(row)
_, _, screen_kwargs = await self._login_with_screen_result(mock_prisma_client, breached=False)
assert screen_kwargs["user_id"] == "reset-user-1"
assert screen_kwargs["password"] == "Str0ng!Passw0rd"
assert screen_kwargs["last_breach_check_at"] == checked_at
assert screen_kwargs["prisma_client"] is mock_prisma_client
@pytest.mark.asyncio
async def test_fresh_breach_hit_restricts_the_current_session(self):
"""A breach found during THIS login must restrict THIS session, not
just the next one."""
row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None)
mock_prisma_client = _prisma_with_user(row)
result, key_kwargs, _ = await self._login_with_screen_result(mock_prisma_client, breached=True)
assert key_kwargs["allowed_routes"] == ["/user/password/change"]
assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True}
assert result.password_reset_required is True
def _sha1_upper(password: str) -> str:
return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
def _client_with_transport(handler) -> AsyncHTTPHandler:
http_handler = AsyncHTTPHandler()
http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
return http_handler
def _client_returning_breach_hit(password: str) -> AsyncHTTPHandler:
body = f"{_sha1_upper(password)[5:]}:42"
return _client_with_transport(lambda request: httpx.Response(200, text=body))
def _client_returning_no_hit() -> AsyncHTTPHandler:
return _client_with_transport(lambda request: httpx.Response(200, text="0000000000000000000000000000000000A:3"))
def _client_never_called() -> AsyncHTTPHandler:
def handler(request: httpx.Request) -> httpx.Response:
raise AssertionError(f"unexpected HTTP call to {request.url}")
return _client_with_transport(handler)
class TestScreenLoginPasswordForBreach:
"""The awaited login-time screen: flags a breached password for a forced
reset, stamps the recheck timestamp, rechecks at most every 24h, returns
the breach verdict so the login can restrict the session it is minting,
and never raises into the login."""
@pytest.mark.asyncio
async def test_breached_password_sets_reset_flag_and_timestamp(self):
password = "Password123!"
mock_prisma_client = _prisma_with_user(None)
breached = await screen_login_password_for_breach(
user_id="reset-user-1",
password=password,
last_breach_check_at=None,
general_settings={},
prisma_client=mock_prisma_client,
client=_client_returning_breach_hit(password),
)
assert breached is True
update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs
assert update_kwargs["where"] == {"user_id": "reset-user-1"}
assert update_kwargs["data"]["password_reset_required"] is True
assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime)
@pytest.mark.asyncio
async def test_clean_password_stamps_timestamp_without_flag(self):
mock_prisma_client = _prisma_with_user(None)
breached = await screen_login_password_for_breach(
user_id="reset-user-1",
password="Str0ng!Passw0rd",
last_breach_check_at=None,
general_settings={},
prisma_client=mock_prisma_client,
client=_client_returning_no_hit(),
)
assert breached is False
update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs
assert "password_reset_required" not in update_kwargs["data"]
assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime)
@pytest.mark.asyncio
async def test_skips_hibp_when_checked_within_24_hours(self):
mock_prisma_client = _prisma_with_user(None)
breached = await screen_login_password_for_breach(
user_id="reset-user-1",
password="Password123!",
last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=23),
general_settings={},
prisma_client=mock_prisma_client,
client=_client_never_called(),
)
assert breached is False
mock_prisma_client.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_rechecks_when_last_check_is_older_than_24_hours(self):
password = "Password123!"
mock_prisma_client = _prisma_with_user(None)
breached = await screen_login_password_for_breach(
user_id="reset-user-1",
password=password,
last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=25),
general_settings={},
prisma_client=mock_prisma_client,
client=_client_returning_breach_hit(password),
)
assert breached is True
assert (
mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"]["password_reset_required"] is True
)
@pytest.mark.asyncio
async def test_skips_hibp_when_check_disabled(self):
mock_prisma_client = _prisma_with_user(None)
breached = await screen_login_password_for_breach(
user_id="reset-user-1",
password="Password123!",
last_breach_check_at=None,
general_settings=_POLICY_NO_BREACH_CHECK,
prisma_client=mock_prisma_client,
client=_client_never_called(),
)
assert breached is False
mock_prisma_client.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_db_failure_never_raises_but_still_reports_the_breach(self):
"""A failed flag write must not fail the login, but the breach verdict
still has to restrict the session being minted right now."""
password = "Password123!"
mock_prisma_client = _prisma_with_user(None)
mock_prisma_client.db.litellm_usertable.update = AsyncMock(side_effect=RuntimeError("db down"))
assert (
await screen_login_password_for_breach(
user_id="reset-user-1",
password=password,
last_breach_check_at=None,
general_settings={},
prisma_client=mock_prisma_client,
client=_client_returning_breach_hit(password),
)
is True
)

144
tests/test_litellm/proxy/auth/test_onboarding.py
View file

@ -8,15 +8,20 @@ Covers the security behavior of:
session key only after the password is written
"""
import hashlib
from datetime import timedelta
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import jwt
import pytest
import respx
from fastapi import HTTPException
import litellm
from litellm.proxy._types import InvitationClaim
from litellm.proxy._types import InvitationClaim, ProxyException
_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False}
# ---------------------------------------------------------------------------
# Helpers
@ -386,7 +391,9 @@ async def test_claim_token_rejects_concurrent_reuse_before_password_write():
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma),
patch("litellm.proxy.proxy_server.master_key", "sk-test"),
patch("litellm.proxy.proxy_server.general_settings", {}),
patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
patch(
"litellm.proxy.proxy_server.generate_key_helper_fn",
new_callable=AsyncMock,
@ -426,7 +433,9 @@ async def test_claim_token_sets_accepted_at_after_password_written():
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma),
patch("litellm.proxy.proxy_server.master_key", "sk-test"),
patch("litellm.proxy.proxy_server.general_settings", {}),
patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
patch("litellm.proxy.proxy_server.premium_user", False),
patch(
"litellm.proxy.proxy_server.generate_key_helper_fn",
@ -454,6 +463,10 @@ async def test_claim_token_sets_accepted_at_after_password_written():
call_kwargs = prisma.db.litellm_usertable.update.call_args
assert call_kwargs.kwargs["where"] == {"user_id": "user-123"}
assert "password" in call_kwargs.kwargs["data"]
# A freshly claimed, policy-screened password lifts any pending forced
# reset and re-arms the login-time breach screen.
assert call_kwargs.kwargs["data"]["password_reset_required"] is False
assert call_kwargs.kwargs["data"]["last_breach_check_at"] is None
# is_accepted was flipped to True on the invitation link
prisma.db.litellm_invitationlink.update.assert_called_once()
@ -483,7 +496,9 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails():
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma),
patch("litellm.proxy.proxy_server.master_key", "sk-test"),
patch("litellm.proxy.proxy_server.general_settings", {}),
patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
patch(
"litellm.proxy.proxy_server.generate_key_helper_fn",
new_callable=AsyncMock,
@ -505,3 +520,124 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails():
}
assert rollback_kwargs["data"]["accepted_at"] is None
assert rollback_kwargs["data"]["is_accepted"] is False
# ---------------------------------------------------------------------------
# POST /onboarding/claim_token - password policy
# ---------------------------------------------------------------------------
def _hibp_url_for(password: str) -> str:
sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
return f"https://api.pwnedpasswords.com/range/{sha1[:5]}"
def _hibp_suffix_for(password: str) -> str:
return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()[5:]
@pytest.mark.asyncio
async def test_claim_token_rejects_short_password_before_consuming_invite():
"""Default policy requires 12 characters; the invite must stay claimable."""
from litellm.proxy.proxy_server import claim_onboarding_link
invite = _make_invite(is_accepted=False)
prisma = _make_prisma(invite, _make_user())
request = _make_claim_request(_make_onboarding_token())
data = InvitationClaim(
invitation_link="invite-abc",
user_id="user-123",
password="Sh0rt!pw",
)
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above
patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above
):
with pytest.raises(ProxyException) as exc_info:
await claim_onboarding_link(data=data, request=request)
assert exc_info.value.code == "400"
assert "at least 12 characters" in exc_info.value.message
prisma.db.litellm_invitationlink.update_many.assert_not_called()
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
@respx.mock
async def test_claim_token_rejects_breached_password_before_consuming_invite():
"""A password found in the HIBP corpus must be rejected and never stored."""
from litellm.proxy.proxy_server import claim_onboarding_link
password = "P@ssword123456"
respx.get(_hibp_url_for(password)).mock(
return_value=httpx.Response(200, text=f"{_hibp_suffix_for(password)}:1387")
)
invite = _make_invite(is_accepted=False)
prisma = _make_prisma(invite, _make_user())
request = _make_claim_request(_make_onboarding_token())
data = InvitationClaim(
invitation_link="invite-abc",
user_id="user-123",
password=password,
)
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above
patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above
):
with pytest.raises(ProxyException) as exc_info:
await claim_onboarding_link(data=data, request=request)
assert exc_info.value.code == "400"
assert "data breaches" in exc_info.value.message
prisma.db.litellm_invitationlink.update_many.assert_not_called()
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
@respx.mock
async def test_claim_token_fails_open_when_hibp_unreachable():
"""An HIBP outage must never block onboarding: the claim proceeds."""
from litellm.proxy.proxy_server import claim_onboarding_link
password = "NewP@ssw0rd-2026"
respx.get(_hibp_url_for(password)).mock(side_effect=httpx.ConnectError("no route to host"))
invite = _make_invite(is_accepted=False)
user = _make_user()
prisma = _make_prisma(invite, user)
request = _make_claim_request(_make_onboarding_token())
data = InvitationClaim(
invitation_link="invite-abc",
user_id="user-123",
password=password,
)
with (
patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam
patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above
patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above
patch("litellm.proxy.proxy_server.premium_user", False), # test-quality-ok: same as above
patch( # test-quality-ok: same as above
"litellm.proxy.proxy_server.generate_key_helper_fn",
new_callable=AsyncMock,
return_value={"token": "sk-generated-key", "user_id": "user-123"},
),
patch( # test-quality-ok: same as above
"litellm.proxy.proxy_server.get_custom_url",
return_value="http://localhost:4000/",
),
patch( # test-quality-ok: same as above
"litellm.proxy.proxy_server.get_disabled_non_admin_personal_key_creation",
return_value=False,
),
patch("litellm.proxy.proxy_server.get_server_root_path", return_value=""), # test-quality-ok: same as above
):
result = await claim_onboarding_link(data=data, request=request)
assert "token" in result
prisma.db.litellm_usertable.update.assert_called_once()

209
tests/test_litellm/proxy/auth/test_password_policy.py
View file

@ -2,22 +2,56 @@
Tests for the configurable password-strength policy in
`litellm.proxy.auth.password_policy`, enforced on every path that persists a
new or changed password for a locally-managed user.
The breach-check (HIBP) tests inject a real AsyncHTTPHandler wrapping an
httpx.MockTransport, so no network is touched and nothing is monkeypatched.
"""
import asyncio
import hashlib
import httpx
import pytest
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.proxy.auth.password_policy import (
DEFAULT_MIN_LENGTH,
MIN_ALLOWED_LENGTH,
PasswordPolicy,
get_password_policy,
validate_password_not_breached,
validate_password_policy,
validate_passwords_bulk,
)
STRONG_PASSWORD = "Str0ng!Passw0rd"
def _sha1_upper(password: str) -> str:
return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()
def _client_with_transport(handler) -> AsyncHTTPHandler:
http_handler = AsyncHTTPHandler()
http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
return http_handler
def _client_never_called() -> AsyncHTTPHandler:
def handler(request: httpx.Request) -> httpx.Response:
raise AssertionError(f"unexpected HTTP call to {request.url}")
return _client_with_transport(handler)
def _client_returning(body: str, status_code: int = 200) -> AsyncHTTPHandler:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(status_code, text=body)
return _client_with_transport(handler)
def test_get_password_policy_defaults_to_pif_baseline():
policy = get_password_policy({})
assert policy == PasswordPolicy(
@ -134,3 +168,178 @@ def test_validate_password_policy_rejects_unicode_letter_as_special_character():
def test_validate_password_policy_accepts_real_special_character_with_unicode_letters():
"""Same base password as the rejection test above, plus an actual symbol."""
assert validate_password_policy("Passwörd1234!", {}) is None
@pytest.mark.asyncio
async def test_breach_check_skipped_when_disabled():
result = await validate_password_not_breached(
password="password12345", # breached in reality, but the check is off
general_settings={"password_policy_check_breached_passwords": False},
client=_client_never_called(),
)
assert result is None
@pytest.mark.asyncio
async def test_rejects_breached_password():
password = "correct horse battery staple"
sha1 = _sha1_upper(password)
body = f"AAAA000000000000000000000000000000A:0\r\n{sha1[5:]}:42\r\nBBBB000000000000000000000000000000B:7"
with pytest.raises(ProxyException) as exc_info:
await validate_password_not_breached(password=password, general_settings={}, client=_client_returning(body))
assert exc_info.value.code == "400"
assert exc_info.value.type == ProxyErrorTypes.validation_error
assert exc_info.value.param == "password"
assert "data breaches" in exc_info.value.message
@pytest.mark.asyncio
async def test_only_sha1_prefix_leaves_the_proxy():
password = "a very secret password"
sha1 = _sha1_upper(password)
captured_requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(200, text="0000000000000000000000000000000000A:1")
result = await validate_password_not_breached(
password=password, general_settings={}, client=_client_with_transport(handler)
)
assert result is None
(request,) = captured_requests
assert request.url.path == f"/range/{sha1[:5]}"
assert sha1[5:] not in str(request.url)
assert request.headers["Add-Padding"] == "true"
assert "litellm" in request.headers["User-Agent"]
@pytest.mark.asyncio
async def test_ignores_padding_entries_with_zero_count():
"""HIBP padding entries (requested via Add-Padding) carry count 0 and must
not be treated as breaches when they collide with the password's suffix."""
password = "a padded-away password"
sha1 = _sha1_upper(password)
result = await validate_password_not_breached(
password=password, general_settings={}, client=_client_returning(f"{sha1[5:]}:0")
)
assert result is None
@pytest.mark.asyncio
async def test_accepts_password_absent_from_breach_corpus():
result = await validate_password_not_breached(
password="a genuinely novel password",
general_settings={},
client=_client_returning("0018A45C4D1DEF81644B54AB7F969B88D65:1\r\n00D4F6E8FA6EECAD2A3AA415EEC418D38EC:2"),
)
assert result is None
@pytest.mark.asyncio
async def test_breach_check_fails_open_on_network_error():
def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("no route to host")
result = await validate_password_not_breached(
password="password12345", # breached, but HIBP is unreachable
general_settings={},
client=_client_with_transport(handler),
)
assert result is None
@pytest.mark.asyncio
async def test_breach_check_fails_open_on_http_error_status():
result = await validate_password_not_breached(
password="password12345",
general_settings={},
client=_client_returning("service unavailable", status_code=503),
)
assert result is None
@pytest.mark.asyncio
async def test_breach_check_fails_open_on_malformed_response_body():
result = await validate_password_not_breached(
password="password12345",
general_settings={},
client=_client_returning(f"{_sha1_upper('password12345')[5:]}:not-a-number"),
)
assert result is None
@pytest.mark.asyncio
async def test_validate_passwords_bulk_screens_concurrently():
"""All HIBP lookups for a batch must be in flight at once: each handler
call stalls until every expected request has arrived, and a handler that
gives up waiting reports the password as breached. Serial awaiting (the
old per-user behavior) leaves each earlier request waiting forever for the
later ones, so every verdict comes back as a breach and the test fails."""
passwords = ("Uniqu3!Passw0rd-a", "Uniqu3!Passw0rd-b", "Uniqu3!Passw0rd-c")
suffix_by_prefix = {_sha1_upper(p)[:5]: _sha1_upper(p)[5:] for p in passwords}
all_arrived = asyncio.Event()
arrivals: list[str] = []
async def handler(request: httpx.Request) -> httpx.Response:
arrivals.append(request.url.path)
if len(arrivals) == len(passwords):
all_arrived.set()
try:
await asyncio.wait_for(all_arrived.wait(), timeout=5)
except TimeoutError:
return httpx.Response(200, text=f"{suffix_by_prefix[request.url.path.rsplit('/', 1)[-1]]}:1")
return httpx.Response(200, text="0000000000000000000000000000000000A:1")
verdicts = await validate_passwords_bulk(passwords, {}, client=_client_with_transport(handler))
assert set(arrivals) == {f"/range/{prefix}" for prefix in suffix_by_prefix}
assert all(verdicts[p] is None for p in passwords)
@pytest.mark.asyncio
async def test_validate_passwords_bulk_deduplicates_lookups():
"""500 users sharing one password must cost exactly one HIBP lookup."""
password = "Sh@red-Passw0rd!"
request_count = 0
def handler(request: httpx.Request) -> httpx.Response:
nonlocal request_count
request_count += 1
return httpx.Response(200, text="0000000000000000000000000000000000A:1")
verdicts = await validate_passwords_bulk((password,) * 500, {}, client=_client_with_transport(handler))
assert request_count == 1
assert verdicts == {password: None}
@pytest.mark.asyncio
async def test_validate_passwords_bulk_mixed_verdicts():
"""Weak passwords are rejected without an HIBP lookup; breached ones get
the breach error; acceptable ones map to None."""
breached = "Br3ached!Passw0rd"
clean = "Cl3an!!Passw0rd42"
weak = "short1!"
breached_sha1 = _sha1_upper(breached)
looked_up_prefixes: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
looked_up_prefixes.append(request.url.path.rsplit("/", 1)[-1])
if request.url.path == f"/range/{breached_sha1[:5]}":
return httpx.Response(200, text=f"{breached_sha1[5:]}:99")
return httpx.Response(200, text="0000000000000000000000000000000000A:1")
verdicts = await validate_passwords_bulk((breached, clean, weak), {}, client=_client_with_transport(handler))
assert _sha1_upper(weak)[:5] not in looked_up_prefixes
assert verdicts[clean] is None
assert "data breaches" in verdicts[breached].message
assert verdicts[breached].code == "400"
assert "12 characters" in verdicts[weak].message
@pytest.mark.asyncio
async def test_validate_passwords_bulk_empty_batch_makes_no_lookups():
verdicts = await validate_passwords_bulk((), {}, client=_client_never_called())
assert verdicts == {}

274
tests/test_litellm/proxy/auth/test_route_checks.py
View file

@ -3,7 +3,6 @@ from datetime import datetime
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
from fastapi import HTTPException, Request
@ -39,7 +38,7 @@ def test_non_admin_config_update_route_rejected():
request.query_params = {}
# Test that calling /config/update route raises HTTPException with 403 status
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER.value,
@ -50,9 +49,8 @@ def test_non_admin_config_update_route_rejected():
)
# Verify the exception is raised with the correct message
assert (
"Only proxy admin can be used to generate, delete, update info for new keys/users/teams"
in str(exc_info.value)
assert "Only proxy admin can be used to generate, delete, update info for new keys/users/teams" in str(
exc_info.value
)
assert "Route=/config/update" in str(exc_info.value)
assert "Your role=internal_user" in str(exc_info.value)
@ -158,7 +156,7 @@ def test_user_banner_update_rejected_for_non_admin():
request = MagicMock(spec=Request)
request.query_params = {}
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER.value,
@ -733,9 +731,7 @@ def test_virtual_key_llm_api_route_includes_passthrough_prefix(route):
valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"])
result = RouteChecks.is_virtual_key_allowed_to_call_route(
route=route, valid_token=valid_token
)
result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
assert result is True
@ -760,9 +756,7 @@ def test_virtual_key_llm_api_routes_allows_google_routes(route):
valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"])
result = RouteChecks.is_virtual_key_allowed_to_call_route(
route=route, valid_token=valid_token
)
result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
assert result is True
@ -832,18 +826,14 @@ def test_google_routes_with_dynamic_model_names_accessible_to_internal_users():
)
# If no exception is raised, the test passes
except Exception as e:
pytest.fail(
f"Internal user should be able to access Google generateContent route. Got error: {str(e)}"
)
pytest.fail(f"Internal user should be able to access Google generateContent route. Got error: {e!s}")
def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names():
"""Test that virtual key works with multiple LiteLLMRoutes member names in allowed_routes"""
# Create a UserAPIKeyAuth with multiple LiteLLMRoutes member names
valid_token = UserAPIKeyAuth(
user_id="test_user", allowed_routes=["openai_routes", "info_routes"]
)
valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes", "info_routes"])
# Test that routes from both groups are allowed
result1 = RouteChecks.is_virtual_key_allowed_to_call_route(
@ -897,13 +887,9 @@ def test_virtual_key_allowed_routes_with_no_member_names_only_explicit():
)
# Test that explicit routes are allowed
result1 = RouteChecks.is_virtual_key_allowed_to_call_route(
route="/chat/completions", valid_token=valid_token
)
result1 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/chat/completions", valid_token=valid_token)
result2 = RouteChecks.is_virtual_key_allowed_to_call_route(
route="/custom/route", valid_token=valid_token
)
result2 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/custom/route", valid_token=valid_token)
assert result1 is True
assert result2 is True
@ -1301,9 +1287,7 @@ def test_virtual_key_without_llm_api_routes_cannot_access_pass_through():
)
assert exc_info.value.status_code == 403
assert "Virtual key is not allowed to call this route" in str(
exc_info.value.detail
)
assert "Virtual key is not allowed to call this route" in str(exc_info.value.detail)
def test_check_passthrough_route_access_key_metadata_exact_match():
@ -1762,9 +1746,7 @@ def test_videos_route_accessible_to_internal_users():
)
# If no exception is raised, the test passes
except Exception as e:
pytest.fail(
f"Internal user should be able to access /v1/videos route. Got error: {str(e)}"
)
pytest.fail(f"Internal user should be able to access /v1/videos route. Got error: {e!s}")
def test_videos_route_with_virtual_key_llm_api_routes():
@ -1786,12 +1768,8 @@ def test_videos_route_with_virtual_key_llm_api_routes():
]
for route in test_routes:
result = RouteChecks.is_virtual_key_allowed_to_call_route(
route=route, valid_token=valid_token
)
assert (
result is True
), f"Virtual key with llm_api_routes should be able to access {route}"
result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
assert result is True, f"Virtual key with llm_api_routes should be able to access {route}"
def test_non_proxy_admin_wildcard_allowed_routes():
@ -1862,9 +1840,7 @@ def test_proxy_admin_viewer_can_access_global_spend_tags():
)
# If no exception is raised, the test passes
except Exception as e:
pytest.fail(
f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {str(e)}"
)
pytest.fail(f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {e!s}")
# Routes returning proxy-wide spend across every team / customer / api_key.
@ -1892,7 +1868,7 @@ def test_internal_user_blocked_from_global_spend_routes(route):
request = MagicMock(spec=Request)
request.query_params = {}
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER.value,
@ -1921,7 +1897,7 @@ def test_internal_user_view_only_blocked_from_global_spend_routes(route):
request = MagicMock(spec=Request)
request.query_params = {}
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
@ -2023,9 +1999,7 @@ def test_proxy_admin_viewer_can_access_audit_logs(route):
request_data={},
)
except Exception as e:
pytest.fail(
f"proxy_admin_viewer should be able to access {route} route. Got error: {str(e)}"
)
pytest.fail(f"proxy_admin_viewer should be able to access {route} route. Got error: {e!s}")
# ── Admin Viewer parity: Logs page endpoints ──────────────────────────────────
@ -2088,9 +2062,7 @@ def test_proxy_admin_viewer_can_access_logs_page_endpoints(route):
request_data={},
)
except Exception as e:
pytest.fail(
f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}"
)
pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}")
@pytest.mark.parametrize(
@ -2200,7 +2172,7 @@ def test_internal_user_blocked_from_admin_viewer_logs_routes(route):
if route not in INTERNAL_USER_BLOCKED_SUBSET:
return
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER.value,
@ -2276,9 +2248,7 @@ def test_proxy_admin_viewer_can_access_settings_read_endpoints(route):
request_data={},
)
except Exception as e:
pytest.fail(
f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}"
)
pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}")
# ── Admin Viewer parity: default-allow GET semantics ─────────────────────────
@ -2477,9 +2447,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
)
local_file = os.path.abspath(local_file)
spec = importlib.util.spec_from_file_location(
"local_enterprise_route_checks", local_file
)
spec = importlib.util.spec_from_file_location("local_enterprise_route_checks", local_file)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.EnterpriseRouteChecks
@ -2490,9 +2458,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with (
patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
),
patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
patch.object(
EnterpriseRouteChecks,
"is_management_routes_disabled",
@ -2508,9 +2474,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with (
patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
),
patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
patch.object(
EnterpriseRouteChecks,
"is_management_routes_disabled",
@ -2526,9 +2490,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with (
patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
),
patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
patch.object(
EnterpriseRouteChecks,
"is_management_routes_disabled",
@ -2539,9 +2501,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
EnterpriseRouteChecks.should_call_route("/v1/chat/completions")
assert exc_info.value.status_code == 403
assert "LLM API routes are disabled for this instance." in str(
exc_info.value.detail
)
assert "LLM API routes are disabled for this instance." in str(exc_info.value.detail)
@patch("litellm.proxy.proxy_server.premium_user", True)
def test_should_embeddings_still_blocked_when_llm_api_disabled(self):
@ -2549,9 +2509,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with (
patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
),
patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True),
patch.object(
EnterpriseRouteChecks,
"is_management_routes_disabled",
@ -2569,9 +2527,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints:
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with (
patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False
),
patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False),
patch.object(
EnterpriseRouteChecks,
"is_management_routes_disabled",
@ -2590,9 +2546,7 @@ def test_route_in_additional_public_routes_wildcard_match():
from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes
with (
patch(
"litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}
),
patch("litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}),
patch("litellm.proxy.proxy_server.premium_user", True),
):
# Wildcard should match subpaths
@ -2684,7 +2638,7 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re
)
# /config/update is still blocked
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER.value,
@ -2772,8 +2726,6 @@ def test_available_roles_accessible_to_non_admin_users(user_role):
# ── _user_is_org_admin tests ──────────────────────────────────────────────────
def _make_org_admin_user(org_id: str) -> LiteLLM_UserTable:
membership = LiteLLM_OrganizationMembershipTable(
user_id="org-admin-user",
@ -2896,9 +2848,7 @@ async def test_add_team_org_context_noop_when_org_id_already_present():
raise AssertionError("must not resolve when organization_id is present")
body = {"team_id": "team-1", "organization_id": "org-explicit"}
out = await add_team_org_context_to_request_body(
route="/team/update", request_body=body, fetch_team_org_id=fetch
)
out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch)
assert out == body
@ -2910,9 +2860,7 @@ async def test_add_team_org_context_noop_for_other_routes():
raise AssertionError("must not resolve for a non-opted-in route")
body = {"team_id": "team-1"}
out = await add_team_org_context_to_request_body(
route="/team/delete", request_body=body, fetch_team_org_id=fetch
)
out = await add_team_org_context_to_request_body(route="/team/delete", request_body=body, fetch_team_org_id=fetch)
assert out == body
@ -2925,9 +2873,7 @@ async def test_add_team_org_context_noop_when_team_has_no_org():
return None
body = {"team_id": "team-1"}
out = await add_team_org_context_to_request_body(
route="/team/update", request_body=body, fetch_team_org_id=fetch
)
out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch)
assert out == body
@ -3198,9 +3144,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath():
# Removing the endpoint should clean up openai_routes
# remove_endpoint_routes takes endpoint_id (UUID portion of
# the route key "{id}:exact:{path}:{methods}")
registered = (
InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
)
registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
endpoint_ids = {k.split(":")[0] for k in registered}
for eid in endpoint_ids:
InitPassThroughEndpointHelpers.remove_endpoint_routes(eid)
@ -3210,9 +3154,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath():
LiteLLMRoutes.openai_routes.value[:] = original_routes
# Clean up any routes registered during this test to avoid
# polluting the module-level _registered_pass_through_routes
registered = (
InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
)
registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes()
for k in registered:
InitPassThroughEndpointHelpers.remove_endpoint_routes(k.split(":")[0])
@ -3243,8 +3185,7 @@ def test_provider_name_substring_not_classified_as_llm_route(route):
from litellm.proxy.auth.route_checks import RouteChecks
assert RouteChecks.is_llm_api_route(route=route) is False, (
f"{route!r} should NOT be classified as an LLM API route — "
"provider-name substring match bypass"
f"{route!r} should NOT be classified as an LLM API route — provider-name substring match bypass"
)
@ -3266,9 +3207,7 @@ def test_legitimate_passthrough_routes_still_classified_as_llm_route(route):
"""Legitimate passthrough routes must still pass is_llm_api_route."""
from litellm.proxy.auth.route_checks import RouteChecks
assert (
RouteChecks.is_llm_api_route(route=route) is True
), f"{route!r} should be classified as an LLM API route"
assert RouteChecks.is_llm_api_route(route=route) is True, f"{route!r} should be classified as an LLM API route"
@pytest.mark.parametrize(
@ -3326,7 +3265,7 @@ def test_internal_user_blocked_from_search_tool_writes(route):
request = MagicMock(spec=Request)
request.query_params = {}
with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info:
with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info:
RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=user_obj,
_user_role=LitellmUserRoles.INTERNAL_USER.value,
@ -3702,12 +3641,7 @@ def test_agent_inference_routes_stay_llm_api(route):
def test_agent_routes_union_still_covers_both_halves(route):
"""Keys configured with allowed_routes=["agent_routes"] must keep both halves."""
assert (
RouteChecks.check_route_access(
route=route, allowed_routes=LiteLLMRoutes.agent_routes.value
)
is True
)
assert RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.agent_routes.value) is True
@pytest.mark.parametrize("route", AGENT_MANAGEMENT_ROUTES)
@ -3761,6 +3695,136 @@ def test_agent_registry_route_gate_open_to_non_admin_roles(user_role, method, ro
valid_token=valid_token,
request_data={},
)
def test_proxy_admin_viewer_user_update_password_param_rejected():
"""The self-service /user/update password carve-out is closed: non-admins
change their own password through /user/password/change, which verifies
the current password. Admin password sets don't pass through this check."""
with pytest.raises(HTTPException) as exc_info:
RouteChecks._check_proxy_admin_viewer_access(
route="/user/update",
_user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
request_data={"password": "hunter2hunter2"},
)
assert exc_info.value.status_code == 403
assert "password" in str(exc_info.value.detail)
def test_proxy_admin_viewer_user_update_user_email_still_allowed():
request = MagicMock(spec=Request)
request.method = "POST"
allowed = RouteChecks._check_proxy_admin_viewer_access(
route="/user/update",
_user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
request_data={"user_email": "viewer@example.com"},
request=request,
)
assert allowed is None
def test_proxy_admin_viewer_can_change_own_password():
request = MagicMock(spec=Request)
request.method = "POST"
allowed = RouteChecks._check_proxy_admin_viewer_access(
route="/user/password/change",
_user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
request_data={"current_password": "a", "new_password": "b"},
request=request,
)
assert allowed is None
@pytest.mark.parametrize(
"user_role",
[
LitellmUserRoles.INTERNAL_USER.value,
LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
],
)
def test_non_admin_roles_can_change_own_password(user_role):
valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role)
request = MagicMock(spec=Request)
request.method = "POST"
request.query_params = {}
allowed = RouteChecks.non_proxy_admin_allowed_routes_check(
user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role),
_user_role=user_role,
route="/user/password/change",
request=request,
valid_token=valid_token,
request_data={"current_password": "a", "new_password": "b"},
)
assert allowed is None
def _password_reset_session_token() -> UserAPIKeyAuth:
"""The UI session key `authenticate_user` mints for a user flagged
`password_reset_required`."""
return UserAPIKeyAuth(
user_id="flagged_user",
allowed_routes=["/user/password/change"],
metadata={"password_reset_required": True},
)
def test_password_reset_session_can_reach_change_password():
result = RouteChecks.is_virtual_key_allowed_to_call_route(
route="/user/password/change",
valid_token=_password_reset_session_token(),
)
assert result is True
@pytest.mark.parametrize(
"route",
[
"/user/info",
"/key/generate",
"/user/update",
"/chat/completions",
],
)
def test_password_reset_session_is_blocked_everywhere_else_with_reset_message(route):
"""Server-side enforcement of the forced reset: a script that logs in via
/v2/login and drives the management API with the session key must get a 403
naming the remediation endpoint, on every route but the change-password one."""
with pytest.raises(HTTPException) as exc_info:
RouteChecks.is_virtual_key_allowed_to_call_route(
route=route,
valid_token=_password_reset_session_token(),
)
assert exc_info.value.status_code == 403
assert "password must be changed" in str(exc_info.value.detail)
assert "/user/password/change" in str(exc_info.value.detail)
def test_restricted_key_without_reset_marker_keeps_generic_message():
"""The reset-specific 403 must not leak onto ordinary allowed_routes keys."""
valid_token = UserAPIKeyAuth(
user_id="test_user",
allowed_routes=["/chat/completions"],
)
with pytest.raises(HTTPException) as exc_info:
RouteChecks.is_virtual_key_allowed_to_call_route(
route="/user/info",
valid_token=valid_token,
)
assert exc_info.value.status_code == 403
assert "password must be changed" not in str(exc_info.value.detail)
assert "not allowed to call this route" in str(exc_info.value.detail)
TEAM_CALLBACK_ROUTES = (
"/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback",
"/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback/langfuse",

623
tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
View file

File diff suppressed because it is too large Load diff

401
tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py Normal file
View file

@ -0,0 +1,401 @@
"""
Tests for POST /user/password/change (litellm/proxy/management_endpoints/password_endpoints.py).
HIBP traffic is intercepted with respx; no test here touches the network.
"""
import hashlib
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import HTTPException
from litellm.proxy._types import UI_TEAM_ID, LitellmTableNames, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA
from litellm.proxy.management_endpoints.password_endpoints import change_password
from litellm.proxy.utils import hash_password, verify_password
CURRENT_PASSWORD = "OldP@ssw0rd-2026"
NEW_PASSWORD = "NewP@ssw0rd-2026"
_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False}
def _make_user_row(password: str | None) -> MagicMock:
user = MagicMock()
user.user_id = "user-123"
user.password = password
return user
def _make_prisma(user: MagicMock | None) -> MagicMock:
prisma = MagicMock()
prisma.db.litellm_usertable.find_first = AsyncMock(return_value=user)
prisma.db.litellm_usertable.update = AsyncMock(return_value=user)
return prisma
def _caller(user_id: str | None = "user-123") -> UserAPIKeyAuth:
return UserAPIKeyAuth(user_id=user_id, team_id=UI_TEAM_ID, metadata=dict(PASSWORD_SESSION_METADATA))
def _sso_session_caller() -> UserAPIKeyAuth:
return UserAPIKeyAuth(user_id="user-123", team_id=UI_TEAM_ID, metadata={})
def _virtual_key_caller() -> UserAPIKeyAuth:
return UserAPIKeyAuth(user_id="user-123", team_id="team-abc", metadata=dict(PASSWORD_SESSION_METADATA))
def _hibp_url_for(password: str) -> str:
sha1 = hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper()
return f"https://api.pwnedpasswords.com/range/{sha1[:5]}"
def _hibp_suffix_for(password: str) -> str:
return hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper()[5:]
@pytest.mark.asyncio
async def test_change_password_success_writes_new_scrypt_hash():
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
response = await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
assert response.user_id == "user-123"
update_kwargs = prisma.db.litellm_usertable.update.call_args.kwargs
assert update_kwargs["where"] == {"user_id": "user-123"}
stored = update_kwargs["data"]["password"]
assert stored != NEW_PASSWORD
assert verify_password(NEW_PASSWORD, stored)
# A successful change lifts any pending forced reset and re-arms the
# login-time breach screen for the new password.
assert update_kwargs["data"]["password_reset_required"] is False
assert update_kwargs["data"]["last_breach_check_at"] is None
@pytest.mark.asyncio
async def test_change_password_rejects_wrong_current_password():
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
assert exc_info.value.status_code == 400
assert "Current password is incorrect" in exc_info.value.detail["error"]
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_change_password_rejects_unchanged_password():
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=CURRENT_PASSWORD),
user_api_key_dict=_caller(),
)
assert exc_info.value.status_code == 400
assert "must be different from the current password" in exc_info.value.detail["error"]
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"caller",
[
pytest.param(_sso_session_caller(), id="sso_dashboard_session"),
pytest.param(_virtual_key_caller(), id="virtual_key_with_forged_metadata"),
],
)
async def test_change_password_rejects_non_password_login_session(caller: UserAPIKeyAuth):
"""Only the session minted by a password login may change the password, so a
stolen virtual key or an SSO session cannot use the endpoint as a
current_password guessing oracle."""
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
user_api_key_dict=caller,
)
assert exc_info.value.status_code == 403
assert "logging in with a password" in exc_info.value.detail["error"]
prisma.db.litellm_usertable.find_first.assert_not_called()
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_change_password_rejects_session_without_user():
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(user=None)
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
user_api_key_dict=_caller(user_id=None),
)
assert exc_info.value.status_code == 400
prisma.db.litellm_usertable.find_first.assert_not_called()
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_change_password_rejects_account_without_password():
"""SSO users and the env-credential admin have no DB password row to change."""
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(password=None))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
assert exc_info.value.status_code == 400
assert "no password set" in exc_info.value.detail["error"]
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_change_password_enforces_min_length():
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(ProxyException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password="Short1!"),
user_api_key_dict=_caller(),
)
assert exc_info.value.code == "400"
assert exc_info.value.type == ProxyErrorTypes.validation_error
assert exc_info.value.param == "password"
assert "at least 12 characters" in exc_info.value.message
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
@respx.mock
async def test_change_password_rejects_breached_password():
"""With the default policy, the new password is screened against HIBP."""
from litellm.proxy._types import ChangePasswordRequest
breached_password = "Password123!"
respx.get(_hibp_url_for(breached_password)).mock(
return_value=httpx.Response(200, text=f"{_hibp_suffix_for(breached_password)}:1")
)
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", {}
),
):
with pytest.raises(ProxyException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=breached_password),
user_api_key_dict=_caller(),
)
assert exc_info.value.code == "400"
assert exc_info.value.type == ProxyErrorTypes.validation_error
assert exc_info.value.param == "password"
assert "data breaches" in exc_info.value.message
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
@respx.mock
async def test_change_password_verifies_current_password_before_hibp_lookup():
"""A caller who fails current-password verification must not trigger any
HIBP traffic. The HIBP check fails open on errors, so an unmocked lookup
could not prove ordering; instead the route is registered and asserted
uncalled."""
from litellm.proxy._types import ChangePasswordRequest
hibp_route = respx.get(_hibp_url_for(NEW_PASSWORD)).mock(return_value=httpx.Response(200, text=""))
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", {}
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
assert exc_info.value.status_code == 400
assert "Current password is incorrect" in exc_info.value.detail["error"]
assert not hibp_route.called
prisma.db.litellm_usertable.update.assert_not_called()
@pytest.mark.asyncio
async def test_change_password_success_emits_redacted_audit_log():
"""A successful change must land in the audit trail as field names only;
the plaintext passwords must never reach the audit call."""
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
audit_mock = AsyncMock()
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
patch( # test-quality-ok: audit sink is a module-level import; no injection seam
"litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock
),
):
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
audit_mock.assert_awaited_once()
audit_kwargs = audit_mock.await_args.kwargs
assert audit_kwargs["object_id"] == "user-123"
assert audit_kwargs["action"] == "updated"
assert audit_kwargs["table_name"] == LitellmTableNames.USER_TABLE_NAME
assert audit_kwargs["after_value"] == '{"fields_changed": ["password"]}'
assert CURRENT_PASSWORD not in str(audit_kwargs)
assert NEW_PASSWORD not in str(audit_kwargs)
@pytest.mark.asyncio
async def test_change_password_failure_emits_no_audit_log():
from litellm.proxy._types import ChangePasswordRequest
prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD)))
audit_mock = AsyncMock()
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", prisma
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
patch( # test-quality-ok: audit sink is a module-level import; no injection seam
"litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock
),
):
with pytest.raises(HTTPException):
await change_password(
data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
audit_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_change_password_requires_db():
from litellm.proxy._types import ChangePasswordRequest
with (
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.prisma_client", None
),
patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam
"litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
),
):
with pytest.raises(HTTPException) as exc_info:
await change_password(
data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD),
user_api_key_dict=_caller(),
)
assert exc_info.value.status_code == 500

3
tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py
View file

@ -10,6 +10,7 @@ Routes covered:
from __future__ import annotations
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock
from .conftest import normalize
@ -510,6 +511,8 @@ def _db_user(monkeypatch, email: str):
user.user_email = email
user.user_role = "internal_user"
user.password = "scrypt:stored"
user.password_reset_required = None
user.last_breach_check_at = datetime.now(timezone.utc)
repo = MagicMock()
repo.return_value.table.find_first = AsyncMock(return_value=user)
monkeypatch.setattr(ps, "prisma_client", MagicMock())

42
tests/test_litellm/proxy/test__types.py
View file

@ -5,11 +5,13 @@ from pydantic import ValidationError
from litellm.proxy._types import (
ROLES_WITHIN_ORG,
ChangePasswordRequest,
GenerateKeyRequest,
KeyRequest,
LiteLLM_AuditLogs,
LiteLLM_TeamMembership,
LitellmUserRoles,
NewUserRequest,
OrganizationMemberUpdateRequest,
ResetSpendRequest,
UpdateKeyRequest,
@ -335,3 +337,43 @@ def test_virtual_key_mapping_counts_as_configured_when_any_issuer_sets_the_claim
)
assert jwt_auth.is_virtual_key_mapping_configured() is is_configured
def test_new_user_request_loudly_rejects_a_password():
"""
/user/new has never persisted a password (the field used to be silently
dropped). Sending one must now fail visibly so the dead path cannot be
revived without going through the password policy.
"""
with pytest.raises(ValidationError, match="invitation link"):
NewUserRequest(user_email="alice@example.com", password="hunter2hunter2")
def test_new_user_request_without_password_still_works():
request = NewUserRequest(user_email="alice@example.com")
assert request.password is None
def test_update_user_request_accepts_a_password():
"""Admins set user passwords through /user/update; the value must survive
model validation so the endpoint can policy-check and hash it."""
request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2")
assert request.password == "hunter2hunter2"
def test_update_user_request_password_hidden_from_repr():
"""management_endpoint_wrapper string-formats endpoint kwargs into Slack
alerts, so the model's repr/str must never contain the plaintext password."""
request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2")
assert "hunter2hunter2" not in repr(request)
assert "hunter2hunter2" not in str(request)
def test_change_password_request_passwords_hidden_from_repr():
"""Any accidental str()/repr() of the request model (debug logs, exception
handlers, a future management_endpoint_wrapper) must never contain either
plaintext password."""
request = ChangePasswordRequest(current_password="hunter2hunter2", new_password="NewP@ssw0rd-2026")
for rendered in (repr(request), str(request)):
assert "hunter2hunter2" not in rendered
assert "NewP@ssw0rd-2026" not in rendered

52
tests/test_litellm/test_check_mcp_operation_boundary.py Normal file
View file

@ -0,0 +1,52 @@
from pathlib import Path
import pytest
from scripts.check_mcp_operation_boundary import main, violations
@pytest.mark.parametrize(
"source",
(
"from mcp.server.auth.middleware.auth_context import auth_context_var as hidden",
"from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode as mode",
"caller = legacy.get_active_auth_context()",
"owners = transport._stateful_session_owners",
"from weakref import WeakKeyDictionary",
"from litellm.proxy._experimental.mcp_server.server import get_auth_context",
),
)
def test_shared_operation_boundary_rejects_ambient_state(source):
assert violations(Path("operations.py"), source)
def test_legacy_adapter_may_resolve_context_but_policy_must_receive_it():
source = "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode"
assert violations(Path("server.py"), source) == ()
assert violations(Path("legacy_callbacks.py"), source) == ()
assert violations(Path("operations.py"), "def execute(context):\n return context.client_ip") == ()
assert violations(Path("mcp_server_manager.py"), "def _mcp_registry_key(server):\n return server.name") == ()
def test_boundary_command_rejects_shared_state_and_accepts_explicit_context(tmp_path, monkeypatch, capsys):
import subprocess
import sys
package = tmp_path / "litellm/proxy/_experimental/mcp_server"
package.mkdir(parents=True)
module = package / "operations.py"
module.write_text("from mcp.server.auth.middleware.auth_context import auth_context_var as hidden\n")
command = [sys.executable, str(Path(__file__).resolve().parents[2] / "scripts/check_mcp_operation_boundary.py")]
monkeypatch.chdir(tmp_path)
assert main() == 1
assert "operations.py:1:" in capsys.readouterr().err
rejected = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False)
assert rejected.returncode == 1
assert "operations.py:1: MCP request/session state belongs in a legacy adapter" in rejected.stderr
module.write_text("def execute(context):\n return context.client_ip\n")
assert main() == 0
assert "MCP operation boundary: passed" in capsys.readouterr().out
accepted = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False)
assert accepted.returncode == 0
assert "MCP operation boundary: passed" in accepted.stdout

152
tests/test_litellm_rust/support/fake_gcs.py Normal file
View file

@ -0,0 +1,152 @@
from __future__ import annotations
import json
import threading
from collections.abc import Mapping
from dataclasses import dataclass
from functools import partial
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from socket import socket
from types import MappingProxyType
from typing import Final, cast
from urllib.parse import unquote, urlsplit
@dataclass(frozen=True, slots=True)
class RecordedRequest:
method: str
path: str
query: str
headers: Mapping[str, str]
body: bytes
class _FakeGcsHandler(BaseHTTPRequestHandler):
def __init__(
self,
request: socket | tuple[bytes, socket],
client_address: tuple[str, int],
server: ThreadingHTTPServer,
*,
fake: FakeGcs,
) -> None:
self._fake: Final = fake
super().__init__(request, client_address, server)
def _handle(self) -> None:
parsed: Final = urlsplit(self.path)
content_length: Final = int(self.headers.get("Content-Length", "0"))
body: Final = self.rfile.read(content_length) if content_length else b""
headers: Final = MappingProxyType(
{name.title(): value for name, value in self.headers.items()}
)
self._fake.record(
RecordedRequest(
method=self.command,
path=parsed.path,
query=parsed.query,
headers=headers,
body=body,
)
)
if self.headers.get("Authorization") != f"Bearer {self._fake.token}":
self._send_json(401, {"error": "unauthorized"})
return
upload_prefix: Final = "/upload/storage/v1/b/"
download_prefix: Final = "/storage/v1/b/"
if parsed.path.startswith(upload_prefix) and parsed.path.endswith("/o"):
self._upload(parsed.path[len(upload_prefix) : -2], parsed.query, body)
return
if parsed.path.startswith(download_prefix):
self._download(parsed.path[len(download_prefix) :], parsed.query)
return
self._send_json(404, {"error": "not found"})
def _upload(self, path: str, query: str, body: bytes) -> None:
values: Final = {
unquote(pair.partition("=")[0]): unquote(pair.partition("=")[2])
for pair in query.split("&")
if pair
}
if not path or values.get("uploadType") != "media" or "name" not in values:
self._send_json(404, {"error": "not found"})
return
self._fake.put_object(path, values["name"], body)
self._send_json(200, {"name": values["name"], "bucket": path})
def _download(self, path: str, query: str) -> None:
bucket, separator, encoded_name = path.partition("/o/")
if not separator or query != "alt=media":
self._send_json(404, {"error": "not found"})
return
name: Final = unquote(encoded_name)
if name.endswith("/server-error") or name == "server-error":
self._send_json(500, {"error": "server error"})
return
body: Final = self._fake.get_object(bucket, name)
if body is None:
self._send_json(404, {"error": "not found"})
return
self._send(200, body, "application/octet-stream")
def _send_json(self, status: int, value: object) -> None:
payload: Final = json.dumps(value).encode()
self._send(status, payload, "application/json")
def _send(self, status: int, body: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: object) -> None:
pass
do_GET = _handle
do_POST = _handle
class FakeGcs:
def __init__(self) -> None:
self._objects: dict[tuple[str, str], bytes] = {} # mutable-ok: fake object store
self._requests: list[RecordedRequest] = [] # mutable-ok: recorded request history
self._server = ThreadingHTTPServer(
("127.0.0.1", 0),
partial(_FakeGcsHandler, fake=self),
)
self._worker = threading.Thread(target=self._server.serve_forever, daemon=True)
self._worker.start()
self.token: Final = "test-token"
@property
def url(self) -> str:
address: Final = cast(tuple[str, int], self._server.server_address)
host, port = address
return f"http://{host}:{port}"
@property
def objects(self) -> Mapping[tuple[str, str], bytes]:
return MappingProxyType(self._objects)
@property
def requests(self) -> tuple[RecordedRequest, ...]:
return tuple(self._requests)
def put(self, bucket: str, name: str, body: bytes) -> None:
self.put_object(bucket, name, body)
def close(self) -> None:
self._server.shutdown()
self._server.server_close()
self._worker.join(timeout=5)
def record(self, request: RecordedRequest) -> None:
self._requests.append(request)
def put_object(self, bucket: str, name: str, body: bytes) -> None:
self._objects[(bucket, name)] = body
def get_object(self, bucket: str, name: str) -> bytes | None:
return self._objects.get((bucket, name))

337
tests/test_litellm_rust/test_cache.py
View file

@ -8,10 +8,12 @@ import time
import uuid
import weakref
from collections.abc import Generator
from pathlib import Path
from types import SimpleNamespace
from typing import Final, Protocol, cast
from urllib.parse import urlparse
import diskcache
import fakeredis
import pytest
import redis
@ -20,10 +22,13 @@ from azure.storage.blob import ContainerClient
import litellm
from litellm.caching.azure_blob_cache import AzureBlobCache
from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache
from litellm.caching.gcs_cache import GCSCache
from litellm.caching.disk_cache import DiskCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cluster_cache import RedisClusterCache
from litellm.rust_bridge import _native
from litellm.types.caching import LiteLLMCacheType
from tests.test_litellm_rust.support.fake_gcs import FakeGcs
from tests.test_litellm_rust.support.isolation import rebound
pytestmark: Final = pytest.mark.requires_rust_extension
@ -31,6 +36,7 @@ pytestmark: Final = pytest.mark.requires_rust_extension
class CacheLookup(Protocol):
def get_cache(self, **kwargs: object) -> object: ...
def flush_cache(self) -> object: ...
def request(key: str = "key") -> dict[str, object]:
@ -50,6 +56,15 @@ def redis_url() -> Generator[str]:
worker.join(timeout=5)
@pytest.fixture
def fake_gcs() -> Generator[FakeGcs]:
server: Final = FakeGcs()
try:
yield server
finally:
server.close()
@pytest.fixture
def azure_blob_facade() -> Generator[Cache]:
account_url: Final = os.environ.get("AZURE_BLOB_CACHE_ACCOUNT_URL")
@ -519,6 +534,328 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None:
assert client.get("second") is not None
await facade.cache.disconnect()
client.close()
async def test_disk_reads_python_entries_and_python_reads_native_entries(tmp_path: Path) -> None:
disk_cache: Final = DiskCache(disk_cache_dir=str(tmp_path))
response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}}
disk_cache.disk_cache.set(
"sync",
{"timestamp": time.time(), "response": json.dumps(response)},
)
disk_cache.disk_cache.set("async", json.dumps({"timestamp": time.time(), "response": response}))
disk_cache.disk_cache.set("raw", json.dumps(response))
disk_cache.disk_cache.set("invalid", "not a cache entry")
disk_cache.disk_cache.set(
"large",
{"timestamp": time.time(), "response": {"text": "x" * 70_000}},
)
binding: Final = _native._CacheTestResolver(
SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
).resolve()
assert binding.lookup(request("sync")) == response
assert await binding.async_lookup(request("async")) == response
assert binding.lookup(request("raw")) == response
assert await binding.async_lookup(request("invalid")) is None
assert binding.lookup(request("large")) == {"text": "x" * 70_000}
await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response)
stored_response: Final = disk_cache.get_cache("native")
assert isinstance(stored_response, dict)
assert stored_response["response"] == response
stored, expire_time = disk_cache.disk_cache.get("native", expire_time=True)
assert stored is not None
assert time.time() < expire_time <= time.time() + 12.0
await binding.async_store(request("no-ttl"), response)
_, no_expiry = disk_cache.disk_cache.get("no-ttl", expire_time=True)
assert no_expiry is None
async def test_disk_entries_survive_a_fresh_handle_and_expire_on_time(tmp_path: Path) -> None:
first: Final = _native._CacheTestResolver(
SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
).resolve()
await first.async_store(request("persistent"), {"value": "persistent"})
await first.async_store({**request("expiring"), "ttl_seconds": 0.3}, {"value": "expiring"})
fresh: Final = _native._CacheTestResolver(
SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
).resolve()
assert fresh.lookup(request("persistent")) == {"value": "persistent"}
assert fresh.lookup(request("expiring")) == {"value": "expiring"}
await asyncio.sleep(0.4)
assert fresh.lookup(request("expiring")) is None
assert fresh.lookup(request("persistent")) == {"value": "persistent"}
def test_disk_facade_registers_and_store_changes_fall_back(tmp_path: Path) -> None:
facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path))
with pytest.raises(TypeError, match="directories must match"):
_native._CacheTestHandle.disk(str(tmp_path / "other"))._bind_facade(facade)
handle: Final = _native._CacheTestHandle.disk(str(tmp_path))
handle._bind_facade(facade)
resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
binding: Final = resolver.resolve()
assert binding.kind == "native"
binding.store(request("native"), {"value": "native"})
assert facade.get_cache(cache_key="native") == {"value": "native"}
with rebound(facade.cache, "disk_cache", diskcache.Cache(str(tmp_path))):
assert resolver.resolve().kind == "python_callback"
assert resolver.resolve().kind == "native"
class CustomDiskCache(DiskCache):
pass
with rebound(facade, "cache", CustomDiskCache(disk_cache_dir=str(tmp_path))):
assert resolver.resolve().kind == "python_callback"
class CustomStore(diskcache.Cache):
pass
custom_facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path))
custom_facade.cache.disk_cache = CustomStore(str(tmp_path))
with pytest.raises(TypeError, match="built-in diskcache store"):
_native._CacheTestHandle.disk(str(tmp_path))._bind_facade(custom_facade)
async def test_disk_native_batch_lookup_and_store_report_partial_hits(tmp_path: Path) -> None:
binding: Final = _native._CacheTestResolver(
SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path)))
).resolve()
requests: Final = [request("hit"), request("miss"), request("disabled")]
requests[2]["controls"] = {
"supported_call_type": True,
"configured": True,
"native_backend": True,
"default_on": True,
"caching": False,
"no_cache": False,
"no_store": False,
"use_cache": False,
}
await binding.async_store_batch(requests, [{"value": 1}, {"value": 2}, {"value": 3}])
partial: Final = await binding.async_lookup_batch(requests)
assert partial == {
"values": [{"value": 1}, {"value": 2}, None],
"missing_indices": [2],
}
async def test_gcs_reads_python_entries_and_writes_python_compatible_objects(
fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False)
monkeypatch.delenv("GCS_BUCKET_NAME", raising=False)
response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None}
fake_gcs.put(
"bucket",
"cache/sync",
json.dumps({"timestamp": time.time(), "response": json.dumps(response)}).encode(),
)
fake_gcs.put("bucket", "cache/async", json.dumps({"timestamp": time.time(), "response": response}).encode())
fake_gcs.put("bucket", "cache/raw", json.dumps(response).encode())
fake_gcs.put("bucket", "cache/invalid", b"not a cache entry")
binding: Final = _native._CacheTestResolver(
SimpleNamespace(
cache=_native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
)
).resolve()
assert binding.lookup(request("sync")) == response
assert await binding.async_lookup(request("async")) == response
assert binding.lookup(request("raw")) == response
assert await binding.async_lookup(request("invalid")) is None
assert binding.lookup(request("missing")) is None
await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response)
stored: Final = fake_gcs.objects[("bucket", "cache/native")]
stored_value: Final = cast(dict[str, object], json.loads(stored))
assert stored_value["response"] == response
assert isinstance(stored_value["timestamp"], float)
upload: Final = next(item for item in fake_gcs.requests if item.method == "POST")
assert upload.path == "/upload/storage/v1/b/bucket/o"
assert upload.query == "uploadType=media&name=cache%2Fnative"
assert upload.headers["Authorization"] == f"Bearer {fake_gcs.token}"
assert upload.headers["Content-Type"] == "application/json"
upload_text: Final = f"{upload.path}?{upload.query}{upload.headers}"
assert "ttl" not in upload_text.lower()
assert "expiry" not in upload_text.lower()
download: Final = next(item for item in fake_gcs.requests if item.path.endswith("/cache%2Fsync"))
assert download.path == "/storage/v1/b/bucket/o/cache%2Fsync"
assert download.query == "alt=media"
binding.store(request("sync2"), response)
assert binding.lookup(request("sync2")) == response
assert GCSCache(bucket_name="bucket", gcs_path="cache").key_prefix == "cache/"
assert GCSCache(bucket_name="bucket", gcs_path="cache/").key_prefix == "cache/"
assert GCSCache(bucket_name="bucket").key_prefix == ""
async def test_gcs_batch_lookup_preserves_order_and_treats_malformed_entries_as_misses(fake_gcs: FakeGcs) -> None:
fake_gcs.put("bucket", "cache/hit", json.dumps({"timestamp": time.time(), "response": {"value": 1}}).encode())
fake_gcs.put("bucket", "cache/invalid", b"not a cache entry")
binding: Final = _native._CacheTestResolver(
SimpleNamespace(
cache=_native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
)
).resolve()
requests: Final = [request("hit"), request("missing"), request("invalid")]
expected: Final = {"values": [{"value": 1}, None, None], "missing_indices": [1, 2]}
assert await binding.async_lookup_batch(requests) == expected
assert binding.lookup_batch(requests) == expected
await binding.async_store_batch([request("first"), request("second")], [{"value": 1}, {"value": 2}])
assert ("bucket", "cache/first") in fake_gcs.objects
assert ("bucket", "cache/second") in fake_gcs.objects
async def test_gcs_facade_binds_only_exact_matching_configuration(
fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False)
monkeypatch.delenv("GCS_BUCKET_NAME", raising=False)
monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent")
facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/")
assert type(facade.cache) is GCSCache
mismatched_bucket: Final = _native._CacheTestHandle.gcs(
"other",
gcs_path="cache",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
with pytest.raises(TypeError, match="buckets must match"):
mismatched_bucket._bind_facade(facade)
mismatched_prefix: Final = _native._CacheTestHandle.gcs(
"bucket",
gcs_path="x",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
with pytest.raises(TypeError, match="key prefixes must match"):
mismatched_prefix._bind_facade(facade)
mismatched_credentials: Final = _native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
path_service_account="sa.json",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
with pytest.raises(TypeError, match="credentials must match"):
mismatched_credentials._bind_facade(facade)
with pytest.raises(TypeError, match="types must match"):
_native._CacheTestHandle.memory()._bind_facade(facade)
matching: Final = _native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
matching._bind_facade(facade)
resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade))
binding: Final = resolver.resolve()
assert binding.kind == "native"
await binding.async_store(request("native"), {"value": "native"})
assert await binding.async_lookup(request("native")) == {"value": "native"}
assert cast(CacheLookup, facade).get_cache(cache_key="native") is None
with rebound(facade.cache, "bucket_name", "other"):
assert resolver.resolve().kind == "python_callback"
with rebound(facade.cache, "key_prefix", "x/"):
assert resolver.resolve().kind == "python_callback"
with rebound(facade.cache, "path_service_account", "sa.json"):
assert resolver.resolve().kind == "python_callback"
def no_get_cache(*args: object, **kwargs: object) -> None:
return None
with rebound(facade.cache, "get_cache", no_get_cache):
assert resolver.resolve().kind == "python_callback"
with rebound(facade, "ttl", 12):
assert resolver.resolve().kind == "python_callback"
class CustomGcs(GCSCache):
pass
with rebound(facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")):
assert resolver.resolve().kind == "python_callback"
custom_facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/")
with rebound(custom_facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")):
with pytest.raises(TypeError, match="types must match"):
matching._bind_facade(custom_facade)
missing_bucket: Final = Cache(type=LiteLLMCacheType.GCS)
with pytest.raises(TypeError, match="requires a configured bucket name"):
matching._bind_facade(missing_bucket)
async def test_gcs_flush_is_a_no_op_and_ping_is_not_implemented(
fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False)
monkeypatch.delenv("GCS_BUCKET_NAME", raising=False)
binding: Final = _native._CacheTestResolver(
SimpleNamespace(
cache=_native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
)
).resolve()
await binding.async_store(request("key"), {"value": "stored"})
await binding.async_flush()
assert ("bucket", "cache/key") in fake_gcs.objects
assert await binding.async_lookup(request("key")) == {"value": "stored"}
with pytest.raises(NotImplementedError):
await binding.ping()
facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/")
with pytest.raises(AttributeError):
await facade.ping()
assert cast(CacheLookup, facade.cache).flush_cache() is None
async def test_gcs_unauthorized_and_server_errors_surface_as_runtime_errors(fake_gcs: FakeGcs) -> None:
wrong_token: Final = _native._CacheTestResolver(
SimpleNamespace(
cache=_native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
endpoint=fake_gcs.url,
token="wrong-token",
)
)
).resolve()
with pytest.raises(RuntimeError):
wrong_token.lookup(request("missing"))
assert not fake_gcs.objects
binding: Final = _native._CacheTestResolver(
SimpleNamespace(
cache=_native._CacheTestHandle.gcs(
"bucket",
gcs_path="cache",
endpoint=fake_gcs.url,
token=fake_gcs.token,
)
)
).resolve()
with pytest.raises(RuntimeError):
binding.lookup(request("server-error"))
assert binding.lookup(request("missing")) is None
async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_natively(

6
tests/unit/models/test_models.py
View file

@ -324,6 +324,8 @@ class TestUser:
assert user_no_models.has_model_access("any-model")
def test_password_hash_excluded_from_serialization(self):
import json
from litellm.proxy._types import LiteLLM_UserTableWithKeyCount
secret = "$2b$12$abcdefghijklmnopqrstuv"
@ -331,12 +333,12 @@ class TestUser:
assert user.password == secret
assert "password" not in user.model_dump()
assert "password" not in user.model_dump_json()
assert "password" not in json.loads(user.model_dump_json())
with_keys = LiteLLM_UserTableWithKeyCount(user_id="u1", user_email="a@b.c", password=secret, key_count=2)
assert with_keys.password == secret
assert "password" not in with_keys.model_dump()
assert "password" not in with_keys.model_dump_json()
assert "password" not in json.loads(with_keys.model_dump_json())
class TestVerificationToken:

110
ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx Normal file
View file

@ -0,0 +1,110 @@
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import ChangePasswordForm from "./ChangePasswordForm";
const mockChangePasswordCall = vi.fn();
const mockToastSuccess = vi.fn();
const mockClearTokenCookies = vi.fn();
let mockPasswordResetRequired = false;
vi.mock("@/components/networking", () => ({
changePasswordCall: (...args: unknown[]) => mockChangePasswordCall(...args),
getProxyBaseUrl: () => "",
}));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
default: () => ({ accessToken: "sk-session-token", passwordResetRequired: mockPasswordResetRequired }),
}));
vi.mock("@/lib/toast", () => ({
toast: {
success: (...args: unknown[]) => mockToastSuccess(...args),
fromError: vi.fn(),
},
}));
vi.mock("@/utils/cookieUtils", () => ({
clearTokenCookies: (...args: unknown[]) => mockClearTokenCookies(...args),
}));
const fillForm = (values: { current: string; next: string; confirm: string }) => {
fireEvent.change(screen.getByLabelText("Current Password"), { target: { value: values.current } });
fireEvent.change(screen.getByLabelText("New Password"), { target: { value: values.next } });
fireEvent.change(screen.getByLabelText("Confirm New Password"), { target: { value: values.confirm } });
};
const submit = () => fireEvent.click(screen.getByRole("button", { name: "Change Password" }));
describe("ChangePasswordForm", () => {
beforeEach(() => {
vi.clearAllMocks();
mockPasswordResetRequired = false;
});
it("sends the current and new password to the change endpoint and resets on success", async () => {
mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." });
render(<ChangePasswordForm />);
fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" });
submit();
expect(await screen.findByLabelText("Current Password")).toHaveValue("");
expect(mockChangePasswordCall).toHaveBeenCalledWith("sk-session-token", "OldP@ssw0rd-2026", "NewP@ssw0rd-2026");
expect(mockToastSuccess).toHaveBeenCalled();
});
it("blocks submission when the confirmation does not match", async () => {
render(<ChangePasswordForm />);
fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "Different-2026" });
submit();
expect(await screen.findByText("New passwords do not match")).toBeInTheDocument();
expect(mockChangePasswordCall).not.toHaveBeenCalled();
});
it("shows the proxy's rejection message unwrapped", async () => {
mockChangePasswordCall.mockRejectedValue(new Error("{'error': 'Current password is incorrect.'}"));
render(<ChangePasswordForm />);
fillForm({ current: "wrong-password", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" });
submit();
expect(await screen.findByText("Current password is incorrect.")).toBeInTheDocument();
expect(mockToastSuccess).not.toHaveBeenCalled();
});
describe("forced password reset", () => {
it("shows the forced-reset warning only when the session is flagged", () => {
mockPasswordResetRequired = true;
render(<ChangePasswordForm />);
expect(screen.getByText(/must be changed before you can use the dashboard/)).toBeInTheDocument();
});
it("hides the forced-reset warning for a normal session", () => {
render(<ChangePasswordForm />);
expect(screen.queryByText(/must be changed before you can use the dashboard/)).not.toBeInTheDocument();
});
it("signs the user out to re-login after a successful forced change", async () => {
mockPasswordResetRequired = true;
mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." });
const replaceMock = vi.fn();
const realLocation = window.location;
Object.defineProperty(window, "location", { configurable: true, value: { replace: replaceMock } });
try {
render(<ChangePasswordForm />);
fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" });
submit();
await waitFor(() => expect(replaceMock).toHaveBeenCalledWith("/ui/login/"));
expect(mockClearTokenCookies).toHaveBeenCalled();
} finally {
Object.defineProperty(window, "location", { configurable: true, value: realLocation });
}
});
});
});

120
ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx Normal file
View file

@ -0,0 +1,120 @@
"use client";
import React, { useState } from "react";
import { CircleAlert } from "lucide-react";
import { z } from "zod/v4";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { Alert, AlertTitle } from "@/components/shared/Alert";
import { PasswordInput } from "@/components/shared/PasswordInput";
import { FormField } from "@/components/shared/form/FormField";
import { Button } from "@/components/ui/button";
import { Card, CardContent } from "@/components/ui/card";
import { FieldGroup } from "@/components/ui/field";
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
import { changePasswordCall, getProxyBaseUrl } from "@/components/networking";
import { extractProxyErrorMessage } from "@/lib/http/client";
import { useZodForm } from "@/lib/forms/useZodForm";
import { toast } from "@/lib/toast";
import { clearTokenCookies } from "@/utils/cookieUtils";
import { getLoginUrl } from "@/utils/returnUrlUtils";
const changePasswordSchema = z
.object({
currentPassword: z.string().min(1, "Current password is required"),
newPassword: z.string().min(1, "New password is required"),
confirmNewPassword: z.string().min(1, "Confirm your new password"),
})
.refine((values) => values.newPassword === values.confirmNewPassword, {
message: "New passwords do not match",
path: ["confirmNewPassword"],
});
type ChangePasswordValues = z.infer<typeof changePasswordSchema>;
export function ChangePasswordForm() {
const { accessToken, passwordResetRequired } = useAuthorized();
const form = useZodForm(changePasswordSchema, {
defaultValues: { currentPassword: "", newPassword: "", confirmNewPassword: "" },
});
const [isPending, setIsPending] = useState(false);
const [submitError, setSubmitError] = useState<string | null>(null);
const handleSubmit = async (values: ChangePasswordValues) => {
if (!accessToken) return;
setSubmitError(null);
setIsPending(true);
try {
await changePasswordCall(accessToken, values.currentPassword, values.newPassword);
if (passwordResetRequired) {
// The session key was minted restricted; only a fresh login lifts it.
toast.success("Password updated. Please log in with your new password.");
clearTokenCookies();
window.location.replace(getLoginUrl(getProxyBaseUrl()));
return;
}
toast.success("Password updated");
form.reset();
} catch (error) {
setSubmitError(extractProxyErrorMessage(error));
} finally {
setIsPending(false);
}
};
return (
<div className="mx-auto mt-10 w-full max-w-md">
<Card>
<CardContent>
<h3 className="text-2xl font-semibold text-foreground">Change Password</h3>
<p className="text-sm text-muted-foreground">
Enter your current password and choose a new one. The new password must meet this proxy&apos;s password
policy.
</p>
{passwordResetRequired && (
<Alert variant="warning" className="mt-4">
<CircleAlert />
<AlertTitle>
Your password must be changed before you can use the dashboard: it was either found in a known data
breach or set by an administrator as a temporary password. After updating it, you will be signed out to
log in again.
</AlertTitle>
</Alert>
)}
<form className="mb-2 mt-8" onSubmit={form.handleSubmit(handleSubmit)}>
<FieldGroup>
<FormField control={form.control} name="currentPassword" label="Current Password">
{({ ref, ...field }) => <PasswordInput {...field} ref={ref} autoComplete="current-password" />}
</FormField>
<FormField control={form.control} name="newPassword" label="New Password">
{({ ref, ...field }) => <PasswordInput {...field} ref={ref} autoComplete="new-password" />}
</FormField>
<FormField control={form.control} name="confirmNewPassword" label="Confirm New Password">
{({ ref, ...field }) => <PasswordInput {...field} ref={ref} autoComplete="new-password" />}
</FormField>
</FieldGroup>
{submitError && (
<Alert variant="error" className="mt-6">
<CircleAlert />
<AlertTitle>{submitError}</AlertTitle>
</Alert>
)}
<div className="mt-8">
<Button type="submit" disabled={isPending}>
{isPending && <UiLoadingSpinner className="size-4" role="img" aria-label="loading" />}
Change Password
</Button>
</div>
</form>
</CardContent>
</Card>
</div>
);
}
export default ChangePasswordForm;

7
ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx Normal file
View file

@ -0,0 +1,7 @@
"use client";
import ChangePasswordForm from "./ChangePasswordForm";
export default function ChangePasswordPage() {
return <ChangePasswordForm />;
}

2
ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts
View file

@ -50,7 +50,9 @@ const useAuthorized = () => {
isViewOnly: isViewOnlySessionRole(decoded?.user_role),
premiumUser: decoded?.premium_user ?? null,
disabledPersonalKeyCreation: decoded?.disabled_non_admin_personal_key_creation ?? null,
loginMethod: decoded?.login_method ?? null,
showSSOBanner: decoded?.login_method === "username_password",
passwordResetRequired: decoded?.password_reset_required === true,
};
};

58
ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
View file

@ -1,4 +1,4 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { render, screen, waitFor } from "@testing-library/react";
import { AuthProvider } from "@/contexts/AuthContext";
import Layout from "./layout";
@ -121,4 +121,60 @@ describe("(dashboard) Layout", () => {
expect(screen.queryByTestId("dashboard-header")).not.toBeInTheDocument();
expect(screen.queryByTestId("sidebar")).not.toBeInTheDocument();
});
describe("forced password reset routing", () => {
const sessionCookie = (claims: Record<string, unknown>) => {
const encode = (part: Record<string, unknown>) =>
btoa(JSON.stringify(part)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
const exp = Math.floor(Date.now() / 1000) + 3600;
return `${encode({ alg: "HS256", typ: "JWT" })}.${encode({ ...claims, exp })}.sig`;
};
afterEach(() => {
document.cookie = "token=; Max-Age=0; Path=/";
});
it("routes a session flagged password_reset_required to the change-password page", async () => {
const flaggedClaims = {
user_id: "flagged-user",
key: "sk-session",
login_method: "username_password",
password_reset_required: true,
};
document.cookie = `token=${sessionCookie(flaggedClaims)}; Path=/`;
render(
<AuthProvider>
<Layout>
<div data-testid="page-content" />
</Layout>
</AuthProvider>,
);
pendingUiConfig.resolve();
await waitFor(() => expect(replaceMock).toHaveBeenCalledWith(expect.stringContaining("/change-password")));
});
it("does not reroute an unflagged session", async () => {
document.cookie = `token=${sessionCookie({
user_id: "normal-user",
key: "sk-session",
login_method: "username_password",
})}; Path=/`;
render(
<AuthProvider>
<Layout>
<div data-testid="page-content" />
</Layout>
</AuthProvider>,
);
pendingUiConfig.resolve();
expect(await screen.findByTestId("page-content")).toBeInTheDocument();
expect(replaceMock).not.toHaveBeenCalled();
});
});
});

13
ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
View file

@ -7,7 +7,7 @@ import LoadingScreen from "@/components/common_components/LoadingScreen";
import { ThemeProvider } from "@/contexts/ThemeContext";
import { useAuth } from "@/contexts/AuthContext";
import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider";
import { useRouter, useSearchParams } from "next/navigation";
import { usePathname, useRouter, useSearchParams } from "next/navigation";
import { DebugWarningBanner } from "@/components/DebugWarningBanner";
import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner";
import { EnvCredentialLoginWarningBanner } from "@/components/EnvCredentialLoginWarningBanner";
@ -149,7 +149,8 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
function LayoutContent({ children }: { children: React.ReactNode }) {
const router = useRouter();
const searchParams = useSearchParams();
const { accessToken, authLoading } = useAuth();
const pathname = usePathname();
const { accessToken, authLoading, passwordResetRequired } = useAuth();
const isInvitationFlow = Boolean(searchParams.get("invitation_id"));
// Legacy invitation links point at /ui/?invitation_id=; the onboarding form now lives at its own
@ -160,6 +161,14 @@ function LayoutContent({ children }: { children: React.ReactNode }) {
}
}, [authLoading, isInvitationFlow, router, searchParams]);
// A session flagged for a forced password reset can only reach the change-password
// endpoint server-side; keep the UI on the matching page.
useEffect(() => {
if (!authLoading && passwordResetRequired && !pathname?.endsWith("/change-password")) {
router.replace(uiHref("change-password"));
}
}, [authLoading, passwordResetRequired, pathname, router]);
if (authLoading || isInvitationFlow) {
return <LoadingScreen />;
}

52
ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx
View file

@ -3,13 +3,25 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils";
import UserDropdown from "./UserDropdown";
let mockUseAuthorizedImpl = () => ({
let mockUseAuthorizedImpl: () => {
userId: string | null;
userEmail: string | null;
userRoleLabel: string;
premiumUser: boolean;
loginMethod?: string | null;
} = () => ({
userId: "test-user-id",
userEmail: "test@example.com",
userRoleLabel: "Admin",
premiumUser: false,
});
const mockRouterPush = vi.fn();
vi.mock("next/navigation", () => ({
useRouter: () => ({ push: mockRouterPush }),
}));
let mockUseDisableShowPromptsImpl = () => false;
let mockGetLocalStorageItemImpl = (key: string): string | null => {
@ -143,6 +155,44 @@ describe("UserDropdown", () => {
expect(mockOnLogout).toHaveBeenCalledTimes(1);
});
it("should navigate to the change-password page for username/password sessions", async () => {
mockUseAuthorizedImpl = () => ({
userId: "test-user-id",
userEmail: "test@example.com",
userRoleLabel: "Admin",
premiumUser: false,
loginMethod: "username_password",
});
const user = userEvent.setup();
renderWithProviders(<UserDropdown onLogout={mockOnLogout} />);
await user.click(getAccountTrigger());
await user.click(await screen.findByText("Change Password"));
expect(mockRouterPush).toHaveBeenCalledWith(expect.stringContaining("change-password"));
});
it("should hide the change-password entry for SSO sessions", async () => {
mockUseAuthorizedImpl = () => ({
userId: "test-user-id",
userEmail: "test@example.com",
userRoleLabel: "Admin",
premiumUser: false,
loginMethod: "sso",
});
const user = userEvent.setup();
renderWithProviders(<UserDropdown onLogout={mockOnLogout} />);
await user.click(getAccountTrigger());
await waitFor(() => {
expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0);
});
expect(screen.queryByText("Change Password")).not.toBeInTheDocument();
});
it("should toggle hide new feature indicators switch", async () => {
const user = userEvent.setup();
renderWithProviders(<UserDropdown onLogout={mockOnLogout} />);

23
ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx
View file

@ -9,7 +9,9 @@ import {
setLocalStorageItem,
} from "@/utils/localStorageUtils";
import { navAccountDisplayName } from "@/components/Navbar/navDisplayName";
import { ChevronDown, ChevronsUpDown, Crown, LogOut, Mail, ShieldCheck, User } from "lucide-react";
import { uiHref } from "@/utils/uiHref";
import { ChevronDown, ChevronsUpDown, Crown, KeyRound, LogOut, Mail, ShieldCheck, User } from "lucide-react";
import { useRouter } from "next/navigation";
import { Avatar, AvatarFallback } from "@/components/ui/avatar";
import { Badge } from "@/components/ui/badge";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
@ -63,7 +65,9 @@ interface UserDropdownProps {
}
const UserDropdown: React.FC<UserDropdownProps> = ({ onLogout, variant = "navbar", collapsed = false }) => {
const { userId, userEmail, userRoleLabel: userRole, premiumUser } = useAuthorized();
const { userId, userEmail, userRoleLabel: userRole, premiumUser, loginMethod } = useAuthorized();
const router = useRouter();
const [open, setOpen] = useState(false);
const disableShowPrompts = useDisableShowPrompts();
const disableBlogPosts = useDisableBlogPosts();
const disableBouncingIcon = useDisableBouncingIcon();
@ -197,7 +201,7 @@ const UserDropdown: React.FC<UserDropdownProps> = ({ onLogout, variant = "navbar
const displayName = navAccountDisplayName(userEmail, userId);
return (
<Popover>
<Popover open={open} onOpenChange={setOpen}>
{variant === "sidebar" ? (
<PopoverTrigger
render={
@ -258,6 +262,19 @@ const UserDropdown: React.FC<UserDropdownProps> = ({ onLogout, variant = "navbar
>
{renderUserInfoSection()}
<Separator />
{loginMethod === "username_password" && (
<button
type="button"
onClick={() => {
setOpen(false);
router.push(uiHref("change-password"));
}}
className="flex w-full items-center gap-2 rounded-sm px-2 py-1.5 text-sm hover:bg-accent"
>
<KeyRound className="size-4" />
Change Password
</button>
)}
<button
type="button"
onClick={onLogout}

43
ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.test.tsx
View file

@ -9,6 +9,7 @@ interface AuthMock {
userRoleLabel: string;
premiumUser: boolean;
accessToken: string;
loginMethod?: string | null;
}
let mockUseAuthorizedImpl: () => AuthMock = () => ({
@ -19,6 +20,12 @@ let mockUseAuthorizedImpl: () => AuthMock = () => ({
accessToken: "test-token",
});
const mockRouterPush = vi.fn();
vi.mock("next/navigation", () => ({
useRouter: () => ({ push: mockRouterPush }),
}));
let mockUseDisableShowPromptsImpl = () => false;
let mockUseDisableBouncingIconImpl = () => false;
let mockHealthDataImpl = (): { litellm_version?: string } | undefined => ({ litellm_version: "1.99.0" });
@ -201,6 +208,42 @@ describe("SidebarAccountMenu", () => {
expect(mockOnLogout).toHaveBeenCalledTimes(1);
});
it("should navigate to the change-password page for username/password sessions", async () => {
mockUseAuthorizedImpl = () => ({
userId: "test-user-id",
userEmail: "test@example.com",
userRoleLabel: "Admin",
premiumUser: false,
accessToken: "test-token",
loginMethod: "username_password",
});
const user = userEvent.setup();
renderWithProviders(<SidebarAccountMenu onLogout={mockOnLogout} />);
await openMenu(user);
await user.click(screen.getByRole("button", { name: /change password/i }));
expect(mockRouterPush).toHaveBeenCalledWith(expect.stringContaining("change-password"));
});
it("should hide the change-password entry for SSO sessions", async () => {
mockUseAuthorizedImpl = () => ({
userId: "test-user-id",
userEmail: "test@example.com",
userRoleLabel: "Admin",
premiumUser: false,
accessToken: "test-token",
loginMethod: "sso",
});
const user = userEvent.setup();
renderWithProviders(<SidebarAccountMenu onLogout={mockOnLogout} />);
await openMenu(user);
expect(screen.queryByRole("button", { name: /change password/i })).not.toBeInTheDocument();
});
it("should toggle hide new feature indicators on", async () => {
const user = userEvent.setup();
renderWithProviders(<SidebarAccountMenu onLogout={mockOnLogout} />);

Some files were not shown because too many files have changed in this diff Show more